import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import statsmodels.api as sm
#from sklearn.linear_model import LinearRegression
import scipy, scipy.stats
pd.options.mode.chained_assignment = None  # default='warn'

#Set years of data to look at: 2007-2018
start = 2007
end = 2018

def dfAppend(strat, s, e):
    """
    Reads data and appends to a central df.
    df    = data frame to append to
    strat = Strategies label {A = Aggressive, C = Conservative}
    start = year to start
    end   = year to end, not inclusive of end date
    """
    df = pd.DataFrame()
    fileRange = np.arange(s, e + 1)     
    
    for i in fileRange:
        if i == s:
            dfAdd = pd.read_csv(strat + str(i) + ".csv", skiprows=range(1,4), sep=',', encoding='iso-8859-1')
        else:
            dfAdd = pd.read_csv(strat + str(i) + ".csv", skiprows=range(1,4), sep=',', encoding='iso-8859-1')
        dfAdd = dfAdd.iloc[:, : -1] #drops last extra column
        dfAdd['Year'] = i
        df = df.append(dfAdd, ignore_index = True)

    return df

def dfResetIndex (df):
    """
    Resets index each time a df is made
    """
    
    return df.reset_index(drop = True)

def calcReturn(df):
    """
    Gives return of individual security from portfolio
    """
   
    return round(df.Return.mean(), 4)

def calcRf (df):
    """
    Returns Rf for FF
    """
    df = dfResetIndex (df)
    return round(df.TB90[1], 4)

def fillMktPrem (df, s, e):
    """
    Fills in MktPrem to DF
    """
    dfNew = df
    dfNew['MKtReturn'] = ""
    dfNew['MktPrem'] = ""
    fileRange = np.arange(s, e)    
    for i in fileRange:
        dfNew['MKtReturn'].loc[dfNew['Year'] == i+1] = (dfNew['TRI'].loc[dfNew['Year'] == i+1].iloc[0] / dfNew['TRI'].loc[dfNew['Year'] == i].iloc[0] - 1)*100
        dfNew['MktPrem'].loc[dfNew['Year'] == i+1] = dfNew['MKtReturn'].loc[dfNew['Year'] == i+1].iloc[0] - dfNew['TB90'].loc[dfNew['Year'] == i+1].iloc[0]
    return dfNew

def calcMktPrem (df):
    """
    Returns Mkt Premium for FF
    """
    df = dfResetIndex(df)
    return round(df.MktPrem[1], 4)

def calcSMB(df):
    """
    Returns SMB for FF
    """
    #Define Quantile
    SQuantile = 0.3
    LQuantile = 0.7
    df["SMB"] = ""
    
    #Assigns stock size based on market cap
    df.SMB[df.MKTCAP <= df.MKTCAP.quantile(SQuantile)] = "SCap"
    df.SMB[(df.MKTCAP > df.MKTCAP.quantile(SQuantile)) & (df.MKTCAP < df.MKTCAP.quantile(LQuantile))] = "MCap"
    df.SMB[df.MKTCAP >= df.MKTCAP.quantile(LQuantile)] = "LCap"
    
    #Calculates average return of stocks in portfolio subset based on size
    SmallCapReturn = df.Return.loc[df["SMB"] == "SCap"].mean()
    LargeCapReturn = df.Return.loc[df["SMB"] == "LCap"].mean()
    
    #Returns SMB based on definition
    SMB = SmallCapReturn - LargeCapReturn
    return round(SMB, 4)

def calcHML (df):
    """
    Returns HML for FF
    Uses inverse of P/B as proxy for Book/Mkt
    """
    #Define Quantile
    SQuantile = 0.3
    LQuantile = 0.7
    df["HML"] = ""
    df["BP"] = df.PB**(-1) #Create Book/MktValue Proxy
    
    #Assigns stock size based on market cap
    df.HML[df.BP <= df.BP.quantile(SQuantile)] = "SValue"
    df.HML[(df.BP > df.BP.quantile(SQuantile)) & (df.BP < df.BP.quantile(LQuantile))] = "MValue"
    df.HML[df.BP >= df.BP.quantile(LQuantile)] = "LValue"
    
    #Calculates average return of stocks in portfolio subset based on size
    SmallValueReturn = df.Return.loc[df["HML"] == "SValue"].mean()
    LargeValueReturn = df.Return.loc[df["HML"] == "LValue"].mean()
    
    #Returns SMB based on definition
    HML = SmallValueReturn - LargeValueReturn
    return round(HML, 4)

def cleanColumns(df):
    """
    Cleans up unnecessary characters
    Cleans up columns, removing the extras
    """
    dfNew = df
   
    try:
        dfNew.columns = dfNew.columns.str.replace(' ','')
    except:
        pass
    
    try:
        dfNew.columns = dfNew.columns.str.replace('/','')
    except:
        pass
    
    dfNew = dfNew.rename(columns={"PCHG12M": "Return"})
    
    dfNew = dfNew[['Symbol',
                'Year',
                'Return',
                'TRI',
                'TB90',
                'MKTCAP',
                'PB'
                ]]
    
    return dfNew   

###Set up Data Frame

#Create empty data frame for the strategies
dfA = pd.DataFrame()
dfC = pd.DataFrame()

#Append the list
dfA = dfAppend("A", start, end) #year 2007-2018
dfC = dfAppend("C", start, end) #year 2008-2018

###Clean up Data Frame and preparing for FF model
#Remove space in columns
dfA = cleanColumns(dfA)
dfC = cleanColumns(dfC)

dfA

###Fill in MktPrem
#This part should only be done once
dfA = fillMktPrem (dfA, start, end)
dfC = fillMktPrem (dfC, start, end)

###Continue Cleanup
#Drop First Year
dfA = dfA.loc[dfA['Year'] != start]
dfC = dfC.loc[dfC['Year'] != start]

dfA = dfA.reset_index(drop = True)
dfC = dfC.reset_index(drop = True)
#Convert all inputs used to numeric

dfA.iloc[:, 2:] = dfA.iloc[:, 2:].convert_objects(convert_numeric=True)
dfC.iloc[:, 2:] = dfC.iloc[:, 2:].convert_objects(convert_numeric=True)

dfA.head(30)

#Create Fama French 3 factor model for Aggressive Strategy
FFA = pd.DataFrame(columns =
                  ["Year",
                   "Return",
                   "Rf",
                   "MktPrem",
                   "SMB",
                   "HML"                    
                  ])
FFAIndex = 0 
for i in range(start+1, end+1):
    FFA.loc[FFAIndex] = [i, 
                    calcReturn(dfA.loc[dfA['Year'] == i]), 
                    calcRf(dfA.loc[dfA['Year'] == i]), 
                    calcMktPrem(dfA.loc[dfA['Year'] == i]), 
                    calcSMB(dfA.loc[dfA['Year'] == i]), 
                    calcHML(dfA.loc[dfA['Year'] == i])
                   ]
    FFAIndex += 1
FFA['Year'] = FFA['Year'].astype(int)

#Create Fama French 3 factor model for Conservative Strategy
FFC = pd.DataFrame(columns =
                  ["Year",
                   "Return",
                   "Rf",
                   "MktPrem",
                   "SMB",
                   "HML"                    
                  ])
FFCIndex = 0 
for i in range(start+1, end+1):
    FFC.loc[FFCIndex] = [i, 
                    calcReturn(dfC.loc[dfC['Year'] == i]), 
                    calcRf(dfC.loc[dfC['Year'] == i]), 
                    calcMktPrem(dfC.loc[dfC['Year'] == i]), 
                    calcSMB(dfC.loc[dfC['Year'] == i]), 
                    calcHML(dfC.loc[dfC['Year'] == i])
                   ]
    FFCIndex += 1
FFC['Year'] = FFC['Year'].astype(int)

FFA

FFC

#Set up regression
Y = FFA.Return.values 
X = FFA[["MktPrem","SMB","HML"]]


model = sm.OLS( Y.astype(float), X.astype(float) )

result = model.fit()
print (result.params)
print(result.summary())

#Set up regression
Y = FFC.Return.values 
X = FFC[["MktPrem","SMB","HML"]]
X = sm.add_constant(X)
X.rename(columns = {"const":"Intercept"}, inplace = True)


model = sm.OLS( Y.astype(float), X.astype(float) )

result = model.fit()
print (result.params)
print(result.summary())