I ran a linear regression on a month of hourly Nox concentration using StatsModels
ols function.
how can I get summary OLS Regression Results on the test data? I was able to predict for new data but I want to get parameters like AIC
and R-squared
:
%matplotlib inline
import pandas as pd
import numpy as np
from statsmodels.formula.api import ols
import statsmodels.api as sm
import matplotlib.pyplot as plt
# read in data
path =r'https://docs.google.com/spreadsheets/d/e/2PACX-1vTDUZROQwAMckLFWFk6ltL5eBxamCcaqzeKSJjrhhKIB0beXNH_MKfNK8LVlIS71pb4FdgC98xzVdCg/pub?output=csv'
df = pd.read_csv(path, skipinitialspace=True)
df = df[1:]
# take care of datetimeindex
Date = pd.to_datetime(df.iloc[:,-1], format='%Y-%m-%d')
Date.name = 'Date'
Time = pd.to_datetime(df.iloc[:,-2], format='%H:%M:%S')
hour = Time.dt.hour
hour.name = "hour"
Weekday = Date.dt.weekday_name
Weekday.name = 'weekday'
month = Date.dt.strftime('%b')
month.name = 'month'
day = Date.dt.strftime('%a')
day.name = 'day'
weekend = Weekday=='Saturday'
weekend.name = 'weekend'
# clean data
tim = Date.dt.date.astype(str) + " " + Time.dt.time.astype(str)
date_time = pd.to_datetime(tim)
date_time.name = 'date_time'
clean_df = df.apply(pd.to_numeric, errors='coerce')
clean_df = pd.concat([clean_df.iloc[:,:-2], Weekday, hour, month, weekend], axis =1)
clean_df.index = date_time
print((clean_df.isnull().sum()/len(clean_df)) *100)
df_nona = clean_df[['No2','Nox', 'WS', 'WD', 'weekday', 'hour', 'month', 'weekend']]
df_nona = df_nona.dropna()
# wind direction to Dummy variables
directions = np.array('N NNE NE ENE E ESE SE SSE S SSW SW WSW W WNW NW NNW N'.split())
bins = np.arange(11.25, 360, 22.5)
binned_wind_direction = directions[np.digitize(df_nona['WD'], bins)]
df_nona["binned_wd"] = binned_wind_direction
WD = pd.get_dummies(df_nona['binned_wd'])
lin_model = pd.concat([df_nona,WD], axis=1 )
lin_model =lin_model.dropna()
lin_model.hour = lin_model.hour.astype('str')
train = lin_model['2016-01-01':'2016-03-01']
test =lin_model['2016-03-02':'2016-03-05']
# run linear regression
train_mode =ols("""Nox ~ weekend
+ hour
+ E
+ ENE
+ ESE
+ N
+ NE
+ NNE
+ NNW
+ NW
+ S
+ SE
+ SSE
+ SSW
+ SW
+ W
+ WNW
+ WSW
+ WS""", data=train).fit()
print(train_mode_summary)
test_dat = train_mode.predict(test)
fig, ax = plt.subplots(figsize=(9,4))
test_dat['2016-03-02':'2016-03-05'].plot(ax=ax, style='r')
df_nona['2016-03-02':'2016-03-05'].Nox.plot( ax=ax, style='k.', alpha=0.4)