#lasso regression

#from pandas import Series, DataFrame

import pandas as pd

import numpy as np

import os

import matplotlib.pylab as plt

from sklearn.model_selection import train_test_split

from sklearn.tree import DecisionTreeClassifier

from sklearn.metrics import classification_report

import sklearn.metrics

from sklearn.linear_model import LassoLarsCV

# standardize predictors to have mean=0 and sd=1

#predictors=predvar.copy()

from sklearn import preprocessing

predictors.loc[:,'CODPERING']=preprocessing.scale(predictors['CODPERING'].astype('float64'))

predictors.loc[:,'CODPRIPER']=preprocessing.scale(predictors['CODPRIPER'].astype('float64'))

predictors.loc[:,'CODULTPER']=preprocessing.scale(predictors['CODULTPER'].astype('float64'))

predictors.loc[:,'CICREL']=preprocessing.scale(predictors['CICREL'].astype('float64'))

predictors.loc[:,'CRDKAPRACU']=preprocessing.scale(predictors['CRDKAPRACU'].astype('float64'))

predictors.loc[:,'PPKAPRACU']=preprocessing.scale(predictors['PPKAPRACU'].astype('float64'))

predictors.loc[:,'CODPER5']=preprocessing.scale(predictors['CODPER5'].astype('float64'))

predictors.loc[:,'RN']=preprocessing.scale(predictors['RN'].astype('float64'))

predictors.loc[:,'MODALIDADC']=preprocessing.scale(predictors['MODALIDADC'].astype('float64'))

predictors.loc[:,'SEXOC']=preprocessing.scale(predictors['SEXOC'].astype('float64'))

# split data into train and test sets

pred_train, pred_test, tar_train, tar_test = train_test_split(predictors, targets,

                                                              test_size=.3, random_state=123)

# specify the lasso regression model

model=LassoLarsCV(cv=10, precompute=False).fit(pred_train,tar_train)

# print variable names and regression coefficients

dict(zip(predictors.columns, model.coef_))

{'CODPERING': 0.0057904161622687605, 'CODPRIPER': 0.6317522521193139, 'CODULTPER': -0.15191539575581153, 'CICREL': 0.07945048661923974, 'CRDKAPRACU': -0.3022282694810491, 'PPKAPRACU': 0.15702206999868978, 'CODPER5': -0.11697786485114721, 'RN': -0.03802582617592532, 'MODALIDADC': 0.017655346467683828, 'SEXOC': 0.10597063961395894}

# plot coefficient progression

m_log_alphas = -np.log10(model.alphas_)

ax = plt.gca()

plt.plot(m_log_alphas, model.coef_path_.T)

plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k',

            label='alpha CV')

plt.ylabel('Regression Coefficients')

plt.xlabel('-log(alpha)')

plt.title('Regression Coefficients Progression for Lasso Paths')

# plot mean square error for each fold

m_log_alphascv = -np.log10(model.cv_alphas_)

plt.figure()

plt.plot(m_log_alphascv, model.mse_path_, ':')

plt.plot(m_log_alphascv, model.mse_path_.mean(axis=-1), 'k',

         label='Average across the folds', linewidth=2)

plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k',

            label='alpha CV')

plt.legend()

plt.xlabel('-log(alpha)')

plt.ylabel('Mean squared error')

plt.title('Mean squared error on each fold')

# MSE from training and test data

from sklearn.metrics import mean_squared_error

train_error = mean_squared_error(tar_train, model.predict(pred_train))

test_error = mean_squared_error(tar_test, model.predict(pred_test))

print ('training data MSE')

print(train_error)

print ('test data MSE')

print(test_error)

training data MSE 10.81377398206078

test data MSE 10.82396461754075

# R-square from training and test data

rsquared_train=model.score(pred_train,tar_train)

rsquared_test=model.score(pred_test,tar_test)

print ('training data R-square')

print(rsquared_train)

print ('test data R-square')

print(rsquared_test)

training data R-square 0.041399684741574516

test data R-square 0.04201223667290355

Results Explanation:

A lasso regression analysis was conducted to identify a subset of variables from a pool of 10 quantitative predictor variables that best predicted a quantitative response variable. All predictor variables were standardized to have a mean of zero and a standard deviation of one.

Data were randomly split into a training set that included 70% of the observations and a test set that included 30% of the observations. The least angle regression algorithm with k=10 fold cross validation was used to estimate the lasso regression model in the training set, and the model was validated using the test set. The change in the cross validation average (mean) squared error at each step was used to identify the best subset of predictor variables.

The MSE values for both training and test data are quite similar, indicating that the model performs consistently across both datasets. However, the R-square values are quite low (around 0.04), suggesting that the model does not explain much of the variance in the data.

Task

This week’s assignment involves running a lasso regression analysis. Lasso regression analysis is a shrinkage and variable selection method for linear regression models. The goal of lasso regression is to obtain the subset of predictors that minimizes prediction error for a quantitative response variable. The lasso does this by imposing a constraint on the model parameters that causes regression coefficients for some variables to shrink toward zero. Variables with a regression coefficient equal to zero after the shrinkage process are excluded from the model. Variables with non-zero regression coefficients variables are most strongly associated with the response variable. Explanatory variables can be either quantitative, categorical or both.

Code

The plot above shows that there is a linear dependence between temp, atemp and cnt features. The correlations below confirm that observation.

There is a strong correlation between temp and atemp, as well as windspeed(mph) and windspeed(ms) features, due to the fact that they represent similar metrics in different measures. In further analysis two of those features must be dropped or applyed with penalty (L2 or Lasso regression).

Results

A lasso regression analysis was conducted to predict a number of total bikes rentals from a pool of 12 categorical and quantitative predictor variables that best predicted a quantitative response variable. Categorical predictors included weather condition and a series of 2 binary categorical variables for holiday and workingday to improve interpretability of the selected model with fewer predictors. Quantitative predictor variables include year, month, temperature, humidity and wind speed.

Data were randomly split into a training set that included 70% of the observations and a test set that included 30% of the observations. The least angle regression algorithm with k=10 fold cross validation was used to estimate the lasso regression model in the training set, and the model was validated using the test set. The change in the cross validation average (mean) squared error at each step was used to identify the best subset of predictor variables.

Of the 12 predictor variables, 10 were retained in the selected model:

atemp: 63.56915200306693

holiday: -282.431748735072

hum: -12.815264427009353

mnth: 0.0

season: 381.77762475080044

temp: 58.035647703871234

weathersit: -514.6381162101678

weekday: 69.84812053893549

windspeed(mph): 0.0

windspeed(ms): -95.71090321577515

workingday: 36.15135752613271

yr: 2091.5182927517903

Train data R-square 0.7899877818517489

Test data R-square 0.8131871527614188

## Aim of the analysis

The is analysis aims to examine whether predictor variables such as Years of previous experience, Years of employment, years of education, Genders and number of supervision are associated with increased or decreased salary. This is consirdered high dimension. Therefore, lasso regression is fitted to the data. 70 % of the data is used as training set and 30 % is used as test set.

The K-fold cross-validation with 10 random folds is used to assess the model accuracy. The model is assessed at each step by adding one predictor at a time and model with lowest mean square error is selected as the best model.

## Import library

#from pandas import Series, DataFrame import pandas as pd import numpy as np import matplotlib.pylab as plt from sklearn.model_selection import train_test_split from sklearn.linear_model import LassoLarsCV from sklearn import linear_model from sklearn.linear_model import LassoCV from sklearn.datasets import make_regression

#Load the dataset salary = pd.read_csv("salary.csv")

#upper-case all DataFrame column names salary.columns = map(str.upper, salary.columns)

# Data Management salary_clean = salary.dropna() salary_clean

salary_clean["GENDER"]=salary_clean["GENDER"].replace("Male", "0") salary_clean["GENDER"]=salary_clean["GENDER"].replace("Female", "1")

salary_clean.dtypes

#select predictor variables and target variable as separate data sets   predvariables= salary_clean[['YEARS_PREVIOUS_EXPERIENCE', 'YEARS_EMPLOYED', 'YEARS_EDUCATION',                     'GENDER', 'NUMBER_SUPERVISED']]

target_variables = salary_clean.SALARY

# standardize predictors to have mean=0 and sd=1

predictors=predvariables.copy() from sklearn import preprocessing predictors['YEARS_PREVIOUS_EXPERIENCE']=preprocessing.scale(predictors['YEARS_PREVIOUS_EXPERIENCE'].astype('int64')) predictors['YEARS_EMPLOYED']=preprocessing.scale(predictors['YEARS_EMPLOYED'].astype('int64')) predictors['YEARS_EDUCATION']=preprocessing.scale(predictors['YEARS_EDUCATION'].astype('int64')) predictors['GENDER']=preprocessing.scale(predictors['GENDER'].astype('object')) predictors['NUMBER_SUPERVISED']=preprocessing.scale(predictors['NUMBER_SUPERVISED'].astype('int64'))

# split data into train and test sets

pred_train, pred_test, tar_train, tar_test = train_test_split(predictors, target_variables, test_size=.3, random_state=123)

# specify the lasso regression model

model=LassoLarsCV(cv=10, precompute=False).fit(pred_train,tar_train)

# print variable names and regression coefficients

dict(zip(predictors.columns, model.coef_))

# plot coefficient progression

m_log_alphas = -np.log10(model.alphas_) ax = plt.gca() plt.plot(m_log_alphas, model.coef_path_.T) plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k', label='alpha CV') plt.ylabel('Regression Coefficients') plt.xlabel('-log(alpha)') plt.title('Regression Coefficients Progression for Lasso Paths')

# plot mean square error for each fold

m_log_alphascv = -np.log10(model.cv_alphas_) plt.figure() plt.plot(m_log_alphascv, model.mse_path_,':')

plt.plot(m_log_alphascv, model.mse_path_, ':') plt.plot(m_log_alphascv, model.mse_path_, ':') plt.plot(m_log_alphascv, model.mse_path_.mean(axis=-1), 'k', label='Average across the folds', linewidth=2) plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k', label='alpha CV') plt.legend() plt.xlabel('-log(alpha)') plt.ylabel('Mean squared error') plt.title('Mean squared error on each fold')

# MSE from training and test data

from sklearn.metrics import mean_squared_error train_error = mean_squared_error(tar_train, model.predict(pred_train)) test_error = mean_squared_error(tar_test, model.predict(pred_test)) print ('training data MSE') print(train_error) print ('test data MSE') print(test_error)

# R-square from training and test data

rsquared_train=model.score(pred_train,tar_train) rsquared_test=model.score(pred_test,tar_test) print ('training data R-square') print(rsquared_train) print ('test data R-square') print(rsquared_test)

# Interpretation

The final model selected the following predictors

YEARS_EMPLOYED: 5067.966710790326,

YEARS_EDUCATION: 5842.7809308716205,

GENDER: -75.40256991367636,

NUMBER_SUPERVISED': 792.8736233279764

Number_Supervised is the most important predictors to be related to the salary followed by years of education, years of employment and gender has negative coefficients meaning Gender is negatively associated with salary, which seem like does not make sense here in this context. Year of previous experience has shrunk to 0, meaning this variable is not associated with increased ore decreased salary at all.

The test mean square error is quite different from the training, suggesting the model is less accurate at predicting the test set. Also, the R-square for training is quite higher of around 87 percent, but only 3% for test set. So it seems not a good model at all

Dear colleagues students,

I performed lasso regression analysis to identify a subset of 11 categorical and quantitative predictor variables that best predict a quantitative response variable measuring personal income of participants. 

Categorical predictors included gender, major depression, general anxiety, tobacco dependency, gambling problems, working situation (if working full day or not), race (white or other race). Quantitative predictor variables include age, educational degree, marital status, alcohol problems. 

Here’s program code:

Data were randomly split into a training set that included 70% of the observations (N=30166) and a test set that included 30% of the observations (N=12927). 

The least angle regression algorithm with k=10 fold cross validation was used to estimate the lasso regression model in the training set, and the model was validated using the test set. 

The change in the cross validation average squared error at each step was used to identify the best subset of predictor variables.

Of the 11 predictor variables, 7 were retained in the selected model. During the estimation process, educational degree, working situation (full time or not) and sex were most strongly associated with current personal income, followed by age, marital status, race and tobacco dependency. 

These 7 variables accounted for 16.8% of the variance in the personal income response variable.

Please review my work, thank you!

Sincerely,

LASSO is a Supervised Machine Learning method. It stands for Least Absolute Selection Shrinkage Operator. LASSO uses the Shrinkage and Selection method, which causes some of the regression coefficients of the explanatory variables to equal zero (0) and picks the most important ones associated to the response variable.

LASSO provides greater prediction accuracy when there is a linear relationship between the explanatory and response variables with small number of observations and high number of predictors. The Tuning parameter is called lambda (or alpha in sklearn) which controls the shrinkage of the model. Higher lambda means more coefficients are set to zero. When lambda is zero, the model is an OLS regression. Least Angle Regression LAR Algorithm is used.

For the number of my explanatory variables, OLS regression is ideal but LASSO regression is used for the purpose of this study.

From the Bias-Variance Tradeoff graph below, it is seen that the more predictors added to the model, the more prediction error decreases for the training set. Consequently, bias is lower.

 Summary of Results

Lasso regression analysis was run to identify a subset of variables from 5 quantitative predictor variables that best predicted a quantitative response variable ‘relectricperperson’. Quantitative predictor variables include ‘incomeperperson’, ‘urbanrate’, ‘employrate’, ‘internertuserate’, and ‘co2emissions’. All predictor variables were standardized to have a mean of zero and a standard deviation of one.

Data were randomly split into a training set that included 70% of the observations (N=88) and a test set that included 30% of the observations (N=39). The least angle regression algorithm with k=10 fold cross validation was used to estimate the lasso regression model in the training set, and the model was validated using the test set. The change in the cross validation average (mean) squared error at each step was used to identify the best subset of predictor variables.

Of the 5 predictor variables, 4 were retained in the selected model. ‘incomeperperson’ and ‘internetuserate’ were most strongly associated with ‘relectricperperson’. No predictor was negatively associated with ‘relectricperperson’. These 4 variables accounted for 50% of the variance in the ‘relectricperperson’ response variable.

A lasso regression analysis was conducted to identify a subset of variables from a pool of 23 categorical and quantitative predictor variables that best predicted a quantitative response variable measuring school connectedness in adolescents. Categorical predictors included gender and a series of 5 binary categorical variables for race and ethnicity (Hispanic, White, Black, Native American and Asian) to improve interpret-ability of the selected model with fewer predictors.

The dictionary list in the output depicts the relationship of each variable with the school connectedness of the person. As we can see in the first figure, each predictor has a value that negatively or positively associates itself with the school connectedness. Among all of them, self esteem(ESTEEM1) and grade(GPA1) point average with the highest positive values, and depression(DEP1), black(BLACK) and violence(VIOL1) with lowest negative values show us their strong associativity with the school connectedness. The other factors that do not have any value or negligible value can be removed to increase the efficiency.

Taking 30% of the dataset as test set, and 70% as training set, we calculated both - the mean square error as well as the R square error for training and test data respectively; the values of which clearly depict the efficiency of Lasso Regression to be greater than normal linear regression method.

Apart from that, we also plotted the log of Regression coefficients to compare the associativity of all the features with school connectedness. Here, we can see that purple line represents the feature that is most associated positively(self esteem) and the red line in the lower quadrant that represents the feature that is most associated negatively(Depression).

A lasso regression analysis was conducted to identify a subset of variables from a pool of 24 categorical and quantitative predictor variables that best predicted a quantitative response variable measuring school connectedness in adolescents. Categorical predictors included gender and a series of 5 binary categorical variables for race and ethnicity (Hispanic, White, Black, Native American and Asian) to improve interpretability of the selected model with fewer predictors. Binary substance use variables were measured with individual questions about whether the adolescent had ever used alcohol, marijuana, cocaine or inhalants. Additional categorical variables included the availability of cigarettes in the home, whether or not either parent was on public assistance and any experience with being expelled from school. Quantitative predictor variables include age, alcohol problems, and a measure of deviance that included such behaviors as vandalism, other property damage, lying, stealing, running away, driving without permission, selling drugs, and skipping school. Another scale for violence, one for depression, and others measuring self-esteem, parental presence, parental activities, family connectedness and grade point average were also included. All predictor variables were standardized to have a mean of zero and a standard deviation of one.

Data were randomly split into a training set that included 70% of the observations (N=3201) and a test set that included 30% of the observations (N=1701). The least angle regression algorithm with k=10 fold cross validation was used to estimate the lasso regression model in the training set, and the model was validated using the test set. The change in the cross validation average (mean) squared error at each step was used to identify the best subset of predictor variables.

Code for Lasso Regression:

from google.colab import drive,files import pandas as pd import numpy as np import matplotlib.pyplot as plt import os import pydotplus import io from IPython.display import Image from sklearn.metrics import confusion_matrix from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split from sklearn.linear_model import LassoLarsCV from sklearn import preprocessing drive.mount('/content/gdrive') corpusName ="_d21b2085472fd467f689f21cd421b13b_tree_addhealth.csv" folderPath = "/content/gdrive/My Drive/MachineLearningForDataAnalysis" corpus = os.path.join(folderPath, corpusName) df_train = pd.read_csv(corpus) print("Dataset shape = ") print(df_train.shape)

Output:  Dataset shape = (6504, 25)

print(df_train.columns)

Output:  Index(['BIO_SEX', 'HISPANIC', 'WHITE', 'BLACK', 'NAMERICAN', 'ASIAN', 'age',       'TREG1', 'ALCEVR1', 'ALCPROBS1', 'marever1', 'cocever1', 'inhever1',       'cigavail', 'DEP1', 'ESTEEM1', 'VIOL1', 'PASSIST', 'DEVIANT1',       'SCHCONN1', 'GPA1', 'EXPEL1', 'FAMCONCT', 'PARACTV', 'PARPRES'],      dtype='object') 

# print(df_train.head(10)) # print(df_train.info()) df_train.dropna(inplace=True) for col in df_train.columns:  df_train[col]=preprocessing.scale(df_train[col].astype('float64'))

x_train, x_test, y_train, y_test = train_test_split(df_train.drop("TREG1",axis=1), df_train.TREG1,                                                              test_size=.3, random_state=123)

model=LassoLarsCV(cv=10, precompute=False).fit(x_train,y_train)

dict(zip(x_train.columns, model.coef_))

m_log_alphas = -np.log10(model.alphas_) ax = plt.gca() plt.plot(m_log_alphas, model.coef_path_.T) plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k',            label='alpha CV') plt.ylabel('Regression Coefficients') plt.xlabel('-log(alpha)') plt.title('Regression Coefficients Progression for Lasso Paths') plt.savefig(folderPath+"/Reg_Coeff.png")

m_log_alphascv = -np.log10(model.cv_alphas_) plt.figure() plt.plot(m_log_alphascv, model.mse_path_, ':') plt.plot(m_log_alphascv, model.mse_path_.mean(axis=-1), 'k',         label='Average across the folds', linewidth=2) plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k',            label='alpha CV') plt.legend() plt.xlabel('-log(alpha)') plt.ylabel('Mean squared error') plt.title('Mean squared error on each fold') plt.savefig(folderPath+"/Mean_Sq_Err.png")

# MSE from training and test data from sklearn.metrics import mean_squared_error train_error = mean_squared_error(y_train, model.predict(x_train)) test_error = mean_squared_error(y_test, model.predict(x_test)) print ('training data MSE') print(train_error)

Output:  training data MSE 0.7103773580520552 

print ('test data MSE') print(test_error)

Output:  test data MSE 0.721859543809532

# R-square from training and test data rsquared_train=model.score(x_train,y_train) rsquared_test=model.score(x_test,y_test) print ('training data R-square') print(rsquared_train)

Output:  training data R-square 0.2745661967781555

print ('test data R-square') print(rsquared_test)

Output:  test data R-square 0.31071521636196964

print("Number of feature Used = {}".format(np.sum(model.coef_!=0)))

Output:  Number of feature Used = 22

print("Coeff = {}".format(out_dict))

Output: Coeff = {'BIO_SEX': 0.025628290838580477, 'HISPANIC': -0.030002808614276108, 'WHITE': 0.09522057486885759, 'BLACK': -0.051547164889418545, 'NAMERICAN': -0.006659984878718197, 'ASIAN': -0.007331382053598405, 'age': 0.03877631368977216, 'ALCEVR1': 0.07576516459955492, 'ALCPROBS1': 0.05009414437125331, 'marever1': 0.24568705171878635, 'cocever1': 0.03243612799667629, 'inhever1': 0.015269660628021415, 'cigavail': 0.08109571776580017, 'DEP1': 0.045763557239754234, 'ESTEEM1': 0.007312169922740399, 'VIOL1': 0.0, 'PASSIST': -0.012550380318644493, 'DEVIANT1': 0.06974156846904282, 'SCHCONN1': -0.038455747385471306, 'GPA1': -0.1040514070440718, 'EXPEL1': 0.006879433295868557, 'FAMCONCT': 0.0, 'PARACTV': -0.038665425414240076, 'PARPRES': -0.01951092512912656}

As we can see VIOL1 and  FAMCONCT are 0 they can be assumed to be related to some other variable.