import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder,MinMaxScaler, StandardScaler
from sklearn.model_selection import train_test_split, ParameterGrid
from sklearn.metrics import accuracy_score, confusion_matrix, mean_squared_error, log_loss
import operator
import json
from IPython import display
import os
import warnings
np.random.seed(0)
warnings.filterwarnings("ignore")
THRESHOLD = 4
Task: To predict whether the user likes the mobile phone or not.
Assumption: If the average rating of mobile >= threshold, then the user likes it, otherwise not.
Missing values:
'Also Known As'(459),'Applications'(421),'Audio Features'(437),'Bezel-less display'(266),'Browser'(449),'Build Material'(338),'Co-Processor'(451),'Display Colour'(457),'Mobile High-Definition Link(MHL)'(472),'Music'(447)
'Email','Fingerprint Sensor Position'(174),'Games'(446),'HDMI'(454),'Heart Rate Monitor'(467),'IRIS Scanner'(467),
'Optical Image Stabilisation'(219),'Other Facilities'(444),'Phone Book'(444),'Physical Aperture'(87),'Quick Charging'(122),'Ring Tone'(444),'Ruggedness'(430),SAR Value(315),'SIM 3'(472),'SMS'(470)', 'Screen Protection'(229),'Screen to Body Ratio (claimed by the brand)'(428),'Sensor'(242),'Software Based Aperture'(473),
'Special Features'(459),'Standby time'(334),'Stylus'(473),'TalkTime'(259), 'USB Type-C'(374),'Video Player'(456),
'Video Recording Features'(458),'Waterproof'(398),'Wireless Charging','USB OTG Support'(159), 'Video ,'Recording'(113),'Java'(471),'Browser'(448)
Very low variance:
'Architecture'(most entries are 64-bit),'Audio Jack','GPS','Loudspeaker','Network','Network Support','Other Sensors'(28),'SIM Size', 'VoLTE'
Multivalued:
'Colours','Custom UI','Model'(1),'Other Sensors','Launch Date'
Not important:
'Bluetooth', 'Settings'(75),'Wi-Fi','Wi-Fi Features'
Doubtful:
'Aspect Ratio','Autofocus','Brand','Camera Features','Fingerprint Sensor'(very few entries are missing),
'Fingerprint Sensor Position', 'Graphics'(multivalued),'Image resolution'(multivalued),'SIM Size','Sim Slot(s)', 'User Available Storage', 'SIM 1', 'SIM 2','Shooting Modes', 'Touch Screen'(24), 'USB Connectivity'
To check:
'Display Type','Expandable Memory','FM Radio'
High Correlation with other features
'SIM Slot(s)' high correlation with SIM1
'Weight' has high high correlation with capacity , screen-to-body ratio
'Height' - screen size is also there
Given a mobile, we can't directly get these features
'Rating Count', 'Review Count'
Keeping:
'Capacity','Flash'(17),'Height'(22),'Internal Memory'(20, require cleaning),'Operating System'(25, require cleaning), 'Pixel Density'(1, clean it),'Processor'(22, clean it), 'RAM'(17, clean), 'Rating','Resolution'(cleaning), 'Screen Resolution','Screen Size', 'Thickness'(22), 'Type','User Replaceable','Weight'(cleaning),'Sim Size'(), 'Other Sensors'(28), 'Screen to Body Ratio (calculated)','Width',
# read data from file
train = pd.read_csv("../train.csv")
test = pd.read_csv("../test.csv")
# check the number of features and data points in train
print("Number of data points in train: %d" % train.shape[0])
print("Number of features in train: %d" % train.shape[1])
# check the number of features and data points in test
print("Number of data points in test: %d" % test.shape[0])
print("Number of features in test: %d" % test.shape[1])
def data_clean(data):
# Let's first remove all missing value features
columns_to_remove = ['Also Known As','Applications','Audio Features','Bezel-less display'
'Browser','Build Material','Co-Processor','Browser'
'Display Colour','Mobile High-Definition Link(MHL)',
'Music', 'Email','Fingerprint Sensor Position',
'Games','HDMI','Heart Rate Monitor','IRIS Scanner',
'Optical Image Stabilisation','Other Facilities',
'Phone Book','Physical Aperture','Quick Charging',
'Ring Tone','Ruggedness','SAR Value','SIM 3','SMS',
'Screen Protection','Screen to Body Ratio (claimed by the brand)',
'Sensor','Software Based Aperture', 'Special Features',
'Standby time','Stylus','TalkTime', 'USB Type-C',
'Video Player', 'Video Recording Features','Waterproof',
'Wireless Charging','USB OTG Support', 'Video Recording','Java']
columns_to_retain = list(set(data.columns)-set(columns_to_remove))
data = data[columns_to_retain]
#Features having very low variance
columns_to_remove = ['Architecture','Audio Jack','GPS','Loudspeaker','Network','Network Support','VoLTE']
columns_to_retain = list(set(data.columns)-set(columns_to_remove))
data = data[columns_to_retain]
# Multivalued:
columns_to_remove = ['Architecture','Launch Date','Audio Jack','GPS','Loudspeaker','Network','Network Support','VoLTE', 'Custom UI']
columns_to_retain = list(set(data.columns)-set(columns_to_remove))
data = data[columns_to_retain]
# Not much important
columns_to_remove = ['Bluetooth', 'Settings','Wi-Fi','Wi-Fi Features']
columns_to_retain = list(set(data.columns)-set(columns_to_remove))
data = data[columns_to_retain]
return data
train = data_clean(train)
test = data_clean(test)
removing all those data points in which more than 15 features are missing
train = train[(train.isnull().sum(axis=1) <= 15)]
# You shouldn't remove data points from test set
#test = test[(test.isnull().sum(axis=1) <= 15)]
# check the number of features and data points in train
print("Number of data points in train: %d" % train.shape[0])
print("Number of features in train: %d" % train.shape[1])
# check the number of features and data points in test
print("Number of data points in test: %d" % test.shape[0])
print("Number of features in test: %d" % test.shape[1])
def for_integer(test):
try:
test = test.strip()
return int(test.split(' ')[0])
except IOError:
pass
except ValueError:
pass
except:
pass
def for_string(test):
try:
test = test.strip()
return (test.split(' ')[0])
except IOError:
pass
except ValueError:
pass
except:
pass
def for_float(test):
try:
test = test.strip()
return float(test.split(' ')[0])
except IOError:
pass
except ValueError:
pass
except:
pass
def find_freq(test):
try:
test = test.strip()
test = test.split(' ')
if test[2][0] == '(':
return float(test[2][1:])
return float(test[2])
except IOError:
pass
except ValueError:
pass
except:
pass
def for_Internal_Memory(test):
try:
test = test.strip()
test = test.split(' ')
if test[1] == 'GB':
return int(test[0])
if test[1] == 'MB':
# print("here")
return (int(test[0]) * 0.001)
except IOError:
pass
except ValueError:
pass
except:
pass
def find_freq(test):
try:
test = test.strip()
test = test.split(' ')
if test[2][0] == '(':
return float(test[2][1:])
return float(test[2])
except IOError:
pass
except ValueError:
pass
except:
pass
def data_clean_2(x):
data = x.copy()
data['Capacity'] = data['Capacity'].apply(for_integer)
data['Height'] = data['Height'].apply(for_float)
data['Height'] = data['Height'].fillna(data['Height'].mean())
data['Internal Memory'] = data['Internal Memory'].apply(for_Internal_Memory)
data['Pixel Density'] = data['Pixel Density'].apply(for_integer)
data['Internal Memory'] = data['Internal Memory'].fillna(data['Internal Memory'].median())
data['Internal Memory'] = data['Internal Memory'].astype(int)
data['RAM'] = data['RAM'].apply(for_integer)
data['RAM'] = data['RAM'].fillna(data['RAM'].median())
data['RAM'] = data['RAM'].astype(int)
data['Resolution'] = data['Resolution'].apply(for_integer)
data['Resolution'] = data['Resolution'].fillna(data['Resolution'].median())
data['Resolution'] = data['Resolution'].astype(int)
data['Screen Size'] = data['Screen Size'].apply(for_float)
data['Thickness'] = data['Thickness'].apply(for_float)
data['Thickness'] = data['Thickness'].fillna(data['Thickness'].mean())
data['Thickness'] = data['Thickness'].round(2)
data['Type'] = data['Type'].fillna('Li-Polymer')
data['Screen to Body Ratio (calculated)'] = data['Screen to Body Ratio (calculated)'].apply(for_float)
data['Screen to Body Ratio (calculated)'] = data['Screen to Body Ratio (calculated)'].fillna(data['Screen to Body Ratio (calculated)'].mean())
data['Screen to Body Ratio (calculated)'] = data['Screen to Body Ratio (calculated)'].round(2)
data['Width'] = data['Width'].apply(for_float)
data['Width'] = data['Width'].fillna(data['Width'].mean())
data['Width'] = data['Width'].round(2)
data['Flash'][data['Flash'].isna() == True] = "Other"
data['User Replaceable'][data['User Replaceable'].isna() == True] = "Other"
data['Num_cores'] = data['Processor'].apply(for_string)
data['Num_cores'][data['Num_cores'].isna() == True] = "Other"
data['Processor_frequency'] = data['Processor'].apply(find_freq)
#because there is one entry with 208MHz values, to convert it to GHz
data['Processor_frequency'][data['Processor_frequency'] > 200] = 0.208
data['Processor_frequency'] = data['Processor_frequency'].fillna(data['Processor_frequency'].mean())
data['Processor_frequency'] = data['Processor_frequency'].round(2)
data['Camera Features'][data['Camera Features'].isna() == True] = "Other"
#simplifyig Operating System to os_name for simplicity
data['os_name'] = data['Operating System'].apply(for_string)
data['os_name'][data['os_name'].isna() == True] = "Other"
data['Sim1'] = data['SIM 1'].apply(for_string)
data['SIM Size'][data['SIM Size'].isna() == True] = "Other"
data['Image Resolution'][data['Image Resolution'].isna() == True] = "Other"
data['Fingerprint Sensor'][data['Fingerprint Sensor'].isna() == True] = "Other"
data['Expandable Memory'][data['Expandable Memory'].isna() == True] = "No"
data['Weight'] = data['Weight'].apply(for_integer)
data['Weight'] = data['Weight'].fillna(data['Weight'].mean())
data['Weight'] = data['Weight'].astype(int)
data['SIM 2'] = data['SIM 2'].apply(for_string)
data['SIM 2'][data['SIM 2'].isna() == True] = "Other"
return data
train = data_clean_2(train)
test = data_clean_2(test)
# check the number of features and data points in train
print("Number of data points in train: %d" % train.shape[0])
print("Number of features in train: %d" % train.shape[1])
# check the number of features and data points in test
print("Number of data points in test: %d" % test.shape[0])
print("Number of features in test: %d" % test.shape[1])
Not very important feature
def data_clean_3(x):
data = x.copy()
columns_to_remove = ['User Available Storage','SIM Size','Chipset','Processor','Autofocus','Aspect Ratio','Touch Screen',
'Bezel-less display','Operating System','SIM 1','USB Connectivity','Other Sensors','Graphics','FM Radio',
'NFC','Shooting Modes','Browser','Display Colour' ]
columns_to_retain = list(set(data.columns)-set(columns_to_remove))
data = data[columns_to_retain]
columns_to_remove = [ 'Screen Resolution','User Replaceable','Camera Features',
'Thickness', 'Display Type']
columns_to_retain = list(set(data.columns)-set(columns_to_remove))
data = data[columns_to_retain]
columns_to_remove = ['Fingerprint Sensor', 'Flash', 'Rating Count', 'Review Count','Image Resolution','Type','Expandable Memory',\
'Colours','Width','Model']
columns_to_retain = list(set(data.columns)-set(columns_to_remove))
data = data[columns_to_retain]
return data
train = data_clean_3(train)
test = data_clean_3(test)
# check the number of features and data points in train
print("Number of data points in train: %d" % train.shape[0])
print("Number of features in train: %d" % train.shape[1])
# check the number of features and data points in test
print("Number of data points in test: %d" % test.shape[0])
print("Number of features in test: %d" % test.shape[1])
# one hot encoding
train_ids = train['PhoneId']
test_ids = test['PhoneId']
cols = list(test.columns)
cols.remove('PhoneId')
cols.insert(0, 'PhoneId')
combined = pd.concat([train.drop('Rating', axis=1)[cols], test[cols]])
print(combined.shape)
print(combined.columns)
combined = pd.get_dummies(combined)
print(combined.shape)
print(combined.columns)
train_new = combined[combined['PhoneId'].isin(train_ids)]
test_new = combined[combined['PhoneId'].isin(test_ids)]
train_new = train_new.merge(train[['PhoneId', 'Rating']], on='PhoneId')
# check the number of features and data points in train
print("Number of data points in train: %d" % train_new.shape[0])
print("Number of features in train: %d" % train_new.shape[1])
# check the number of features and data points in test
print("Number of data points in test: %d" % test_new.shape[0])
print("Number of features in test: %d" % test_new.shape[1])
train_new.head()
test_new.head()
submission = pd.DataFrame({'PhoneId':test_new['PhoneId'], 'Class':[0]*test_new.shape[0]})
submission = submission[['PhoneId', 'Class']]
submission.head()
submission.to_csv("submission.csv", index=False)
from sklearn.model_selection import train_test_split as tt
train_new['Rating']= train_new['Rating'].map(lambda x: 1 if x>= 4.0 else 0)
pd.plotting.scatter_matrix(train_new)
train_new.groupby('Rating').mean()
x_train=train_new.drop('Rating',axis = 1)
train_id = train_new['PhoneId']
x_train = train_new.drop('PhoneId',axis = 1)
y_train = train_new['Rating']
test_id = test_new['PhoneId']
test_new = test_new.drop('PhoneId',axis = 1)
x_train.describe()
y_train = y_train.drop(index = [165])
x_train = x_train.drop(index = [165])
x_train.describe()
plt.plot(x_train.T,'*')
plt.xticks(rotation = 'vertical')
plt.show()
from sklearn import preprocessing
min_max_scaler = preprocessing.MinMaxScaler()
x_train.describe()
test_new.describe()
test_new.boxplot(column = ['Capacity'])
test_new['Internal Memory'] = test_new['Internal Memory'].map(lambda x:0 if x < test_new['Internal Memory'].quantile(0.5) else 1)
x_train = pd.DataFrame(min_max_scaler.fit_transform(x_train))
test_new = pd.DataFrame(min_max_scaler.fit_transform(test_new))
# x_train['RAM'] = test_new['RAM'].map(lambda x: 0.5 if x == 512 else x)
# x_bin_train = x_train.apply(pd.qcut,q =0.5,retbins = True,labels=[0,1])
# x_bin_test = test_new.apply(pd.qcut,q= 0.5,retbins = True,labels=[0,1])
x_bin_train = x_train.apply(pd.cut, bins = 2 ,labels=[0,1],include_lowest = True)
x_bin_test = test_new.apply(pd.cut, bins = 2 ,labels=[0,1],include_lowest = True)
type(x_bin_test)
pd.DataFrame(x_bin_train).describe()
plt.plot(x_bin_test.T,'*')
plt.xticks(rotation = 'vertical')
plt.show()
x_bin_train = np.asarray(x_bin_train)
x_bin_test = np.asarray(x_bin_test)
class MPneuron():
def __init__(self):
self.b = 0
def loss(self,y,y_pred):
return(np.dot((y-y_pred),(y-y_pred).T))
def model(self,X):
return(np.sum(X)>=self.b)
def predict(self,X):
y = []
for i in X:
y.append(self.model(i))
return(np.array(y))
def fit(self,X,Y):
self.acc = {}
self.losses = {}
for b in range(X.shape[1] + 1):
self.b = b
y_pred = self.predict(X)
self.acc[b] = accuracy_score(Y,y_pred)
self.losses[b] = self.loss(Y,y_pred)
best_b = max(self.acc,key= self.acc.get)
self.b = best_b
print(self.acc)
print('The best b is :',self.b)
print('The optimal Accuracy is :',self.acc[self.b])
print('The optimal Loss is :',self.losses[self.b])
MPn = MPneuron()
MPn.fit(x_bin_train,y_train)
x,y = zip(*list(MPn.losses.items()))
plt.plot(x,y)
y_test=MPn.predict(np.asarray(x_bin_test))
y_test.astype(int)
submission = pd.DataFrame({'PhoneId':test_id, 'Class':y_test.astype(int)})
submission = submission[['PhoneId', 'Class']]
submission.to_csv("submission2.csv", index=False)
!ls