I'm now study machine learning, I want to apply the Logistic Regression to handle the data.
The dataSet link: enter link description here
However it reports error like this, and I'm not sure how to make values match.
Traceback (most recent call last):
File "C:/Users/Minglang.Tuo20/PycharmProjects/clothes/Main.py", line 8, in <module>
class main():
File "C:/Users/Minglang.Tuo20/PycharmProjects/clothes/Main.py", line 28, in main
new_methods.Logistic_Regression()
File "C:\Users\Minglang.Tuo20\PycharmProjects\clothes\Meachine_Learning_Methods.py", line 19, in Logistic_Regression
self.file_category['Logistic Regression'] = lr.predict(self.X_train)
File "C:\Users\Minglang.Tuo20\PycharmProjects\clothes\venv\lib\site-packages\pandas\core\frame.py", line 3040, in __setitem__
self._set_item(key, value)
File "C:\Users\Minglang.Tuo20\PycharmProjects\clothes\venv\lib\site-packages\pandas\core\frame.py", line 3116, in _set_item
value = self._sanitize_column(key, value)
File "C:\Users\Minglang.Tuo20\PycharmProjects\clothes\venv\lib\site-packages\pandas\core\frame.py", line 3764, in _sanitize_column
value = sanitize_index(value, self.index)
File "C:\Users\Minglang.Tuo20\PycharmProjects\clothes\venv\lib\site-packages\pandas\core\internals\construction.py", line 748, in sanitize_index
"Length of values "
ValueError: Length of values (16492) does not match length of index (20615)
And here I print the raw csv.file and it has 20615 rows.
Review Text ... Sentiment
0 Absolutely wonderful - silky and sexy and comf... ... True
1 Love this dress! it's sooo pretty. i happene... ... True
3 I love, love, love this jumpsuit. it's fun, fl... ... True
4 This shirt is very flattering to all due to th... ... True
5 I love tracy reese dresses, but this one is no... ... False
... ... ... ...
23478 I was surprised at the positive reviews for th... ... False
23479 So i wasn't sure about ordering this skirt bec... ... True
23480 ... True
23481 I was very happy to snag this dress at such a ... ... True
23485 This dress in a lovely platinum is feminine an... ... True
[20615 rows x 6 columns]
For my program:
Firstly I use pandas to read the csv.file and use vectorizer to analyze the raw text.
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
class handle_data():
def __init__(self):
'''read the data from csv'''
self.file = pd.read_csv("DataSet.csv")
self.file_category = self.file[['Review Text','Rating','Class Name','Age']]
self.file_category.head()
'''vectorizer and analyze'''
self.vectorizer = CountVectorizer()
self.analyze = self.vectorizer.build_analyzer()
'''train-data and test-data'''
self.X_train = None
self.Y_train = None
self.X_test = None
self.Y_test = None
def CleanData(self):
'''clean the data and classification of words'''
self.file_category['Review Text'] = self.file_category['Review Text'].fillna('')
self.file_category['Word Counts'] = self.file_category['Review Text'].apply(self.wordcounts)
self.file_category.head()
def wordcounts(self,string):
'''calculate the number of emotion(字段解析器)'''
c = {}
if self.analyze(string):
d = {}
words = self.vectorizer.fit_transform([string]).toarray()
vocabulary = self.vectorizer.vocabulary_
for key,value in vocabulary.items():
d[value] = key
#print("vocabulary: ")
#print(d)
for index,i in enumerate(words[0]):
c[d[index]] = i
#print("words[0]")
#print(c)
return c
Then based on the Sentiment_Classifier like this, depending the 'Rating" we can get the conclusion whether the clouth has sentiment index.
class sentiment_classifier():
def classification(self,file_category):
'''classify the satisfaction of customers'''
file_category = file_category[file_category['Rating']!=3]
file_category['Sentiment'] = file_category['Rating']>=4
file_category.head()
return file_category
Next, I split the data to the train-data and test-data:
def train_test_data(self,file_category):
'''split the data to train and test'''
train_data, test_data = train_test_split(file_category,train_size=0.8,random_state=0)
X_train = self.vectorizer.fit_transform(train_data['Review Text'])
self.X_train = X_train
Y_train = train_data['Sentiment']
self.Y_train = Y_train
X_test = self.vectorizer.fit_transform(test_data['Review Text'])
self.X_test= X_test
Y_test = test_data["Sentiment"]
self.Y_test = Y_test
Finally, I use the Logistic Regression to construct the predict model. However, It report error when my code to handle the data by the Logistic Regression:
import datetime as dt
from sklearn.linear_model import LogisticRegression
class meachine_learning_methods():
'''The class contains 4 main methods to train the data, including Logistic_regression, Naive_Bayes, Support Vector Machine and Neural Network'''
def __init__(self,X_train,Y_train,file_category):
self.X_train = X_train
self.Y_train = Y_train
self.file_category = file_category.copy()
def Logistic_Regression(self):
'''The method of Logistic_Regression'''
start = dt.datetime.now()
lr = LogisticRegression()
lr.fit(self.X_train,self.Y_train)
self.file_category['Logistic Regression'] = lr.predict(self.X_train)
print('Elapsed time: ', str(dt.datetime.now() - start))
In addition, The main.class like this:
from Handle_Data import handle_data
import warnings
from Meachine_Learning_Methods import meachine_learning_methods
from Sentiment_Classifier import sentiment_classifier
class main():
warnings.filterwarnings('ignore')
#Read the data
#Handle Data
new_file = handle_data()
new_file.CleanData()
#print(new_file.file_category)
#Classification the dataset
new_classifiers = sentiment_classifier()
new_file.file_category = new_classifiers.classification(new_file.file_category)
print(new_file.file_category)
#Train the dataSet
new_file.train_test_data(new_file.file_category)
new_methods = meachine_learning_methods(new_file.X_train,new_file.Y_train,new_file.file_category)
new_methods.Logistic_Regression()
Please help me to handle the bug and thanks!