Update svm_model.py

svm_model.py

@@ -15,7 +15,7 @@ except AttributeError:
 else:
     ssl._create_default_https_context = _create_unverified_https_context
 # print(f"nltk version: {nltk.__version__}")
-nltk.download('stopwords')
+# nltk.download('stopwords')
 
 # 
 class SVMModel:
@@ -89,13 +89,21 @@ class SVM:
         self.data_folder = '.'
         
         print(f"Start loading data")
-        self._load_data()
+        # self._load_data()
         
         print(f"Setting vectorizer")
-        self.vectorizer = TfidfVectorizer(max_features=4000, min_df=7, max_df=0.8, stop_words=stopwords.words('english'))
+        # self.vectorizer = TfidfVectorizer(max_features=4000, min_df=7, max_df=0.8, stop_words=stopwords.words('english'))
+        # parmas_dict = np.load("svm_vectorizer.npy", allow_pickle=True).item()
+        # print(f"parmas_dict: {parmas_dict.keys()}")
         
-        print(f"Start preprocessing data")
-        self._preprocess_data()
+        
+        # self.vectorizer.set_params(**parmas_dict)
+        
+        import pickle
+        self.vectorizer = pickle.load(open("tfidf.pickle", "rb"))
+        
+        # print(f"Start preprocessing data")
+        # self._preprocess_data()
         
         # self.setup_model()
         self.setup_model_ours()
@@ -132,8 +140,15 @@ class SVM:
     
     def _preprocess_data(self, ):
         self.X_train = self.vectorizer.fit_transform(self.x_train).toarray()
+        
+        import pickle
+        # self.vectorizer_params = self.vectorizer.get_params()
+        # np.save("svm_vectorizer.npy", self.vectorizer_params)
+        pickle.dump(self.vectorizer, open("tfidf.pickle", "wb"))
+        
         self.X_test = self.vectorizer.transform(self.x_test).toarray()
         
+        # self.X_train = self.vectorizer.transform