Similarity matrix fix, another entity fix, more print stuff and a bit of model fine tune

database/query.py

@@ -19,7 +19,7 @@ class DatabaseService:
         start_date: Optional[datetime] = None,
         end_date: Optional[datetime] = None,
         topic: Optional[str] = None,
-        entities: Optional[List[str]] = None,
+        entities: Optional[List[tuple[str,str]]] = None,
         limit: int = 10
     ) -> List[Dict[str, any]]:
         try:
@@ -31,9 +31,6 @@ class DatabaseService:
                 dbname=self.DB_NAME
             ) as conn:
                 with conn.cursor() as cursor:
-                    # Enable unaccent extension if not already enabled
-                    cursor.execute("CREATE EXTENSION IF NOT EXISTS unaccent;")
-
                     # Base query with date range and topic filters
                     base_query = sql.SQL('''
                         WITH filtered_articles AS (
@@ -124,7 +121,7 @@ class DatabaseService:
 
                     # Fallback: Retry with fewer filters if no results
                     if not articles:
-                        print("No articles found with all filters. Relaxing filters...")
+                        print("No articles found with entities...")
                         fallback_query = sql.SQL('''
                             SELECT
                                 content,

database/query_processor.py

@@ -30,6 +30,7 @@ class QueryProcessor:
             # Query processing
             query_embedding = self.embedding_model.encode(query).tolist()
             entities = self.nlp_model.extract_entities(query)
+            print(f"Extracted entities: {entities}")
             
             # Database search
             articles = await self._execute_semantic_search(
@@ -44,6 +45,7 @@ class QueryProcessor:
                 return {"message": "No articles found", "articles": []}
 
             # Summary generation
+            print("Starting summary generation")
             summary_data = self._generate_summary(articles)
             return {
                 "summary": summary_data["summary"],
@@ -113,15 +115,19 @@ class QueryProcessor:
                     "summary": "No content available for summarization",
                     "key_sentences": []
                 }
-
+            
+            print("Starting first summary generation")
             embeddings = self.embedding_model.encode(sentences)
-            similarity_matrix = np.inner(embeddings, embeddings)
+            print("Embeddings generated first summary")
+            similarity_matrix = self.embedding_model.similarity(embeddings, embeddings).numpy
             centrality_scores = degree_centrality_scores(similarity_matrix, threshold=None)
             
             top_indices = np.argsort(-centrality_scores)[:10]
             key_sentences = [sentences[idx].strip() for idx in top_indices]
             combined_text = ' '.join(key_sentences)
             
+            print(f"First summary done with: {len(key_sentences)} sentences")
+
             return {
                 "summary": self.summarization_model.summarize(combined_text),
                 "key_sentences": key_sentences

models/summarization.py

@@ -19,9 +19,9 @@ class SummarizationModel:
             inputs,
             max_length=max_length,
             min_length=min_length,
-            num_beams=5,
+            num_beams=4,
             no_repeat_ngram_size=3,
-            early_stopping=False
+            early_stopping=True,
         )
         
         return self.tokenizer.decode(summary_ids[0], skip_special_tokens=True)