Update app.py

app.py

@@ -47,10 +47,13 @@ class DatasetSearcher:
             
             # Store column information
             self.columns = list(self.df.columns)
-            self.text_columns = [col for col in self.columns 
-                               if self.df[col].dtype == 'object' 
-                               and not any(term in col.lower() 
-                                         for term in ['embed', 'vector', 'encoding'])]
+            # Identify searchable columns
+            self.text_columns = []
+            for col in self.columns:
+                if col.lower() not in ['embed', 'vector', 'encoding']:
+                    sample_val = self.df[col].iloc[0] if not self.df.empty else None
+                    if isinstance(sample_val, (str, int, float, list, dict)) or sample_val is None:
+                        self.text_columns.append(col)
             
             # Update session state columns
             st.session_state['search_columns'] = self.text_columns
@@ -66,16 +69,32 @@ class DatasetSearcher:
     def prepare_features(self):
         """Prepare text embeddings for semantic search"""
         try:
-            # Combine text columns for embedding
-            combined_text = self.df[self.text_columns].fillna('').agg(' '.join, axis=1)
+            # Process text columns and handle different data types
+            processed_texts = []
+            for _, row in self.df.iterrows():
+                row_texts = []
+                for col in self.text_columns:
+                    value = row[col]
+                    if isinstance(value, (list, dict)):
+                        # Convert lists or dicts to string representation
+                        row_texts.append(str(value))
+                    elif isinstance(value, (int, float)):
+                        # Convert numbers to strings
+                        row_texts.append(str(value))
+                    elif value is None:
+                        row_texts.append('')
+                    else:
+                        # Handle string values
+                        row_texts.append(str(value))
+                processed_texts.append(' '.join(row_texts))
             
             # Create embeddings in batches to manage memory
             batch_size = 32
             all_embeddings = []
             
             with st.spinner("Preparing search features..."):
-                for i in range(0, len(combined_text), batch_size):
-                    batch = combined_text[i:i+batch_size].tolist()
+                for i in range(0, len(processed_texts), batch_size):
+                    batch = processed_texts[i:i+batch_size]
                     embeddings = self.text_model.encode(batch)
                     all_embeddings.append(embeddings)
             
@@ -98,10 +117,23 @@ class DatasetSearcher:
         search_columns = [column] if column and column != "All Fields" else self.text_columns
         keyword_scores = np.zeros(len(self.df))
         
+        query_lower = query.lower()
         for col in search_columns:
             if col in self.df.columns:
-                matches = self.df[col].fillna('').str.lower().str.count(query.lower())
-                keyword_scores += matches
+                for idx, value in enumerate(self.df[col]):
+                    if isinstance(value, (list, dict)):
+                        # Search in string representation of lists or dicts
+                        text = str(value).lower()
+                    elif isinstance(value, (int, float)):
+                        # Convert numbers to strings for searching
+                        text = str(value).lower()
+                    elif value is None:
+                        text = ''
+                    else:
+                        # Handle string values
+                        text = str(value).lower()
+                    
+                    keyword_scores[idx] += text.count(query_lower)
         
         # Combine scores
         combined_scores = 0.5 * similarities + 0.5 * (keyword_scores / max(1, keyword_scores.max()))