added support for ts php and ruby

Chunker.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from CodeParser import CodeParser
-from Utils import count_tokens
+from utils import count_tokens
 
 
 

CodeParser.py

@@ -5,6 +5,12 @@ from tree_sitter import Language, Parser, Node
 from typing import Union, List
 import logging
 
+def return_simple_line_numbers_with_code(code: str) -> str:
+    code_lines = code.split('\n')
+    code_with_line_numbers = [f"Line {i + 1}: {line}" for i, line in enumerate(code_lines)]
+    joined_lines = "\n".join(code_with_line_numbers)
+    return joined_lines
+
 
 class CodeParser:
     # Added a CACHE_DIR class attribute for caching
@@ -17,7 +23,11 @@ class CodeParser:
             "py": "python",
             "js": "javascript",
             "jsx": "javascript",
-            "css": "css"
+            "css": "css",
+            "ts": "typescript",
+            "tsx": "typescript",
+            "php": "php",
+            "rb": "ruby"
         }
         if file_extensions is None:
             self.language_names = []
@@ -28,37 +38,65 @@ class CodeParser:
         self._install_parsers()
 
     def _install_parsers(self):
-        logging.basicConfig(level=logging.INFO)  # Configure logging
-
-        # Ensure cache directory exists
-        if not os.path.exists(self.CACHE_DIR):
-            os.makedirs(self.CACHE_DIR)
-
-        # Configure logging to output to the terminal
         logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-        for language in self.language_names:
-            repo_path = os.path.join(self.CACHE_DIR, f"tree-sitter-{language}")
-
-            if not os.path.exists(repo_path):
-                clone_command = f"git clone https://github.com/tree-sitter/tree-sitter-{language} {repo_path}"
-                result = subprocess.run(
-                    clone_command,
-                    shell=True,
-                    stdout=subprocess.PIPE,  # Capture standard output
-                    stderr=subprocess.PIPE  # Capture standard error
-                )
-
-                # Check if cloning was successful
-                if result.returncode != 0:
-                    logging.error(
-                        f"Failed to clone repository for {language}. Command: '{clone_command}'. Error: {result.stderr.decode('utf-8')}")
-                    raise Exception(f"Failed to clone repository for {language}")
-
-            build_path = os.path.join(self.CACHE_DIR, f"build/{language}.so")
-            Language.build_library(build_path, [repo_path])
-
-            self.languages[language] = Language(build_path, language)
+        try:
+            # Ensure cache directory exists
+            if not os.path.exists(self.CACHE_DIR):
+                os.makedirs(self.CACHE_DIR)
+
+            for language in self.language_names:
+                repo_path = os.path.join(self.CACHE_DIR, f"tree-sitter-{language}")
+
+                # Check if the repository exists and contains necessary files
+                if not os.path.exists(repo_path) or not self._is_repo_valid(repo_path, language):
+                    try:
+                        if os.path.exists(repo_path):
+                            logging.info(f"Updating existing repository for {language}")
+                            update_command = f"cd {repo_path} && git pull"
+                            subprocess.run(update_command, shell=True, check=True)
+                        else:
+                            logging.info(f"Cloning repository for {language}")
+                            clone_command = f"git clone https://github.com/tree-sitter/tree-sitter-{language} {repo_path}"
+                            subprocess.run(clone_command, shell=True, check=True)
+                    except subprocess.CalledProcessError as e:
+                        logging.error(f"Failed to clone/update repository for {language}. Error: {e}")
+                        continue
+
+                try:
+                    build_path = os.path.join(self.CACHE_DIR, f"build/{language}.so")
+                    
+                    # Special handling for TypeScript
+                    if language == 'typescript':
+                        ts_dir = os.path.join(repo_path, 'typescript')
+                        tsx_dir = os.path.join(repo_path, 'tsx')
+                        if os.path.exists(ts_dir) and os.path.exists(tsx_dir):
+                            Language.build_library(build_path, [ts_dir, tsx_dir])
+                        else:
+                            raise FileNotFoundError(f"TypeScript or TSX directory not found in {repo_path}")
+                    if language == 'php':
+                        php_dir = os.path.join(repo_path, 'php')
+                        Language.build_library(build_path, [php_dir])
+                    else:
+                        Language.build_library(build_path, [repo_path])
+                    
+                    self.languages[language] = Language(build_path, language)
+                    logging.info(f"Successfully built and loaded {language} parser")
+                except Exception as e:
+                    logging.error(f"Failed to build or load language {language}. Error: {str(e)}")
+
+        except Exception as e:
+            logging.error(f"An unexpected error occurred during parser installation: {str(e)}")
+
+    def _is_repo_valid(self, repo_path: str, language: str) -> bool:
+        """Check if the repository contains necessary files."""
+        if language == 'typescript':
+            return (os.path.exists(os.path.join(repo_path, 'typescript', 'src', 'parser.c')) and
+                     os.path.exists(os.path.join(repo_path, 'tsx', 'src', 'parser.c')))
+        elif language == 'php':
+            return os.path.exists(os.path.join(repo_path, 'php', 'src', 'parser.c'))
+        else:
+            return os.path.exists(os.path.join(repo_path, 'src', 'parser.c'))
 
     def parse_code(self, code: str, file_extension: str) -> Union[None, Node]:
         language_name = self.language_extension_map.get(file_extension)
@@ -112,6 +150,31 @@ class CodeParser:
                 'function_declaration': 'Function',
                 'arrow_function': 'Arrow Function',
                 'statement_block': 'Block',
+            },
+            'ts': {
+                'import_statement': 'Import',
+                'export_statement': 'Export',
+                'class_declaration': 'Class',
+                'function_declaration': 'Function',
+                'arrow_function': 'Arrow Function',
+                'statement_block': 'Block',
+                'interface_declaration': 'Interface',
+                'type_alias_declaration': 'Type Alias',
+            },
+            'php': {
+                'namespace_definition': 'Namespace',
+                'class_declaration': 'Class',
+                'method_declaration': 'Method',
+                'function_definition': 'Function',
+                'interface_declaration': 'Interface',
+                'trait_declaration': 'Trait',
+            },
+            'rb': {
+                'class': 'Class',
+                'method': 'Method',
+                'module': 'Module',
+                'singleton_class': 'Singleton Class',
+                'begin': 'Begin Block',
             }
         }
 
@@ -119,6 +182,8 @@ class CodeParser:
             return node_types[file_extension]
         elif file_extension == "jsx":
             return node_types["js"]
+        elif file_extension == "tsx":
+            return node_types["ts"]
         else:
             raise ValueError("Unsupported file type")
         
@@ -135,6 +200,17 @@ class CodeParser:
             'js': {
                 'comment': 'Comment',
                 'decorator': 'Decorator',  # Broadened category
+            },
+            'ts': {
+                'comment': 'Comment',
+                'decorator': 'Decorator',
+            },
+            'php': {
+                'comment': 'Comment',
+                'attribute': 'Attribute',
+            },
+            'rb': {
+                'comment': 'Comment',
             }
         }
 
@@ -263,11 +339,4 @@ class CodeParser:
             self.map_line_to_node_type(child, line_to_node_type, depth + 1)
 
         return line_to_node_type
-    
-    def print_simple_line_numbers_with_code(self, code: str):
-
-        code_lines = code.split('\n')
-
-        for i, line in enumerate(code_lines):
-            print(f"Line {i + 1}: {line}")
-
+    

requirements.txt

@@ -0,0 +1,68 @@
+aiofiles==23.2.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.3.0
+attrs==23.2.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+contourpy==1.2.0
+cycler==0.12.1
+fastapi==0.110.0
+ffmpy==0.3.2
+filelock==3.13.1
+fonttools==4.49.0
+fsspec==2024.2.0
+gradio==4.19.2
+gradio_client==0.10.1
+h11==0.14.0
+httpcore==1.0.4
+httpx==0.27.0
+huggingface-hub==0.20.3
+idna==3.6
+importlib_resources==6.1.2
+Jinja2==3.1.3
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.8.3
+mdurl==0.1.2
+numpy==1.26.4
+orjson==3.9.15
+packaging==23.2
+pandas==2.2.1
+pillow==10.2.0
+pydantic==2.6.2
+pydantic_core==2.16.3
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+referencing==0.33.0
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.0
+rpds-py==0.18.0
+ruff==0.2.2
+semantic-version==2.10.0
+setuptools==69.1.1
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.36.3
+tomlkit==0.12.0
+toolz==0.12.1
+tqdm==4.66.2
+tree-sitter==0.20.4
+typer==0.9.0
+typing_extensions==4.10.0
+tzdata==2024.1
+urllib3==2.2.1
+uvicorn==0.27.1
+websockets==11.0.3

test_code_chunker.py

@@ -1,13 +1,11 @@
 import unittest
 from unittest.mock import patch
 from Chunker import Chunker, CodeChunker
-from utils import load_json
 import tiktoken
-import json
-import os
+from utils import load_json
 
-                                
 
+                        
 # Mocking the count_tokens function as it's external and not the focus of these tests
 def mock_count_tokens(string: str, encoding_name='gpt-4') -> int:
     """Returns the number of tokens in a text string."""
@@ -18,7 +16,7 @@ def mock_count_tokens(string: str, encoding_name='gpt-4') -> int:
 # Python Test Class
 class TestCodeChunkerPython(unittest.TestCase):
     def setUp(self):
-        self.patcher = patch('app.util.TextChunker.Chunker.count_tokens', side_effect=mock_count_tokens)
+        self.patcher = patch('utils.count_tokens', side_effect=mock_count_tokens)
         self.mock_count_tokens = self.patcher.start()
         self.code_chunker = CodeChunker(file_extension='py')
         self.mock_codebase = load_json('mock_codefiles.json')