New Heuristics-based minified file detection

src/codegen/sdk/codebase/codebase_context.py

@@ -66,8 +66,10 @@
     ".*/tests/static/chunk-.*.js",
     ".*/ace/.*.js",
     "src/vs/platform/contextview/browser/contextMenuService.ts",
+    "*/semver.js",
     "*/compiled/*",
     "*.min.js",
+    "*@*.js",
 ]
 
 
@@ -213,7 +215,7 @@
             syncs[SyncType.ADD].append(self.to_absolute(filepath))
         logger.info(f"> Parsing {len(syncs[SyncType.ADD])} files in {self.projects[0].subdirectories or 'ALL'} subdirectories with {self.extensions} extensions")
         self._process_diff_files(syncs, incremental=False)
         files: list[SourceFile] = self.get_nodes(NodeType.FILE)
         logger.info(f"> Found {len(files)} files")
         logger.info(f"> Found {len(self.nodes)} nodes and {len(self.edges)} edges")
         if self.config.track_graph:
@@ -243,8 +245,8 @@
             elif diff.change_type == ChangeType.Modified:
                 files_to_sync[filepath] = SyncType.REPARSE
             elif diff.change_type == ChangeType.Renamed:
                 files_to_sync[diff.rename_from] = SyncType.DELETE
                 files_to_sync[diff.rename_to] = SyncType.ADD
             elif diff.change_type == ChangeType.Removed:
                 files_to_sync[filepath] = SyncType.DELETE
             else:
@@ -281,10 +283,10 @@
                 files_to_write.append((sync.path, sync.old_content))
                 modified_files.add(sync.path)
             elif sync.change_type == ChangeType.Renamed:
                 files_to_write.append((sync.rename_from, sync.old_content))
                 files_to_remove.append(sync.rename_to)
                 modified_files.add(sync.rename_from)
                 modified_files.add(sync.rename_to)
             elif sync.change_type == ChangeType.Added:
                 files_to_remove.append(sync.path)
                 modified_files.add(sync.path)

src/codegen/sdk/core/file.py

@@ -34,6 +34,7 @@
 from codegen.sdk.topological_sort import pseudo_topological_sort
 from codegen.sdk.tree_sitter_parser import get_parser_by_filepath_or_extension, parse_file
 from codegen.sdk.typescript.function import TSFunction
+from codegen.sdk.utils import is_minified_js
 from codegen.shared.decorators.docs import apidoc, noapidoc
 from codegen.visualizations.enums import VizNode
 
@@ -45,8 +46,6 @@
 
 logger = logging.getLogger(__name__)
 
-MINIFIED_FILE_THRESHOLD = 500
-
 
 @apidoc
 class File(Editable[None]):
@@ -581,8 +580,8 @@ def from_content(cls, filepath: str | PathLike | Path, content: str, ctx: Codeba
         path = ctx.to_absolute(filepath)
 
         # Sanity check to ensure file is not a minified file
-        if any(len(line) >= MINIFIED_FILE_THRESHOLD for line in content.split("\n")):
-            logger.info(f"File {filepath} is a minified file (Line length < {MINIFIED_FILE_THRESHOLD}). Skipping...", extra={"filepath": filepath})
+        if is_minified_js(content):
+            logger.info(f"File {filepath} is a minified file. Skipping...", extra={"filepath": filepath})
             return None
 
         ts_node = parse_file(path, content)

src/codegen/sdk/utils.py

@@ -1,6 +1,7 @@
 import os
 import re
 import shutil
+import statistics
 from collections.abc import Iterable
 from contextlib import contextmanager
 from xml.dom.minidom import parseString
@@ -83,7 +84,7 @@
 
 def find_first_function_descendant(node: TSNode) -> TSNode:
     type_names = [function_type.value for function_type in TSFunctionTypeNames]
     return find_first_descendant(node=node, type_names=type_names, max_depth=2)
 
 
 def find_index(target: TSNode, siblings: list[TSNode]) -> int:
@@ -103,7 +104,7 @@
     while node is not None and (max_depth is None or depth <= max_depth):
         if node.type in type_names:
             return node
         node = node.parent
         depth += 1
     return None
 
@@ -149,9 +150,9 @@
 
     # Want to prevent it from matching with part of the match within a comment
     else:
         if not ts_match.children:
             return ts_match
         comments = find_all_descendants(ts_match, "comment")
         # see if any of these comments partially overlaps with the match
         if any(comment.start_byte < start_byte < comment.end_byte or comment.start_byte < end_byte < comment.end_byte for comment in comments):
             return None
@@ -245,3 +246,59 @@
     if len(input) > max_chars:
         return input[:max_chars] + f"...(truncated from {len(input)} characters)."
     return input
+
+
+def is_minified_js(content):
+    """Analyzes a string to determine if it contains minified JavaScript code.
+
+    Args:
+        content: String containing JavaScript code to analyze
+
+    Returns:
+        bool: True if the content appears to be minified JavaScript, False otherwise
+    """
+    try:
+        # Skip empty content
+        if not content.strip():
+            return False
+
+        # Characteristics of minified JS files
+        lines = content.split("\n")
+
+        # 1. Check for average line length (minified files have very long lines)
+        line_lengths = [len(line) for line in lines if line.strip()]
+        if not line_lengths:  # Handle empty content case
+            return False
+
+        avg_line_length = statistics.mean(line_lengths)
+
+        # 2. Check for semicolon-to-newline ratio (minified often has ; instead of newlines)
+        semicolons = content.count(";")
+        newlines = len(lines) - 1
+        semicolon_ratio = semicolons / max(newlines, 1)  # Avoid division by zero
+
+        # 3. Check whitespace ratio (minified has low whitespace)
+        whitespace_chars = len(re.findall(r"[\s]", content))
+        total_chars = len(content)
+        whitespace_ratio = whitespace_chars / total_chars if total_chars else 0
+
+        # 4. Check for common minification patterns
+        has_common_patterns = bool(re.search(r"[\w\)]\{[\w:]+\}", content))  # Condensed object notation
+
+        # 5. Check for short variable names (common in minified code)
+        variable_names = re.findall(r"var\s+(\w+)", content)
+        avg_var_length = statistics.mean([len(name) for name in variable_names]) if variable_names else 0
+
+        # Decision logic - tuned threshold values
+        is_minified = (
+            (avg_line_length > 250)  # Very long average line length
+            and (semicolon_ratio > 0.8 or has_common_patterns)  # High semicolon ratio or minification patterns
+            and (whitespace_ratio < 0.08)  # Very low whitespace ratio
+            and (avg_var_length < 3 or not variable_names)  # Extremely short variable names or no vars
+        )
+
+        return is_minified
+
+    except Exception as e:
+        print(f"Error analyzing content: {e}")
+        return False

tests/unit/codegen/sdk/codebase/file/test_file.py

@@ -1,3 +1,4 @@
+import os
 import sys
 
 import pytest
@@ -214,7 +215,14 @@ def test_files_in_subdirectories_case_sensitivity(tmpdir) -> None:
 
 
 def test_minified_file(tmpdir) -> None:
-    with get_codebase_session(tmpdir=tmpdir, files={"file1.min.js": "console.log(123)", "file2.js": f"console.log(1{'0' * 1000})"}) as codebase:
+    with get_codebase_session(
+        tmpdir=tmpdir,
+        files={
+            "file1.min.js": "console.log(123)",
+            "file2.js": open(f"{os.path.dirname(__file__)}/example.min.js").read(),
+        },
+        programming_language=ProgrammingLanguage.TYPESCRIPT,
+    ) as codebase:
         # This should match the `*.min.js` pattern
         file1 = codebase.ctx.get_file("file1.min.js")
         assert file1 is None