dropped previous history

.gitignore

@@ -0,0 +1,24 @@
+.DS_Store
+Thumbs.db
+
+# tipycal python ignores
+__pycache__/
+*.py[cod]
+*$py.class
+.python-version
+
+.Python
+build/
+dist/
+sdist/
+var/
+
+.idea/
+
+.env
+*.log
+
+# project specific ignore list
+common/settings/__init__.py
+project-crawler/*.json
+visual-inspector/html/

README.md

@@ -0,0 +1,40 @@
+# Sherlock Code Inspector
+
+The source code inspector is back in town
+
+## ToDo list
+
+* [x] parse directory tree, gather size, loc, etc
+* [x] generate cluster chart from json
+* [x] parse sources for imports
+* [x] parse sourced for nested blocks
+* [x] show inspector metadata when selecting a file
+* [x] aggregate analysis metadata up to project root
+* [x] detect indentation width
+* [x] dockerize
+* [x] a smarter import resolution
+* [ ] improve coupling based on usage of top-level symbols
+* [ ] improve UI layout
+
+Next sprint
+* [ ] run analysis as a service
+* [ ] add support for cloning a git repo and run analysis on it
+* [ ] filtering
+
+* [ ] render a project as static portable html
+* [ ] complexity trends
+* [ ] symbolic wordcloud
+
+* [x] parse c/c++/obj-c files
+* [x] parse python files
+* [x] parse java/kotlin files
+* [ ] add swift parser
+* [ ] add C# parser
+* [ ] add go parser
+* [ ] add javascript parser
+
+* [ ] external libraries report
+
+* [ ] add a view to display aggregated data
+* [ ] add an admin (or restricted) view for adding a new git repo url
+

project-crawler/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3-slim
+
+RUN apt-get update && \
+    apt-get install -y entr git && \
+    pip install --upgrade pip
+
+RUN mkdir /main
+WORKDIR /main
+
+# prepare the machine
+COPY ./etc/ ./etc/
+RUN pip install -r ./etc/requirements.txt
+
+# pick all source files
+COPY ./*.py ./
+COPY ./common/ ./common/
+COPY ./entrypoint.sh /
+
+RUN mkdir ./root
+RUN mkdir ./out
+VOLUME ["/main/root"]
+VOLUME ["/main/out"]
+
+ENTRYPOINT [ "/entrypoint.sh" ]

project-crawler/build.sh

@@ -0,0 +1,3 @@
+set -e
+docker build -t sherlock-crawler .
+

project-crawler/code_parser.py

@@ -0,0 +1,15 @@
+from datetime import datetime
+
+import ast
+import os
+import re
+import timeit
+
+
+
+def parse_swift_imports(source_code):
+    import_refs = re.compile(r'\s*?import\s+(?:(?:typealias|struct|class|enum|protocol|let|var|func)\s+)?([/\w\.\-\+]+)\s*?')
+    imports = import_refs.findall(source_code)
+
+    return imports
+

project-crawler/code_parser_interface.py

@@ -0,0 +1,86 @@
+import abc
+import re
+
+class CodeParserInterface(metaclass=abc.ABCMeta):
+    """Interface definition for sherlock code parser"""
+
+    @property
+    def supported_extensions(self):
+        return self._supported_extensions
+
+    @property
+    def decision_markers(self):
+        return self._decision_markers
+
+    @abc.abstractmethod
+    def remove_comments_and_literals(self, source_code, messages): pass
+
+    @abc.abstractmethod
+    def compute_nested_complexity(self, source_code, messages): pass
+
+    @abc.abstractmethod
+    def parse_imports(self, source_code, messages): pass
+
+    @abc.abstractmethod
+    def resolve_imports(self, root): pass
+
+    def count_effective_lines_of_code(self, source_code, messages):
+        blank_lines = 0
+        effective_lines = 0
+        all_lines = 0
+        for line in source_code.split('\n'):
+            all_lines += 1
+            if line.strip():
+                effective_lines += 1
+            else:
+                blank_lines += 1
+
+        if 300 <= effective_lines < 500:
+            messages.append('[info] Arguably many lines of code, this may be ok for now.')
+        elif 500 <= effective_lines < 1000:
+            messages.append('[warning] Quite many lines of code, plan on refactoring.')
+        elif 1000 <= effective_lines:
+            messages.append('[error] Way too many lines of code, refactoring is required.')
+
+        return {
+            'loc': all_lines,
+            'sloc': effective_lines,
+            'blank_lines': blank_lines,
+        }
+
+    def compute_decision_complexity(self, source_code, messages):
+        decisions = re.compile(r'(?:^|\W)('+'|'.join(self.decision_markers)+')(?:$|\W)')
+        found = decisions.findall(source_code)
+        decision_count = len(found)
+
+        if 40 <= decision_count < 60:
+            messages.append('[info] Arguably many decisions, it is ok if it makes other files less complicated.')
+        elif 60 <= decision_count < 100:
+            messages.append('[warning] Quite many decisions, consider adding more unit tests and review the entire file.')
+        elif 100 <= decision_count:
+            messages.append('[error] Way too many decisions, full code coverage is required.')
+
+        return decision_count
+
+    def inspect(self, source_code, messages):
+        meta = {}
+        clean_source_code = self.remove_comments_and_literals(source_code, messages)
+
+        line_stats = self.count_effective_lines_of_code(clean_source_code, messages)
+        meta.update(line_stats)
+
+        meta['nested_complexity'] = self.compute_nested_complexity(clean_source_code, messages)
+        meta['decision_complexity'] = self.compute_decision_complexity(clean_source_code, messages)
+        meta['imports'] = self.parse_imports(source_code, messages)
+
+        meta['cognitive_complexity'] = sum([
+            meta.get('nested_complexity') or 1,
+            meta.get('decision_complexity'),
+            len(meta.get('imports', [])),
+        ])
+
+        return meta
+
+
+if __name__ == '__main__':
+    print(f'{__file__} is a true module, you can only import it.')

project-crawler/common/settings/__init__.py

@@ -0,0 +1 @@
+PROJECT_ROOT = "/Users/fleorin/src/sherlock-codes"

project-crawler/common/utils.py

@@ -0,0 +1,31 @@
+import pprint
+
+
+def static_var(variable, value):
+    def decorate(function):
+        setattr(function, variable, value)
+        return function
+    return decorate
+
+
+def print_stage(text, row_size=80):
+    """Pretty banner stage printing helper"""
+    filler=' '*(row_size-4-len(text))
+    print(f"{'*'*row_size}");
+    print(f"* {text}{filler} *")
+    print(f"{'*'*row_size}");
+
+
+def pp(x, pretext=None):
+    pp = pprint.PrettyPrinter(indent=4)
+
+    if pretext:
+        print(f'{pretext} >>>')
+
+    pp.pprint(x)
+
+    if pretext:
+        print(f'<<<')
+
+if __name__ == '__main__':
+    print(f'{__file__} is a pure python module, import it.')