Restructure database and stabilize back-end logic

.gitignore

@@ -4,6 +4,6 @@
 engine/repos/
 venv/
 web/credentials.py
-**/*.json
-*.err
-*.log
+clones_*.json
+qa/*.err
+qa/*.log

clear_db.sh

@@ -2,5 +2,5 @@
 
 set -e
 
-psql -f web/prepare_tables.pgsql code_duplication
+psql -f web/prepare_tables.pgsql cyclone
 rm -rf engine/repos/

cli/__init__.py

@@ -0,0 +1 @@
+"""Package containing implementation of the application's CLI (command line interface)."""

cli/__main__.py

@@ -1,3 +1,9 @@
+"""
+Intended entry point of the application's CLI.
+
+Please use the following command to run the CLI: `python3 -m cli`
+"""
+
 from cli.app import main
 
 if __name__ == "__main__":

cli/app.py

@@ -1,19 +1,18 @@
+"""Module containing the CLI's core logic."""
+
 import sys
 import os
 from datetime import datetime
-from engine.preprocessing.args_handler import handle_args
+from .args_handler import handle_args
 from engine.preprocessing.module_parser import get_modules_from_dir
 from engine.algorithms.algorithm_runner import run_two_repos, IODINE
 from engine.utils.benchmark import time_snap
 from fastlog import log
-from engine.errors.UserInputError import UserInputError
+from engine.errors.user_input import UserInputError
 
 
 def main():
-    """
-    Entry point of the application.
-    """
-
+    """Entry point of the application."""
     try:
         # Parse command line arguments
         repos = handle_args(sys.argv)
@@ -46,7 +45,8 @@ def main():
         # Create output directory if it doesn't exist and print output
         output_path = os.getcwd()
         now = datetime.now()
-        output_filename = "clones_" + f"{now.year}-{now.month}-{now.day}_{now.hour}-{now.minute}-{now.second}" + ".json"
+        output_filename = "clones_" + \
+            f"{now.year}-{now.month}-{now.day}_{now.hour}-{now.minute}-{now.second}" + ".json"
         os.makedirs(output_path, exist_ok=True)
         with open(os.path.join(output_path, output_filename), "w") as output_file:
             output_file.write(clones.json())

cli/args_handler.py

@@ -0,0 +1,75 @@
+"""Module containing functions for handling command-line arguments supplied by the user."""
+
+from os.path import isdir
+from engine.errors.user_input import UserInputError
+from engine.preprocessing.repoinfo import RepoInfo
+
+
+_USAGE_TEXT = """\
+Usage:
+    python3 -m cli <first repository> <second repository> - Repository comparison mode
+    python3 -m cli <repository>                           - Single repository mode
+
+Valid repository path formats:
+    Short GitHub repository path                - username/repository
+    Full remote repository path                 - https://github.com/username/repository
+    Absolute or relative local directory path   - /home/user/directory"""
+
+
+def repo_path_to_local_path(repo_path):
+    """
+    Convert a repository path into a local file system path.
+
+    This repository path is extended (compared to the repository path
+    used by RepoInfo) by adding support for existing local directories.
+
+    The process typically includes checking for a local directory or
+    parsing a full or short repository URL and then cloning the repository.
+
+    Arguments:
+        repo_path {string} -- Path of a remote repository or a local directory.
+
+    Returns:
+        string -- Path of a local directory equivalent to the one
+                  specified by the repository path.
+                  In case of local directories, the paths are equal.
+
+    """
+    if isdir(repo_path):
+        return repo_path
+
+    info = RepoInfo.parse_repo_info(repo_path)
+
+    if info and info.clone_or_pull():
+        return info.dir
+    else:
+        return None
+
+
+def handle_args(argv):
+    """
+    Check the command line arguments and handles them.
+
+    If there is any problem, an error message will be printed
+    and the script will exit with a non-zero exit code.
+    If everything goes right, tuple of local repository paths will be returned.
+
+    Arguments:
+        argv -- List of command line arguments.
+
+    Returns:
+        tuple[string] -- Tuple of local repository paths.
+
+    """
+    if len(argv) == 1 or (len(argv) == 2 and argv[1] in ['-h', '--help', '--usage']):
+        # Special case where the usage text is printed using the built-in
+        # print function instead of the logging library because
+        # the app exits right after the message is displayed.
+        print(_USAGE_TEXT)
+        raise UserInputError(None, 0)
+
+    if len(argv) < 2 or len(argv) > 3:
+        raise UserInputError(
+            f"Invalid number of command line arguments: {len(argv) - 1}")
+
+    return tuple(repo_path_to_local_path(a) for a in argv[1:])

engine/__init__.py

@@ -0,0 +1 @@
+"""Package containing the entire code clone detection engine."""

engine/algorithms/__init__.py

@@ -1,3 +1,5 @@
+"""Package containing all implemented clone detection algorithms."""
+
 OXYGEN = "oxygen"
 CHLORINE = "chlorine"
 IODINE = "iodine"

engine/algorithms/algorithm_runner.py

@@ -1,13 +1,15 @@
+"""Module used for algorithm abstraction by providing a common interface."""
+
 from .oxygen.oxygen import oxygen
 from .chlorine.chlorine import chlorine_single_repo, chlorine_two_repos
 from .iodine.iodine import iodine
-from ..errors.UserInputError import UserInputError
+from engine.errors.user_input import UserInputError
 from . import OXYGEN, IODINE, CHLORINE
 
 
 def run_single_repo(modules, algorithm):
     """
-    Runs the specified code clone detection algorithm on a single repository.
+    Run the specified code clone detection algorithm on a single repository.
 
     Arguments:
         modules {list[list[TreeNode]]} -- List of the repo's modules.
@@ -18,8 +20,8 @@ def run_single_repo(modules, algorithm):
 
     Returns:
         DetectionResult -- Result of the code clone detection.
-    """
 
+    """
     if algorithm == OXYGEN:
         return oxygen(modules)
     elif algorithm == CHLORINE:
@@ -30,7 +32,7 @@ def run_single_repo(modules, algorithm):
 
 def run_two_repos(modules1, modules2, algorithm):
     """
-    Runs the specified code clone detection algorithm on two repositores.
+    Run the specified code clone detection algorithm on two repositores.
 
     Arguments:
         modules1 {list[list[TreeNode]]} -- List of first repo's modules.
@@ -42,8 +44,8 @@ def run_two_repos(modules1, modules2, algorithm):
 
     Returns:
         DetectionResult -- Result of the code clone detection.
-    """
 
+    """
     if algorithm == CHLORINE:
         return chlorine_two_repos(modules1, modules2)
     elif algorithm == IODINE:

engine/algorithms/chlorine/__init__.py

@@ -0,0 +1 @@
+"""Package containing the Chlorine algorithm and its helper functions."""

engine/algorithms/chlorine/chlorine.py

@@ -1,8 +1,10 @@
+"""Module containing implementation of the Chlorine algorithm."""
+
 from collections import defaultdict
+from itertools import chain
 from ...utils.benchmark import time_snap
-from ...utils.list_tools import flatten
-from ...results.DetectedClone import DetectedClone
-from ...results.DetectionResult import DetectionResult
+from ...results.detected_clone import DetectedClone
+from ...results.detection_result import DetectionResult
 
 # Minimum weight of a single node used in comparison.
 _MIN_NODE_WEIGHT = 50
@@ -24,6 +26,7 @@ def _get_skeleton_recursive(node):
 def _can_be_compared(node1, node2):
     """
     First get rid of nodes with a weight below the specified threshold.
+
     Checks if two nodes can be possible compared with each other.
     In order to be comparable, the nodes must have an equal value
     and they must have the exact same number of children.
@@ -34,6 +37,7 @@ def _can_be_compared(node1, node2):
 
     Returns:
         bool -- True if nodes can be compared, False if they cannot.
+
     """
     return \
         node1.weight >= _MIN_NODE_WEIGHT and \
@@ -44,8 +48,9 @@ def _can_be_compared(node1, node2):
 
 def _type1_compare(node1, node2):
     """
-    Compares two nodes and returns the weight of their matching subtrees
-    and a skeleton string representing their common syntax tree skeleton.
+    Compare two nodes and return the weight of their matching subtree.
+
+    Also return a string representing their common syntax tree skeleton.
 
     Arguments:
         node1 {TreeNode} -- First node.
@@ -54,8 +59,8 @@ def _type1_compare(node1, node2):
     Returns:
         int -- Weight of the matching subtrees.
         string -- Common skeleton of the two nodes.
-    """
 
+    """
     combined_weight = node1.weight + node2.weight
 
     if not _can_be_compared(node1, node2):
@@ -80,17 +85,16 @@ def _type1_compare(node1, node2):
 
 def _compare_internal(n1, n2, ignore_set, match_dict, skeleton_weight_dict):
     """
-    Common logic shared by single-repo analysis and
-    two repository comparison mode.
+    Run common logic shared by single-repo analysis and 2-repo comparison mode.
 
     Arguments:
         n1 {TreeNode} -- First node.
         n2 {TreeNode} -- Second node.
         ignore_set {set[TreeNode]} -- Set of nodes to ignore.
         match_dict {dict[string: set[TreeNode]]} -- Origin nodes of matches.
         skeleton_weight_dict {dict[string: int]} -- Skeleton weights.
-    """
 
+    """
     if not _can_be_compared(n1, n2):
         return
 
@@ -109,13 +113,13 @@ def _compare_internal(n1, n2, ignore_set, match_dict, skeleton_weight_dict):
 
 def _dict_to_result(match_dict, skeleton_weight_dict):
     """
-    Compiles the detection result together from the input dictionaries.
+    Compile the detection result together from the input dictionaries.
 
     Arguments:
         match_dict {dict[string: set[TreeNode]]} -- Origin nodes of matches.
         skeleton_weight_dict {dict[string: int]} -- Skeleton weights.
-    """
 
+    """
     clones = []
 
     for k, v in match_dict.items():
@@ -128,8 +132,9 @@ def _dict_to_result(match_dict, skeleton_weight_dict):
 
 def chlorine_single_repo(modules):
     """
-    Finds all clones satisfying the settings at the top of this source file
-    in a single repository given its modules.
+    Find all clones in a single repository given its modules.
+
+    Clones must satisfy the settings at the top of this source file.
     Detected code clones are printed on STDOUT, including the common skeleton,
     path to each clones (source file path, line number, column offset),
     size of each clone (number of nodes in its syntax tree) and their
@@ -140,8 +145,8 @@ def chlorine_single_repo(modules):
 
     Returns:
         DetectionResult -- Result of the code clone detection.
-    """
 
+    """
     time_snap("Function started")
 
     nodes = [m[0] for m in modules]
@@ -186,7 +191,8 @@ def chlorine_single_repo(modules):
 
 def chlorine_two_repos(modules1, modules2):
     """
-    Finds code clones between two repositories given their module lists.
+    Find code clones between two repositories given their module lists.
+
     Clones must satisfy rules defined at the top of this source file.
     Detected clones are printed on STDOUT.
     See `find_clones_in_repo(repo_url)` for details on output format.
@@ -197,12 +203,12 @@ def chlorine_two_repos(modules1, modules2):
 
     Returns:
         DetectionResult -- Result of the code clone detection.
-    """
 
+    """
     time_snap("Function started")
 
     repo1_nodes = [m[0] for m in modules1]
-    repo2_nodes = flatten(modules2)
+    repo2_nodes = list(chain.from_iterable(modules2))
 
     time_snap("Module lists optimized")
 

engine/algorithms/iodine/__init__.py

@@ -0,0 +1 @@
+"""Package containing the Iodine algorithm and its helper functions."""

engine/algorithms/iodine/anti_unification.py

@@ -1,18 +1,18 @@
-from ...nodes.PatternNode import PatternNode
+from ...nodes.pattern import PatternNode
 
 
 def anti_unify(list1, list2, index1, index2, worktable):
     """
-    Creates a tree of PatternNodes from two lists
+    Create a tree of PatternNodes from two lists.
 
     Arguments:
         list1 {list of TreeNodes} -- first tree to be compared
         list2 {list of TreeNodes} -- second tree to be compared
         index1 {int} -- index of current TreeNode to be compared from list1
         index2 {int} -- index of current TreeNode to be compared from list2
         worktable {2D boolean array} -- keeps track of which two nodes have been checked together
-    """
 
+    """
     # mark the pair as checked/True
     worktable[index1].set(True, index2)
     # determine if subtrees are the same (and lengths same)

engine/algorithms/iodine/iodine.py

@@ -1,11 +1,13 @@
+"""Module containing the Iodine algorithm's interface."""
+
 from .pattern_collection import pattern_collection
-from ...results.DetectedClone import DetectedClone
-from ...results.DetectionResult import DetectionResult
+from ...results.detected_clone import DetectedClone
+from ...results.detection_result import DetectionResult
 
 
 def iodine(module_list_1, module_list_2):
     """
-    Finds clones between the two modules by comparing all possible subtrees of
+    Find clones between the two modules by comparing all possible subtrees of
     their methods. Returns the results.
 
     Arguments:
@@ -14,6 +16,7 @@ def iodine(module_list_1, module_list_2):
 
     Returns:
         DetectionResult -- Result of the code clone detection.
+
     """
     clusters = []
     for module_tree_1 in module_list_1: