Adds RightTyper support

README.md

@@ -33,6 +33,7 @@
 | [Type4Py](https://github.com/saltudelft/type4py)                      |                                                                      |
 | [GPT](https://openai.com)                                             |                                                                      |
 | [Ollama](https://ollama.ai)                                           |                                                                      |
+| [RightTyper](https://github.com/RightTyper/RightTyper)                |                                                                      |
 
 ---
 

src/main_runner.py

@@ -22,6 +22,7 @@
     ScalpelRunner,
     Type4pyRunner,
     LLMRunner,
+    RightTyperRunner,
 )
 from utils import FileHandler
 
@@ -65,10 +66,11 @@ def get_args():
             "hityper",
             "type4py",
             "hityperdl",
+            "righttyper",
         ],
         help=(
             "List of runners to execute. Choices are:"
-            "headergen, pyright, scalpel, jedi, hityper, type4py, hityperdl"
+            "headergen, pyright, scalpel, jedi, hityper, type4py, hityperdl, righttyper"
         ),
     )
     parser.add_argument(
@@ -164,6 +166,15 @@ def main():
                 "config": config,
             },
         ),
+        "righttyper": (
+            RightTyperRunner,
+            {
+                "debug": args.debug,
+                "nocache": args.nocache,
+                "custom_benchmark_dir": args.custom_benchmark_dir,
+                "config": config,
+            },
+        ),
         # PySonar2Runner,
         # PytypeRunner,
         # PyreRunner,

src/runner_class.py

@@ -457,3 +457,22 @@ def spawn_docker_instance(self):
             ],
         )
         return container
+
+
+class RightTyperRunner(TypeEvalPyRunner):
+    def __init__(
+        self,
+        host_results_path,
+        config,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
+        super().__init__(
+            "righttyper",
+            "./target_tools/righttyper",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
+        )
+        self.config = config

src/target_tools/righttyper/Dockerfile

@@ -0,0 +1,24 @@
+# Pull the Python base image
+FROM python:3.12-slim-bullseye
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+#ENV PYTHONUNBUFFERED 1
+
+# Set work directory
+WORKDIR /app
+
+# Install dependencies
+RUN apt-get update \
+    && apt-get -y install gcc
+RUN apt-get -y install git
+
+COPY requirements.txt /app/requirements.txt
+
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+
+COPY src /tmp/src
+
+# Keep the container alive
+CMD ["bash"]

src/target_tools/righttyper/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/RightTyper/RightTyper.git@for_TypeEvalPy

src/target_tools/righttyper/src/runner.py

@@ -0,0 +1,97 @@
+import argparse
+import json
+import logging
+from pathlib import Path
+from sys import stdout
+
+import translator
+import utils
+
+# Create a logger
+logger = logging.getLogger("runner")
+logger.setLevel(logging.DEBUG)
+
+file_handler = logging.FileHandler("/tmp/righttyper_log.log")
+file_handler.setLevel(logging.DEBUG)
+
+console_handler = logging.StreamHandler(stdout)
+console_handler.setLevel(logging.DEBUG)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+logger.addHandler(file_handler)
+logger.addHandler(console_handler)
+
+
+def list_python_files(folder_path):
+    return sorted([
+        path
+        for path in folder_path.rglob("*.py")
+        if (path.parent / (path.stem + "_gt.json")).exists()
+    ])
+
+
+def process_file(file_path, benchmark_path):
+    import subprocess
+    import sys
+    # using Python 3.10 syntax avoids typing.Self, which TypeEvalPy doesn't expect
+    subprocess.run(
+        [sys.executable, "-m", "righttyper",
+         "--no-output-files", "--json-output",
+         "--no-sampling", "--use-top-pct=100", "--no-simplify-type-sets",
+         "--python-version=3.10",
+         "--root", benchmark_path, file_path],
+        cwd=file_path.parent,
+        check=False,    # many snippets exit with an exception, etc.
+    )
+    (file_path.parent / "righttyper.log").unlink()
+
+
+def main_runner(args):
+    benchmark_path = Path(args.bechmark_path)
+    python_files = list_python_files(benchmark_path)
+    error_count = 0
+    for file in python_files:
+        try:
+            logger.info(file)
+            # Run the inference here and gather results in /tmp/results
+            process_file(file, benchmark_path)
+            result_file = file.parent / "righttyper.json"
+            with result_file.open("r") as f:
+                result = json.load(f)
+
+            # save RightTyper's data
+            result_file.rename(file.parent / (file.stem + "_rt.json"))
+
+            # Translate the results into TypeEvalPy format
+            translated = translator.process_annotations(result, file.parent, strip_generics=True)
+
+            # Save translated file to the same folder /tmp/results
+            json_file_path = str(file).replace(".py", "_result.json")
+            with open(json_file_path, "w") as json_file:
+                json.dump(translated, json_file, indent=4)
+
+        except Exception as e:
+            logger.info(f"Command returned non-zero exit status: {e} for file: {file}")
+            error_count += 1
+
+    logger.info(f"Runner finished with errors:{error_count}")
+
+
+if __name__ == "__main__":
+    is_running_in_docker = utils.is_running_in_docker()
+    if is_running_in_docker:
+        print("Python is running inside a Docker container")
+        parser = argparse.ArgumentParser()
+        parser.add_argument(
+            "--bechmark_path",  # misspelled in the original
+            help="Specify the benchmark path",
+            default="/tmp/micro-benchmark",
+        )
+
+        args = parser.parse_args()
+        main_runner(args)
+    else:
+        print("Python is not running inside a Docker container")
+        file_path = ""
+        process_file(file_path)