feat(app): add auth and expose rpc server sidecar (#762)

Co-authored-by: Rok Roškar <roskarr@ethz.ch>

git_services/Dockerfile.init

@@ -11,8 +11,6 @@ RUN apt-get update && \
 ADD . /git_services/
 WORKDIR /git_services
 
-ENV USER_ID 1000
-ENV GROUP_ID 1000
 USER 1000:1000
 # Add poetry to path
 ENV PATH "/home/jovyan/.local/bin:$PATH"

git_services/Dockerfile.sidecar

@@ -11,19 +11,12 @@ RUN apt-get update && \
 ADD . /git_services/
 WORKDIR /git_services
 
-ENV USER_ID 1000
-ENV GROUP_ID 1000
 USER 1000:1000
 # Add poetry to path
 ENV PATH "/home/jovyan/.local/bin:$PATH"
 
 RUN curl -sSL https://install.python-poetry.org | python3 - && \
     poetry install --no-dev
 
-ENV HOST="0.0.0.0"
-# Note: This will return a 200 as soon as the server is up,
-#       even if this is an invalid rpc request.
-HEALTHCHECK CMD curl http://$HOST:4000
-
 ENTRYPOINT ["tini", "-g", "--"]
-CMD ["poetry", "run", "python", "-m", "git_services.sidecar.rpc_server"]
+CMD ["poetry", "run", "gunicorn", "-c", "git_services/sidecar/gunicorn.conf.py"]

git_services/git_services/cli/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 import shlex
-from subprocess import Popen, PIPE
+import os
 
 
 class GitCommandError(Exception):
@@ -21,6 +21,12 @@ def __init__(self, repo_directory: Path) -> None:
             raise RepoDirectoryDoesNotExistError
 
     def _execute_command(self, command: str, **kwargs):
+        # NOTE: When running in gunicorn with gevent Popen and PIPE from subprocess do not work
+        # and the gevent equivalents have to be used
+        if os.environ.get("RUNNING_WITH_GEVENT"):
+            from gevent.subprocess import Popen, PIPE
+        else:
+            from subprocess import Popen, PIPE
         args = shlex.split(command)
         res = Popen(args, stdout=PIPE, stderr=PIPE, cwd=self.repo_directory, **kwargs)
         stdout, stderr = res.communicate()

git_services/git_services/sidecar/app.py

@@ -0,0 +1,42 @@
+import os
+from urllib.parse import urljoin
+
+from flask import Flask
+from jsonrpc.backend.flask import api, Blueprint
+
+from git_services.sidecar.rpc_server import autosave, status
+from git_services.sidecar.config import config_from_env
+
+
+def health_endpoint():
+    """Health endpoint for probes."""
+    return {"status": "running"}
+
+
+def get_app():
+    """Setup flask app"""
+    os.environ["RUNNING_WITH_GEVENT"] = "true"
+    config = config_from_env()
+    app = Flask(__name__)
+    health_bp = Blueprint("health", __name__)
+    health_bp.route("/")(health_endpoint)
+    jsonrpc_bp = api.as_blueprint()
+    api.dispatcher.add_method(status, "git/get_status")
+    api.dispatcher.add_method(autosave, "autosave/create")
+    app.register_blueprint(jsonrpc_bp, url_prefix=urljoin(config.url_prefix, "jsonrpc"))
+    app.register_blueprint(health_bp, url_prefix=urljoin(config.url_prefix, "health"))
+
+    if config.sentry.enabled:
+        import sentry_sdk
+        from sentry_sdk.integrations.flask import FlaskIntegration
+
+        sentry_sdk.init(
+            dsn=config.sentry.dsn,
+            environment=config.sentry.environment,
+            integrations=[FlaskIntegration()],
+            traces_sample_rate=config.sentry.sample_rate,
+        )
+    return app
+
+
+app = get_app()

git_services/git_services/sidecar/config.py

@@ -1,12 +1,29 @@
+from typing import Any, Callable, Text, Union
+
 from dataclasses import dataclass
 import dataconf
 
 from git_services.cli.sentry import SentryConfig
 
 
+def _parse_value_as_numeric(val: Any, parse_to: Callable) -> Union[float, int]:
+    output = parse_to(val)
+    if type(output) is not float and type(output) is not int:
+        raise ValueError(
+            f"parse_to should convert to float or int, it returned type {type(output)}"
+        )
+    return output
+
+
 @dataclass
 class Config:
     sentry: SentryConfig
+    port: Union[Text, int] = 4000
+    host: Text = "0.0.0.0"
+    url_prefix: Text = "/"
+
+    def __post_init__(self):
+        self.port = _parse_value_as_numeric(self.port, int)
 
 
 def config_from_env() -> Config:

git_services/git_services/sidecar/gunicorn.conf.py

@@ -0,0 +1,6 @@
+from git_services.sidecar.config import config_from_env
+
+_config = config_from_env()
+bind = f"{_config.host}:{_config.port}"
+wsgi_app = "git_services.sidecar.app:app"
+worker_class = "gevent"

git_services/git_services/sidecar/rpc_server.py

@@ -1,20 +1,30 @@
-from jsonrpc import JSONRPCResponseManager, dispatcher
 import os
-import requests
-from werkzeug.wrappers import Request, Response
-from werkzeug.serving import run_simple
+from contextlib import contextmanager
 from pathlib import Path
 from subprocess import PIPE, Popen
 import shlex
 
+import requests
+
 from git_services.cli import GitCLI
-from git_services.cli.sentry import setup_sentry
-from git_services.sidecar.config import config_from_env
 
 
-@dispatcher.add_method
-def status(path: str = ".", **kwargs):
-    """Execute \"git status\" on the repository."""
+def status(path: str = os.environ.get("MOUNT_PATH", "."), **kwargs):
+    """Execute \"git status --porcelain=v2 --branch\" on the repository.
+
+    Args:
+        path (str): The location of the repository, defaults to the environment variable
+        called 'MOUNT_PATH' if that is not defined then it will default to '.'.
+
+    Returns:
+        dict: A dictionary with several keys:
+        'clean': boolean indicating if the repository is clean
+        'ahead': integer indicating how many commits the local repo is ahead of the remote
+        'behind': integer indicating how many commits the local repo is behind of the remote
+        'branch': string with the name of the current branch
+        'commit': string with the current commit SHA
+        'status': string with the 'raw' result from running git status in the repository
+    """
     cli = GitCLI(Path(path))
     status = cli.git_status("--porcelain=v2 --branch")
 
@@ -54,10 +64,21 @@ def status(path: str = ".", **kwargs):
     }
 
 
-@dispatcher.add_method
 def autosave(**kwargs):
-    """Create an autosave branch with uncommitted work."""
-    try:
+    """Create an autosave branch with uncommitted work and push it to the remote."""
+
+    @contextmanager
+    def _shutdown_git_proxy_when_done():
+        """Inform the git-proxy it can shut down.
+        The git-proxy will wait for this in order to shutdown.
+        If this "shutdown" call does not happen then the proxy will ignore SIGTERM signals
+        and shutdown after a specific long period (i.e. 10 minutes)."""
+        try:
+            yield None
+        finally:
+            requests.get(f"http://localhost:{git_proxy_health_port}/shutdown")
+
+    with _shutdown_git_proxy_when_done():
         git_proxy_health_port = os.getenv("GIT_PROXY_HEALTH_PORT", "8081")
         repo_path = os.environ.get("MOUNT_PATH")
         status_result = status(path=repo_path)
@@ -108,22 +129,3 @@ def autosave(**kwargs):
         cli.git_reset(f"--soft {current_branch}")
         cli.git_checkout(f"{current_branch}")
         cli.git_branch(f"-D {autosave_branch_name}")
-    finally:
-        # INFO: Inform the proxy it can shut down
-        # NOTE: Do not place return, break or continue here, otherwise
-        # the exception from try will be completely discarded.
-        requests.get(f"http://localhost:{git_proxy_health_port}/shutdown")
-
-
-@Request.application
-def application(request):
-    """Listen for incoming requests on /jsonrpc"""
-    response = JSONRPCResponseManager.handle(request.data, dispatcher)
-    return Response(response.json, mimetype="application/json")
-
-
-if __name__ == "__main__":
-    config = config_from_env()
-    setup_sentry(config.sentry)
-
-    run_simple(os.getenv("HOST"), 4000, application)