backend: use sandbox for evaluating JS/Python code

Author: usrbinsam

CodeChallenge/api/questions.py

@@ -3,7 +3,7 @@
 from hmac import compare_digest as str_cmp
 
 import requests
-from flask import Blueprint, current_app, jsonify, request, redirect, url_for
+from flask import Blueprint, current_app, jsonify, request, redirect, url_for, abort
 from flask_jwt_extended import get_current_user, jwt_required
 
 from .. import core
@@ -119,7 +119,7 @@ def answer_next_question():
 @bp.route("/history", methods=["GET"])
 @jwt_required
 def history():
-    """ Returns all past questions and answers for the currrent user"""
+    """ Returns all past questions and answers for the current user"""
 
     u = get_current_user()
 
@@ -148,7 +148,7 @@ def reset_all():
         u = get_current_user()
         u.rank = 0
 
-        for ans in Answer.query.filter_by(user_id= u.id):  # type: Answer
+        for ans in Answer.query.filter_by(user_id=u.id):  # type: Answer
             db.session.delete(ans)
 
         db.session.commit()
@@ -183,9 +183,18 @@ def answer_eval():
         return jsonify(status="error",
                        reason="missing 'text' property in JSON body"), 400
 
+    try:
+        language = request.json["language"]
+    except KeyError:
+        return jsonify(status="error",
+                       reason="missing 'language' property in JSON body"), 400
+
     # designated output variable for evaluation
-    code += ";output"
-    r = requests.post(current_app.config["DUKTAPE_API"], json={"code": code})
+    if language == "js":
+        code += ";output"
+
+    r = requests.post(current_app.config["SANDBOX_API_URL"],
+                      json={"code": code, "language": language})
 
     if not r.ok:
         if r.status_code >= 500:
@@ -196,7 +205,7 @@ def answer_eval():
         eval_data = r.json()
     except ValueError:
         return jsonify(status="error",
-                       reason="response from duktape API was not JSON"), 500
+                       reason="response from sandbox API was not JSON"), 500
 
     eval_error = eval_data["error"]
     eval_output = str(eval_data["output"])

CodeChallenge/config.py

@@ -33,9 +33,14 @@ class DefaultConfig:
     # no trailing /
     EXTERNAL_URL = "https://hackcwhq.com"
 
-    DUKTAPE_API = "http://localhost:5001/js/eval"
+    SANDBOX_API_URL = "http://sandbox.cwhq-apps.com:3000/"
 
     ALLOW_RESET = False
+    MAX_VOTES = 2
+
+    # number of days to leave CodeChallenge open
+    # past the final rank
+    CHALLENGE_ENDS = 1
 
     @property
     def ROOT_DIR(self):

docker/Dockerfile

@@ -1,19 +1,17 @@
 FROM alpine:latest
 LABEL maintainer="Sam Hoffman <sam@codewizardshq.com>"
 
+ENV LANGUAGE=""
+ENV CODEFILE=""
+
 RUN apk --update add --no-cache python3 bash curl binutils libc-dev python3-dev gcc
 RUN pip3 install -U pip
-RUN pip3 install flask flask-cors pyduktape gunicorn
-
-RUN adduser -D -H webapp
-RUN mkdir -p "/var/www/flaskapp"
-
-# default Flask port
-EXPOSE 5000/tcp
+RUN pip3 install pyduktape
 
-USER webapp
+RUN adduser -D -H sandbox-user
+USER sandbox-user
 
-COPY "start.sh" "/tmp/start.sh"
-COPY "app.py" "/var/www/flaskapp/app.py"
+COPY "main.py" "/tmp/main.py"
+VOLUME "/mnt/code"
 
-ENTRYPOINT [ "/tmp/start.sh" ]
+ENTRYPOINT [ "/tmp/main.py" ]

docker/app.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+
+import json
+import os
+import sys
+from subprocess import Popen, PIPE
+from typing import Tuple
+
+from pyduktape import DuktapeContext
+
+CODE_FOLDER = "/mnt/code/"
+
+CLIOutput = Tuple[str, str]  # stdout/stderr
+
+
+def echo(output="", error="") -> None:
+    print(json.dumps({
+        "output": output,
+        "error": error
+    }))
+
+
+def exec_python(code: str) -> CLIOutput:
+    p = Popen("/usr/bin/python3", stdin=PIPE, stderr=PIPE, stdout=PIPE)
+    stdout, stderr = p.communicate(code.encode("utf8"))
+
+    return stdout.decode("utf8"), stderr.decode("utf8")
+
+
+def exec_js(code: str) -> CLIOutput:
+    ctx = DuktapeContext()
+
+    output = ""
+    error = ""
+
+    try:
+        output = ctx.eval_js(code)
+    except Exception as e:
+        error = str(e)
+
+    return output, error
+
+
+def main() -> int:
+    language = os.getenv("LANGUAGE")
+    filename = os.getenv("CODEFILE")
+    code = ""
+
+    with open(os.path.join(CODE_FOLDER, filename), "r") as contents:
+        code = contents.read()
+
+    stdout, stderr = "", ""
+
+    if language not in ("python", "js"):
+        echo(error=f"unsupported language {language!r}")
+        return -1
+
+    if language == "python":
+        stdout, stderr = exec_python(code)
+
+    elif language == "js":
+        stdout, stderr = exec_js(code)
+
+    echo(stdout, stderr)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docker/main.py

@@ -8,6 +8,13 @@
 
 app = CodeChallenge.create_app("DefaultConfig")
 
+NOW = datetime.now(timezone.utc)
+
+CC_CLOSED = (NOW - timedelta(days=5)).timestamp()
+CC_2D_PRIOR = (NOW - timedelta(days=2)).timestamp()
+CC_4D_PRIOR = (NOW - timedelta(days=4)).timestamp()
+CC_2D_FUTURE = (NOW + timedelta(days=2)).timestamp()
+
 
 @pytest.fixture(scope="module")
 def client_challenge_today():
@@ -39,12 +46,9 @@ def client_challenge_today():
 
 @pytest.fixture(scope="module")
 def client_challenge_future():
-    now = datetime.now(timezone.utc)
-    future = now + timedelta(days=2)
-
     app.config["TESTING"] = True
     app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
-    app.config["CODE_CHALLENGE_START"] = future.timestamp()
+    app.config["CODE_CHALLENGE_START"] = CC_2D_FUTURE
 
     with app.test_client() as client:
         with app.app_context():
@@ -54,12 +58,9 @@ def client_challenge_future():
 
 @pytest.fixture(scope="module")
 def client_challenge_past():
-    now = datetime.now(timezone.utc)
-    past = now - timedelta(days=2)
-
     app.config["TESTING"] = True
     app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
-    app.config["CODE_CHALLENGE_START"] = past.timestamp()
+    app.config["CODE_CHALLENGE_START"] = CC_2D_PRIOR
 
     with app.test_client() as client:
         with app.app_context():
@@ -69,12 +70,9 @@ def client_challenge_past():
 
 @pytest.fixture(scope="module")
 def client_challenge_lastq():
-    now = datetime.now(timezone.utc)
-    past = now - timedelta(days=4)
-
     app.config["TESTING"] = True
     app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
-    app.config["CODE_CHALLENGE_START"] = past.timestamp()
+    app.config["CODE_CHALLENGE_START"] = CC_4D_PRIOR
 
     with app.test_client() as client:
         with app.app_context():
@@ -276,46 +274,36 @@ def test_answer_exceed_attempts(client_challenge_past):
             assert "X-RateLimit-Remaining" in retval.headers
 
 
-@pytest.mark.skipif(not os.getenv("DUKTAPE_API"), reason="envvar DUKTAPE_API is not set")
+@pytest.mark.skipif(not os.getenv("SANDBOX_API_URL"), reason="envvar SANDBOX_API_URL is not set")
 def test_answer_finalq_wrong(client_challenge_lastq):
     login(client_challenge_lastq,
           "cwhqsam",
           "supersecurepassword")
 
     rv = client_challenge_lastq.post("/api/v1/questions/final", json=dict(
-        text="var output; output = 11"
+        text="var output; output = 11",
+        language="js"
     ))
 
     assert rv.status_code == 200
     assert rv.json["correct"] is False
 
 
-@pytest.mark.skipif(not os.getenv("DUKTAPE_API"), reason="envvar DUKTAPE_API is not set")
+@pytest.mark.skipif(not os.getenv("SANDBOX_API_URL"), reason="envvar SANDBOX_API_URL is not set")
 def test_answer_finalq_right(client_challenge_lastq):
     login(client_challenge_lastq,
           "cwhqsam",
           "supersecurepassword")
 
     rv = client_challenge_lastq.post("/api/v1/questions/final", json=dict(
-        text="var output; output = 10"
+        text="var output; output = 10",
+        language="js"
     ))
 
     assert rv.status_code == 200
     assert rv.json["correct"] is True
 
 
-def test_reset_all(client_challenge_past):
-    retval = client_challenge_past.delete("/api/v1/questions/reset")
-
-    assert retval.status_code == 200
-
-    retval = client_challenge_past.get("/api/v1/users/hello")
-    data = retval.get_json()
-
-    assert retval.status_code == 200
-    assert data["rank"] == 0
-
-
 def test_leaderboard(client_challenge_past):
 
     # register a bunch of fake users