set up basic autograder

.gitignore

@@ -15,39 +15,6 @@ Untitled.ipynb
 # override the global
 !.vscode/settings.json
 
-## Terraform ##
-
-# Local .terraform directories
-**/.terraform/*
-
-# .tfstate files
-*.tfstate
-*.tfstate.*
-
-# Crash log files
-crash.log
-
-# Exclude all .tfvars files, which are likely to contain sentitive data, such as
-# password, private keys, and other secrets. These should not be part of version
-# control as they are data points which are potentially sensitive and subject
-# to change depending on the environment.
-#
-*.tfvars
-
-# Ignore override files as they are usually used to override resources locally and so
-# are not checked in
-override.tf
-override.tf.json
-*_override.tf
-*_override.tf.json
-
-# Include override files you do wish to add to version control using negated pattern
-#
-# !example_override.tf
-
-# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
-# example: *tfplan*
-
-# Ignore CLI configuration files
-.terraformrc
-terraform.rc
+extras/autograder/results/
+extras/autograder/submission/
+autograder.zip

Makefile

@@ -30,3 +30,20 @@ update_packages:
 	./extras/scripts/update_lectures.sh
 
 	echo "Please update homework notebooks separately, in python-public-policy-assignments"
+
+# based on https://gradescope-autograders.readthedocs.io/en/latest/manual_docker/
+autograde:
+	mkdir -p ./extras/autograder/results ./extras/autograder/results
+
+	docker run --rm \
+		-v ./extras/autograder:/autograder \
+		gradescope/autograder-base \
+		/bin/bash -c "/autograder/source/setup.sh && /autograder/source/run_autograder"
+
+	cat ./extras/autograder/results/results.json
+
+build_autograder:
+	# https://stackoverflow.com/a/17351814/358804
+	git archive -o ./extras/autograder.zip HEAD:./extras/autograder/source
+
+	echo "Now upload extras/autograder.zip to Gradescope."

extras/autograder/source/requirements.txt

@@ -0,0 +1 @@
+gradescope_utils

extras/autograder/source/run_autograder

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# based on
+# https://github.com/gradescope/autograder_samples/blob/master/python/src/run_autograder
+
+set -ex
+
+cd /autograder/source
+
+python3 run_tests.py

extras/autograder/source/run_tests.py

@@ -0,0 +1,9 @@
+"""based on https://github.com/gradescope/autograder_samples/blob/master/python/src/run_tests.py"""
+
+import unittest
+from gradescope_utils.autograder_utils.json_test_runner import JSONTestRunner
+
+if __name__ == '__main__':
+    suite = unittest.defaultTestLoader.discover('tests')
+    with open('/autograder/results/results.json', 'w') as f:
+        JSONTestRunner(visibility='visible', stream=f).run(suite)

extras/autograder/source/setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -ex
+
+apt-get install -y python3 python3-pip
+
+pip3 install -r /autograder/source/requirements.txt

extras/autograder/source/tests/test_files.py

@@ -0,0 +1,18 @@
+"""https://github.com/gradescope/gradescope-utils/tree/master/gradescope_utils/autograder_utils#readme"""
+
+from unittest import TestCase
+import os
+from gradescope_utils.autograder_utils.decorators import number
+
+
+class TestFiles(TestCase):
+    @number("1.1")
+    def test_notebook_and_py_file(self):
+        """There should be exactly one notebook and one Python file submitted"""
+
+        files = os.listdir("/autograder/submission")
+        extensions = [os.path.splitext(filename)[1] for filename in files]
+        extensions.sort()
+        self.assertListEqual(
+            extensions, [".ipynb", ".py"], f"Files submitted: {', '.join(files)}"
+        )

extras/environment.yml

@@ -7,6 +7,9 @@ dependencies:
   - black
   - nbqa
   - notebook=7.*
+  - pip
+  - pip:
+      - gradescope_utils
 
   # extensions
   - jupyter-resource-usage

extras/lib/school.py

@@ -148,13 +148,15 @@ class SchoolText:
     ".zoom.us/rec",
     "- [google colab](https://colab.research.google.com/)",
     "anaconda",
+    "autograder",  # matches "grader"
     "built around it",  # referring to Colab
     "columbia's graduate school of architecture",  # bio
     "conda activate",
     "conda config",
     "create the environment",
     "dictreader",
     "for row in reader",
+    "gradescope_utils",  # matches "gradescope"
     "hannahkates/nyu-python-public-policy",
     "https://community.canvaslms.com/t5/canvas-basics-guide/what-are-grading-schemes/ta-p/41",
     "jupyterhub_url",

extras/terraform/.gitignore

@@ -0,0 +1,34 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+
+# Exclude all .tfvars files, which are likely to contain sentitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+#
+*.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+#
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

meta/instructor_guide.md

@@ -124,6 +124,14 @@ Most of the issues are around Plotly rendering. Things that have been hit repeat
 
 - Comments in [`environment.yml`](https://github.com/afeld/python-public-policy/blob/main/extras/environment.yml)
 - [Student troubleshooting guide](../assignments.md#common-issues)
+
+## Autograder
+
+Requires [Docker](https://www.docker.com/). Put files in `extras/autograder/submission/`, then run:
+
+```sh
+make autograde
+```
 {%- endif %}
 
 ## Contacts

pyproject.toml

@@ -1,2 +1,7 @@
 [tool.black]
 line-length = 100
+
+[tool.pytest.ini_options]
+addopts = [
+    "--ignore=extras/autograder/",
+]