Add: initial work on a regression

The idea is that when-ever we find anything passing validation
where it shouldn't, we add an entry.

To make this somewhat sane, a custom regression-language was
created where you can target the API in a simple human-readable
way. The drawback is that it can be difficult to see what the
real calls are that are being made. This is a balance between
being able to motivate people to write regressions and being
very verbose in case the regression fails.

.Dockerignore

@@ -1,5 +1,6 @@
 __pycache__/
 *.pyc
+.coverage
 /.env
 /BaNaNaS
 /data

.github/workflows/testing.yml

@@ -22,10 +22,14 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Flake8
+    - name: Flake8 bananas_api
       uses: TrueBrain/actions-flake8@master
       with:
         path: bananas_api
+    - name: Flake8 regression_runner
+      uses: TrueBrain/actions-flake8@master
+      with:
+        path: regression_runner
 
   black:
     name: Black
@@ -37,8 +41,36 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: 3.8
-    - name: Black
+    - name: Set up packages
       run: |
         python -m pip install --upgrade pip
         pip install black
-        black -l 120 --check bananas_api
+    - name: Black
+      run: |
+        black -l 120 --check bananas_api regression_runner
+
+  regression:
+    name: Regression
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Set up packages
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r regression_runner/requirements.txt
+    - name: Install tusd
+      run: |
+        wget -q https://github.com/tus/tusd/releases/download/v1.1.0/tusd_linux_amd64.tar.gz
+        mkdir -p tusd
+        tar xf tusd_linux_amd64.tar.gz -C tusd
+        sudo mv tusd/tusd_linux_amd64/tusd /usr/bin/tusd
+        rm -rf tusd_linux_amd64.tar.gz tusd
+    - name: Regression
+      run: |
+        make regression

.gitignore

@@ -1,6 +1,8 @@
 __pycache__/
 *.pyc
+.coverage
 /.env
 /BaNaNaS
 /data
+/htmlcov
 /local_storage

Makefile

@@ -0,0 +1,13 @@
+all:
+	noop
+
+coverage:
+	COVERAGE_FILE="$(shell pwd)/.coverage" python -m regression_runner --coverage regression/*.yaml
+	coverage report -m
+	coverage html
+
+regression:
+	python -m regression_runner regression/*.yaml
+
+
+.PHONY: all coverage regression

regression/000_success_newgrf.yaml

@@ -0,0 +1,38 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: valid.grf
+- file-upload: readme.txt
+- file-upload: changelog.txt
+- api: new-package/update
+  name: "test--test"
+  version: "v1"
+  license: "GPL v2"
+  description: "My Description"
+  url: "https://www.openttd.org/"
+  tags:
+  - "tag1"
+  - "tag2"
+  compatibility:
+  - name: master
+    conditions:
+    - ">= 1.0.0"
+    - "< 1.10.0"
+- api: new-package/info
+  name: "test--test"
+  version: "v1"
+  license: "GPL v2"
+  description: "My Description"
+  url: "https://www.openttd.org/"
+  tags:
+  - "tag1"
+  - "tag2"
+  compatibility:
+  - name: master
+    conditions:
+    - ">= 1.0.0"
+    - "< 1.10.0"
+  md5sum-partial: "c81c3d77"
+  content-type: "newgrf"
+  unique-id: "4e4d4c01"
+- api: new-package/publish

regression/002_success_new_version.yaml

@@ -0,0 +1,28 @@
+steps:
+- api: user/login
+- api: new-package/start
+- api: new-package/update
+  name: "test--test"
+  version: "v2"
+  license: "GPL v2"
+  description: "My Description"
+  url: "https://www.openttd.org/"
+  tags:
+  - "tag1"
+  - "tag2"
+- file-upload: valid2.grf
+- file-upload: readme.txt
+- file-upload: changelog.txt
+- api: new-package/info
+  name: "test--test"
+  version: "v2"
+  license: "GPL v2"
+  description: "My Description"
+  url: "https://www.openttd.org/"
+  tags:
+  - "tag1"
+  - "tag2"
+  md5sum-partial: "fc03854a"
+  content-type: "newgrf"
+  unique-id: "4e4d4c01"
+- api: new-package/publish

regression/010_same_newgrf.yaml

@@ -0,0 +1,10 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: valid.grf
+- api: new-package/update
+  name: "test"
+  version: "v2"
+  license: "GPL v2"
+- api: new-package/publish
+  error: "There is already an entry with the same md5sum-partial for this package; this most likely means you are uploading the exact same content."

regression/011_same_version.yaml

@@ -0,0 +1,10 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: valid.grf
+- api: new-package/update
+  name: "test"
+  version: "v1"
+  license: "GPL v2"
+- api: new-package/publish
+  error: "There is already an entry with the same version for this package."

regression/012_not_author.yaml

@@ -0,0 +1,11 @@
+steps:
+- api: user/login
+  username: regression2
+- api: new-package/start
+- file-upload: valid3.grf
+- api: new-package/update
+  name: "test"
+  version: "v3"
+  license: "GPL v2"
+- api: new-package/publish
+  error: "You do not have permission to upload a new version for this package."

regression/020_newgrf_and_scenario.yaml

@@ -0,0 +1,7 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: valid.grf
+- file-upload: valid2.grf
+- api: new-package/publish
+  error: "More than one newgrf files was detected, where only one was expected."

regression/021_two_newgrfs.yaml

@@ -0,0 +1,7 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: valid3.grf
+- file-upload: scenario.scn
+- api: new-package/publish
+  error: "More than one Content Type was detected, where only one was expected. For example, you are uploading both a NewGRF and a Scenario in the same package. This is not possible."

regression/100_success_base_sounds.yaml

@@ -0,0 +1,22 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: nosound.obs
+- file-upload: nosound.cat
+- file-upload: license.txt
+- file-upload: readme.txt
+  name: readme_nl.txt
+- file-upload: changelog.txt
+  name: changelog_nl.txt
+- api: new-package/update
+  name: "test"
+  version: "v1"
+  license: "Custom"
+- api: new-package/info
+  name: "test"
+  version: "v1"
+  license: "Custom"
+  md5sum-partial: "b16a57bf"
+  content-type: "base-sounds"
+  unique-id: "4e554c4c"
+- api: new-package/publish

regression/101_success_base_music.yaml

@@ -0,0 +1,21 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: noteon-noteoff.obm
+- file-upload: noteon-noteoff.mid
+- file-upload: readme.txt
+  name: readme_nl_NL.txt
+- file-upload: changelog.txt
+  name: changelog_nl_NL.txt
+- api: new-package/update
+  name: "test"
+  version: "v1"
+  license: "GPL v2"
+- api: new-package/info
+  name: "test"
+  version: "v1"
+  license: "GPL v2"
+  md5sum-partial: "c2c91e76"
+  content-type: "base-music"
+  unique-id: "4e4f4e4f"
+- api: new-package/publish

regression/102_success_base_graphics.yaml

@@ -0,0 +1,25 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: small.obg
+- file-upload: valid.grf
+- file-upload: valid.grf
+  name: valid1.grf
+- file-upload: valid2.grf
+- file-upload: valid3.grf
+- file-upload: valid.grf
+  name: valid4.grf
+- file-upload: valid.grf
+  name: valid5.grf
+- api: new-package/update
+  name: "test"
+  version: "v1"
+  license: "GPL v2"
+- api: new-package/info
+  name: "test"
+  version: "v1"
+  license: "GPL v2"
+  md5sum-partial: "2ee06136"
+  content-type: "base-graphics"
+  unique-id: "534d4c4c"
+- api: new-package/publish

regression/110_invalid_cat.yaml

@@ -0,0 +1,8 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: nosound.obs
+- file-upload: noteon-noteoff.mid
+  name: nosound.cat
+- api: new-package/publish
+  error: "nosound.cat: Invalid cat header."

regression/111_invalid_md5.yaml

@@ -0,0 +1,16 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: small.obg
+- file-upload: valid.grf
+- file-upload: valid.grf
+  name: valid1.grf
+- file-upload: valid.grf
+  name: valid2.grf
+- file-upload: valid3.grf
+- file-upload: valid.grf
+  name: valid4.grf
+- file-upload: valid.grf
+  name: valid5.grf
+- api: new-package/publish
+  error: "valid2.grf: The md5sum doesn't match the one mentioned in small.obg."

regression/112_invalid_midi.yaml

@@ -0,0 +1,8 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: noteon-noteoff.obm
+- file-upload: nosound.cat
+  name: noteon-noteoff.mid
+- api: new-package/publish
+  error: "noteon-noteoff.mid: Invalid MIDI header."

regression/120_missing_md5.yaml

@@ -0,0 +1,17 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: small-missing-md5.obg
+- file-upload: valid.grf
+- file-upload: valid.grf
+  name: valid1.grf
+- file-upload: valid.grf
+  name: valid2.grf
+- file-upload: valid.grf
+  name: valid3.grf
+- file-upload: valid.grf
+  name: valid4.grf
+- file-upload: valid.grf
+  name: valid5.grf
+- api: new-package/publish
+  error: "small-missing-md5.obg: Option md5s:valid5.grf is missing."

regression/121_missing_shortname.yaml

@@ -0,0 +1,17 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: small-missing-shortname.obg
+- file-upload: valid.grf
+- file-upload: valid.grf
+  name: valid1.grf
+- file-upload: valid.grf
+  name: valid2.grf
+- file-upload: valid.grf
+  name: valid3.grf
+- file-upload: valid.grf
+  name: valid4.grf
+- file-upload: valid.grf
+  name: valid5.grf
+- api: new-package/publish
+  error: "small-missing-shortname.obg: Option metadata:shortname is missing."

regression/200_success_ai_and_utf8.yaml

@@ -0,0 +1,19 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: main.nut
+- file-upload: info-ai.nut
+  name: info.nut
+- file-upload: utf8_with_bom.nut
+- api: new-package/update
+  name: "test"
+  version: "v1"
+  license: "GPL v2"
+- api: new-package/info
+  name: "test"
+  version: "v1"
+  license: "GPL v2"
+  md5sum-partial: "82b67279"
+  content-type: "ai"
+  unique-id: "52454752"
+- api: new-package/publish

regression/201_success_ai_library.yaml

@@ -0,0 +1,18 @@
+steps:
+- api: user/login
+- api: new-package/start
+- file-upload: main.nut
+- file-upload: library-ai.nut
+  name: library.nut
+- api: new-package/update
+  name: "test"
+  version: "v1"
+  license: "GPL v2"
+- api: new-package/info
+  name: "test"
+  version: "v1"
+  license: "GPL v2"
+  md5sum-partial: "f529a430"
+  content-type: "ai-library"
+  unique-id: "52454752"
+- api: new-package/publish