Add: first version of content-api

It serves both the webserver as "tusd", the resumable upload part
of the API.

This current version only works with the files on local storage,
which means you need all the files of BaNaNaS in "local_storage"
folder.

This repository in combination with content-server allows you to
run your own BaNaNaS from scratch.

.Dockerignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+/BaNaNaS
+/data
+/local_storage

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+/BaNaNaS
+/data
+/local_storage

.version

@@ -0,0 +1 @@
+dev

Dockerfile

@@ -1,2 +1,48 @@
 FROM python:3.8-slim
 
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        git \
+        wget \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN wget -q https://github.com/tus/tusd/releases/download/v1.1.0/tusd_linux_amd64.tar.gz \
+    && mkdir -p /tusd \
+    && tar xf tusd_linux_amd64.tar.gz -C /tusd \
+    && mv /tusd/tusd_linux_amd64/tusd /usr/bin/tusd \
+    && rm -rf tusd_linux_amd64.tar.gz /tusd \
+    && apt-get remove -y wget
+
+WORKDIR /code
+
+COPY requirements.txt \
+        LICENSE \
+        README.md \
+        .version \
+        /code/
+COPY licenses /code/licenses
+
+# TODO -- Should be done by the python process
+RUN mkdir -p \
+        /code/BaNaNaS/AI \
+        /code/BaNaNaS/AI\ Library \
+        /code/BaNaNaS/Base\ Graphics \
+        /code/BaNaNaS/Base\ Music \
+        /code/BaNaNaS/Base\ Sounds \
+        /code/BaNaNaS/Game\ Script \
+        /code/BaNaNaS/Game\ Script\ Library \
+        /code/BaNaNaS/Heightmap \
+        /code/BaNaNaS/NewGRF \
+        /code/BaNaNaS/Scenario
+
+RUN pip install -r requirements.txt
+
+# Validate that what was installed was what was expected
+RUN pip freeze 2>/dev/null > requirements.installed \
+    && diff -u --strip-trailing-cr requirements.txt requirements.installed 1>&2 \
+    || ( echo "!! ERROR !! requirements.txt defined different packages or versions for installation" \
+        && exit 1 ) 1>&2
+
+COPY content_api /code/content_api
+
+ENTRYPOINT ["python", "-m", "content_api"]
+CMD ["--storage", "local"]

README.md

@@ -0,0 +1,35 @@
+# Content API
+
+This is the HTTP API for OpenTTD's content service, called BaNaNaS.
+It works together with [https://github.com/OpenTTD/content-server](content-server), which serves the in-game client.
+
+## Development
+
+This API is written in Python 3.8 with aiohttp, and makes strong use of asyncio.
+
+### Running a local server
+
+To start it, you are advised to first create a virtualenv:
+
+```bash
+python3 -m venv .env
+.env/bin/pip install -r requirements.txt
+```
+
+Next, you can start the HTTP server by running:
+
+```bash
+.env/bin/python -m content_api --web-port 8080 --storage local
+```
+
+This will start the API on port 8080 for you to work with locally.
+
+### Running via docker
+
+```bash
+docker build openttd/content-api:local
+mkdir -p $(pwd)/../content-common
+docker run --rm -p 127.0.0.1:8080:80 -v $(pwd)/../content-common/local_storage:/code/local_storage -v $(pwd)/../content-common/BaNaNaS:/code/BaNaNaS openttd/content-api:local --storage local --index local
+```
+
+The mount assumes that [https://github.com/OpenTTD/content-server](content-server) and this repository has the same parent folder on your disk, as both servers need to read the same local storage.

content_api/__main__.py

@@ -0,0 +1,117 @@
+import asyncio
+import click
+import importlib
+import logging
+
+from aiohttp import web
+from aiohttp.web_log import AccessLogger
+
+from .helpers import sentry
+from .helpers.content_save import (
+    set_index_instance,
+    set_commit_graceperiod,
+)
+from .new_upload.session import set_cleanup_graceperiod
+from .new_upload.session_publish import set_storage_instance
+from .web_routes import (
+    common,
+    discover,
+    fallback,
+    new,
+    update,
+)
+
+log = logging.getLogger(__name__)
+
+CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}
+
+
+class ErrorOnlyAccessLogger(AccessLogger):
+    def log(self, request, response, time):
+        # Only log if the status was not successful
+        if not (200 <= response.status < 400):
+            super().log(request, response, time)
+
+
+async def _run_tusd(tusd_port, web_port):
+    command = (
+        f"tusd"
+        f" --port {tusd_port}"
+        f" --hooks-http http://127.0.0.1:{web_port}/new-package/tusd-internal"
+        f" --hooks-enabled-events pre-create,post-create,post-finish"
+    )
+    tusd_proc = await asyncio.create_subprocess_shell(
+        command, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL
+    )
+    await tusd_proc.wait()
+
+
+@click.command(context_settings=CONTEXT_SETTINGS)
+@click.option("--web-port", help="Port of the web server", default=80, show_default=True)
+@click.option("--tusd-port", help="Port of the tus server", default=1080, show_default=True)
+@click.option("--sentry-dsn", help="Sentry DSN")
+@click.option(
+    "--sentry-environment", help="Environment we are running in (for Sentry)", default="development",
+)
+@click.option("--storage", type=click.Choice(["local"], case_sensitive=False), required=True)
+@click.option("--index", type=click.Choice(["local"], case_sensitive=False), required=True)
+@click.option(
+    "--commit-graceperiod", help="Graceperiod between commits to disk (in seconds)", default=60 * 5, show_default=True
+)
+@click.option(
+    "--cleanup-graceperiod", help="Graceperiod between cleanup of new uploads", default=60 * 15, show_default=True
+)
+@click.option("--validate", help="Only validate BaNaNaS files and exit", is_flag=True)
+def main(
+    web_port,
+    tusd_port,
+    sentry_dsn,
+    sentry_environment,
+    storage,
+    index,
+    commit_graceperiod,
+    cleanup_graceperiod,
+    validate,
+):
+    sentry.setup_sentry(sentry_dsn, sentry_environment)
+
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO
+    )
+
+    set_commit_graceperiod(commit_graceperiod)
+    set_cleanup_graceperiod(cleanup_graceperiod)
+
+    storage = storage.lower()
+    storage_module = importlib.import_module(f"content_api.storage.{storage}")
+    storage_instance = getattr(storage_module, "Storage")()
+    set_storage_instance(storage_instance)
+
+    index = index.lower()
+    index_module = importlib.import_module(f"content_api.index.{index}")
+    index_instance = getattr(index_module, "Index")()
+    set_index_instance(index_instance)
+
+    webapp = web.Application()
+    webapp.add_routes(common.routes)
+    webapp.add_routes(discover.routes)
+    webapp.add_routes(new.routes)
+    webapp.add_routes(update.routes)
+    # Always make sure "fallback" comes last. It has a catch-all rule.
+    webapp.add_routes(fallback.routes)
+
+    index_instance.load_all()
+
+    if validate:
+        return
+
+    loop = asyncio.get_event_loop()
+
+    # Start tusd as part of the application
+    loop.create_task(_run_tusd(tusd_port, web_port))
+    # Start aiohttp server
+    web.run_app(webapp, port=web_port, access_log_class=ErrorOnlyAccessLogger)
+
+
+if __name__ == "__main__":
+    main(auto_envvar_prefix="CONTENT_API")

content_api/helpers/api_schema.py

@@ -0,0 +1,151 @@
+from marshmallow import (
+    fields,
+    Schema,
+    validate,
+    validates,
+    validates_schema,
+)
+from marshmallow.exceptions import ValidationError
+from marshmallow_enum import EnumField
+
+from .content_storage import get_indexed_package
+from .enums import (
+    Availability,
+    ContentType,
+    License,
+    Status,
+)
+
+DEPENDENCY_CHECK = True
+
+
+def set_dependency_check(state):
+    global DEPENDENCY_CHECK
+    DEPENDENCY_CHECK = state
+
+
+class OrderedSchema(Schema):
+    class Meta:
+        ordered = True
+
+
+class ReplacedBy(OrderedSchema):
+    unique_id = fields.String(data_key="unique-id", validate=validate.Length(min=8, max=8))
+
+
+class Global(OrderedSchema):
+    read_only = ["archived", "replaced_by"]
+
+    name = fields.String()
+    archived = fields.Boolean()
+    replaced_by = fields.Nested(ReplacedBy(), data_key="replaced-by", allow_none=True)
+    description = fields.String()
+    url = fields.String()
+    tags = fields.List(fields.String(validate=validate.Length(max=32)))
+
+
+class Author(OrderedSchema):
+    display_name = fields.String(data_key="display-name")
+    openttd = fields.String(allow_none=True)
+    github = fields.String(allow_none=True)
+
+
+class Authors(OrderedSchema):
+    authors = fields.List(fields.Nested(Author))
+
+
+class Dependency(OrderedSchema):
+    content_type = EnumField(ContentType, data_key="content-type", by_value=True)
+    unique_id = fields.String(data_key="unique-id", validate=validate.Length(min=8, max=8))
+    md5sum_partial = fields.String(data_key="md5sum-partial", validate=validate.Length(min=8, max=8))
+
+    @validates_schema
+    def validate_dependency(self, data, **kwargs):
+        if not DEPENDENCY_CHECK:
+            return
+
+        # Check the unique-id exists
+        package = get_indexed_package(data["content_type"], data["unique_id"])
+        if package is None:
+            raise ValidationError(
+                f"Package with unique-id '{data['unique_id']}' does not exist for {data['content_type'].value}."
+            )
+
+        # Check there is any version with that md5sum-partial
+        for version in package["versions"]:
+            if version["md5sum_partial"] == data["md5sum_partial"]:
+                break
+        else:
+            raise ValidationError(
+                f"No version with md5sum-partial '{data['md5sum_partial']}' exist for "
+                f"{data['content_type'].value} with unique-id '{data['unique_id']}'."
+            )
+
+
+class Compatability(OrderedSchema):
+    name = fields.String()
+    conditions = fields.List(fields.String(), validate=validate.Length(min=1, max=2))
+
+    @validates("conditions")
+    def validate_conditions(self, data, **kwargs):
+        if len(data) == 1:
+            if not data[0].startswith((">= ", "< ")):
+                raise ValidationError(
+                    f"Condition can only mark the first client-version this version does or doesn't work for;"
+                    f" expected '>= VERSION' or '< VERSION', got '{data[0]}'."
+                )
+        else:
+            if not data[0].startswith(">= "):
+                raise ValidationError(
+                    f"First condition can only mark the first client-version this version does work for;"
+                    f" expected '>= VERSION', got '{data[0]}'."
+                )
+            if not data[1].startswith("< "):
+                raise ValidationError(
+                    f"Second condition can only mark the first client-version this version doesn't work for;"
+                    f" expected '< VERSION', got '{data[0]}'."
+                )
+
+
+class VersionMinimized(Global):
+    read_only = ["upload_date", "md5sum_partial", "filesize", "license", "availability"]
+    read_only_for_new = ["upload_date", "md5sum_partial", "filesize", "availability"]
+
+    version = fields.String()
+    license = EnumField(License, by_value=True)
+    upload_date = fields.DateTime(data_key="upload-date", format="iso")
+    md5sum_partial = fields.String(data_key="md5sum-partial", validate=validate.Length(min=8, max=8))
+    filesize = fields.Integer()
+    availability = EnumField(Availability, by_value=True)
+    dependencies = fields.List(fields.Nested(Dependency()))
+    compatibility = fields.List(fields.Nested(Compatability()))
+
+
+class Package(Global):
+    read_only = ["content_type", "unique_id", "archived", "replaced_by"]
+
+    content_type = EnumField(ContentType, data_key="content-type", by_value=True)
+    unique_id = fields.String(data_key="unique-id", validate=validate.Length(min=8, max=8))
+    authors = fields.List(fields.Nested(Author))
+    versions = fields.List(fields.Nested(VersionMinimized))
+
+
+class Version(VersionMinimized):
+    read_only = ["content_type", "unique_id"]
+
+    content_type = EnumField(ContentType, data_key="content-type", by_value=True)
+    unique_id = fields.String(data_key="unique-id", validate=validate.Length(min=8, max=8))
+
+
+class UploadStatusFiles(OrderedSchema):
+    uuid = fields.String()
+    filename = fields.String()
+    filesize = fields.Integer()
+    errors = fields.List(fields.String())
+
+
+class UploadStatus(Version):
+    files = fields.List(fields.Nested(UploadStatusFiles))
+    warnings = fields.List(fields.String)
+    errors = fields.List(fields.String)
+    status = EnumField(Status, by_value=True)