Add: first version of the content_server rewritten in Python

It serves the TCP and HTTP connections from the OpenTTD client,
and requires the BaNaNaS repository to be checked out.

This current version only works with the files on local storage,
which means you need all the files of BaNaNaS in "local_storage"
folder.

This repository in combination with content-api allows you to
run your own BaNaNaS from scratch.

.Dockerignore

@@ -0,0 +1,4 @@
+__pycache__/
+*.pyc
+/BaNaNaS
+/local_storage

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+*.pyc
+/BaNaNaS
+/local_storage

.version

@@ -0,0 +1 @@
+dev

Dockerfile

@@ -1,2 +1,26 @@
 FROM python:3.8-slim
 
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        git \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /code
+
+COPY requirements.txt \
+        LICENSE \
+        README.md \
+        .version \
+        /code/
+
+RUN pip install -r requirements.txt
+
+# Validate that what was installed was what was expected
+RUN pip freeze 2>/dev/null > requirements.installed \
+    && diff -u --strip-trailing-cr requirements.txt requirements.installed 1>&2 \
+    || ( echo "!! ERROR !! requirements.txt defined different packages or versions for installation" \
+        && exit 1 ) 1>&2
+
+COPY content_server /code/content_server
+
+ENTRYPOINT ["python", "-m", "content_server"]
+CMD []

README.md

@@ -0,0 +1,36 @@
+# Content Server
+
+This is the server serving the in-game client for OpenTTD's content service, called BaNaNaS.
+It works together with [https://github.com/OpenTTD/content-api](content-api), which serves the HTTP API.
+
+## Development
+
+This API is written in Python 3.8 with aiohttp, and makes strong use of asyncio.
+
+### Running a local server
+
+To start it, you are advised to first create a virtualenv:
+
+```bash
+python3 -m venv .env
+.env/bin/pip install -r requirements.txt
+```
+
+Next, you can start the HTTP server by running:
+
+```bash
+.env/bin/python -m content_server --web-port 8081 --storage local
+```
+
+This will start the HTTP part of this server on port 8081 and the content server part on port 3978 for you to work with locally.
+You will either have to modify the client to use `localhost` as content server, or change your `hosts` file to change the IP of `binaries.openttd.org` and `content.openttd.org` to point to `127.0.0.1`.
+
+### Running via docker
+
+```bash
+docker build openttd/content-server:local
+mkdir -p $(pwd)/../content-common
+docker run --rm -p 127.0.0.1:8081:80 -p 127.0.0.1:3978:3978 -v $(pwd)/../content-common/local_storage:/code/local_storage -v $(pwd)/../content-common/BaNaNaS:/code/BaNaNaS openttd/content-server:local --storage local --index local
+```
+
+The mount assumes that [https://github.com/OpenTTD/content-api](content-api) and this repository has the same parent folder on your disk, as both servers need to read the same local storage.

content_server/__main__.py

@@ -0,0 +1,91 @@
+import asyncio
+import click
+import importlib
+import logging
+
+from aiohttp import web
+from aiohttp.web_log import AccessLogger
+
+from . import web_routes
+from .application.content_server import Application
+from .helpers import sentry
+from .openttd import tcp_content
+
+log = logging.getLogger(__name__)
+
+CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}
+
+
+class ErrorOnlyAccessLogger(AccessLogger):
+    def log(self, request, response, time):
+        # Only log if the status was not successful
+        if not (200 <= response.status < 400):
+            super().log(request, response, time)
+
+
+async def run_server(application, bind, port):
+    loop = asyncio.get_event_loop()
+
+    server = await loop.create_server(
+        lambda: tcp_content.OpenTTDProtocolTCPContent(application),
+        host=bind,
+        port=port,
+        reuse_port=True,
+        start_serving=True,
+    )
+    log.info(f"Listening on {bind}:{port} ...")
+
+    return server
+
+
+@click.command(context_settings=CONTEXT_SETTINGS)
+@click.option(
+    "--bind", help="The IP to bind the server to", multiple=True, default=["::", "127.0.0.1"], show_default=True
+)
+@click.option("--content-port", help="Port of the content server", default=3978, show_default=True)
+@click.option("--web-port", help="Port of the web server", default=80, show_default=True)
+@click.option("--sentry-dsn", help="Sentry DSN")
+@click.option(
+    "--sentry-environment", help="Environment we are running in (for Sentry)", default="development",
+)
+@click.option("--storage", type=click.Choice(["local"], case_sensitive=False), required=True)
+@click.option("--index", type=click.Choice(["local"], case_sensitive=False), required=True)
+@click.option("--validate", help="Only validate BaNaNaS files and exit", is_flag=True)
+def main(bind, content_port, web_port, sentry_dsn, sentry_environment, storage, index, validate):
+    sentry.setup_sentry(sentry_dsn, sentry_environment)
+
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO
+    )
+
+    storage = storage.lower()
+    storage_module = importlib.import_module(f"content_server.storage.{storage}")
+    storage_instance = getattr(storage_module, "Storage")()
+
+    index = index.lower()
+    index_module = importlib.import_module(f"content_server.index.{index}")
+    index_instance = getattr(index_module, "Index")()
+
+    app_instance = Application(storage_instance, index_instance)
+
+    if validate:
+        return
+
+    log.info(f"Starting content_server with {storage} storage ...")
+
+    loop = asyncio.get_event_loop()
+    server = loop.run_until_complete(run_server(app_instance, bind, content_port))
+
+    web_routes.CONTENT_SERVER_APPLICATION = app_instance
+
+    webapp = web.Application()
+    webapp.add_routes(web_routes.routes)
+
+    web.run_app(webapp, port=web_port, access_log_class=ErrorOnlyAccessLogger)
+
+    log.info(f"Shutting down content_server ...")
+    server.close()
+
+
+if __name__ == "__main__":
+    main(auto_envvar_prefix="CONTENT_SERVER")

content_server/application/content_server.py

@@ -0,0 +1,153 @@
+import logging
+import random
+
+from collections import defaultdict
+
+from ..openttd.protocol.enums import ContentType
+
+log = logging.getLogger(__name__)
+
+
+class Application:
+    def __init__(self, storage, index):
+        super().__init__()
+
+        self.storage = storage
+        self.index = index
+        self.protocol = None
+
+        self._md5sum_mapping = defaultdict(lambda: defaultdict(dict))
+
+        self._id_mapping = defaultdict(lambda: defaultdict(dict))
+        self._by_content_id = {}
+        self._by_content_type = defaultdict(list)
+        self._by_unique_id = defaultdict(dict)
+        self._by_unique_id_and_md5sum = defaultdict(lambda: defaultdict(dict))
+
+        self._reload_md5sum_mapping()
+        self._reload()
+
+    def _send_content_entry(self, source, content_entry):
+        source.protocol.send_PACKET_CONTENT_SERVER_INFO(
+            content_type=content_entry.content_type,
+            content_id=content_entry.content_id,
+            filesize=content_entry.filesize,
+            name=content_entry.name,
+            version=content_entry.version,
+            url=content_entry.url,
+            description=content_entry.description,
+            unique_id=content_entry.unique_id,
+            md5sum=content_entry.md5sum,
+            dependencies=content_entry.dependencies,
+            tags=content_entry.tags,
+        )
+
+    def receive_PACKET_CONTENT_CLIENT_INFO_LIST(self, source, content_type, openttd_version):
+        version_major = (openttd_version >> 28) & 0xF
+        version_minor = (openttd_version >> 24) & 0xF
+        version_patch = (openttd_version >> 20) & 0xF
+        version = (version_major, version_minor, version_patch)
+
+        for content_entry in self._by_content_type[content_type]:
+            if content_entry.min_version and version < content_entry.min_version:
+                continue
+            if content_entry.max_version and version >= content_entry.max_version:
+                continue
+
+            self._send_content_entry(source, content_entry)
+
+    def receive_PACKET_CONTENT_CLIENT_INFO_EXTID(self, source, content_infos):
+        for content_info in content_infos:
+            content_entry = self._by_unique_id[content_info.content_type].get(content_info.unique_id)
+            if content_entry:
+                self._send_content_entry(source, content_entry)
+
+    def receive_PACKET_CONTENT_CLIENT_INFO_EXTID_MD5(self, source, content_infos):
+        for content_info in content_infos:
+            content_entry = (
+                self._by_unique_id_and_md5sum[content_info.content_type]
+                .get(content_info.unique_id, {})
+                .get(content_info.md5sum)
+            )
+            if content_entry:
+                self._send_content_entry(source, content_entry)
+
+    def receive_PACKET_CONTENT_CLIENT_INFO_ID(self, source, content_infos):
+        for content_info in content_infos:
+            content_entry = self._by_content_id.get(content_info.content_id)
+            if content_entry:
+                self._send_content_entry(source, content_entry)
+
+    def receive_PACKET_CONTENT_CLIENT_CONTENT(self, source, content_infos):
+        for content_info in content_infos:
+            content_entry = self._by_content_id[content_info.content_id]
+
+            try:
+                stream = self.storage.get_stream(content_entry)
+            except Exception:
+                log.exception("Error with storage, aborting for this client ...")
+                return
+
+            source.protocol.send_PACKET_CONTENT_SERVER_CONTENT(
+                content_type=content_entry.content_type,
+                content_id=content_entry.content_id,
+                filesize=content_entry.filesize,
+                filename=f"{content_entry.name} - {content_entry.version}",
+                stream=stream,
+            )
+
+    def get_by_content_id(self, content_id):
+        return self._by_content_id.get(content_id)
+
+    def get_by_unique_id_and_md5sum(self, content_type, unique_id, md5sum):
+        return self._by_unique_id_and_md5sum[content_type].get(unique_id, {}).get(md5sum)
+
+    def _reload_md5sum_mapping(self):
+        for content_type in ContentType:
+            if content_type == ContentType.CONTENT_TYPE_END:
+                continue
+
+            for unique_id_str in self.storage.list_folder(content_type):
+                unique_id = bytes.fromhex(unique_id_str)
+
+                for filename in self.storage.list_folder(content_type, unique_id_str):
+                    md5sum, _, _ = filename.partition(".")
+
+                    md5sum_partial = bytes.fromhex(md5sum[0:8])
+                    md5sum = bytes.fromhex(md5sum)
+
+                    self._md5sum_mapping[content_type][unique_id][md5sum_partial] = md5sum
+
+    def _reload(self):
+        self.index.load_all(
+            self._by_content_id,
+            self._by_content_type,
+            self._by_unique_id,
+            self._by_unique_id_and_md5sum,
+            self._md5sum_mapping,
+            self._get_next_content_id,
+        )
+
+        for content_entry in self._by_content_id.values():
+            content_entry.calculate_dependencies(self)
+
+    def _get_next_content_id(self, content_type, unique_id, md5sum):
+        # Cache the result for the livetime of this process. This means that
+        # if we reload, we keep the content_ids as they were. This avoids
+        # clients having content_ids that are no longer valid.
+        # If this process cycles, this mapping is lost. That can lead to some
+        # clients having to restart their OpenTTD, but as this is expected to
+        # be a rare event, it should be fine.
+        content_id = self._id_mapping[content_type][unique_id].get(md5sum)
+        if content_id is not None:
+            return content_id
+
+        # Pick a random content_id and check if it is not used. The total
+        # amount of available content at the time of writing is around the
+        # 5,000 entries. So the chances of hitting an existing number is
+        # very small, and this should be done pretty quick.
+        while True:
+            content_id = random.randrange(0, 2 ** 31)
+            if content_id not in self._by_content_id:
+                self._id_mapping[content_type][unique_id][md5sum] = content_id
+                return content_id

content_server/helpers/content_type.py

@@ -0,0 +1,26 @@
+from ..openttd.protocol.enums import ContentType
+
+
+content_type_folder_name_mapping = {
+    ContentType.CONTENT_TYPE_BASE_GRAPHICS: "base-graphics",
+    ContentType.CONTENT_TYPE_NEWGRF: "newgrf",
+    ContentType.CONTENT_TYPE_AI: "ai",
+    ContentType.CONTENT_TYPE_AI_LIBRARY: "ai-library",
+    ContentType.CONTENT_TYPE_SCENARIO: "scenario",
+    ContentType.CONTENT_TYPE_HEIGHTMAP: "heightmap",
+    ContentType.CONTENT_TYPE_BASE_SOUNDS: "base-sounds",
+    ContentType.CONTENT_TYPE_BASE_MUSIC: "base-music",
+    ContentType.CONTENT_TYPE_GAME: "game-script",
+    ContentType.CONTENT_TYPE_GAME_LIBRARY: "game-script-library",
+}
+
+
+def get_folder_name_from_content_type(content_type):
+    return content_type_folder_name_mapping[content_type]
+
+
+def get_content_type_from_name(content_type_name):
+    for content_type, name in content_type_folder_name_mapping.items():
+        if name == content_type_name:
+            return content_type
+    raise Exception("Unknown content_type: ", content_type_name)

content_server/helpers/sentry.py

@@ -0,0 +1,18 @@
+import logging
+import sentry_sdk
+
+log = logging.getLogger(__name__)
+
+
+def setup_sentry(sentry_dsn, environment):
+    if not sentry_dsn:
+        return
+
+    # Release is expected to be in the file '.version'
+    with open(".version") as f:
+        release = f.readline().strip()
+
+    sentry_sdk.init(sentry_dsn, release=release, environment=environment)
+    log.info(
+        "Sentry initialized with release='%s' and environment='%s'", release, environment,
+    )