From c21984e276663683d095f8b3a0d4384656ea3f6d Mon Sep 17 00:00:00 2001 From: Patric Stout Date: Wed, 26 Feb 2020 15:29:09 +0100 Subject: [PATCH 1/4] Add: first version of the bananas_server rewritten in Python It serves the TCP and HTTP connections from the OpenTTD client. This repository in combination with bananas-api allows you to run your own BaNaNaS from scratch. --- .Dockerignore | 5 + .github/workflows/deployment.yml | 40 +++ .github/workflows/publish.yml | 69 ++++ .github/workflows/testing.yml | 4 +- .gitignore | 5 + .version | 1 + Dockerfile | 34 ++ LICENSE | 339 ++++++++++++++++++ README.md | 56 +++ bananas_server/__init__.py | 0 bananas_server/__main__.py | 102 ++++++ bananas_server/application/__init__.py | 0 bananas_server/application/bananas_server.py | 149 ++++++++ bananas_server/helpers/__init__.py | 0 bananas_server/helpers/click.py | 41 +++ bananas_server/helpers/content_type.py | 26 ++ bananas_server/helpers/sentry.py | 26 ++ bananas_server/index/__init__.py | 0 bananas_server/index/github.py | 58 +++ bananas_server/index/local.py | 274 ++++++++++++++ bananas_server/index/schema.py | 37 ++ bananas_server/openttd/__init__.py | 0 bananas_server/openttd/protocol/__init__.py | 0 bananas_server/openttd/protocol/enums.py | 29 ++ bananas_server/openttd/protocol/exceptions.py | 18 + bananas_server/openttd/protocol/read.py | 46 +++ bananas_server/openttd/protocol/source.py | 21 ++ bananas_server/openttd/protocol/write.py | 35 ++ bananas_server/openttd/receive.py | 175 +++++++++ bananas_server/openttd/send.py | 81 +++++ bananas_server/openttd/tcp_content.py | 57 +++ bananas_server/storage/__init__.py | 0 bananas_server/storage/local.py | 65 ++++ bananas_server/storage/s3.py | 80 +++++ bananas_server/web_routes.py | 94 +++++ requirements.base | 8 + requirements.txt | 24 ++ 37 files changed, 1997 insertions(+), 2 deletions(-) create mode 100644 .Dockerignore create mode 100644 .github/workflows/deployment.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .gitignore create mode 100644 .version create mode 100644 LICENSE create mode 100644 README.md create mode 100644 bananas_server/__init__.py create mode 100644 bananas_server/__main__.py create mode 100644 bananas_server/application/__init__.py create mode 100644 bananas_server/application/bananas_server.py create mode 100644 bananas_server/helpers/__init__.py create mode 100644 bananas_server/helpers/click.py create mode 100644 bananas_server/helpers/content_type.py create mode 100644 bananas_server/helpers/sentry.py create mode 100644 bananas_server/index/__init__.py create mode 100644 bananas_server/index/github.py create mode 100644 bananas_server/index/local.py create mode 100644 bananas_server/index/schema.py create mode 100644 bananas_server/openttd/__init__.py create mode 100644 bananas_server/openttd/protocol/__init__.py create mode 100644 bananas_server/openttd/protocol/enums.py create mode 100644 bananas_server/openttd/protocol/exceptions.py create mode 100644 bananas_server/openttd/protocol/read.py create mode 100644 bananas_server/openttd/protocol/source.py create mode 100644 bananas_server/openttd/protocol/write.py create mode 100644 bananas_server/openttd/receive.py create mode 100644 bananas_server/openttd/send.py create mode 100644 bananas_server/openttd/tcp_content.py create mode 100644 bananas_server/storage/__init__.py create mode 100644 bananas_server/storage/local.py create mode 100644 bananas_server/storage/s3.py create mode 100644 bananas_server/web_routes.py create mode 100644 requirements.base create mode 100644 requirements.txt diff --git a/.Dockerignore b/.Dockerignore new file mode 100644 index 0000000..67bc949 --- /dev/null +++ b/.Dockerignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +/.env +/BaNaNaS +/local_storage diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml new file mode 100644 index 0000000..d052e78 --- /dev/null +++ b/.github/workflows/deployment.yml @@ -0,0 +1,40 @@ +name: Deployment + +on: + deployment: + +jobs: + deploy_to_aws: + name: Deploy to AWS + runs-on: ubuntu-latest + + steps: + - name: Deployment in progress + uses: openttd/actions/deployments-update@v1 + with: + github-token: ${{ secrets.DEPLOYMENT_TOKEN }} + state: in_progress + description: "Deployment of ${{ github.event.deployment.payload.version }} to ${{ github.event.deployment.environment }} started" + + - name: Deploy on AWS + uses: openttd/actions/deploy-aws@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ secrets.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: BananasServer + + - if: success() + name: Deployment successful + uses: openttd/actions/deployments-update@v1 + with: + github-token: ${{ secrets.DEPLOYMENT_TOKEN }} + state: success + description: "Successfully deployed ${{ github.event.deployment.payload.version }} on ${{ github.event.deployment.environment }}" + + - if: failure() || cancelled() + name: Deployment failed + uses: openttd/actions/deployments-update@v1 + with: + github-token: ${{ secrets.DEPLOYMENT_TOKEN }} + state: failure diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..88c954c --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,69 @@ +name: Publish image + +on: + push: + branches: + - master + tags: + - '*' + repository_dispatch: + types: + - publish_latest_tag + - publish_master + +jobs: + publish_image: + name: Publish image + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - if: github.event_name == 'repository_dispatch' + name: Repository dispatch + uses: openttd/actions/checkout-dispatch@v1 + + - name: Checkout tags and submodules + uses: openttd/actions/checkout@v1 + with: + with-tags: true + + - name: Set variables + id: vars + uses: openttd/actions/docker-vars@v1 + with: + docker-hub-username: ${{ secrets.DOCKER_USERNAME }} + + - name: Build + uses: openttd/actions/docker-build@v1 + with: + name: ${{ steps.vars.outputs.name }} + tag: ${{ steps.vars.outputs.tag }} + tags: ${{ steps.vars.outputs.tags }} + version: ${{ steps.vars.outputs.version }} + date: ${{ steps.vars.outputs.date }} + + - if: steps.vars.outputs.dry-run == 'false' + name: Publish + id: publish + uses: openttd/actions/docker-publish@v1 + with: + docker-hub-username: ${{ secrets.DOCKER_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_PASSWORD }} + name: ${{ steps.vars.outputs.name }} + tag: ${{ steps.vars.outputs.tag }} + + # Dorpsgek only runs on production, not on staging. + - if: steps.vars.outputs.dry-run == 'false' && steps.vars.outputs.environment == 'production' + name: Trigger deployment + uses: openttd/actions/deployments-create@v1 + with: + ref: ${{ steps.vars.outputs.sha }} + environment: ${{ steps.vars.outputs.environment }} + version: ${{ steps.vars.outputs.version }} + date: ${{ steps.vars.outputs.date }} + docker-tag: ${{ steps.publish.outputs.remote-tag }} + github-token: ${{ secrets.DEPLOYMENT_TOKEN }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6418205..3da2919 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -25,7 +25,7 @@ jobs: - name: Flake8 uses: TrueBrain/actions-flake8@master with: - path: content_server + path: bananas_server black: name: Black @@ -41,4 +41,4 @@ jobs: run: | python -m pip install --upgrade pip pip install black - black -l 120 --check content_server + black -l 120 --check bananas_server diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..67bc949 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +/.env +/BaNaNaS +/local_storage diff --git a/.version b/.version new file mode 100644 index 0000000..38f8e88 --- /dev/null +++ b/.version @@ -0,0 +1 @@ +dev diff --git a/Dockerfile b/Dockerfile index 1bbba1a..3b36c3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,2 +1,36 @@ FROM python:3.8-slim +ARG BUILD_DATE="" +ARG BUILD_VERSION="dev" + +LABEL maintainer="truebrain@openttd.org" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=${BUILD_DATE} +LABEL org.label-schema.version=${BUILD_VERSION} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /code + +COPY requirements.txt \ + LICENSE \ + README.md \ + .version \ + /code/ +# Needed for Sentry to know what version we are running +RUN echo "${BUILD_VERSION}" > /code/.version + +RUN pip --no-cache-dir install -r requirements.txt + +# Validate that what was installed was what was expected +RUN pip freeze 2>/dev/null > requirements.installed \ + && diff -u --strip-trailing-cr requirements.txt requirements.installed 1>&2 \ + || ( echo "!! ERROR !! requirements.txt defined different packages or versions for installation" \ + && exit 1 ) 1>&2 + +COPY bananas_server /code/bananas_server + +ENTRYPOINT ["python", "-m", "bananas_server"] +CMD ["--bind", "0.0.0.0", "--storage", "local", "--index", "local"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..440855f --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# BaNaNaS Server + +[![GitHub License](https://img.shields.io/github/license/OpenTTD/bananas-server)](https://github.com/OpenTTD/bananas-server/blob/master/LICENSE) +[![GitHub Tag](https://img.shields.io/github/v/tag/OpenTTD/bananas-server?include_prereleases&label=stable)](https://github.com/OpenTTD/bananas-server/releases) +[![GitHub commits since latest release](https://img.shields.io/github/commits-since/OpenTTD/bananas-server/latest/master)](https://github.com/OpenTTD/bananas-server/commits/master) + +[![GitHub Workflow Status (Testing)](https://img.shields.io/github/workflow/status/OpenTTD/bananas-server/Testing/master?label=master)](https://github.com/OpenTTD/bananas-server/actions?query=workflow%3ATesting) +[![GitHub Workflow Status (Publish Image)](https://img.shields.io/github/workflow/status/OpenTTD/bananas-server/Publish%20image?label=publish)](https://github.com/OpenTTD/bananas-server/actions?query=workflow%3A%22Publish+image%22) +[![GitHub Workflow Status (Deployments)](https://img.shields.io/github/workflow/status/OpenTTD/bananas-server/Deployment?label=deployment)](https://github.com/OpenTTD/bananas-server/actions?query=workflow%3A%22Deployment%22) + +[![GitHub deployments (Staging)](https://img.shields.io/github/deployments/OpenTTD/bananas-server/staging?label=staging)](https://github.com/OpenTTD/bananas-server/deployments) +[![GitHub deployments (Production)](https://img.shields.io/github/deployments/OpenTTD/bananas-server/production?label=production)](https://github.com/OpenTTD/bananas-server/deployments) + +This is the server serving the in-game client for OpenTTD's content service, called BaNaNaS. +It works together with [bananas-api](https://github.com/OpenTTD/bananas-api), which serves the HTTP API. + +## Development + +This API is written in Python 3.8 with aiohttp, and makes strong use of asyncio. + +### Running a local server + +#### Dependencies + +- Python3.8 or higher. + +#### Preparing your venv + +To start it, you are advised to first create a virtualenv: + +```bash +python3 -m venv .env +.env/bin/pip install -r requirements.txt +``` + +#### Starting a local server + +Next, you can start the HTTP server by running: + +```bash +.env/bin/python -m bananas_server --web-port 8081 --storage local --index local +``` + +This will start the HTTP part of this server on port 8081 and the content server part on port 3978 for you to work with locally. +You will either have to modify the client to use `localhost` as content server, or change your `hosts` file to change the IP of `binaries.openttd.org` and `content.openttd.org` to point to `127.0.0.1`. + +### Running via docker + +```bash +docker build -t openttd/bananas-server:local . +export BANANAS_COMMON=$(pwd)/../bananas-common +mkdir -p "${BANANAS_COMMON}/local_storage" "${BANANAS_COMMON}/BaNaNaS" +docker run --rm -p 127.0.0.1:8081:80 -p 127.0.0.1:3978:3978 -v "${BANANAS_COMMON}/local_storage:/code/local_storage" -v "${BANANAS_COMMON}/BaNaNaS:/code/BaNaNaS" openttd/bananas-server:local +``` + +The mount assumes that [bananas-api](https://github.com/OpenTTD/bananas-api) and this repository has the same parent folder on your disk, as both servers need to read the same local storage. diff --git a/bananas_server/__init__.py b/bananas_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bananas_server/__main__.py b/bananas_server/__main__.py new file mode 100644 index 0000000..98aa87e --- /dev/null +++ b/bananas_server/__main__.py @@ -0,0 +1,102 @@ +import asyncio +import click +import logging + +from aiohttp import web +from aiohttp.web_log import AccessLogger + +from . import web_routes +from .application.bananas_server import Application +from .helpers.click import ( + click_additional_options, + import_module, +) +from .helpers.sentry import click_sentry +from .index.github import click_index_github +from .index.local import click_index_local +from .storage.local import click_storage_local +from .storage.s3 import click_storage_s3 +from .openttd import tcp_content + +log = logging.getLogger(__name__) + +CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} + + +class ErrorOnlyAccessLogger(AccessLogger): + def log(self, request, response, time): + # Only log if the status was not successful + if not (200 <= response.status < 400): + super().log(request, response, time) + + +async def run_server(application, bind, port): + loop = asyncio.get_event_loop() + + server = await loop.create_server( + lambda: tcp_content.OpenTTDProtocolTCPContent(application), + host=bind, + port=port, + reuse_port=True, + start_serving=True, + ) + log.info(f"Listening on {bind}:{port} ...") + + return server + + +@click_additional_options +def click_logging(): + logging.basicConfig( + format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO + ) + + +@click.command(context_settings=CONTEXT_SETTINGS) +@click_logging # Should always be on top, as it initializes the logging +@click_sentry +@click.option( + "--bind", help="The IP to bind the server to", multiple=True, default=["::1", "127.0.0.1"], show_default=True +) +@click.option("--content-port", help="Port of the content server", default=3978, show_default=True) +@click.option("--web-port", help="Port of the web server", default=80, show_default=True) +@click.option( + "--storage", + type=click.Choice(["local", "s3"], case_sensitive=False), + required=True, + callback=import_module("bananas_server.storage", "Storage"), +) +@click_storage_local +@click_storage_s3 +@click.option( + "--index", + type=click.Choice(["local", "github"], case_sensitive=False), + required=True, + callback=import_module("bananas_server.index", "Index"), +) +@click_index_local +@click_index_github +@web_routes.click_web_routes +@click.option("--validate", help="Only validate BaNaNaS files and exit", is_flag=True) +def main(bind, content_port, web_port, storage, index, validate): + app_instance = Application(storage(), index()) + + if validate: + return + + loop = asyncio.get_event_loop() + server = loop.run_until_complete(run_server(app_instance, bind, content_port)) + + web_routes.BANANAS_SERVER_APPLICATION = app_instance + + webapp = web.Application() + webapp.add_routes(web_routes.routes) + + web.run_app(webapp, host=bind, port=web_port, access_log_class=ErrorOnlyAccessLogger) + + log.info(f"Shutting down bananas_server ...") + server.close() + + +if __name__ == "__main__": + main(auto_envvar_prefix="BANANAS_SERVER") diff --git a/bananas_server/application/__init__.py b/bananas_server/application/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bananas_server/application/bananas_server.py b/bananas_server/application/bananas_server.py new file mode 100644 index 0000000..854adbc --- /dev/null +++ b/bananas_server/application/bananas_server.py @@ -0,0 +1,149 @@ +import logging +import random + +from collections import defaultdict + +from ..openttd.protocol.enums import ContentType + +log = logging.getLogger(__name__) + + +class Application: + def __init__(self, storage, index): + super().__init__() + + self.storage = storage + self.index = index + self.protocol = None + + self._md5sum_mapping = defaultdict(lambda: defaultdict(dict)) + + self._id_mapping = defaultdict(lambda: defaultdict(dict)) + self._by_content_id = {} + self._by_content_type = defaultdict(list) + self._by_unique_id = defaultdict(dict) + self._by_unique_id_and_md5sum = defaultdict(lambda: defaultdict(dict)) + + self.reload() + + def _send_content_entry(self, source, content_entry): + source.protocol.send_PACKET_CONTENT_SERVER_INFO( + content_type=content_entry.content_type, + content_id=content_entry.content_id, + filesize=content_entry.filesize, + name=content_entry.name, + version=content_entry.version, + url=content_entry.url, + description=content_entry.description, + unique_id=content_entry.unique_id, + md5sum=content_entry.md5sum, + dependencies=content_entry.dependencies, + tags=content_entry.tags, + ) + + def receive_PACKET_CONTENT_CLIENT_INFO_LIST(self, source, content_type, openttd_version): + version_major = (openttd_version >> 28) & 0xF + version_minor = (openttd_version >> 24) & 0xF + version_patch = (openttd_version >> 20) & 0xF + version = (version_major, version_minor, version_patch) + + for content_entry in self._by_content_type[content_type]: + if content_entry.min_version and version < content_entry.min_version: + continue + if content_entry.max_version and version >= content_entry.max_version: + continue + + self._send_content_entry(source, content_entry) + + def receive_PACKET_CONTENT_CLIENT_INFO_EXTID(self, source, content_infos): + for content_info in content_infos: + content_entry = self._by_unique_id[content_info.content_type].get(content_info.unique_id) + if content_entry: + self._send_content_entry(source, content_entry) + + def receive_PACKET_CONTENT_CLIENT_INFO_EXTID_MD5(self, source, content_infos): + for content_info in content_infos: + content_entry = ( + self._by_unique_id_and_md5sum[content_info.content_type] + .get(content_info.unique_id, {}) + .get(content_info.md5sum) + ) + if content_entry: + self._send_content_entry(source, content_entry) + + def receive_PACKET_CONTENT_CLIENT_INFO_ID(self, source, content_infos): + for content_info in content_infos: + content_entry = self._by_content_id.get(content_info.content_id) + if content_entry: + self._send_content_entry(source, content_entry) + + def receive_PACKET_CONTENT_CLIENT_CONTENT(self, source, content_infos): + for content_info in content_infos: + content_entry = self._by_content_id[content_info.content_id] + + try: + stream = self.storage.get_stream(content_entry) + except Exception: + log.exception("Error with storage, aborting for this client ...") + return + + source.protocol.send_PACKET_CONTENT_SERVER_CONTENT( + content_type=content_entry.content_type, + content_id=content_entry.content_id, + filesize=content_entry.filesize, + filename=f"{content_entry.name} - {content_entry.version}", + stream=stream, + ) + + def get_by_content_id(self, content_id): + return self._by_content_id.get(content_id) + + def get_by_unique_id_and_md5sum(self, content_type, unique_id, md5sum): + return self._by_unique_id_and_md5sum[content_type].get(unique_id, {}).get(md5sum) + + def reload_md5sum_mapping(self): + for content_type in ContentType: + if content_type == ContentType.CONTENT_TYPE_END: + continue + + for unique_id_str in self.storage.list_folder(content_type): + unique_id = bytes.fromhex(unique_id_str) + + for filename in self.storage.list_folder(content_type, unique_id_str): + md5sum, _, _ = filename.partition(".") + + md5sum_partial = bytes.fromhex(md5sum[0:8]) + md5sum = bytes.fromhex(md5sum) + + self._md5sum_mapping[content_type][unique_id][md5sum_partial] = md5sum + + def reload(self): + self.index.reload(self) + + def clear(self): + self._by_content_id.clear() + self._by_content_type.clear() + self._by_unique_id.clear() + self._by_unique_id_and_md5sum.clear() + self._md5sum_mapping.clear() + + def _get_next_content_id(self, content_type, unique_id, md5sum): + # Cache the result for the livetime of this process. This means that + # if we reload, we keep the content_ids as they were. This avoids + # clients having content_ids that are no longer valid. + # If this process cycles, this mapping is lost. That can lead to some + # clients having to restart their OpenTTD, but as this is expected to + # be a rare event, it should be fine. + content_id = self._id_mapping[content_type][unique_id].get(md5sum) + if content_id is not None: + return content_id + + # Pick a random content_id and check if it is not used. The total + # amount of available content at the time of writing is around the + # 5,000 entries. So the chances of hitting an existing number is + # very small, and this should be done pretty quick. + while True: + content_id = random.randrange(0, 2 ** 31) + if content_id not in self._by_content_id: + self._id_mapping[content_type][unique_id][md5sum] = content_id + return content_id diff --git a/bananas_server/helpers/__init__.py b/bananas_server/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bananas_server/helpers/click.py b/bananas_server/helpers/click.py new file mode 100644 index 0000000..b31f4f2 --- /dev/null +++ b/bananas_server/helpers/click.py @@ -0,0 +1,41 @@ +import importlib + + +def import_module(prefix, class_name): + def _callback(value): + value = value.lower() + module = importlib.import_module(f"{prefix}.{value}") + class_ = getattr(module, class_name) + return class_ + + def callback(ctx, param, value): + if isinstance(value, tuple): + return [_callback(v) for v in value] + else: + return _callback(value) + + return callback + + +def click_additional_options(additional_func): + def decorator(func): + additional_params = [] + for param in getattr(additional_func, "__click_params__", []): + additional_params.append(param.name) + + def inner_decorator(**kwargs): + additional_kwargs = {param: kwargs[param] for param in additional_params} + additional_func(**additional_kwargs) + + # Remove the kwargs that are consumed by the additional_func + [kwargs.pop(kwarg) for kwarg in additional_kwargs] + + func(**kwargs) + + inner_decorator.__click_params__ = getattr(func, "__click_params__", []) + getattr( + additional_func, "__click_params__", [] + ) + inner_decorator.__doc__ = func.__doc__ + return inner_decorator + + return decorator diff --git a/bananas_server/helpers/content_type.py b/bananas_server/helpers/content_type.py new file mode 100644 index 0000000..4ab702d --- /dev/null +++ b/bananas_server/helpers/content_type.py @@ -0,0 +1,26 @@ +from ..openttd.protocol.enums import ContentType + + +content_type_folder_name_mapping = { + ContentType.CONTENT_TYPE_BASE_GRAPHICS: "base-graphics", + ContentType.CONTENT_TYPE_NEWGRF: "newgrf", + ContentType.CONTENT_TYPE_AI: "ai", + ContentType.CONTENT_TYPE_AI_LIBRARY: "ai-library", + ContentType.CONTENT_TYPE_SCENARIO: "scenario", + ContentType.CONTENT_TYPE_HEIGHTMAP: "heightmap", + ContentType.CONTENT_TYPE_BASE_SOUNDS: "base-sounds", + ContentType.CONTENT_TYPE_BASE_MUSIC: "base-music", + ContentType.CONTENT_TYPE_GAME: "game-script", + ContentType.CONTENT_TYPE_GAME_LIBRARY: "game-script-library", +} + + +def get_folder_name_from_content_type(content_type): + return content_type_folder_name_mapping[content_type] + + +def get_content_type_from_name(content_type_name): + for content_type, name in content_type_folder_name_mapping.items(): + if name == content_type_name: + return content_type + raise Exception("Unknown content_type: ", content_type_name) diff --git a/bananas_server/helpers/sentry.py b/bananas_server/helpers/sentry.py new file mode 100644 index 0000000..3548fc8 --- /dev/null +++ b/bananas_server/helpers/sentry.py @@ -0,0 +1,26 @@ +import click +import logging +import sentry_sdk + +from ..helpers.click import click_additional_options + +log = logging.getLogger(__name__) + + +@click_additional_options +@click.option("--sentry-dsn", help="Sentry DSN.") +@click.option( + "--sentry-environment", help="Environment we are running in.", default="development", +) +def click_sentry(sentry_dsn, sentry_environment): + if not sentry_dsn: + return + + # Release is expected to be in the file '.version' + with open(".version") as f: + release = f.readline().strip() + + sentry_sdk.init(sentry_dsn, release=release, environment=sentry_environment) + log.info( + "Sentry initialized with release='%s' and environment='%s'", release, sentry_environment, + ) diff --git a/bananas_server/index/__init__.py b/bananas_server/index/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bananas_server/index/github.py b/bananas_server/index/github.py new file mode 100644 index 0000000..30d496a --- /dev/null +++ b/bananas_server/index/github.py @@ -0,0 +1,58 @@ +import click +import git +import logging +import os + +from .local import Index as LocalIndex +from ..helpers.click import click_additional_options + +log = logging.getLogger(__name__) + +_github_url = None + + +class Index(LocalIndex): + def __init__(self): + super().__init__() + + try: + self._git = git.Repo(self._folder) + except git.exc.NoSuchPathError: + self._git = git.Repo.init(self._folder) + + # Make sure the origin is set correctly + if "origin" not in self._git.remotes: + self._git.create_remote("origin", _github_url) + origin = self._git.remotes.origin + if origin.url != _github_url: + origin.set_url(_github_url) + + def _fetch_latest(self): + log.info("Updating index to latest version from GitHub") + + origin = self._git.remotes.origin + + # Checkout the latest master, removing and commits/file changes local + # might have. + origin.fetch() + origin.refs.master.checkout(force=True, B="master") + for file_name in self._git.untracked_files: + os.unlink(f"{self._folder}/{file_name}") + + def reload(self, application): + self._fetch_latest() + super().reload(application) + + +@click_additional_options +@click.option( + "--index-github-url", + help="Repository URL on GitHub. (index=github only)", + default="https://github.com/OpenTTD/BaNaNaS", + show_default=True, + metavar="URL", +) +def click_index_github(index_github_url): + global _github_url + + _github_url = index_github_url diff --git a/bananas_server/index/local.py b/bananas_server/index/local.py new file mode 100644 index 0000000..fee0998 --- /dev/null +++ b/bananas_server/index/local.py @@ -0,0 +1,274 @@ +import click +import logging +import os +import yaml + +from .schema import ContentEntry as ContentEntryTest +from ..helpers.click import click_additional_options +from ..helpers.content_type import get_content_type_from_name +from ..helpers.content_type import get_folder_name_from_content_type +from ..openttd.protocol.enums import ContentType + +log = logging.getLogger(__name__) + +_folder = None + + +class ContentEntry: + def __init__( + self, + content_id, + content_type, + filesize, + name, + version, + url, + description, + unique_id, + md5sum, + dependencies, + min_version, + max_version, + tags, + ): + super().__init__() + + self.content_id = content_id + self.content_type = content_type + self.filesize = filesize + self.name = name + self.version = version + self.url = url + self.description = description + self.unique_id = unique_id + self.md5sum = md5sum + self.raw_dependencies = dependencies + self.dependencies = None + self.min_version = min_version + self.max_version = max_version + self.tags = tags + + def calculate_dependencies(self, application): + dependencies = [] + + for dependency in self.raw_dependencies: + (content_type, unique_id, md5sum) = dependency + dep_content_entry = application.get_by_unique_id_and_md5sum(content_type, unique_id, md5sum) + if dep_content_entry is None: + log.error("Invalid dependency: %r", dependency) + continue + + dependencies.append(dep_content_entry.content_id) + + self.dependencies = dependencies + + def __repr__(self): + return ( + f"ContentEntry(content_id={self.content_id!r}, " + f"content_type={self.content_type!r}, " + f"filesize={self.filesize!r}, " + f"name={self.name!r}, " + f"version={self.version!r}, " + f"url={self.url!r}, " + f"description={self.description!r}, " + f"unique_id={self.unique_id!r}, " + f"md5sum={self.md5sum!r}, " + f"dependencies={self.dependencies!r}, " + f"min_version={self.min_version!r}, " + f"max_version={self.max_version!r}, " + f"tags={self.tags!r})" + ) + + +class Index: + def __init__(self): + self._folder = _folder + + def _read_content_entry_version(self, content_type, unique_id, data, md5sum_mapping, get_next_content_id): + unique_id = bytes.fromhex(unique_id) + + md5sum_partial = bytes.fromhex(data["md5sum-partial"]) + md5sum = md5sum_mapping[content_type][unique_id][md5sum_partial] + + content_id = get_next_content_id(content_type, unique_id, md5sum) + + dependencies = [] + for dependency in data.get("dependencies", []): + dep_content_type = get_content_type_from_name(dependency["content-type"]) + dep_unique_id = bytes.fromhex(dependency["unique-id"]) + + dep_md5sum_partial = bytes.fromhex(dependency["md5sum-partial"]) + dep_md5sum = md5sum_mapping[dep_content_type][dep_unique_id][dep_md5sum_partial] + + dependencies.append((dep_content_type, dep_unique_id, dep_md5sum)) + + min_version = None + max_version = None + for com in data.get("compatibility", {}): + if com["name"] != "master": + continue + + for conditions in com["conditions"]: + if conditions.startswith(">="): + min_version = [int(p) for p in conditions[2:].split(".")] + elif conditions.startswith("<"): + max_version = [int(p) for p in conditions[1:].split(".")] + else: + raise Exception("Invalid compatibility flag", com) + + # Validate the object to make sure all fields are within set limits. + ContentEntryTest().load( + { + "content-type": content_type, + "content-id": content_id, + "filesize": data["filesize"], + "name": data["name"], + "version": data["version"], + "url": data.get("url", ""), + "description": data.get("description", ""), + "unique-id": unique_id, + "md5sum": md5sum, + "min-version": min_version, + "max-version": max_version, + "tags": data.get("tags", []), + "raw-dependencies": dependencies, + } + ) + + # Calculate if this entry wouldn't exceed the OpenTTD packet size if + # we would transmit this over the wire. + size = 1 + 4 + 4 # content-type, content-id, filesize + size += len(data["name"]) + 2 + size += len(data["version"]) + 2 + size += len(data.get("url", "")) + 2 + size += len(data.get("description", "")) + 2 + size += len(unique_id) + 2 + size += len(md5sum) + 2 + size += len(dependencies) * 4 + size += 1 + for tag in data.get("tags", []): + size += len(tag) + 2 + + if size > 1400: + raise Exception("Entry would exceed OpenTTD packet size.") + + content_entry = ContentEntry( + content_type=content_type, + content_id=content_id, + filesize=data["filesize"], + name=data["name"], + version=data["version"], + url=data.get("url", ""), + description=data.get("description", ""), + unique_id=unique_id, + md5sum=md5sum, + dependencies=dependencies, + min_version=min_version, + max_version=max_version, + tags=data.get("tags", []), + ) + + return content_entry + + def _read_content_entry(self, content_type, folder_name, unique_id, md5sum_mapping, get_next_content_id): + folder_name = f"{folder_name}/{unique_id}" + + with open(f"{folder_name}/global.yaml") as f: + global_data = yaml.safe_load(f.read()) + + # If this entry is blacklisted, we won't be finding anything useful + if global_data.get("blacklisted"): + return [], [] + + content_entries = [] + archived_content_entries = [] + for version in os.listdir(f"{folder_name}/versions"): + with open(f"{folder_name}/versions/{version}") as f: + version_data = yaml.safe_load(f.read()) + + # Extend the version data with global data with fields not set + for key, value in global_data.items(): + if key not in version_data: + version_data[key] = value + + try: + content_entry = self._read_content_entry_version( + content_type, unique_id, version_data, md5sum_mapping, get_next_content_id + ) + except Exception: + log.exception(f"Failed to load entry {folder_name}/versions/{version}. Skipping.") + continue + + if version_data["availability"] == "new-games": + content_entries.append(content_entry) + else: + archived_content_entries.append(content_entry) + + return content_entries, archived_content_entries + + def reload(self, application): + application.clear() + application.reload_md5sum_mapping() + + self.load_all( + application._by_content_id, + application._by_content_type, + application._by_unique_id, + application._by_unique_id_and_md5sum, + application._md5sum_mapping, + application._get_next_content_id, + ) + + for content_entry in application._by_content_id.values(): + content_entry.calculate_dependencies(application) + + def load_all( + self, by_content_id, by_content_type, by_unique_id, by_unique_id_and_md5sum, md5sum_mapping, get_next_content_id + ): + for content_type in ContentType: + if content_type == ContentType.CONTENT_TYPE_END: + continue + + counter_entries = 0 + counter_archived = 0 + + content_type_folder_name = get_folder_name_from_content_type(content_type) + folder_name = f"{self._folder}/{content_type_folder_name}" + + if not os.path.isdir(folder_name): + continue + + for unique_id in os.listdir(folder_name): + content_entries, archived_content_entries = self._read_content_entry( + content_type, folder_name, unique_id, md5sum_mapping, get_next_content_id + ) + for content_entry in content_entries: + counter_entries += 1 + by_content_id[content_entry.content_id] = content_entry + by_unique_id_and_md5sum[content_type][content_entry.unique_id][content_entry.md5sum] = content_entry + + by_content_type[content_type].append(content_entry) + by_unique_id[content_type][content_entry.unique_id] = content_entry + + for content_entry in archived_content_entries: + counter_archived += 1 + by_content_id[content_entry.content_id] = content_entry + by_unique_id_and_md5sum[content_type][content_entry.unique_id][content_entry.md5sum] = content_entry + + log.info( + "Loaded %d entries and %d archived for %s", counter_entries, counter_archived, content_type_folder_name + ) + + +@click_additional_options +@click.option( + "--index-local-folder", + help="Folder to use for index storage. (index=local only)", + type=click.Path(dir_okay=True, file_okay=False), + default="BaNaNaS", + show_default=True, +) +def click_index_local(index_local_folder): + global _folder + + _folder = index_local_folder diff --git a/bananas_server/index/schema.py b/bananas_server/index/schema.py new file mode 100644 index 0000000..cd09fc0 --- /dev/null +++ b/bananas_server/index/schema.py @@ -0,0 +1,37 @@ +from marshmallow import ( + fields, + Schema, + validate, +) +from marshmallow_enum import EnumField + +from ..helpers.content_type import ContentType + + +class ContentEntry(Schema): + class Meta: + ordered = True + + content_id = fields.Integer(data_key="content-id") + unique_id = fields.Raw(data_key="unique-id", validate=validate.Length(min=4, max=4)) + content_type = EnumField(ContentType, data_key="content-type", by_value=True) + filesize = fields.Integer() + # Most of these limits are limitations in the OpenTTD client. + name = fields.String(validate=validate.Length(max=31)) + version = fields.String(validate=validate.Length(max=15)) + url = fields.String(validate=validate.Length(max=95)) + description = fields.String(validate=validate.Length(max=511)) + tags = fields.List(fields.String(validate=validate.Length(max=31))) + md5sum = fields.Raw(validate=validate.Length(min=16, max=16)) + min_version = fields.List(fields.Integer(), data_key="min-version", missing=None) + max_version = fields.List(fields.Integer(), data_key="max-version", missing=None) + raw_dependencies = fields.List( + fields.Tuple( + ( + EnumField(ContentType, by_value=True), + fields.Raw(validate=validate.Length(min=4, max=4)), + fields.Raw(validate=validate.Length(min=16, max=16)), + ) + ), + data_key="raw-dependencies", + ) diff --git a/bananas_server/openttd/__init__.py b/bananas_server/openttd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bananas_server/openttd/protocol/__init__.py b/bananas_server/openttd/protocol/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bananas_server/openttd/protocol/enums.py b/bananas_server/openttd/protocol/enums.py new file mode 100644 index 0000000..725e97b --- /dev/null +++ b/bananas_server/openttd/protocol/enums.py @@ -0,0 +1,29 @@ +import enum + + +# Copy from OpenTTD/src/network/core/tcp_content.h +class PacketTCPContentType(enum.IntEnum): + PACKET_CONTENT_CLIENT_INFO_LIST = 0 # Queries the content server for a list of info of a given content type + PACKET_CONTENT_CLIENT_INFO_ID = 1 # Queries the content server for information about a list of internal IDs + PACKET_CONTENT_CLIENT_INFO_EXTID = 2 # Queries the content server for information about a list of external IDs + PACKET_CONTENT_CLIENT_INFO_EXTID_MD5 = ( + 3 # Queries the content server for information about a list of external IDs and MD5 + ) + PACKET_CONTENT_SERVER_INFO = 4 # Reply of content server with information about content + PACKET_CONTENT_CLIENT_CONTENT = 5 # Request a content file given an internal ID + PACKET_CONTENT_SERVER_CONTENT = 6 # Reply with the content of the given ID + PACKET_CONTENT_END = 7 # Must ALWAYS be on the end of this list!! (period) + + +class ContentType(enum.IntEnum): + CONTENT_TYPE_BASE_GRAPHICS = 1 # The content consists of base graphics + CONTENT_TYPE_NEWGRF = 2 # The content consists of a NewGRF + CONTENT_TYPE_AI = 3 # The content consists of an AI + CONTENT_TYPE_AI_LIBRARY = 4 # The content consists of an AI library + CONTENT_TYPE_SCENARIO = 5 # The content consists of a scenario + CONTENT_TYPE_HEIGHTMAP = 6 # The content consists of a heightmap + CONTENT_TYPE_BASE_SOUNDS = 7 # The content consists of base sounds + CONTENT_TYPE_BASE_MUSIC = 8 # The content consists of base music + CONTENT_TYPE_GAME = 9 # The content consists of a game script + CONTENT_TYPE_GAME_LIBRARY = 10 # The content consists of a GS library + CONTENT_TYPE_END = 11 # Helper to mark the end of the types diff --git a/bananas_server/openttd/protocol/exceptions.py b/bananas_server/openttd/protocol/exceptions.py new file mode 100644 index 0000000..c5985ee --- /dev/null +++ b/bananas_server/openttd/protocol/exceptions.py @@ -0,0 +1,18 @@ +class PacketInvalid(Exception): + """There was an error with this packet. This is a base exception.""" + + +class PacketInvalidSize(PacketInvalid): + """The size of this packet is not as announced.""" + + +class PacketInvalidType(PacketInvalid): + """The type of this packet is not valid.""" + + +class PacketInvalidData(PacketInvalid): + """The packet contains invalid data.""" + + +class PacketTooBig(PacketInvalid): + """The packet is too big to transmit.""" diff --git a/bananas_server/openttd/protocol/read.py b/bananas_server/openttd/protocol/read.py new file mode 100644 index 0000000..4d7d87c --- /dev/null +++ b/bananas_server/openttd/protocol/read.py @@ -0,0 +1,46 @@ +import struct + +from .exceptions import PacketInvalidData + + +def validate_length(data, length): + if len(data) < length: + raise PacketInvalidData("packet too short") + + +def read_uint8(data): + validate_length(data, 1) + value = struct.unpack("". + # To have it a bit easier in the logic, convert those instances to an + # IPv4Address. + if isinstance(self.ip, ipaddress.IPv6Address) and self.ip.ipv4_mapped: + self.ip = self.ip.ipv4_mapped + + def __repr__(self): + return f"Source(addr={self.addr!r}, ip={self.ip!r}, port={self.port!r})" diff --git a/bananas_server/openttd/protocol/write.py b/bananas_server/openttd/protocol/write.py new file mode 100644 index 0000000..ab1e326 --- /dev/null +++ b/bananas_server/openttd/protocol/write.py @@ -0,0 +1,35 @@ +import struct + +from .exceptions import PacketTooBig + +SEND_MTU = 1460 + + +def write_init(type): + return b"\x00\x00" + struct.pack(" SEND_MTU: + raise PacketTooBig(len(data)) + return struct.pack(" 2: + length, _ = read_uint16(data) + + if len(data) < length: + break + + queue.put_nowait(data[0:length]) + data = data[length:] + + return data + + def receive_packet(self, source, data): + # Check length of packet + length, data = read_uint16(data) + if length != len(data) + 2: + raise PacketInvalidSize(len(data) + 2, length) + + # Check if type is in range + type, data = read_uint8(data) + if type >= PacketTCPContentType.PACKET_CONTENT_END: + raise PacketInvalidType(type) + + # Check if we expect this packet + type = PacketTCPContentType(type) + func = getattr(self, f"receive_{type.name}", None) + if func is None: + raise PacketInvalidType(type) + + # Process this packet + kwargs = func(source, data) + return type, kwargs + + @staticmethod + def receive_PACKET_CONTENT_CLIENT_INFO_LIST(source, data): + content_type, data = read_uint8(data) + openttd_version, data = read_uint32(data) + + if content_type >= ContentType.CONTENT_TYPE_END: + raise PacketInvalidData("invalid ContentType", content_type) + + content_type = ContentType(content_type) + + if len(data) != 0: + raise PacketInvalidData("more bytes than expected; remaining: ", len(data)) + + return {"content_type": content_type, "openttd_version": openttd_version} + + @staticmethod + def _receive_client_info(data, count, has_content_id=False, has_content_type_and_unique_id=False, has_md5sum=False): + content_infos = [] + for _ in range(count): + content_info = {} + + if has_content_id: + content_id, data = read_uint32(data) + content_info["content_id"] = content_id + + if has_content_type_and_unique_id: + content_type, data = read_uint8(data) + if content_type >= ContentType.CONTENT_TYPE_END: + raise PacketInvalidData("invalid ContentType", content_type) + content_type = ContentType(content_type) + content_info["content_type"] = content_type + + unique_id, data = read_uint32(data) + if content_type == ContentType.CONTENT_TYPE_NEWGRF: + # OpenTTD client sends NewGRFs byte-swapped for some reason. + # So we swap it back here, as nobody needs to know the + # protocol is making a boo-boo. + content_info["unique_id"] = unique_id.to_bytes(4, "big") + elif content_type in (ContentType.CONTENT_TYPE_SCENARIO, ContentType.CONTENT_TYPE_HEIGHTMAP): + # We store Scenarios / Heightmaps byte-swapped (to what OpenTTD expects). + # This is because otherwise folders are named 01000000, 02000000, which + # makes sorting a bit odd, and in general just difficult to read. + content_info["unique_id"] = unique_id.to_bytes(4, "big") + else: + content_info["unique_id"] = unique_id.to_bytes(4, "little") + + if has_md5sum: + md5sum = bytearray() + for _ in range(16): + md5sum_snippet, data = read_uint8(data) + md5sum.append(md5sum_snippet) + md5sum = bytes(md5sum) + content_info["md5sum"] = md5sum + + content_infos.append(ContentInfo(**content_info)) + + return content_infos, data + + @classmethod + def receive_PACKET_CONTENT_CLIENT_INFO_ID(cls, source, data): + count, data = read_uint16(data) + + content_infos, data = cls._receive_client_info(data, count, has_content_id=True) + + if len(data) != 0: + raise PacketInvalidData("more bytes than expected; remaining: ", len(data)) + + return {"content_infos": content_infos} + + @classmethod + def receive_PACKET_CONTENT_CLIENT_INFO_EXTID(cls, source, data): + count, data = read_uint8(data) + + content_infos, data = cls._receive_client_info(data, count, has_content_type_and_unique_id=True) + + if len(data) != 0: + raise PacketInvalidData("more bytes than expected; remaining: ", len(data)) + + return {"content_infos": content_infos} + + @classmethod + def receive_PACKET_CONTENT_CLIENT_INFO_EXTID_MD5(cls, source, data): + count, data = read_uint8(data) + + content_infos, data = cls._receive_client_info( + data, count, has_content_type_and_unique_id=True, has_md5sum=True + ) + + if len(data) != 0: + raise PacketInvalidData("more bytes than expected; remaining: ", len(data)) + + return {"content_infos": content_infos} + + @classmethod + def receive_PACKET_CONTENT_CLIENT_CONTENT(cls, source, data): + count, data = read_uint16(data) + + content_infos, data = cls._receive_client_info(data, count, has_content_id=True) + + if len(data) != 0: + raise PacketInvalidData("more bytes than expected; remaining: ", len(data)) + + return {"content_infos": content_infos} diff --git a/bananas_server/openttd/send.py b/bananas_server/openttd/send.py new file mode 100644 index 0000000..4916ef6 --- /dev/null +++ b/bananas_server/openttd/send.py @@ -0,0 +1,81 @@ +import struct + +from .protocol.enums import ( + ContentType, + PacketTCPContentType, +) +from .protocol.write import ( + SEND_MTU, + write_init, + write_string, + write_uint8, + write_uint32, + write_presend, +) + + +class OpenTTDProtocolSend: + def send_PACKET_CONTENT_SERVER_INFO( + self, content_type, content_id, filesize, name, version, url, description, unique_id, md5sum, dependencies, tags + ): + data = write_init(PacketTCPContentType.PACKET_CONTENT_SERVER_INFO) + + data = write_uint8(data, content_type.value) + data = write_uint32(data, content_id) + + data = write_uint32(data, filesize) + data = write_string(data, name) + data = write_string(data, version) + data = write_string(data, url) + data = write_string(data, description) + + if content_type == ContentType.CONTENT_TYPE_NEWGRF: + # OpenTTD client sends NewGRFs byte-swapped for some reason. + # So we swap it back here, as nobody needs to know the + # protocol is making a boo-boo. + data = write_uint32(data, struct.unpack(">I", unique_id)[0]) + elif content_type in (ContentType.CONTENT_TYPE_SCENARIO, ContentType.CONTENT_TYPE_HEIGHTMAP): + # We store Scenarios / Heightmaps byte-swapped (to what OpenTTD expects). + # This is because otherwise folders are named 01000000, 02000000, which + # makes sorting a bit odd, and in general just difficult to read. + data = write_uint32(data, struct.unpack(">I", unique_id)[0]) + else: + data = write_uint32(data, struct.unpack(" Date: Mon, 20 Apr 2020 22:38:58 +0200 Subject: [PATCH 2/4] Fix: processed feedback given in pull-request --- bananas_server/application/bananas_server.py | 50 ++++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/bananas_server/application/bananas_server.py b/bananas_server/application/bananas_server.py index 854adbc..665aa5a 100644 --- a/bananas_server/application/bananas_server.py +++ b/bananas_server/application/bananas_server.py @@ -41,6 +41,34 @@ def _send_content_entry(self, source, content_entry): tags=content_entry.tags, ) + def _safe_name(self, name): + new_name = "" + + for letter in name: + if ( + (letter >= "a" and letter <= "z") + or (letter >= "A" and letter <= "Z") + or (letter >= "0" and letter <= "9") + or letter == "." + ): + new_name += letter + elif new_name[-1] != "_": + new_name += "_" + + return new_name + + def _safe_filename(self, name, version): + return self._safe_name(name) + "-" + self._safe_name(version) + + def get_by_content_id(self, content_id): + return self._by_content_id.get(content_id) + + def get_by_unique_id(self, content_type, unique_id): + return self._by_unique_id[content_type].get(unique_id) + + def get_by_unique_id_and_md5sum(self, content_type, unique_id, md5sum): + return self._by_unique_id_and_md5sum[content_type].get(unique_id, {}).get(md5sum) + def receive_PACKET_CONTENT_CLIENT_INFO_LIST(self, source, content_type, openttd_version): version_major = (openttd_version >> 28) & 0xF version_minor = (openttd_version >> 24) & 0xF @@ -57,29 +85,29 @@ def receive_PACKET_CONTENT_CLIENT_INFO_LIST(self, source, content_type, openttd_ def receive_PACKET_CONTENT_CLIENT_INFO_EXTID(self, source, content_infos): for content_info in content_infos: - content_entry = self._by_unique_id[content_info.content_type].get(content_info.unique_id) + content_entry = self.get_by_unique_id(content_info.content_type, content_info.unique_id) if content_entry: self._send_content_entry(source, content_entry) def receive_PACKET_CONTENT_CLIENT_INFO_EXTID_MD5(self, source, content_infos): for content_info in content_infos: - content_entry = ( - self._by_unique_id_and_md5sum[content_info.content_type] - .get(content_info.unique_id, {}) - .get(content_info.md5sum) + content_entry = self.get_by_unique_id_and_md5sum( + content_info.content_type, content_info.unique_id, content_info.md5sum ) if content_entry: self._send_content_entry(source, content_entry) def receive_PACKET_CONTENT_CLIENT_INFO_ID(self, source, content_infos): for content_info in content_infos: - content_entry = self._by_content_id.get(content_info.content_id) + content_entry = self.get_by_content_id(content_info.content_id) if content_entry: self._send_content_entry(source, content_entry) def receive_PACKET_CONTENT_CLIENT_CONTENT(self, source, content_infos): for content_info in content_infos: - content_entry = self._by_content_id[content_info.content_id] + content_entry = self.get_by_content_id(content_info.content_id) + if not content_entry: + continue try: stream = self.storage.get_stream(content_entry) @@ -91,16 +119,10 @@ def receive_PACKET_CONTENT_CLIENT_CONTENT(self, source, content_infos): content_type=content_entry.content_type, content_id=content_entry.content_id, filesize=content_entry.filesize, - filename=f"{content_entry.name} - {content_entry.version}", + filename=self._safe_filename(content_entry.name, content_entry.version), stream=stream, ) - def get_by_content_id(self, content_id): - return self._by_content_id.get(content_id) - - def get_by_unique_id_and_md5sum(self, content_type, unique_id, md5sum): - return self._by_unique_id_and_md5sum[content_type].get(unique_id, {}).get(md5sum) - def reload_md5sum_mapping(self): for content_type in ContentType: if content_type == ContentType.CONTENT_TYPE_END: From 4c3e056682d437b79ad4408a9f7ed9c81bf58abe Mon Sep 17 00:00:00 2001 From: Patric Stout Date: Mon, 20 Apr 2020 23:06:23 +0200 Subject: [PATCH 3/4] Fix: some more fiddling with safe_name routine --- bananas_server/application/bananas_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bananas_server/application/bananas_server.py b/bananas_server/application/bananas_server.py index 665aa5a..8835a7a 100644 --- a/bananas_server/application/bananas_server.py +++ b/bananas_server/application/bananas_server.py @@ -52,10 +52,11 @@ def _safe_name(self, name): or letter == "." ): new_name += letter - elif new_name[-1] != "_": + elif new_name and new_name[-1] != "_": new_name += "_" - return new_name + return new_name.strip("._") + def _safe_filename(self, name, version): return self._safe_name(name) + "-" + self._safe_name(version) From 1f8edbdd439cf38a2805eb46a5787c34344f52f6 Mon Sep 17 00:00:00 2001 From: Patric Stout Date: Mon, 20 Apr 2020 23:14:26 +0200 Subject: [PATCH 4/4] Fix: run black before commit you sillybean --- bananas_server/application/bananas_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bananas_server/application/bananas_server.py b/bananas_server/application/bananas_server.py index 8835a7a..bceb6ba 100644 --- a/bananas_server/application/bananas_server.py +++ b/bananas_server/application/bananas_server.py @@ -57,7 +57,6 @@ def _safe_name(self, name): return new_name.strip("._") - def _safe_filename(self, name, version): return self._safe_name(name) + "-" + self._safe_name(version)