Feature: tag what region a NewGRF / Heightmap / Scenario is about

Dockerfile

@@ -41,6 +41,9 @@ COPY requirements.txt \
         clients-development.yaml \
         clients-production.yaml \
         clients-staging.yaml \
+        region-un-m49.csv \
+        region-iso-3166-1.json \
+        region-iso-3166-2.json \
         /code/
 COPY licenses /code/licenses
 # Needed for Sentry to know what version we are running

README.md

@@ -71,3 +71,11 @@ This means that for clients, you need to contact two endpoints:
 In production the Load Balancer redirects the URLs to the right ports, but during development this is something to keep in mind.
 
 [bananas-frontend-cli](https://github.com/OpenTTD/bananas-frontend-cli) for example allows you to define the web-endpoint and the tusd-endpoint.
+
+## Regions
+
+To unify the way authors indicate what region their content is about, we have a built-in list of supported regions.
+This is a combination of the [UN M49](https://unstats.un.org/unsd/methodology/m49/overview) list and ISO 3166-1 / 3166-2 list.
+
+- The 3166-1 / 3166-2 list is easiest found in the Debian `iso-codes` package, after which it is located in `/usr/share/iso-codes/json/iso_3166-[12].json`.
+- The UN M49 can be found [here](https://unstats.un.org/unsd/methodology/m49/overview).

bananas_api/helpers/api_schema.py

@@ -22,6 +22,7 @@
     Status,
     TerrainType,
 )
+from .regions import REGIONS
 
 DEPENDENCY_CHECK = True
 
@@ -77,6 +78,14 @@ def _format_error(self, value, message):
         return super()._format_error(value.decode(), message)
 
 
+class ValidateRegion(validate.Validator):
+    def __call__(self, value):
+        if value not in REGIONS.keys():
+            raise ValidationError("Invalid region.")
+
+        return value
+
+
 class OrderedSchema(Schema):
     class Meta:
         ordered = True
@@ -96,6 +105,7 @@ class Global(OrderedSchema):
     description = fields.String(validate=ValidateBytesLength(max=511))
     url = fields.String(validate=ValidateURL())
     tags = fields.List(fields.String(validate=ValidateBytesLength(max=31)))
+    regions = fields.List(fields.String(validate=ValidateRegion()), validate=validate.Length(max=10))
 
 
 class Author(OrderedSchema):
@@ -276,3 +286,9 @@ class ConfigLicense(OrderedSchema):
 class ConfigBranch(OrderedSchema):
     name = fields.String()
     description = fields.String()
+
+
+class ConfigRegion(OrderedSchema):
+    code = fields.String()
+    name = fields.String()
+    parent = fields.String()

bananas_api/helpers/regions.py

@@ -0,0 +1,148 @@
+import json
+import os
+import unicodedata
+
+REGIONS = {}
+
+folder = os.getcwd()
+if not os.path.exists(f"{folder}/region-un-m49.csv"):
+    folder = os.environ["PYTHONPATH"]
+if not os.path.exists(f"{folder}/region-un-m49.csv"):
+    raise Exception("Unable to locate region files. Please run from the root of the project.")
+
+with open(f"{folder}/region-un-m49.csv") as fp:
+    # Skip the CSV header.
+    next(fp)
+
+    for line in fp.readlines():
+        line = line.strip().split(";")
+
+        if len(line) != 15:
+            raise Exception("Invalid line; is the UN M49 CSV file corrupted?")
+
+        (
+            global_code,
+            global_name,
+            region_code,
+            region_name,
+            sub_region_code,
+            sub_region_name,
+            intermediate_region_code,
+            intermediate_region_name,
+            country,
+            _,
+            country_code,
+            _,
+            _,
+            _,
+            _,
+        ) = line
+
+        if global_code != "001":
+            raise Exception("Invalid global code; is this an UN M49 CSV file from earth?")
+
+        if not country_code:
+            continue
+
+        # If the intermediate region is set, ignore the sub region.
+        # This is because in these cases the sub region tends to be a long
+        # name and doesn't add information to the (intermediate) region.
+        if intermediate_region_code:
+            sub_region_code = intermediate_region_code
+            sub_region_name = intermediate_region_name
+
+        # Prefix UN specific codes with UN-. This to give them a visual
+        # difference from the ISO codes.
+        if region_code:
+            region_code = f"UN-{region_code}"
+        if sub_region_code:
+            sub_region_code = f"UN-{sub_region_code}"
+
+        REGIONS[country_code] = {
+            "name": country,
+            "parent": sub_region_code,
+        }
+        if sub_region_code:
+            REGIONS[sub_region_code] = {
+                "name": sub_region_name,
+                "parent": region_code,
+            }
+        if region_code:
+            REGIONS[region_code] = {
+                "name": region_name,
+                "parent": None,
+            }
+        if global_code:
+            REGIONS[global_code] = {
+                "name": global_name,
+                "parent": None,
+            }
+
+
+with open(f"{folder}/region-iso-3166-1.json") as fp:
+    data = json.load(fp)
+
+    for country in data["3166-1"]:
+        country_code = country["alpha_2"]
+        country_name = country.get("common_name", country["name"]).split(",")[0].strip()
+
+        # Debian has a much better friendly name for many countries.
+        # So use that name instead of the official one the UN is using.
+        # Example:
+        # Official name: United Kingdom of Great Britain and Northern Ireland
+        # Debian's name: United Kingdom
+        if country_code in REGIONS:
+            REGIONS[country_code]["name"] = country_name
+        elif country_code == "TW":
+            # Taiwan is not in the UN dataset, but is in the 3166-1 dataset.
+            REGIONS[country_code] = {
+                "name": country_name,
+                "parent": "UN-030",  # Eastern Asia
+            }
+
+
+with open(f"{folder}/region-iso-3166-2.json") as fp:
+    data = json.load(fp)
+
+    for country in data["3166-2"]:
+        subdivision_code = country["code"]
+        # Normalize all names to be within ASCII. This makes searching in-game easier.
+        subdivision_name = unicodedata.normalize("NFKD", country["name"]).encode("ascii", "ignore").decode()
+
+        # There are several ways to denote aliases; strip those out.
+        subdivision_name = subdivision_name.lstrip("/")
+        subdivision_name = subdivision_name.split("/")[0].strip()
+        subdivision_name = subdivision_name.split("(")[0].strip()
+        subdivision_name = subdivision_name.split("[")[0].strip()
+        subdivision_name = subdivision_name.split(",")[0].strip()
+
+        country_code = subdivision_code[:2]
+
+        REGIONS[subdivision_code] = {
+            "name": subdivision_name,
+            "parent": country_code,
+        }
+
+REGIONS["UN-MARS"] = {
+    "name": "Mars",
+    "parent": None,
+}
+
+# According to wikipedia (https://en.wikipedia.org/wiki/ISO_3166-2:GB) these
+# are part of ISO 3166-2, but the ISO doesn't mention them. So we insert them.
+REGIONS["GB-ENG"] = {
+    "name": "England",
+    "parent": "GB",
+}
+REGIONS["GB-NIR"] = {
+    "name": "Northern Ireland",
+    "parent": "GB",
+}
+REGIONS["GB-SCT"] = {
+    "name": "Scotland",
+    "parent": "GB",
+}
+REGIONS["GB-WLS"] = {
+    "name": "Wales",
+    "parent": "GB",
+}

bananas_api/new_upload/session.py

@@ -45,7 +45,16 @@
 
 log = logging.getLogger(__name__)
 
-KEYS_ALLOWED_TO_UPDATE = {"version", "license", "dependencies", "compatibility", "name", "description", "url"}
+KEYS_ALLOWED_TO_UPDATE = {
+    "version",
+    "license",
+    "dependencies",
+    "compatibility",
+    "name",
+    "description",
+    "url",
+    "regions",
+}
 TIMER_TIMEOUT = 60 * 15
 
 _timer = defaultdict(lambda: None)

bananas_api/new_upload/session_publish.py

@@ -35,6 +35,13 @@
 
 _storage_instance = None
 
+# Only for certain content-type it makes sense to have a region.
+CONTENT_TYPE_WITH_REGION = (
+    ContentType.NEWGRF,
+    ContentType.HEIGHTMAP,
+    ContentType.SCENARIO,
+)
+
 
 def _tar_add_file_from_string(tar, arcname, content):
     content = content.encode()
@@ -173,6 +180,9 @@ def create_package(session):
         for field in ("name", "description", "url"):
             if field in raw_data:
                 version_data[field] = raw_data[field]
+
+        if "regions" in raw_data and session["content_type"] in CONTENT_TYPE_WITH_REGION:
+            version_data["regions"] = sorted(raw_data["regions"])
     else:
         # This is a new package, so construct it from the ground up. Run it
         # through the validator, just to make sure we are adding valid
@@ -190,6 +200,9 @@ def create_package(session):
             if field in raw_data:
                 package_data[field] = raw_data[field]
 
+        if "regions" in raw_data and session["content_type"] in CONTENT_TYPE_WITH_REGION:
+            package_data["regions"] = sorted(raw_data["regions"])
+
         package = Package().load(package_data)
 
         index_package(package, index_versions=False)

bananas_api/new_upload/session_validation.py

@@ -1,5 +1,6 @@
 from ..helpers.api_schema import Classification
 from ..helpers.enums import License
+from ..helpers.regions import REGIONS
 
 
 def validate_is_valid_package(session, data):
@@ -89,6 +90,12 @@ def validate_new_package(session):
         session["warnings"].append("URL is not yet set for this package; although not mandatory, highly advisable.")
 
 
+def get_region_codes(codes, region):
+    codes.add(region)
+    if REGIONS[region]["parent"]:
+        get_region_codes(codes, REGIONS[region]["parent"])
+
+
 def validate_packet_size(session, package):
     # Calculate if this entry wouldn't exceed the OpenTTD packet size if
     # we would transmit this over the wire.
@@ -111,5 +118,11 @@ def validate_packet_size(session, package):
         else:
             raise ValueError("Unknown type for classification value")
 
+    codes = set()
+    for region in session.get("regions", package.get("regions", [])):
+        get_region_codes(codes, region)
+    for code in codes:
+        size += len(REGIONS[code]["name"]) + 2
+
     if size > 1400:
         session["errors"].append("Entry would exceed OpenTTD packet size; trim down on your description.")

bananas_api/tool_reclassify/reclassify.py

@@ -74,7 +74,7 @@ def reclassify_and_update_metadata(index_folder, storage_folder, category, uniqu
     Index(index_folder).store_version(f"{category}/{unique_id}", data)
 
     data["errors"] = []
-    validate_packet_size(data, {})
+    validate_packet_size(data, global_data)
     if data["errors"]:
         result["error"] = True
         result["message"] = "error while validating session: " + ", ".join(data["errors"])

bananas_api/web_routes/config.py

@@ -3,12 +3,14 @@
 from ..helpers.api_schema import (
     ConfigBranch,
     ConfigLicense,
+    ConfigRegion,
     ConfigUserAudience,
 )
 from ..helpers.enums import (
     Branch,
     License,
 )
+from ..helpers.regions import REGIONS
 from ..helpers.user_session import (
     get_user_method,
     get_user_methods,
@@ -71,3 +73,14 @@ async def config_branches(request):
     for branch, description in BRANCHES.items():
         branches.append(ConfigBranch().dump({"name": branch.value, "description": description}))
     return web.json_response(branches)
+
+
+@routes.get("/config/regions")
+async def config_regions(request):
+    regions = []
+    for code, region in sorted(REGIONS.items(), key=lambda x: x[0]):
+        data = {"code": code, "name": region["name"]}
+        if "parent" in region:
+            data["parent"] = region["parent"]
+        regions.append(ConfigRegion().dump(data))
+    return web.json_response(regions)