Add: classify scenarios with the information we can extract from it

bananas_api/helpers/api_schema.py

@@ -13,12 +13,14 @@
 from .enums import (
     Availability,
     Branch,
+    Climate,
     ContentType,
     License,
     NewGRFSet,
     Palette,
     Resolution,
     Shape,
+    Size,
     Status,
     TerrainType,
 )
@@ -212,6 +214,8 @@ class Classification(OrderedSchema):
     shape = EnumField(Shape, by_value=True)
     resolution = EnumField(Resolution, by_value=True)
     terrain_type = EnumField(TerrainType, by_value=True, data_key="terrain-type")
+    size = EnumField(Size, by_value=True)
+    climate = EnumField(Climate, by_value=True)
 
 
 class VersionMinimized(Global):

bananas_api/helpers/enums.py

@@ -109,3 +109,17 @@ class TerrainType(Enum):
     FLAT = "flat"
     HILLY = "hilly"
     MOUNTAINOUS = "mountainous"
+
+
+class Climate(Enum):
+    TEMPERATE = "temperate"
+    SUB_ARCTIC = "sub-arctic"
+    SUB_TROPICAL = "sub-tropical"
+    TOYLAND = "toyland"
+
+
+class Size(Enum):
+    SMALL = "small"
+    NORMAL = "normal"
+    LARGE = "large"
+    HUGE = "huge"

bananas_api/new_upload/classifiers/scenario.py

@@ -0,0 +1,71 @@
+from ...helpers.api_schema import Classification
+from ...helpers.enums import (
+    Climate,
+    Shape,
+    Size,
+    TerrainType,
+)
+from ..readers.scenario import (
+    Landscape,
+    Scenario,
+)
+
+
+def classify_scenario(scenario: Scenario) -> Classification:
+    classification = {}
+
+    surface = scenario.map_size[0] * scenario.map_size[1]
+    if surface < 256 * 256:
+        classification["size"] = Size.SMALL
+    elif surface == 256 * 256:
+        classification["size"] = Size.NORMAL
+    elif surface <= 1024 * 1024:
+        classification["size"] = Size.LARGE
+    else:
+        classification["size"] = Size.HUGE
+
+    aspect_ratio = scenario.map_size[0] / scenario.map_size[1]
+    if aspect_ratio < 1:
+        aspect_ratio = 1 / aspect_ratio
+
+    if aspect_ratio < 1.2:
+        classification["shape"] = Shape.SQUARE
+    elif aspect_ratio < 2.5:
+        classification["shape"] = Shape.RECTANGLE
+    else:
+        classification["shape"] = Shape.NARROW
+
+    # Only keep the lower heights and skip sea-level.
+    small_histogram = [0] * 16
+    for i in range(1, 15):
+        small_histogram[i] = scenario.histogram[i]
+    for i in range(15, 256):
+        small_histogram[15] += scenario.histogram[i]
+    # Normalize the histogram.
+    small_histogram = [value * 100 / (surface - scenario.histogram[0]) for value in small_histogram]
+
+    # Find all elevations with a certain surface amount. One hill doesn't
+    # make a mountain (yes, this is meant ironically).
+    common_elevations = [i for i, v in enumerate(small_histogram) if v >= 1]
+    # And check the difference between the highest and lowest elevation.
+    height_difference = max(common_elevations) - min(common_elevations)
+
+    if height_difference > 9:
+        classification["terrain-type"] = TerrainType.MOUNTAINOUS
+    elif height_difference > 4:
+        classification["terrain-type"] = TerrainType.HILLY
+    elif height_difference > 1:
+        classification["terrain-type"] = TerrainType.FLAT
+    else:
+        classification["terrain-type"] = TerrainType.VERY_FLAT
+
+    if scenario.landscape == Landscape.TEMPERATE:
+        classification["climate"] = Climate.TEMPERATE
+    elif scenario.landscape == Landscape.ARCTIC:
+        classification["climate"] = Climate.SUB_ARCTIC
+    elif scenario.landscape == Landscape.TROPIC:
+        classification["climate"] = Climate.SUB_TROPICAL
+    elif scenario.landscape == Landscape.TOYLAND:
+        classification["climate"] = Climate.TOYLAND
+
+    return Classification().load(classification)

bananas_api/new_upload/readers/scenario.py

@@ -1,3 +1,4 @@
+import enum
 import hashlib
 import io
 import lzma
@@ -8,6 +9,14 @@
 from .helpers import binreader
 
 
+@enum.unique
+class Landscape(enum.IntEnum):
+    TEMPERATE = 0
+    ARCTIC = 1
+    TROPIC = 2
+    TOYLAND = 3
+
+
 class PlainFile:
     @staticmethod
     def open(f):
@@ -67,6 +76,9 @@ class Scenario:
     @ivar map_size: Map size
     @type map_size: (C{int}, c{int})
 
+    @ivar histogram: List of 256 entries indicating how often that height level occurs
+    @type histogram: C{list} of C{int}
+
     @ivar newgrf: List of NewGRF as (grf-id, md5sum, version, filename)
     @type newgrf: C{list} of (C{int}, C{str}, C{int}, C{str})
 
@@ -75,17 +87,29 @@ class Scenario:
 
     @ivar gs: List of game scripts as (short-id, md5sum, version, name)
     @type gs: C{list} of (C{None}, C{None}, C{int}, C{str})
+
+    @ivar landscape: Landscape type
+    @type landscape: C{int}
     """
 
     package_type = PackageType.SCENARIO
 
     def __init__(self):
         self.md5sum = None
         self.savegame_version = None
+        self.is_patchpack = None
         self.map_size = (None, None)
+        self.histogram = None
         self.newgrf = []
         self.ai = []
         self.gs = []
+        self.landscape = None
+
+        # In old savegames, heightmap was in MAPT. In newer in MAPH.
+        # So we assume MAPH, but if that means histogram remains empty,
+        # we use MAPT. That way, the new purpose of those bits in MAPT
+        # aren't influencing the histogram.
+        self._histogram_old = None
 
     def read(self, fp):
         """
@@ -99,7 +123,9 @@ def read(self, fp):
         reader = binreader.BinaryReader(fp, md5sum)
 
         compression = reader.read(4)
-        self.savegame_version = reader.uint16(be=True)
+        savegame_version = reader.uint16(be=True)
+        self.savegame_version = savegame_version & 0xFFF
+        self.is_patchpack = savegame_version & 0x8000 != 0
         reader.uint16()
 
         decompressor = UNCOMPRESS.get(compression)
@@ -117,12 +143,66 @@ def read(self, fp):
                 raise ValidationException("Invalid savegame.")
 
             type = reader.uint8()
+
+            # JGRPP's extended header
+            if (type & 0x0F) == 0x0F:
+                ext_flags = reader.uint32(be=True)
+                type = reader.uint8()
+            else:
+                ext_flags = 0
+
             if (type & 0x0F) == 0x00:
                 size = type << 20 | reader.uint24(be=True)
 
+                # JGRPP's big RIFF flag
+                if ext_flags & 0x01:
+                    size += reader.uint32(be=True) << 28
+
+                # New savegame format, with the height of the tile in MAPH.
+                if tag == b"MAPH":
+                    self.histogram = [0] * 256
+                    while size > 0:
+                        data = reader.read(min(size, 8192))
+                        for h in data:
+                            self.histogram[h] += 1
+                        size -= min(size, 8192)
+
+                # Old savegame format, with the height of the tile in MAPT.
+                if tag == b"MAPT":
+                    self._histogram_old = [0] * 256
+                    while size > 0:
+                        data = reader.read(min(size, 8192))
+                        for h in data:
+                            self._histogram_old[h & 0xF] += 1
+                        size -= min(size, 8192)
+
+                # JGRPP-based map chunk containing the height of the tile.
+                if tag == b"WMAP":
+                    self.histogram = [0] * 256
+
+                    # First chunk is 8 bytes per tile for the whole map.
+                    chunk_map_size = self.map_size[0] * self.map_size[1] * 8
+                    if chunk_map_size > size:
+                        raise ValidationException("Invalid savegame.")
+
+                    # Read this first chunk.
+                    while chunk_map_size > 0:
+                        data = reader.read(min(chunk_map_size, 8192))
+
+                        for i in range(0, len(data), 8):
+                            # Height is the second byte in the tile.
+                            h = data[i + 1]
+                            self.histogram[h] += 1
+
+                        chunk_map_size -= min(chunk_map_size, 8192)
+                        size -= min(size, 8192)
+
+                    # There can be other tile-data after the first chunk; skip this.
+                    while size > 0:
+                        reader.skip(min(size, 8192))
+                        size -= min(size, 8192)
+
                 if tag in (
-                    b"MAPT",
-                    b"MAPH",
                     b"MAPO",
                     b"MAP2",
                     b"M3LO",
@@ -171,11 +251,35 @@ def read(self, fp):
                 raise ValidationException("Invalid savegame.")
 
         try:
-            reader.uint8()
+            v = reader.uint8()
         except ValidationException:
             pass
         else:
-            raise ValidationException("Junk at the end of file.")
+            # Savegames before r20090 could contain a junk-block of 128 KiB of all zeros.
+            if v == 0:
+                for _ in range(128 * 1024 - 1):
+                    try:
+                        v = reader.uint8()
+                    except ValidationException:
+                        # In case it is not a block of 128 KiB, it is junk.
+                        raise ValidationException("Junk at the end of file.")
+
+                    # In case it is not a zero, it is junk.
+                    if v != 0:
+                        raise ValidationException("Junk at the end of file.")
+            else:
+                raise ValidationException("Junk at the end of file.")
+
+        if self.map_size[0] is None:
+            raise ValidationException("Scenario is missing essential chunks (MAPS).")
+        if self.histogram is None:
+            if self._histogram_old is None:
+                raise ValidationException("Scenario is missing essential chunks (MAPT / MAPH).")
+            else:
+                self.histogram = self._histogram_old
+
+        if self.map_size[0] * self.map_size[1] == self.histogram[0]:
+            raise ValidationException("Map is completely empty.")
 
         self.md5sum = md5sum.digest()
 
@@ -255,7 +359,7 @@ def read_item(self, tag, fields, index, data):
         reader = binreader.BinaryReader(io.BytesIO(data))
 
         # Only look at those chunks that have data we are interested in.
-        if tag not in (b"MAPS", b"NGRF", b"AIPL", b"GSDT"):
+        if tag not in (b"MAPS", b"NGRF", b"AIPL", b"GSDT", b"PATS"):
             return
 
         table = self.read_table(fields, reader)
@@ -272,6 +376,8 @@ def read_item(self, tag, fields, index, data):
         elif tag == b"GSDT":
             if table["is_random"] == 0 and table["name"]:
                 self.gs.append((None, None, table["version"], table["name"]))
+        elif tag == b"PATS":
+            self.landscape = table["game_creation.landscape"]
 
     def read_item_without_header(self, tag, index, data):
         """
@@ -310,3 +416,35 @@ def read_item_without_header(self, tag, index, data):
             is_random = reader.uint8() != 0
             if not is_random and len(name) > 0:
                 self.gs.append((None, None, version, name))
+        elif tag == b"PATS":
+            # Before version 97, landscape was part not part of OPTS.
+            if self.savegame_version < 97:
+                return
+
+            # Various of difficulty-related settings.
+            reader.skip(19)
+
+            # Subsidy-duration setting.
+            if self.savegame_version >= 292:
+                reader.skip(2)
+
+            # Difficulty-level setting.
+            if self.savegame_version < 178:
+                reader.skip(1)
+
+            # Competitor-start-time and competitor-speed settings.
+            if self.savegame_version >= 97 and self.savegame_version < 110:
+                reader.skip(2)
+
+            reader.skip(1)  # town-name setting.
+            self.landscape = Landscape(reader.uint8())
+        elif tag == b"OPTS":
+            if self.savegame_version < 4:
+                reader.skip(17 * 2)  # Difficulty-custom settings.
+            else:
+                reader.skip(18 * 2)
+            reader.skip(1)  # Difficulty-level setting.
+            reader.skip(1)  # Currency setting.
+            reader.skip(1)  # Units setting.
+            reader.skip(1)  # Town-name setting.
+            self.landscape = Landscape(reader.uint8())

bananas_api/new_upload/validate.py

@@ -9,6 +9,7 @@
 )
 from .classifiers.heightmap import classify_heightmap
 from .classifiers.newgrf import classify_newgrf
+from .classifiers.scenario import classify_scenario
 from .exceptions import (
     BaseSetDoesntMentionFileException,
     BaseSetMentionsFileThatIsNotThereException,
@@ -57,6 +58,7 @@
 CLASSIFIERS = {
     PackageType.HEIGHTMAP: classify_heightmap,
     PackageType.NEWGRF: classify_newgrf,
+    PackageType.SCENARIO: classify_scenario,
 }
 
 PACKAGE_TYPE_PAIRS = {

bananas_api/tool_reclassify/__main__.py

@@ -39,7 +39,7 @@ def update_progress(unique_id, version, md5sum_partial, name, error, message, cl
     default="local_storage",
     show_default=True,
 )
-@click.argument("category", type=click.Choice(["heightmap", "newgrf"]))
+@click.argument("category", type=click.Choice(["heightmap", "newgrf", "scenario"]))
 @click.argument("unique_id", type=str, required=False)
 def main(index_local_folder, storage_local_folder, category, unique_id):
     global TOTAL_ENTRIES

bananas_api/tool_reclassify/reclassify.py

@@ -3,17 +3,20 @@
 
 from .heightmap import load_heightmap
 from .newgrf import load_newgrf
+from .scenario import load_scenario
 
 from ..helpers.enums import Availability
 from ..index.common_disk import Index
 from ..new_upload.classifiers.heightmap import classify_heightmap
 from ..new_upload.classifiers.newgrf import classify_newgrf
+from ..new_upload.classifiers.scenario import classify_scenario
 from ..new_upload.session_validation import validate_packet_size
 
 
 CLASSIFICATION_TO_FUNCTION = {
     "heightmap": (load_heightmap, classify_heightmap),
     "newgrf": (load_newgrf, classify_newgrf),
+    "scenario": (load_scenario, classify_scenario),
 }