Merge branch 'ezfe-improv-world-loading'

.circleci/config.yml

@@ -16,7 +16,7 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-          - v1-dependencies-{{ checksum "requirements.txt" }}
+          - v1-dependencies-{{ checksum "requirements-dev.txt" }}
           # fallback to using the latest cache if no exact match is found
           - v1-dependencies-
 
@@ -25,19 +25,26 @@ jobs:
           command: |
             python3 -m venv venv
             . venv/bin/activate
-            pip install -r requirements.txt
+            pip install -r requirements-dev.txt
 
       - save_cache:
           paths:
             - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
+          key: v1-dependencies-{{ checksum "requirements-dev.txt" }}
 
       - run:
           name: run tests
           command: |
             . venv/bin/activate
             python -m unittest discover -v -s tests
 
+      - run:
+          name: run stylecheck
+          command: |
+            . venv/bin/activate
+            python -m black --check src
+            python -m black --check tests
+
       - store_artifacts:
           path: test-reports
           destination: test-reports

README.md

@@ -1,6 +1,6 @@
 # Amulet Map Editor
 
-<a href="https://circleci.com/gh/Podshot/Amulet-Map-Editor"><img alt="CircleCI" src="https://circleci.com/gh/Podshot/Amulet-Map-Editor.svg"></a>
+<a href="https://circleci.com/gh/Amulet-Team/Amulet-Map-Editor"><img alt="CircleCI" src="https://circleci.com/gh/Amulet-Team/Amulet-Map-Editor.svg"></a>
 
 A new Minecraft world editor that aims to be flexible, extendable, and support most editions
 of Minecraft.

requirements-dev.txt

@@ -2,6 +2,6 @@
 Sphinx==1.7.4
 sphinx-autodoc-typehints==1.3.0
 sphinx_rtd_theme==0.3.1
-black==18.4a2
+black==18.9b0
 PyInstaller==3.4
 pre_commit==1.11.1

src/api/world_loader.py

@@ -54,8 +54,10 @@ def identify(self, directory: str) -> Tuple[str, str]:
         for name, module in self._identifiers.items():
             if module.identify(directory):
                 return name, module.FORMAT
+            elif __debug__:
+                print(f"{name} rejected the world")
 
-        raise ModuleNotFoundError("Could not find a valid format loader")
+        raise ModuleNotFoundError("Could not find a matching format loader")
 
     def load_world(self, directory: str) -> World:
         """
@@ -64,11 +66,13 @@ def load_world(self, directory: str) -> World:
         :param directory: The directory of the world
         :return: The loaded world
         """
-        for module in self._identifiers.values():
+        for name, module in self._identifiers.items():
             if module.identify(directory):
                 return module.load(directory)
+            elif __debug__:
+                print(f"{name} rejected the world")
 
-        raise ModuleNotFoundError()
+        raise ModuleNotFoundError("Could not find a matching format loader")
 
 
 loader = _WorldLoader()

src/command_line/command_handler.py

@@ -115,6 +115,11 @@ def _load_commands(self, reload=False):
                 self._modules.append(module)
             except ImportError:
                 self._retry_modules.append(os.path.basename(cmd)[:-3])
+            except NotImplementedError:
+                _io.print(
+                    f"Couldn't import {os.path.basename(cmd)[:-3]} since it's unimplemented",
+                    color="red",
+                )
             except Exception as e:
                 _io.print(
                     f"Couldn't import {os.path.basename(cmd)[:-3]} due to error: {e}",

src/formats/anvil/anvil_world.py

@@ -33,17 +33,17 @@ def load_chunk(
             if not self.load_region(rx, rz):
                 raise Exception()
 
-        cx &= 0x1f
-        cz &= 0x1f
+        cx &= 0x1F
+        cz &= 0x1F
 
         chunk_offset = self._loaded_regions[key]["offsets"][
-            (cx & 0x1f) + (cz & 0x1f) * 32
+            (cx & 0x1F) + (cz & 0x1F) * 32
         ]
         if chunk_offset == 0:
             raise Exception()
 
         sector_start = chunk_offset >> 8
-        number_of_sectors = chunk_offset & 0xff
+        number_of_sectors = chunk_offset & 0xFF
 
         if number_of_sectors == 0:
             raise Exception()
@@ -93,8 +93,8 @@ def load_region(self, rx: int, rz: int) -> bool:
             self._loaded_regions[key] = {}
 
             file_size = path.getsize(filename)
-            if file_size & 0xfff:
-                file_size = (file_size | 0xfff) + 1
+            if file_size & 0xFFF:
+                file_size = (file_size | 0xFFF) + 1
                 fp.truncate(file_size)
 
             if not file_size:
@@ -122,7 +122,7 @@ def load_region(self, rx: int, rz: int) -> bool:
 
             for offset in offsets:
                 sector = offset >> 8
-                count = offset & 0xff
+                count = offset & 0xFF
 
                 for i in range(sector, sector + count):
                     if i >= len(free_sectors):

src/formats/anvil2/anvil2_world.py

@@ -8,6 +8,7 @@
 import numpy
 
 from api import WorldFormat
+from math import log, ceil
 from nbt import nbt
 from os import path
 
@@ -32,17 +33,17 @@ def load_chunk(
         if not self.load_region(rx, rz):
             raise Exception()
 
-        cx &= 0x1f
-        cz &= 0x1f
+        cx &= 0x1F
+        cz &= 0x1F
 
         chunk_offset = self._loaded_regions[key]["offsets"][
-            (cx & 0x1f) + (cz & 0x1f) * 32
+            (cx & 0x1F) + (cz & 0x1F) * 32
         ]
         if chunk_offset == 0:
             raise Exception()
 
         sector_start = chunk_offset >> 8
-        number_of_sectors = chunk_offset & 0xff
+        number_of_sectors = chunk_offset & 0xFF
 
         if number_of_sectors == 0:
             raise Exception()
@@ -91,8 +92,8 @@ def load_region(self, rx: int, rz: int) -> bool:
         self._loaded_regions[key] = {}
 
         file_size = path.getsize(filename)
-        if file_size & 0xfff:
-            file_size = (file_size | 0xfff) + 1
+        if file_size & 0xFFF:
+            file_size = (file_size | 0xFFF) + 1
             fp.truncate(file_size)
 
         if not file_size:
@@ -120,7 +121,7 @@ def load_region(self, rx: int, rz: int) -> bool:
 
         for offset in offsets:
             sector = offset >> 8
-            count = offset & 0xff
+            count = offset & 0xFF
 
             for i in range(sector, sector + count):
                 if i >= len(free_sectors):
@@ -133,6 +134,44 @@ def load_region(self, rx: int, rz: int) -> bool:
         return True
 
 
+def _decode_long_array(long_array: array_like, size: int) -> ndarray:
+    """
+    Decode an long array (from BlockStates or Heightmaps)
+    :param long_array: Encoded long array
+    :size uint: The expected size of the returned array
+    :return: Decoded array as numpy array
+    """
+    long_array = numpy.array(long_array, dtype=">q")
+    bits_per_block = (len(long_array) * 64) // size
+    binary_blocks = numpy.unpackbits(
+        long_array[::-1].astype(">i8").view("uint8")
+    ).reshape(-1, bits_per_block)
+    return binary_blocks.dot(2 ** numpy.arange(binary_blocks.shape[1] - 1, -1, -1))[
+        ::-1  # Undo the bit-shifting that Minecraft does with the palette indices
+    ][:size]
+
+def _encode_long_array(data_array: array_like, palette_size: int) -> ndarray:
+    """
+    Encode an array of data to a long array (from BlockStates or Heightmaps).
+    :param long_array: Data to encode
+    :palette_size uint: Must be at least 4
+    :return: Encoded array as numpy array
+    """
+    data_array = numpy.array(data_array, dtype=">i2")
+    bits_per_block = max(4, int(ceil(log(palette_size, 2))))
+    binary_blocks = (
+        numpy.unpackbits(data_array.astype(">i2").view("uint8"))
+        .reshape(-1, 16)[:, (16 - bits_per_block) :][::-1]
+        .reshape(-1)
+    )
+    binary_blocks = numpy.pad(
+        binary_blocks, ((64 - (len(data_array) * bits_per_block)) % 64, 0), "constant"
+    ).reshape(-1, 64)
+    return binary_blocks.dot(
+        2 ** numpy.arange(binary_blocks.shape[1] - 1, -1, -1, dtype=">q")
+    )[::-1]
+
+
 class Anvil2World(WorldFormat):
     def __init__(self, directory: str, definitions: str):
         self._directory = directory
@@ -151,7 +190,7 @@ def load(cls, directory: str, definitions: str) -> World:
 
         return World(directory, root_tag, wrapper)
 
-    def __read_palette(self, palette: nbt.TAG_List) -> list:
+    def _read_palette(self, palette: nbt.TAG_List) -> list:
         blockstates = []
         for entry in palette:
             name = entry["Name"].value
@@ -179,22 +218,10 @@ def get_blocks(self, cx: int, cz: int) -> Union[numpy.ndarray, NotImplementedErr
             lower = section["Y"].value << 4
             upper = (section["Y"].value + 1) << 4
 
-            palette = self.__read_palette(section["Palette"])
-
-            blockstate_array = section["BlockStates"].value
-            blockstate_array = numpy.array(blockstate_array, dtype=">q")
-            bits_per_block = len(blockstate_array) // 64
-            binary_blocks = numpy.unpackbits(
-                blockstate_array[::-1].astype(">i8").view("uint8")
-            ).reshape(-1, bits_per_block)
-            before_palette = binary_blocks.dot(
-                2 ** numpy.arange(binary_blocks.shape[1] - 1, -1, -1)
-            )[
-                ::-1
-            ]  # Undo the bit-shifting that Minecraft does with the palette indices
+            palette = self._read_palette(section["Palette"])
 
             _blocks = numpy.asarray(palette, dtype="object")[
-                before_palette
+                _decode_long_array(section["BlockStates"].value, 4096)
             ]  # Mask the decoded long array with the entries from the palette
 
             uniques = numpy.append(

src/utils/format_utils.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from os.path import exists, join
+
+from nbt import nbt
+
+
+def check_all_exist(in_dir: str, *args: str) -> bool:
+    """
+    Check that all files exist in a parent directory
+
+    :param in_dir: The parent directory
+    :param *args: file or folder names to look for
+    :return: Boolean value indicating whether all were found
+    """
+
+    for child in args:
+        if not exists(join(in_dir, child)):
+            print(f"Didn't find {child}")
+            return False
+        else:
+            print(f"Found {child}")
+
+    return True
+
+
+def check_one_exists(in_dir: str, *args: str) -> bool:
+    """
+    Check that at least one file exists in a parent directory
+
+    :param in_dir: The parent directory
+    :param *args: file or folder names to look for
+    :return: Boolean value indicating whether at least one was found
+    """
+
+    for child in args:
+        if exists(join(in_dir, child)):
+            print(f"Found {child}")
+            return True
+
+    return False
+
+
+def load_leveldat(in_dir: str):
+    """
+    Load the root tag of the level.dat file in the directory
+
+    :param in_dir: The world directory containing the level.dat file
+    :return: The NBT root tag
+    """
+
+    fp = open(join(in_dir, "level.dat"), "rb")
+    root_tag = nbt.NBTFile(fileobj=fp)
+    fp.close()
+
+    return root_tag
+
+
+def check_version_leveldat(root_tag, min: int = None, max: int = None) -> bool:
+    """
+    Check the Version tag from the provided level.dat NBT structure
+
+    :param root_tag: the root level.dat tag
+    :param min: The lowest acceptable value (optional)
+    :param max: The highest acceptable value (optional)
+    :return: Whether the version tag falls in the correct range
+    """
+
+    version_found: int = root_tag.get("Data", nbt.TAG_Compound()).get(
+        "Version", nbt.TAG_Compound()
+    ).get("Id", nbt.TAG_Int(-1)).value
+
+    min_qualifies: bool = True
+    if min is not None:
+        min_qualifies = version_found >= min
+
+    max_qualifies: bool = True
+    if max is not None:
+        max_qualifies = version_found <= max
+
+    if __debug__:
+        min_text: str = f"{min} <= " if min is not None else ""
+        max_text: str = f" >= {max}" if max is not None else ""
+        print(f"Checking {min_text}{version_found}{max_text}")
+
+    return min_qualifies and max_qualifies

src/utils/world_utils.py

@@ -88,7 +88,7 @@ def from_nibble_array(arr: ndarray) -> ndarray:
     new_arr = zeros((shape[0], shape[1], shape[2] * 2), dtype=uint8)
 
     new_arr[:, :, ::2] = arr
-    new_arr[:, :, ::2] &= 0xf
+    new_arr[:, :, ::2] &= 0xF
     new_arr[:, :, 1::2] = arr
     new_arr[:, :, 1::2] >>= 4