diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8af0bd5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.pyc +venv/ +dist/ +build/ +*.ini +_*/ \ No newline at end of file diff --git a/FORMATS.md b/FORMATS.md new file mode 100644 index 0000000..b8c69bf --- /dev/null +++ b/FORMATS.md @@ -0,0 +1,291 @@ +Starbound data formats +====================== + +This document is intended to describe Starbound's various data structures. + +* [File formats](#file-formats) +* [SBON](#sbon) +* [Celestial data](#celestial-data) +* [World data](#world-data) + + +File formats +------------ + +Starbound uses regular JSON and Lua files for some things, but this +document will only focus on the custom file formats. + +* [BTreeDB5](#btreedb5) +* [SBAsset6](#sbasset6) +* [SBVJ01](#sbvj01) + +### BTreeDB5 + +A B-tree database format which enables quick scanning and updating. +It's used by Starbound to save world and universe data. + +#### Header + +The header consists of 512 bytes, representing the following fields: + +| Field # | Type | Description +| ------: | ----------- | ----------- +| 1 | `char[8]` | The string "BTreeDB5" +| 2 | `int32` | Byte size of blocks (see below) +| 3 | `char[16]` | The name of the database (null padded) +| 4 | `int32` | Byte size of index keys +| 5 | `bool` | Whether to use root node #2 instead +| 6 | `int32` | Free node #1 block index +| – | `byte[4]` | Unknown +| 7 | `int32` | Offset in file of end of free block #1 +| 8 | `int32` | Root node #1 block index +| 9 | `boolean` | Whether root node #1 is a leaf +| 10 | `int32` | Free node #2 block index +| – | `byte[4]` | Unknown +| 11 | `int32` | Offset in file of end of free block #2 +| 12 | `int32` | Root node #2 block index +| 13 | `boolean` | Whether root node #2 is a leaf +| – | `byte[445]` | Unused bytes + +In the BTreeDB4 format there was also a "free node is dirty" boolean +which is not accounted for above. It may be one of the "Unknown" +values. + +#### Blocks + +The most primitive structure in the BTreeDB5 format is the block. It's +a chunk of bytes of a fixed size (defined in the header) which plays a +certain role in the database. + +A lot of fields in the BTreeDB5 format references blocks by their index +which means an offset of `header_size + index * block_size`. + +##### Root nodes + +The root node is the entry point when scanning for a specific key. The +root node can be either an index block or a leaf block, depending on +how large the database is. Usually, it will be an index block. + +Since BTreeDB5 databases are meant to be updated on the fly, the root +node may alternate to allow for transactional updates to the index. +If the *alternate root block index* flag is true, the alternate root +index should be used for entry instead when scanning for a key. + +##### Index block + +The index block always starts with the characters `II`. + +##### Leaf block + +The leaf block always starts with the characters `LL`. + +##### Free block + +The free block always starts with the characters `FF`. + +Free blocks may be either brand new blocks (after growing the file), or +reclaimed blocks that were no longer in use. + +#### Scanning for a key + +This section will contain information on how to retrieve a value from a +BTreeDB5 database. + +### SBAsset6 + +A large file that contains an index pointing to many small files within +it. The main asset file (`packed.pak`) and mods are of this type. + +#### Header + +The header for SBAsset6 is very straightforward: + +| Field # | Type | Description +| ------: | --------- | ----------- +| 1 | `char[8]` | The string "SBAsset6" +| 2 | `uint64` | Metadata offset + +The metadata offset points to another location in the file where the +metadata can be read. Seek to that point in the file and find: + +| Field # | Type | Description +| ------: | ----------- | ----------- +| 1 | `char[5]` | The string "INDEX" +| 2 | SBON map | Information about the file +| 3 | SBON varint | Number of files in the index +| 4 + 3n | SBON string | SBON UTF-8 encoded string +| 5 + 3n | `uint64` | Offset where file starts +| 6 + 3n | `uint64` | Length of file + +Once the index has been parsed into memory, it can be used to seek to +various files in the SBAsset6 file. + +### SBVJ01 + +Versioned JSON-like data. Used for player data files and the like. The +data structures themselves use a custom binary form of JSON which will +be referred to as "SBON" in this document. + +The file structure is simply the string `"SBVJ01"` followed by a single +versioned JSON object (see below). + + +SBON +---- + +(I'm calling this "Starbound Binary Object Notation", but don't know +what the Starbound developers call it internally.) + +This format is similar to other binary formats for JSON (e.g., BSON). +SBON is used in most file formats to represent complex data such as +metadata and entities. + +### Data types + +* Variable length integer (also known as [VLQ][vlq]) +* Bytes (varint for length + the bytes) +* String (bytes with UTF-8 encoding) +* List (varint for count, dynamic for values) +* Map (varint for count, string/dynamic pairs for entries) +* Dynamic (byte for type + value) + * `0x01`: Nil value + * `0x02`: Double-precision float (a.k.a. `double`) + * `0x03`: Boolean + * `0x04`: Signed varint (see below) + * `0x05`: String + * `0x06`: List + * `0x07`: Map + +#### Varint + +A variable length (in bytes) integer, also known as [VLQ][vlq]. As long as the most significant bit is set read the next byte and concatenate its 7 other bits with the 7 bits of the previous bytes. The resulting string of bits is the binary representation of the number. + +The purposes of this data type is to allow (common) lower values 0...127 to only use up one byte, 128...16383 two bytes, and so on. + +#### Signed varint + +A signed varint is just a regular varint, except that the least significant bit (the very last bit in the data stream) is used to represent the sign of the number. If the bit is 1, the number should be considered negative and also have one subtracted from its value (because there is no negative 0). If the bit is 0, the number is positive. In both cases, the least significant bit should not be considered part of the number. + +### Versioned JSON + +Starbound has a data structure known as "versioned JSON" which consists +of SBON. In addition to arbitrary data, it also holds a name and a +version. + +Most complex data structures are represented as versioned JSON and may +have Lua scripts that upgrade older versions to the current one. + +| Field | Type | Description +| ------------- | ------------ | ----------- +| Name | SBON string | The name or type of the data structure. +| Is versioned? | `bool` | Flag indicating that there’s a version. +| Version | `int32` | Version (only if previous field is `true`). +| Data | SBON dynamic | The data itself, usually a map. + + +Celestial data +-------------- + +Celestial files are BTreeDB5 databases that contain generated +information about the universe. + +Little is currently known about this format as the keys are hashes of +some key that has not yet been reverse engineered. + + +World data +---------- + +World files (this includes the player ship) are BTreeDB5 databases that +contain the metadata, entities, and tile regions of the world. + +### Regions + +Regions are indexed by *type*, *X*, and *Y* values. The *type* value is +`1` for tile data and `2` for entity data. There's a special key, +{0, 0, 0} which points to the world metadata. All values are gzip +deflated and must be inflated before they can be read. + +The BTreeDB5 key for a region is represented in binary as a byte for +*type* followed by two shorts for the *X* and *Y* coordinates. + +The X axis goes from left to right and the Y axis goes from down to up. + +#### World metadata + +Once inflated, the {0, 0, 0} value starts with two integers (8 bytes) +holding the number of tiles along the X and Y axes, followed by an SBON +data structure containing all the world's metadata. + +#### Tile data + +The {1, X, Y} value contains three bytes followed by the data for 32×32 +tiles. The purpose of the three bytes is currently unknown. + +A single tile is made up of 30 bytes of binary data: + +| Field # | Bytes | Type | Description +| -------: | ----: | -------- | ----------- +| 1 | 1–2 | `int16` | Foreground material¹ +| 2 | 3 | `uint8` | Foreground hue shift +| 3 | 4 | `uint8` | Foreground color variant +| 4 | 5–6 | `int16` | Foreground mod +| 5 | 7 | `uint8` | Foreground mod hue shift +| 6 | 8–9 | `int16` | Background material¹ +| 7 | 10 | `uint8` | Background hue shift +| 8 | 11 | `uint8` | Background color variant +| 9 | 12–13 | `int16` | Background mod +| 10 | 14 | `uint8` | Background mod hue shift +| 11 | 15 | `uint8` | Liquid +| 12 | 16–19 | `float` | Liquid level +| 13 | 20–23 | `float` | Liquid pressure +| 14 | 24 | `bool` | Liquid is infinite +| 15 | 25 | `uint8` | Collision map² +| 16 | 26–27 | `uint16` | Dungeon ID³ +| 17 | 28 | `uint8` | "Biome"⁴ +| 18 | 29 | `uint8` | "Environment Biome"⁴ +| 19 | 30 | `bool` | Indestructible (tree/vine base) +| 20 | 31 | `unknown`| Unknown? + +¹ Refers to a material by its id. Additional constants: + +| Constant | Meaning +| -------: | ------- +| -36 | Unknown (seen on ships) +| -9 | Unknown (possibly dungeon related) +| -8 | Unknown (possibly dungeon related) +| -7 | Unknown (possibly dungeon related) +| -3 | Not placeable +| -2 | Not generated (or outside world bounds) +| -1 | Empty + +² Used by the game to block the player's movement. Constants: + +| Constant | Meaning +| -------: | ------- +| 1 | Empty space +| 2 | Platform (floor that player can pass through) +| 3 | Dynamic (e.g., closed door) +| 5 | Solid + +³ Dungeon info is stored in the world metadata. Additional constants: + +| Constant | Meaning +| -------: | ------- +| 65,531 | Tile removed by player +| 65,532 | Tile placed by player +| 65,533 | Microdungeon +| 65,535 | Not associated with anything + +⁴ Unverified, simply quoting from this page: http://seancode.com/galileo/format/tile.html + +#### Entity data + +Values for {2, X, Y} keys are a sequence of various entities in the +region at (X, Y). Each entity is a versioned JSON object. + +The data, once inflated, consists of an SBON varint for the count and +then the versioned JSON objects, one after the other. + + +[vlq]: https://en.wikipedia.org/wiki/Variable-length_quantity diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c3a2575 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Blixt +Copyright (c) 2018 txxia + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..8f68d3d --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +Starbound Map Viewer +==================== +This tool is built on top of @blixt's awesome work: [Starbound Utilities](https://github.com/blixt/py-starbound). + +![MapViewer](./screenshots/MapViewer.PNG) + +Feel free to contribute either via submitting pull requests or writing +up issues with suggestions and/or bugs. + +Release +------- +``` +pyinstaller mapviewer.spec +``` + +License +------- + +[MIT License](./LICENSE) diff --git a/assets/shader/map.fs.glsl b/assets/shader/map.fs.glsl new file mode 100644 index 0000000..4bafc0a --- /dev/null +++ b/assets/shader/map.fs.glsl @@ -0,0 +1,161 @@ +#version 330 + +// Config +#define GRID_INTENSITY 0.0001 +// #define DEBUG_TILE_ID +// #define DEBUG_REGION_ID + +// Macros +#define FDIV(a, b) (float(a) / float(b)) + +#define RGB16_R(u16) (FDIV(((u16) >> 11) & 0x1FU, 31)) +#define RGB16_G(u16) (FDIV(((u16) >> 5) & 0x3FU, 63)) +#define RGB16_B(u16) (FDIV((u16) & 0x1FU, 31)) +#define RGB16(u16) (vec3(RGB16_R(u16), RGB16_G(u16), RGB16_B(u16))) + +#define HUE8(u8) (FDIV(u8, 360)) + +// Constants +#define GRID_DIM 7 +#define GRID_DIM_INV FDIV(1, GRID_DIM) +#define REGION_COUNT (GRID_DIM*GRID_DIM) + +#define REGION_DIM 32 +#define REGION_TILES ((REGION_DIM)*(REGION_DIM)) +#define TILE_SIZE 32 +#define REGION_BYTES ((REGION_TILES)*(TILE_SIZE)) +#define LAYERS_PER_REGION 2 + +// Tile Data +#define FG_MAT(d) ((d)[0].x >> 16) +#define FG_HUE(d) (HUE8(((d)[0].x >> 8) & 0xFFU)) +#define FG_VAR(d) ((d)[0].x & 0xFU) + +#define BG_MAT(d) ((d)[0].z >> 16) +#define BG_HUE(d) (HUE8(((d)[0].z >> 8) & 0xFFU)) +#define BG_VAR(d) ((d)[0].z & 0xFFU) + +#define COLL(d) (((d)[1].z >> 8) & 0xFFU) +#define COLL_EMPTY 1U +#define COLL_PLATFORM 2U +#define COLL_DYNAMIC 3U +#define COLL_SOLID 5U + +uniform vec2 iResolution; +uniform float iTime; +uniform mat3 iFragProjection; +uniform bool iRegionValid[REGION_COUNT]; +uniform usamplerBuffer iRegionLayer[REGION_COUNT * LAYERS_PER_REGION]; +uniform struct Config { + bool showGrid; +} iConfig; + +out vec4 outputColor; + +// http://lolengine.net/blog/2013/07/27/rgb-to-hsv-in-glsl +vec3 rgb2hsv(vec3 c) +{ + vec4 K = vec4(0.0, -1.0 / 3.0, 2.0 / 3.0, -1.0); + vec4 p = mix(vec4(c.bg, K.wz), vec4(c.gb, K.xy), step(c.b, c.g)); + vec4 q = mix(vec4(p.xyw, c.r), vec4(c.r, p.yzx), step(p.x, c.r)); + + float d = q.x - min(q.w, q.y); + float e = 1.0e-10; + return vec3(abs(q.z + (q.w - q.y) / (6.0 * d + e)), d / (q.x + e), q.x); +} +vec3 hsv2rgb(vec3 c) { + vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0); + vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www); + return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y); +} + +// Get the linear tile id in a region, given its 2D coordinate. +int getTileId(in vec2 normalCoord01){ + // transform to [0..TILE_SIZE]^2 + ivec2 t = ivec2(normalCoord01 * TILE_SIZE) + 1; + return (t.y - 1) * REGION_DIM + t.x - 1; +} + +// Get region number in [0, REGION_COUNT) based on pixel coordinate in [0, 1]^2. +int getRegionId(in vec2 normalCoord01) { + ivec2 r01 = ivec2(normalCoord01 * float(GRID_DIM)); + return r01.y * GRID_DIM + r01.x; +} + +void getTile(in int regionId, in int id, inout uvec4 tileData[2]){ + int regionBase = regionId * 2; + tileData[0] = texelFetch(iRegionLayer[regionBase], id); + tileData[1] = texelFetch(iRegionLayer[regionBase+1], id); +} + +vec3 invalidRegionColor(in vec2 normalCoord01){ + return vec3(0.0, 0.5, 0.7) * sqrt(sin(normalCoord01.y * iResolution.y)); +} + +vec3 tileColor(in int regionId, in int tileId) { + uvec4 tileData[2]; + getTile(regionId, tileId, tileData); + + uint fgId = FG_MAT(tileData); + float fgHueShift01 = FG_HUE(tileData); + uint bgId = BG_MAT(tileData); + float bgHueShift01 = BG_HUE(tileData); + float coll = float(COLL(tileData) != COLL_EMPTY); + + return mix( + hsv2rgb(vec3(bgHueShift01, 1.0, 0.2)), + hsv2rgb(vec3(fgHueShift01, 1.0, coll)), + coll + ); +} + +vec3 gridColor(in vec2 normalCoord01) { + vec2 grid_fract = fract(normalCoord01 * GRID_DIM); + vec2 grid_proximity = 1.0 / abs(round(grid_fract) - grid_fract); + vec3 grid = vec3(GRID_INTENSITY * (grid_proximity.x * grid_proximity.y)); + return grid; +} + +void mainImage(out vec4 fragColor, in vec2 fragCoord){ + // r in [-1, +1]^2 + vec3 r01aug = iFragProjection * vec3(fragCoord, 1.0); + vec2 r01 = r01aug.xy / r01aug.z; + vec2 r = r01.xy * 2.0 - 1.0; + vec3 pixel; + + int regionId = getRegionId(r01); + if (iRegionValid[regionId]) { + vec2 regionCoord01 = mod(r01, GRID_DIM_INV) * GRID_DIM; + int tileId = getTileId(regionCoord01); + pixel = tileColor(regionId, tileId); + + #ifdef DEBUG_TILE_ID + pixel = vec3( + FDIV(mod(tileId, REGION_DIM), REGION_DIM), + FDIV(FDIV(tileId, REGION_DIM), REGION_DIM), + 0.0 + ); + #endif + } else { + pixel = invalidRegionColor(r01); + } + if (iConfig.showGrid){ + pixel += gridColor(r01); + } + + #ifdef DEBUG_REGION_ID + pixel = vec3( + FDIV(mod(regionId, GRID_DIM), GRID_DIM), + FDIV(FDIV(regionId, GRID_DIM), GRID_DIM), + 0.0 + ); + #endif + + fragColor = vec4(pixel, 1.0); +} + +void main() +{ + vec2 fragCoord = gl_FragCoord.xy; + mainImage(outputColor, fragCoord); +} \ No newline at end of file diff --git a/assets/shader/map.vs.glsl b/assets/shader/map.vs.glsl new file mode 100644 index 0000000..cab54ad --- /dev/null +++ b/assets/shader/map.vs.glsl @@ -0,0 +1,6 @@ +#version 330 +layout(location = 0) in vec2 position; +void main() +{ + gl_Position = vec4(position, 0.0, 1.0); +} \ No newline at end of file diff --git a/glfw3.dll b/glfw3.dll new file mode 100644 index 0000000..ebf2933 Binary files /dev/null and b/glfw3.dll differ diff --git a/map/__init__.py b/map/__init__.py new file mode 100644 index 0000000..8df5b52 --- /dev/null +++ b/map/__init__.py @@ -0,0 +1,2 @@ +from .model import * +from .renderer import WorldRenderer diff --git a/map/model.py b/map/model.py new file mode 100644 index 0000000..5f3a816 --- /dev/null +++ b/map/model.py @@ -0,0 +1,104 @@ +import numpy as np + +from utils import cache + +REGION_DIM = 32 +TILES_PER_REGION = REGION_DIM * REGION_DIM +TILE_SIZE = 31 +REGION_BYTES = TILES_PER_REGION * TILE_SIZE + + +class WorldView: + """ + Represents a view to the world map. + """ + + def __init__(self, world, center_region, grid_dim=5): + """ + :param world: world object + :param center_region: center tile coordinate of the view, as tuple + :param grid_dim: number of cells on any side of the grid + """ + self._on_region_updated = [] + + assert world is not None + self._world = world + + self._grid_dim = None + self.grid_dim = grid_dim + + self._region_grid = [[None for _ in range(grid_dim)] for _ in range(grid_dim)] + + self._center_region = np.zeros(2) + self.center_region = center_region + + @property + def world(self): + return self._world + + @property + def region_grid(self): + return self._region_grid + + @property + def center_region(self): + return self._center_region + + @center_region.setter + def center_region(self, value): + assert isinstance(value, np.ndarray) + assert value.size == 2 + assert value.dtype == np.int + if any(self._center_region != value): + self._center_region = value + self._update_regions() + + @property + def grid_dim(self): + return self._grid_dim + + @grid_dim.setter + def grid_dim(self, value): + assert type(value) == int + assert 0 < value + if self._grid_dim != value: + self._grid_dim = value + + def on_region_updated(self, callback): + self._on_region_updated.append(callback) + + def get_location(self, coord01): + """ + Find the location indicated by the given coordinate in the current view + :param coord01: coordinate in the current view + :return: (region_coord, tile_coord) + """ + region_coord = self.center_region - self.grid_dim // 2 + (coord01 * self.grid_dim).astype(np.int) + tile_coord = (coord01 % (1 / self.grid_dim) * self.grid_dim * REGION_DIM).astype(np.int) + return region_coord, tile_coord + + @cache.memoized_method(maxsize=1024) + def get_region(self, region_coord): + """ + :param region_coord: region coordinate + :return: regions at the given location, None if not available + """ + try: + return self.world.get_raw_tiles(region_coord[0], region_coord[1]) + except KeyError or RuntimeError: + return None + + def _update_regions(self): + """ + Updates the grid of size grid_dim^2 containing regions centered at the current view. + """ + # FIXME handle wrapping around + # Retrieve visible regions as a grid, each item is a byte array of 1024 tiles (31 bytes per tile) + for y in range(self.grid_dim): + ry = self.center_region[1] + y - 1 + for x in range(self.grid_dim): + rx = self.center_region[0] + x - 1 + self.region_grid[y][x] = self.get_region((rx, ry)) + + for cb in self._on_region_updated: + cb() diff --git a/map/renderer.py b/map/renderer.py new file mode 100644 index 0000000..78880eb --- /dev/null +++ b/map/renderer.py @@ -0,0 +1,325 @@ +import ctypes +from functools import lru_cache + +import OpenGL.GL as gl +import numpy as np +from OpenGL.GL import shaders + +from utils.resource import asset_path +from .model import TILES_PER_REGION + +''' +Per tile data (31+1 bytes): +------------------------------------- +h short 2 2 foreground_material +B uchar 1 3 foreground_hue_shift +B uchar 1 4 foreground_variant + +h short 2 6 foreground_mod +B uchar 1 7 foreground_mod_hue_shift +--> pad 1 byte here <-- + +h short 2 9 background_material +B uchar 1 10 background_hue_shift +B uchar 1 11 background_variant + +h short 2 13 background_mod +B uchar 1 14 background_mod_hue_shift +B uchar 1 15 liquid +-------------------------------------- +f float 4 19 liquid_level + +f float 4 23 liquid_pressure + +B uchar 1 24 liquid_infinite +B uchar 1 25 collision +H ushrt 2 27 dungeon_id + +B uchar 1 28 biome +B uchar 1 29 biome_2 +? bool 1 30 indestructible +x pad 1 31 (padding) +''' + +QUAD_VERTS_BL = 0 +QUAD_VERTS_BR = 1 +QUAD_VERTS_TR = 2 +QUAD_VERTS_TL = 3 +QUAD_VERTS = np.array([ + [-1, -1], + [1, -1], + [1, 1], + [-1, 1] +], np.float32) + +QUAD_IDX = np.array([ + [0, 1, 2], + [0, 2, 3] +], np.uint16) + +LAYERS_PER_REGION = 2 + + +class WorldRendererConfig: + def __init__(self): + self.showGrid = True + + +class WorldRenderer: + def __init__(self, view, grid_dim): + self._view = None + self.view = view + + self._grid_dim = grid_dim + + self.config = WorldRendererConfig() + self.vertices = np.copy(QUAD_VERTS) + self.indices = np.copy(QUAD_IDX) + self.region_validity = [True] * self.region_count + self.region_layers = tuple(range(self.region_layer_count)) + + # Setting up VAO + self.vao = gl.glGenVertexArrays(1) + gl.glBindVertexArray(self.vao) + + # Quad vertex buffer + self.vbo = gl.glGenBuffers(1) + gl.glBindBuffer(gl.GL_ARRAY_BUFFER, self.vbo) + gl.glBufferData(gl.GL_ARRAY_BUFFER, + self.vertices.nbytes, + self.vertices, + gl.GL_DYNAMIC_DRAW) + gl.glEnableVertexAttribArray(0) + gl.glVertexAttribPointer(0, 2, gl.GL_FLOAT, gl.GL_FALSE, 0, None) + + # Finish setting up VAO + gl.glBindVertexArray(0) + + # Quad index buffer + self.ebo = gl.glGenBuffers(1) + gl.glBindBuffer(gl.GL_ELEMENT_ARRAY_BUFFER, self.ebo) + gl.glBufferData(gl.GL_ELEMENT_ARRAY_BUFFER, + QUAD_IDX.nbytes, + QUAD_IDX, + gl.GL_STATIC_DRAW) + # Regions data + self.regions_texs = gl.glGenTextures(self.region_layer_count) + self.regions_tbos = gl.glGenBuffers(self.region_layer_count) + for tbo in self.regions_tbos: + gl.glBindBuffer(gl.GL_TEXTURE_BUFFER, tbo) + gl.glBufferData(gl.GL_TEXTURE_BUFFER, TILES_PER_REGION * 16, None, gl.GL_DYNAMIC_DRAW) + + # Create shaders + vs_src, fs_src = WorldRenderer.load_shaders() + self.vs = shaders.compileShader(vs_src, gl.GL_VERTEX_SHADER) + self.fs = shaders.compileShader(fs_src, gl.GL_FRAGMENT_SHADER) + self.program = shaders.compileProgram(self.vs, self.fs) + + # initial callback + self._update_region_textures() + + @property + def view(self): + return self._view + + @view.setter + def view(self, value): + if self._view != value: + self._view = value + if value is not None: + self._view.on_region_updated(self._update_region_textures) + self._update_region_textures() + + @property + def grid_dim(self): + return self._grid_dim + + @property + def region_count(self): + return self.grid_dim * self.grid_dim + + @property + def region_layer_count(self): + return self.region_count * LAYERS_PER_REGION + + def draw(self, rect, resolution, time): + """ + Draw a frame of the world view. + :param rect: array of 4 floats (min_x, min_y, max_x, max_y) + representing region in [-1, +1]^2 to draw the map + :param resolution: size of the framebuffer + :param time: time since the application start + """ + gl.glClearColor(0, 0, 0, 1) + gl.glClear(gl.GL_COLOR_BUFFER_BIT) + + gl.glUseProgram(self.program) + + # textures + for i, (tex, tbo) in enumerate(zip(self.regions_texs, self.regions_tbos)): + gl.glActiveTexture(gl.GL_TEXTURE0 + i) + gl.glBindTexture(gl.GL_TEXTURE_BUFFER, tex) + gl.glTexBuffer(gl.GL_TEXTURE_BUFFER, gl.GL_RGBA32UI, tbo) + + # uniforms + rect_resolution = ( + resolution[0] * ((rect[2] - rect[0]) / 2), + resolution[1] * ((rect[3] - rect[1]) / 2) + ) + # Compute fragment projection from window space to rect space [-1,1]^2 + rect01 = tuple((r + 1) * 0.5 for r in rect) + rect_in_frag_space = tuple(r * resolution[i % 2] for i, r in enumerate(rect01)) + frag_projection = _project_rect(rect_in_frag_space).astype(np.float32) + + gl.glUniform2f(gl.glGetUniformLocation(self.program, "iResolution"), rect_resolution[0], rect_resolution[1]) + gl.glUniform1f(gl.glGetUniformLocation(self.program, "iTime"), time) + gl.glUniformMatrix3fv(gl.glGetUniformLocation(self.program, "iFragProjection"), 1, True, frag_projection) + gl.glUniform1iv(gl.glGetUniformLocation(self.program, "iRegionValid"), self.region_count, self.region_validity) + gl.glUniform1iv(gl.glGetUniformLocation(self.program, "iRegionLayer"), + self.region_layer_count, self.region_layers) + gl.glUniform1i(gl.glGetUniformLocation(self.program, "iConfig.showGrid"), self.config.showGrid) + + gl.glBindVertexArray(self.vao) + + # quad + self.vertices[QUAD_VERTS_BL][:] = rect[:2] # min_x, min_y + self.vertices[QUAD_VERTS_BR][:] = (rect[2], rect[1]) # max_x, min_y + self.vertices[QUAD_VERTS_TR][:] = rect[2:] # max_x, max_y + self.vertices[QUAD_VERTS_TL][:] = (rect[0], rect[3]) # min_x, max_y + gl.glBufferSubData(gl.GL_ARRAY_BUFFER, + offset=0, + size=self.vertices.nbytes, + data=self.vertices) + + gl.glBindBuffer(gl.GL_ELEMENT_ARRAY_BUFFER, self.ebo) + gl.glDrawElements(gl.GL_TRIANGLES, self.indices.size, gl.GL_UNSIGNED_SHORT, None) + gl.glBindVertexArray(0) + + def _update_region_textures(self): + regions = sum(self.view.region_grid, []) \ + if self.view is not None \ + else [None] * self.region_count + + for i, r in enumerate(regions): + self.region_validity[i] = r is not None + + layers = sum((_region_to_layers(r) for r in regions), []) + + # upload each layer to a texture buffer + for layer, tbo in zip(layers, self.regions_tbos): + if layer is None: + continue + layer_ctypes = np.frombuffer(layer).ctypes + gl.glBindBuffer(gl.GL_TEXTURE_BUFFER, tbo) + buf = gl.glMapBuffer(gl.GL_TEXTURE_BUFFER, gl.GL_WRITE_ONLY) + ctypes.memmove(buf, layer_ctypes, len(layer)) + gl.glUnmapBuffer(gl.GL_TEXTURE_BUFFER) + gl.glBindBuffer(gl.GL_TEXTURE_BUFFER, 0) + + @staticmethod + def load_shaders(): + with open(asset_path('shader/map.vs.glsl'), 'r') as vs: + vs_src = vs.read() + with open(asset_path('shader/map.fs.glsl'), 'r') as fs: + fs_src = fs.read() + return vs_src, fs_src + + +@lru_cache(maxsize=512) +def _region_to_layers(r): + if r is None: + return [None, None] + else: + # pad 1 byte to the index 8 of each tile, making each tile 32 bytes in size + r = _pad_region(r, tile_size=31, offset=8) + # slice each region into 2 layers, due to the size limit of texture buffer + return _slice_tiles(r, tile_size=32, slices=2) + + +def _slice_tiles(region, *, tile_size, slices): + """ + :param region: byte string containing the region data + :param tile_size: size of a tile in bytes + :param slices: number of slices per tile + :return: list containing `slices` elements + + >>> _slice_tiles(b'\\0\\1\\2\\3' * 2, tile_size=4, slices=2) + [bytearray(b'\\x00\\x01\\x00\\x01'), bytearray(b'\\x02\\x03\\x02\\x03')] + """ + assert tile_size % slices == 0 + region_size = len(region) + tile_count = region_size // tile_size + slice_size = tile_size // slices + layer_size = region_size // slices + layers = [bytearray(b'\0' * layer_size) for _ in range(slices)] + for t in range(0, tile_count): + dst_start = t * slice_size + for s in range(slices): + src_start = t * tile_size + s * slice_size + layers[s][dst_start:dst_start + slice_size] = region[src_start: src_start + slice_size] + return layers + + +def _pad_region(region, *, tile_size, offset): + """ + Insert padding to the `offset`th (zero-based) byte of every tile + + >>> _pad_region(b'\\0\\1\\2\\3' * 2, tile_size=4, offset=2) + bytearray(b'\\x00\\x01\\x00\\x02\\x03\\x00\\x01\\x00\\x02\\x03') + """ + assert len(region) % tile_size == 0 + assert tile_size >= offset + tile_count = len(region) // tile_size + new_tile_size = tile_size + 1 + padded_region = bytearray(b'\0' * tile_count * new_tile_size) + for i in range(tile_count): + base = i * new_tile_size + r = base - i + padded_region[base: base + offset] = region[r:r + offset] + padded_region[base + offset + 1: base + new_tile_size] = region[r + offset:r + tile_size] + return padded_region + + +def _project_rect(rect): + """ + Returns project from a rect in window space ([0,w], [0,h]) to [0,1]^2 + :param rect: 4 floats of (min_x, min_y, max_x, max_y) + :return: 3x3 numpy matrix + + >>> _project_rect([0, 0, 10, 20]) # scaling only + array([[0.1 , 0. , 0. ], + [0. , 0.05, 0. ], + [0. , 0. , 1. ]]) + >>> _project_rect([3, 4, 4, 5]) # translation only + array([[ 1., 0., -3.], + [ 0., 1., -4.], + [ 0., 0., 1.]]) + >>> _project_rect([0, 6, 10, 16]) # translation + scaling + array([[ 0.1, 0. , 0. ], + [ 0. , 0.1, -0.6], + [ 0. , 0. , 1. ]]) + >>> np.matmul(_project_rect([0, 6, 10, 16]), [5, 11, 1]) # projecting the center of the rect + array([0.5, 0.5, 1. ]) + """ + rw = rect[2] - rect[0] + rh = rect[3] - rect[1] + # translate to origin + translation = [ + [1, 0, -rect[0]], + [0, 1, -rect[1]], + [0, 0, 1] + ] + # scale down + scaling = [ + [1 / rw, 0, 0], + [0, 1 / rh, 0], + [0, 0, 1] + ] + proj = np.matmul(scaling, translation) + return proj + + +if __name__ == '__main__': + import doctest + + doctest.testmod() diff --git a/mapviewer.py b/mapviewer.py new file mode 100644 index 0000000..04467d4 --- /dev/null +++ b/mapviewer.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- +import logging +import os + +if 'PYGLFW_LIBRARY' not in os.environ: + os.environ['PYGLFW_LIBRARY'] = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'glfw3.dll') + +import OpenGL.GL as gl +import numpy as np +import glfw +import imgui +from imgui.integrations.glfw import GlfwRenderer + +from starbound import GameDirectory +from map import WorldRenderer, WorldView, REGION_DIM +from utils.config import CONFIG + +CONFIG_SECTION = 'map_viewer' +CONFIG_GAME_ROOT = 'starbound_root' + +POPUP_SETTINGS = 'Settings' +POPUP_SELECT_WORLD = 'Select World' + + +class G: + """ + Global runtime-only data + """ + gui_show_help_overlay = False + gui_config_changed = False + + framebuffer_size = np.zeros(2) + mouse_in_map_normal01 = np.zeros(2) + + +class WorldViewer(object): + + def __init__(self): + if not CONFIG.has_section(CONFIG_SECTION): + CONFIG.add_section(CONFIG_SECTION) + self.config = CONFIG[CONFIG_SECTION] + self.gamedata = GameDirectory() + self.gamedata.game_root = self.config.get(CONFIG_GAME_ROOT) + + self.world_coord = None + self.world = None + self.player_start = np.zeros(2, dtype=np.int) + self.world_size = np.zeros(2, dtype=np.int) + self.world_size_in_regions = np.zeros(2, dtype=np.int) + + # TODO this implementation caps at 9x9 grid, may need to look into alternatives + ''' + https://www.khronos.org/opengl/wiki/Array_Texture + gl.glTexSubImage1Dui + https://www.reddit.com/r/opengl/comments/4u8qyv/opengl_limited_number_of_textures_how_can_you/ + ''' + self.grid_dim = 7 + self.view = None + self.world_renderer = WorldRenderer(self.view, grid_dim=self.grid_dim) + + self.io = imgui.get_io() + self.set_styles() + + def change_world(self, world_coord): + try: + self.world = self.gamedata.get_world(world_coord) + self.world_coord = world_coord + logging.info("Changed world to %s (%s)", world_coord, self.world) + except Exception as e: + logging.error(e) + self.world = None + self.world_coord = None + if self.world is not None: + self.player_start = np.array(self.world.metadata['playerStart'], dtype=np.int) + self.world_size = np.array((self.world.width, self.world.height), dtype=np.int) + self.world_size_in_regions = np.ceil(self.world_size / REGION_DIM).astype(np.int) + + logging.info('World size in regions: {}'.format(self.world_size_in_regions)) + center_region = np.floor(self.player_start / REGION_DIM).astype(np.int) + self.world_renderer.view = self.view = WorldView(self.world, + center_region=center_region, + grid_dim=self.grid_dim) + else: + self.world_renderer.view = self.view = None + + def render(self, framebuffer_size): + """ + :param framebuffer_size: tuple of 2 ints indicating framesize + """ + G.framebuffer_size = framebuffer_size + aspect = float(framebuffer_size[1]) / framebuffer_size[0] + map_rect_normal = np.array((-1, 1 - 2 / max(aspect, 1), 1, 1)) + map_rect_normal01 = (map_rect_normal + 1) * 0.5 + map_rect = np.array(( + map_rect_normal01[0] * framebuffer_size[0], + map_rect_normal01[1] * framebuffer_size[1], + map_rect_normal01[2] * framebuffer_size[0], + map_rect_normal01[3] * framebuffer_size[1] + )) + map_rect_size = map_rect[2:4] - map_rect[0:2] + mouse = np.array(( + self.io.mouse_pos[0], + framebuffer_size[1] - self.io.mouse_pos[1] + )) + G.mouse_in_map_normal01 = (mouse - map_rect[:2]) / map_rect_size + + imgui.new_frame() + self.show_map_controller_window() + self.show_tooltip() + if G.gui_show_help_overlay: + self.show_help_overlay() + self.world_renderer.draw(map_rect_normal, framebuffer_size, glfw.get_time()) + + # imgui.show_test_window() + # self.show_debug_window() + + imgui.render() + + def show_map_controller_window(self): + imgui.set_next_window_position(0, G.framebuffer_size[0]) + imgui.set_next_window_size(G.framebuffer_size[0], G.framebuffer_size[1] - G.framebuffer_size[0]) + + if imgui.begin("Map", closable=False, flags=imgui.WINDOW_NO_RESIZE | + imgui.WINDOW_NO_MOVE | + imgui.WINDOW_NO_COLLAPSE | + imgui.WINDOW_NO_TITLE_BAR): + if self.view is not None: + center_x, center_y = self.view.center_region + if imgui.button("<") or self.get_keys_on_map(glfw.KEY_A, glfw.KEY_LEFT): + center_x = max(center_x - 1, 1) + imgui.same_line() + if imgui.button(">") or self.get_keys_on_map(glfw.KEY_D, glfw.KEY_RIGHT): + center_x = min(center_x + 1, self.world_size_in_regions[0]) + imgui.same_line() + _, center_x = imgui.slider_int("X", center_x, + min_value=1, + max_value=self.world_size_in_regions[0]) + if imgui.button("^") or self.get_keys_on_map(glfw.KEY_W, glfw.KEY_UP): + center_y = min(center_y + 1, self.world_size_in_regions[1]) + imgui.same_line() + if imgui.button("v") or self.get_keys_on_map(glfw.KEY_S, glfw.KEY_DOWN): + center_y = max(center_y - 1, 1) + imgui.same_line() + _, center_y = imgui.slider_int("Y", center_y, + min_value=1, + max_value=self.world_size_in_regions[1]) + self.view.center_region = np.array((center_x, center_y)) + + imgui.separator() + + _, self.world_renderer.config.showGrid = imgui.checkbox("Grid", self.world_renderer.config.showGrid) + + imgui.separator() + if imgui.tree_node("World Info", flags=imgui.TREE_NODE_DEFAULT_OPEN): + if self.world is not None: + imgui.label_text('Coordinates', self.world_coord) + imgui.label_text('Size', str(np.array((self.world.width, self.world.height)))) + imgui.label_text('PlayerStart', str(self.player_start)) + else: + imgui.text('Select a world to start') + imgui.tree_pop() + + imgui.separator() + if imgui.button("Settings.."): + imgui.open_popup(POPUP_SETTINGS) + self.popup_settings() + imgui.same_line() + if imgui.button("Select World.."): + imgui.open_popup(POPUP_SELECT_WORLD) + self.popup_select_world() + imgui.same_line() + _, G.gui_show_help_overlay = imgui.checkbox("Usage", G.gui_show_help_overlay) + + imgui.end() + + def show_tooltip(self): + if self.world is None or self.view is None: + return + mouse = self.io.mouse_pos + if 0 <= mouse.x <= G.framebuffer_size[0] and 0 <= mouse.y <= G.framebuffer_size[0]: + region_coord, tile_coord = self.view.get_location(G.mouse_in_map_normal01) + + imgui.begin_tooltip() + imgui.push_item_width(60) + imgui.text(str(region_coord * REGION_DIM + tile_coord)) + imgui.label_text('region', str(region_coord)) + imgui.label_text('tile', str(tile_coord)) + imgui.pop_item_width() + imgui.end_tooltip() + + def show_help_overlay(self): + imgui.set_next_window_position(0, 0, imgui.ALWAYS) + if imgui.begin("Help", closable=False, + flags=imgui.WINDOW_NO_MOVE | + imgui.WINDOW_NO_TITLE_BAR | + imgui.WINDOW_NO_RESIZE | + imgui.WINDOW_ALWAYS_AUTO_RESIZE | + imgui.WINDOW_NO_SAVED_SETTINGS | + imgui.WINDOW_NO_FOCUS_ON_APPEARING | + imgui.WINDOW_NO_INPUTS): + imgui.text("Usage") + imgui.separator() + imgui.bullet_text("Navigate with WASD/arrow-keys (click on the map first)") + # TODO imgui.bullet_text("Right-click on a tile to see details") + imgui.end() + + def popup_settings(self): + if imgui.begin_popup_modal(POPUP_SETTINGS, flags=imgui.WINDOW_ALWAYS_AUTO_RESIZE)[0]: + changed, self.config[CONFIG_GAME_ROOT] = imgui.input_text("Game Root", + self.config.get(CONFIG_GAME_ROOT, ''), + 255) + G.gui_config_changed |= changed + + imgui.separator() + if imgui.button('OK'): + if G.gui_config_changed: + G.gui_config_changed = False + logging.info('Detected config change, saving the file...') + CONFIG.save() + self.on_config_updated() + imgui.close_current_popup() + imgui.end_popup() + + def popup_select_world(self): + if imgui.begin_popup(POPUP_SELECT_WORLD): + for world_coord in self.gamedata.world_list: + _, selected = imgui.selectable(world_coord) + if selected: + self.change_world(world_coord) + imgui.end_popup() + + def show_debug_window(self): + imgui.label_text("time", '{:.1f}'.format(glfw.get_time())) + imgui.label_text("fps", '{:.1f}'.format(self.io.framerate)) + imgui.label_text("mouse", '{:.1f}, {:.1f}'.format(self.io.mouse_pos.x, self.io.mouse_pos.y)) + imgui.label_text('mouse in map', str(G.mouse_in_map_normal01)) + + def get_keys_on_map(self, *keys): + return not imgui.is_window_focused() and \ + not self.io.want_capture_keyboard and \ + any(self.io.keys_down[k] for k in keys) + + def set_styles(self): + imgui.push_style_var(imgui.STYLE_WINDOW_ROUNDING, 0) + + def on_config_updated(self): + self.gamedata.game_root = self.config[CONFIG_GAME_ROOT] + + +def impl_glfw_init(): + width, height = 400, 600 + window_name = "Starbound World Viewer" + + logging.info("GLFW version: %s", glfw.get_version_string()) + + if not glfw.init(): + logging.fatal("Could not initialize OpenGL context") + exit(1) + + # OS X supports only forward-compatible core profiles from 3.2 + glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 3) + glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 3) + glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE) + + glfw.window_hint(glfw.OPENGL_FORWARD_COMPAT, gl.GL_TRUE) + + # Create a windowed mode window and its OpenGL context + window = glfw.create_window( + int(width), int(height), window_name, None, None + ) + glfw.make_context_current(window) + + if not window: + glfw.terminate() + logging.fatal("Could not initialize Window") + exit(1) + + glfw.set_window_aspect_ratio(window, 2, 3) + return window + + +def main(): + window = impl_glfw_init() + impl = GlfwRenderer(window) + viewer = WorldViewer() + while not glfw.window_should_close(window): + glfw.poll_events() + impl.process_inputs() + framebuffer_size = glfw.get_framebuffer_size(window) + gl.glViewport(0, 0, framebuffer_size[0], framebuffer_size[1]) + viewer.render(framebuffer_size) + glfw.swap_buffers(window) + impl.shutdown() + imgui.shutdown() + glfw.terminate() + + +if __name__ == "__main__": + logging.basicConfig(format='%(pathname)s:%(lineno)d:\n%(levelname)7s | %(message)s ', level=logging.DEBUG) + main() diff --git a/mapviewer.spec b/mapviewer.spec new file mode 100644 index 0000000..cda4f88 --- /dev/null +++ b/mapviewer.spec @@ -0,0 +1,32 @@ +# -*- mode: python -*- + +block_cipher = None + + +a = Analysis(['mapviewer.py'], + pathex=['D:\\workspace\\starbound-map'], + binaries=[], + datas=[('glfw3.dll', '.')], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher) + +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + a.binaries, + Tree('assets'), + a.zipfiles, + a.datas, + name='StarboundMapViewer', + version='version.txt', + debug=False, + strip=False, + upx=True, + runtime_tmpdir=None, + console=False ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c4b42f6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +glfw==1.6.0 +imgui==0.1.0 +numpy==1.14.3 +PyOpenGL==3.1.0 diff --git a/screenshots/MapViewer.PNG b/screenshots/MapViewer.PNG new file mode 100644 index 0000000..e4ed564 Binary files /dev/null and b/screenshots/MapViewer.PNG differ diff --git a/starbound/__init__.py b/starbound/__init__.py new file mode 100644 index 0000000..787f779 --- /dev/null +++ b/starbound/__init__.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- + +import hashlib +import io +import struct +import zlib +from collections import namedtuple + +from . import sbon +from .btreedb5 import BTreeDB5 +from .directory import * +from .sbasset6 import SBAsset6 + +# Override range with xrange when running Python 2.x. +try: + range = xrange +except: + pass + + +class CelestialChunks(BTreeDB5): + def get(self, key): + key = hashlib.sha256(key.encode('utf-8')).digest() + data = super(CelestialChunks, self).get(key) + data = zlib.decompress(data) + stream = io.BytesIO(data) + return read_versioned_json(stream) + + def read_header(self): + super(CelestialChunks, self).read_header() + assert self.name == 'Celestial2', 'Invalid header' + + +TILE_STRUCT = '>hBBhBhBBhBBffBBHBB?x' +Tile = namedtuple('Tile', [ + 'foreground_material', + 'foreground_hue_shift', + 'foreground_variant', + 'foreground_mod', + 'foreground_mod_hue_shift', + 'background_material', + 'background_hue_shift', + 'background_variant', + 'background_mod', + 'background_mod_hue_shift', + 'liquid', + 'liquid_level', + 'liquid_pressure', + 'liquid_infinite', + 'collision', + 'dungeon_id', + 'biome', + 'biome_2', + 'indestructible', +]) + +VersionedJSON = namedtuple('VersionedJSON', ['name', 'version', 'data']) + + +class World(BTreeDB5): + def get(self, layer, x, y): + # World keys are based on a layer followed by X and Y coordinates. + data = super(World, self).get(struct.pack('>BHH', layer, x, y)) + return zlib.decompress(data) + + def get_entities(self, x, y): + stream = io.BytesIO(self.get(2, x, y)) + count = sbon.read_varint(stream) + return [read_versioned_json(stream) for _ in range(count)] + + def get_tiles(self, x, y): + stream = io.BytesIO(self.get(1, x, y)) + # TODO: Figure out what this means. + unknown = stream.read(3) + # There are 1024 (32x32) tiles in a region. + return [self.read_tile(stream) for _ in range(1024)] + + def get_raw_tiles(self, x, y): + stream = io.BytesIO(self.get(1, x, y)) + # TODO: Figure out what this means. + unknown = stream.read(3) + return stream.read(31 * 1024) + + def read_header(self): + super(World, self).read_header() + assert self.name == 'World4', 'Not a World4 file' + + def read_metadata(self): + # World metadata is held at a special layer/x/y combination. + stream = io.BytesIO(self.get(0, 0, 0)) + self.width, self.height = struct.unpack('>ii', stream.read(8)) + name, version, data = read_versioned_json(stream) + assert name == 'WorldMetadata', 'Invalid world data' + self.metadata = data + self.metadata_version = version + + @classmethod + def read_tile(cls, stream): + values = struct.unpack(TILE_STRUCT, stream.read(31)) + return Tile(*values) + + +def read_sbvj01(stream): + assert stream.read(6) == b'SBVJ01', 'Invalid header' + return read_versioned_json(stream) + + +def read_versioned_json(stream): + name = sbon.read_string(stream) + # The object only has a version if the following bool is true. + if stream.read(1) == b'\x00': + version = None + else: + version, = struct.unpack('>i', stream.read(4)) + data = sbon.read_dynamic(stream) + return VersionedJSON(name, version, data) + + +def write_sbvj01(stream, vj): + stream.write(b'SBVJ01') + write_versioned_json(stream, vj) + + +def write_versioned_json(stream, vj): + sbon.write_string(stream, vj.name) + if vj.version is None: + stream.write(struct.pack('>b', 0)) + else: + stream.write(struct.pack('>bi', 1, vj.version)) + sbon.write_dynamic(stream, vj.data) diff --git a/starbound/btreedb5.py b/starbound/btreedb5.py new file mode 100644 index 0000000..dfbea90 --- /dev/null +++ b/starbound/btreedb5.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- + +import binascii +import io +import struct + +from starbound import sbon + + +# Override range with xrange when running Python 2.x. +try: + range = xrange +except: + pass + + +HEADER = '>8si16si?ixxxxii?ixxxxii?445x' +HEADER_SIZE = struct.calcsize(HEADER) +# Constants for the different block types. +FREE = b'FF' +INDEX = b'II' +LEAF = b'LL' + + +class BTreeDB5(object): + def __init__(self, stream): + self.stream = stream + + def get(self, key): + if not hasattr(self, 'key_size'): + self.read_header() + assert len(key) == self.key_size, 'Invalid key length' + # Traverse the B-tree until we reach a leaf. + offset = HEADER_SIZE + self.block_size * self.root_block + entry_size = self.key_size + 4 + s = self.stream + while True: + s.seek(offset) + block_type = s.read(2) + if block_type != INDEX: + break + # Read the index header and scan for the closest key. + lo, (_, hi, block) = 0, struct.unpack('>Bii', s.read(9)) + offset += 11 + while lo < hi: + mid = (lo + hi) // 2 + s.seek(offset + entry_size * mid) + if key < s.read(self.key_size): + hi = mid + else: + lo = mid + 1 + if lo > 0: + s.seek(offset + entry_size * (lo - 1) + self.key_size) + block, = struct.unpack('>i', s.read(4)) + offset = HEADER_SIZE + self.block_size * block + assert block_type == LEAF, 'Did not reach a leaf' + # Scan leaves for the key, then read the data. + reader = LeafReader(self) + num_keys, = struct.unpack('>i', reader.read(4)) + for i in range(num_keys): + cur_key = reader.read(self.key_size) + length = sbon.read_varint(reader) + if key == cur_key: + return reader.read(length) + reader.seek(length, 1) + # None of the keys in the leaf node matched. + raise KeyError(binascii.hexlify(key)) + + def read_header(self): + self.stream.seek(0) + data = struct.unpack(HEADER, self.stream.read(HEADER_SIZE)) + assert data[0] == b'BTreeDB5', 'Invalid header' + self.block_size = data[1] + self.name = data[2].rstrip(b'\0').decode('utf-8') + self.key_size = data[3] + self.use_other_root = data[4] + self.free_block_1 = data[5] + self.free_block_1_end = data[6] + self.root_block_1 = data[7] + self.root_block_1_is_leaf = data[8] + self.free_block_2 = data[9] + self.free_block_2_end = data[10] + self.root_block_2 = data[11] + self.root_block_2_is_leaf = data[12] + + @property + def root_block(self): + return self.root_block_2 if self.use_other_root else self.root_block_1 + + @property + def root_block_is_leaf(self): + if self.use_other_root: + return self.root_block_2_is_leaf + else: + return self.root_block_1_is_leaf + + def swap_root(self): + self.use_other_root = not self.use_other_root + + +class LeafReader(object): + def __init__(self, db): + # The stream offset must be right after an "LL" marker. + self.db = db + self.offset = 2 + + def read(self, size=-1): + if size < 0: + raise NotImplemented('Can only read specific amount') + with io.BytesIO() as data: + for length in self._traverse(size): + data.write(self.db.stream.read(length)) + return data.getvalue() + + def seek(self, offset, whence=0): + if whence != 1 or offset < 0: + raise NotImplemented('Can only seek forward relatively') + for length in self._traverse(offset): + self.db.stream.seek(length, 1) + + def _traverse(self, length): + block_end = self.db.block_size - 4 + while True: + if self.offset + length <= block_end: + yield length + self.offset += length + break + delta = block_end - self.offset + yield delta + block, = struct.unpack('>i', self.db.stream.read(4)) + assert block >= 0, 'Could not traverse to next block' + self.db.stream.seek(HEADER_SIZE + self.db.block_size * block) + assert self.db.stream.read(2) == LEAF, 'Did not reach a leaf' + self.offset = 2 + length -= delta diff --git a/starbound/directory.py b/starbound/directory.py new file mode 100644 index 0000000..2c3e484 --- /dev/null +++ b/starbound/directory.py @@ -0,0 +1,60 @@ +import glob +import logging +import mmap +import os + +import starbound + + +class GameDirectory: + WORLD_PATH = 'storage/universe/{coordinates}.world' + + def __init__(self): + self._game_root = None + self._world_list = [] + + @property + def game_root(self): + return self._game_root + + @game_root.setter + def game_root(self, value): + if self._game_root != value: + self._game_root = value + self.synchronize() + + @property + def world_list(self): + return self._world_list + + def get_file(self, relpath): + return os.path.join(self.game_root, relpath) + + def get_files(self, relpath): + return glob.glob(self.get_file(relpath)) + + def get_world(self, coord): + world_file = self.get_file(GameDirectory.WORLD_PATH.format(coordinates=coord)) + if not os.path.isfile(world_file): + logging.warning("Failed to load world, not a file: %s", world_file) + return None + else: + world_fd = open(world_file, 'rb') + world_mm = mmap.mmap(world_fd.fileno(), 0, access=mmap.ACCESS_READ) + world = starbound.World(world_mm) + world.read_metadata() + logging.debug('Loaded world [%s] at %s', coord, world_file) + return world + + def synchronize(self): + if not self.valid(): + return + self._sync_world_list() + + def valid(self): + return self.game_root is not None and os.path.isdir(self.game_root) + + def _sync_world_list(self): + world_files = self.get_files(GameDirectory.WORLD_PATH.format(coordinates='*')) + self._world_list[:] = (os.path.splitext(os.path.basename(f))[0] for f in world_files) + logging.debug("Updated world list, found %d worlds", len(self.world_list)) diff --git a/starbound/sbasset6.py b/starbound/sbasset6.py new file mode 100644 index 0000000..dd3dae7 --- /dev/null +++ b/starbound/sbasset6.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +from collections import namedtuple +import struct + +from starbound import sbon + + +# Override range with xrange when running Python 2.x. +try: + range = xrange +except: + pass + + +HEADER = '>8sQ' +HEADER_SIZE = struct.calcsize(HEADER) + + +IndexEntry = namedtuple('IndexEntry', ['offset', 'length']) + + +class SBAsset6(object): + def __init__(self, stream): + self.stream = stream + + def get(self, path): + if not hasattr(self, 'index'): + self.read_index() + offset, length = self.index[path] + self.stream.seek(offset) + return self.stream.read(length) + + def read_header(self): + self.stream.seek(0) + data = struct.unpack(HEADER, self.stream.read(HEADER_SIZE)) + assert data[0] == b'SBAsset6', 'Invalid header' + self.metadata_offset = data[1] + # Read the metadata as well. + self.stream.seek(self.metadata_offset) + assert self.stream.read(5) == b'INDEX', 'Invalid index data' + self.metadata = sbon.read_map(self.stream) + self.file_count = sbon.read_varint(self.stream) + # Store the offset of where the file index starts. + self.index_offset = self.stream.tell() + + def read_index(self): + if not hasattr(self, 'index_offset'): + self.read_header() + self.stream.seek(self.index_offset) + self.index = {} + for i in range(self.file_count): + path = sbon.read_string(self.stream) + offset, length = struct.unpack('>QQ', self.stream.read(16)) + self.index[path] = IndexEntry(offset, length) diff --git a/starbound/sbon.py b/starbound/sbon.py new file mode 100644 index 0000000..6de0113 --- /dev/null +++ b/starbound/sbon.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- + +import struct +import sys + + +if sys.version >= '3': + _int_type = int + _str_type = str + def _byte(x): + return bytes((x,)) + def _items(d): + return d.items() +else: + _int_type = (int, long) + _str_type = basestring + range = xrange + def _byte(x): + return chr(x) + def _items(d): + return d.iteritems() + + +def read_bytes(stream): + length = read_varint(stream) + return stream.read(length) + + +def read_dynamic(stream): + type_id = ord(stream.read(1)) + if type_id == 1: + return None + elif type_id == 2: + return struct.unpack('>d', stream.read(8))[0] + elif type_id == 3: + return stream.read(1) != b'\0' + elif type_id == 4: + return read_varint_signed(stream) + elif type_id == 5: + return read_string(stream) + elif type_id == 6: + return read_list(stream) + elif type_id == 7: + return read_map(stream) + raise ValueError('Unknown dynamic type 0x%02X' % type_id) + + +def read_list(stream): + length = read_varint(stream) + return [read_dynamic(stream) for _ in range(length)] + + +def read_map(stream): + length = read_varint(stream) + value = dict() + for _ in range(length): + key = read_string(stream) + value[key] = read_dynamic(stream) + return value + + +def read_string(stream): + return read_bytes(stream).decode('utf-8') + + +def read_varint(stream): + """Read while the most significant bit is set, then put the 7 least + significant bits of all read bytes together to create a number. + + """ + value = 0 + while True: + byte = ord(stream.read(1)) + if not byte & 0b10000000: + return value << 7 | byte + value = value << 7 | (byte & 0b01111111) + + +def read_varint_signed(stream): + value = read_varint(stream) + # Least significant bit represents the sign. + if value & 1: + return -(value >> 1) - 1 + else: + return value >> 1 + + +def write_bytes(stream, value): + write_varint(stream, len(value)) + stream.write(value) + + +def write_dynamic(stream, value): + if value is None: + stream.write(b'\x01') + elif isinstance(value, float): + stream.write(b'\x02') + stream.write(struct.pack('>d', value)) + elif isinstance(value, bool): + stream.write(b'\x03\x01' if value else b'\x03\x00') + elif isinstance(value, _int_type): + stream.write(b'\x04') + write_varint_signed(stream, value) + elif isinstance(value, _str_type): + stream.write(b'\x05') + write_string(stream, value) + elif isinstance(value, list): + stream.write(b'\x06') + write_list(stream, value) + elif isinstance(value, dict): + stream.write(b'\x07') + write_map(stream, value) + else: + raise ValueError('Cannot write value %r' % (value,)) + + +def write_list(stream, value): + write_varint(stream, len(value)) + for v in value: + write_dynamic(stream, v) + + +def write_map(stream, value): + write_varint(stream, len(value)) + for k, v in _items(value): + write_string(stream, k) + write_dynamic(stream, v) + + +def write_string(stream, value): + write_bytes(stream, value.encode('utf-8')) + + +def write_varint(stream, value): + buf = _byte(value & 0b01111111) + value >>= 7 + while value: + buf = _byte(value & 0b01111111 | 0b10000000) + buf + value >>= 7 + stream.write(buf) + + +def write_varint_signed(stream, value): + write_varint(stream, (-(value + 1) << 1 | 1) if value < 0 else (value << 1)) diff --git a/utils/cache.py b/utils/cache.py new file mode 100644 index 0000000..901e92c --- /dev/null +++ b/utils/cache.py @@ -0,0 +1,24 @@ +import functools +import weakref + + +# https://stackoverflow.com/questions/33672412/python-functools-lru-cache-with-class-methods-release-object +def memoized_method(*lru_args, **lru_kwargs): + def decorator(func): + @functools.wraps(func) + def wrapped_func(self, *args, **kwargs): + # We're storing the wrapped method inside the instance. If we had + # a strong reference to self the instance would never die. + self_weak = weakref.ref(self) + + @functools.wraps(func) + @functools.lru_cache(*lru_args, **lru_kwargs) + def cached_method(*args, **kwargs): + return func(self_weak(), *args, **kwargs) + + setattr(self, func.__name__, cached_method) + return cached_method(*args, **kwargs) + + return wrapped_func + + return decorator diff --git a/utils/config.py b/utils/config.py new file mode 100644 index 0000000..7d32a11 --- /dev/null +++ b/utils/config.py @@ -0,0 +1,25 @@ +import atexit +import configparser + +CONFIG_FILE = 'config.ini' + + +class Config(configparser.ConfigParser): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def load(self): + self.read(CONFIG_FILE) + + def save(self): + with open(CONFIG_FILE, 'w') as f: + self.write(f) + + +CONFIG = Config() +CONFIG.load() + + +@atexit.register +def save_config(): + CONFIG.save() diff --git a/utils/resource.py b/utils/resource.py new file mode 100644 index 0000000..27ffdb4 --- /dev/null +++ b/utils/resource.py @@ -0,0 +1,16 @@ +import logging +import os +import sys + + +def asset_path(relative_path): + """ Get absolute path to resource, works for dev and for PyInstaller """ + try: + # PyInstaller creates a temp folder and stores path in _MEIPASS + base_path = sys._MEIPASS + logging.info("Detected PyInstaller environment") + except Exception: + base_path = os.path.abspath("./assets") + logging.info("Detected dev environment") + logging.debug("asset path: %s", base_path) + return os.path.join(base_path, relative_path) diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..79f39ee --- /dev/null +++ b/version.txt @@ -0,0 +1,44 @@ +# UTF-8 +# +VSVersionInfo( + ffi=FixedFileInfo( + # filevers and prodvers should be always a tuple with four items: (1, 2, 3, 4) + # Set not needed items to zero 0. + # filevers=(0, 0, 1, 0), + # prodvers=(0, 0, 1, 0), + # Contains a bitmask that specifies the valid bits 'flags'r + mask=0x3f, + # Contains a bitmask that specifies the Boolean attributes of the file. + flags=0x0, + # The operating system for which this file was designed. + # 0x4 - NT and there is no need to change it. + OS=0x40004, + # The general type of file. + # 0x1 - the file is an application. + fileType=0x1, + # The function of the file. + # 0x0 - the function is not defined for this fileType + subtype=0x0, + # Creation date and timestamp. + # date=(0, 0) + ), + kids=[ + StringFileInfo( + [ + StringTable( + u'040904B0', + [ + # StringStruct(u'CompanyName', u'Microsoft Corporation'), + StringStruct(u'FileDescription', u'Starbound Map Viewer'), + StringStruct(u'FileVersion', u''), + #StringStruct(u'InternalName', u'cmd'), + StringStruct(u'LegalCopyright', u'© Blixt, txxia'), + #StringStruct(u'OriginalFilename', u'Cmd.Exe'), + StringStruct(u'ProductName', u'Starbound Map Viewer'), + StringStruct(u'ProductVersion', u'0.0.1') + ]) + ]), + # En-US + VarFileInfo([VarStruct(u'Translation', [1033, 1200])]) + ] +) \ No newline at end of file