180 dmb/dmbreader.py
View file
@@ -1,7 +1,10 @@
from .dmb import Tile, Proc, WorldData, Var, Instance, Resource, Type, RawString
from .dmb import Tile, Mob, Proc, WorldData, Var, Instance, Resource, Type, RawString
from .dmbwriter import DmbWriter
from .tree import ObjectTree
from .util import ModularInt, mod8
from . import value
from . import value, constants
from .crypt import byond32
from blist import *
import numpy as np
import re
import io
import struct
@@ -12,28 +15,6 @@ class DmbFileError(ValueError):
pass


class StringDecoder:
def __init__(self, reader, key, count):
self.reader = reader
self.key = key
self.len = count

def __iter__(self):
return self

def __next__(self):
if self.len <= 0:
raise StopIteration
else:
self.len -= 1
val = (self.reader._uint8() ^ self.key) & 0xFF
self.key += 9
return val

def dump(self):
return RawString(bytearray(self.reader._nbytes(self.len)), self.key)


class TileGenerator:
def __init__(self, reader, count):
self.reader = reader
@@ -45,18 +26,20 @@ def __init__(self, reader, count):
def gen(self, count):
while self.len > 0 and count > 0:
if self.curr is None or self.rle_count == 0:
self.curr = Tile(self.reader._uarch(), self.reader._uarch(), self.reader._uarch())
self.curr = (self.reader._uarch(), self.reader._uarch(), self.reader._uarch())
self.rle_count = self.reader._uint8()
self.rle_count -= 1
self.len -= 1
count -= 1
self.calls += 1
yield self.curr
yield Tile(self.curr[0], self.curr[1], self.curr[2])


class Dmb:
def __init__(self, dmbname, throttle=False, verbose=False, lazy_resolve=False):
self.lazy_resolve = lazy_resolve
def __init__(self, dmbname, throttle=False, verbose=False, string_mode=constants.string_mode_strings, check_string_crc=False, fully_populate_types=False):
self.string_mode = constants.string_mode_strings
self.check_string_crc = check_string_crc
self.fully_populate_types = fully_populate_types
self.reader = open(dmbname, 'rb')
self.bit32 = False
self.throttle = throttle
@@ -66,9 +49,9 @@ def __init__(self, dmbname, throttle=False, verbose=False, lazy_resolve=False):
if verbose:
print("Compiled with byond {0} (requires {1} server, {2} client)".format(self.world.world_version, self.world.min_server, self.world.min_client))

flags = self._uint32()
self.flags = self._uint32()
self.world.map_x, self.world.map_y, self.world.map_z = (self._uint16(), self._uint16(), self._uint16())
self.bit32 = (flags & 0x40000000) > 0
self.bit32 = (self.flags & 0x40000000) > 0

if verbose:
print("{0}-bit dmb".format(32 if self.bit32 else 16))
@@ -82,43 +65,49 @@ def __init__(self, dmbname, throttle=False, verbose=False, lazy_resolve=False):
type_count = self._uarch()
if verbose:
print("{0} types".format(type_count))
self.types = [t for t in self._typegen(type_count)]
self.types = blist([t for t in self._typegen(type_count)])

mob_count = self._uarch()
if verbose:
print("{0} mobs".format(mob_count))
[m for m in self._mobgen(mob_count)] # skip mobs
self.mobs = blist([m for m in self._mobgen(mob_count)])

self.strcrc = np.uint32(0)
string_count = self._uarch()
if verbose:
print("{0} strings".format(string_count))
self.strings = [s for s in self._stringgen(string_count)]
self._uint32() # CRC
self.strings = blist([s for s in self._stringgen(string_count)])
crc = self._uint32() # CRC
if self.check_string_crc:
if crc != self.strcrc:
raise DmbFileError("String table CRC mismatch (expected: {0}, got: {1})".format(crc, self.strcrc))
elif verbose:
print("String table CRC check passed.")

data_count = self._uarch()
if verbose:
print("{0} data".format(data_count))
self.data = [d for d in self._datagen(data_count)]
self.data = blist([d for d in self._datagen(data_count)])

proc_count = self._uarch()
if verbose:
print("{0} procs".format(proc_count))
self.procs = [p for p in self._procgen(proc_count)]
self.procs = blist([p for p in self._procgen(proc_count)])

var_count = self._uarch()
if verbose:
print("{0} vars".format(var_count))
self.vars = [v for v in self._vargen(var_count)]
self.variables = blist([v for v in self._vargen(var_count)])

argproc_count = self._uarch()
if verbose:
print("{0} argprocs".format(argproc_count))
self.argprocs = [v for v in self._argprocgen(argproc_count)]
self.argprocs = blist([v for v in self._argprocgen(argproc_count)])

instance_count = self._uarch()
if verbose:
print("{0} instances".format(instance_count))
self.instances = [i for i in self._instancegen(instance_count)]
self.instances = blist([i for i in self._instancegen(instance_count)])

mappop_count = self._uint32()
if verbose:
@@ -128,19 +117,35 @@ def __init__(self, dmbname, throttle=False, verbose=False, lazy_resolve=False):
self._parse_extended_data()

res_count = self._uarch()
# print("{0} resources".format(res_count), file=sys.stderr)
self.resources = [r for r in self._resourcegen(res_count)]
print("{0} resources".format(res_count))
self.resources = blist([r for r in self._resourcegen(res_count)])

self.reader.close()
self.reader = None
self.tree = ObjectTree()

self._populate_types()
if self.fully_populate_types:
self.tree.populate_variables(self)

def __del__(self):
if self.reader is not None:
self.reader.close()

def insert_string(self, string):
print("Inserted string '{0}'".format(string))
def_string = ""
if self.string_mode == constants.string_mode_byte_strings:
def_string = b''
while len(self.strings) >= 0xFF00 and len(self.strings) < 0xFFFF:
self.strings.append(def_string)
if isinstance(string, str) and self.string_mode == constants.string_mode_byte_strings:
string = RawString(string, 0, mode=constants.raw_string_mode_string, lazy=True).encode()
elif isinstance(string, (bytes, bytearray)) and self.string_mode == constants.string_mode_strings:
string = RawString(string, 0, mode=constants.raw_string_mode_decrypted, lazy=True).decode()
self.strings.append(string)
return self.strings.len - 1

def _shift_coords(self, move_count, x, y, z):
x += move_count

@@ -167,16 +172,18 @@ def _populate_map(self, count):
tile = self.tile(x, y, z)
for move_count, instanceid in self._mappopgen(count):
if move_count > 0:
tile = self.tile(self._shift_coords(move_count, x, y, z))
x, y, z = self._shift_coords(move_count, x, y, z)
tile = self.tile(x, y, z)
tile.instances.append(self._resolve_instance(instanceid))

def _populate_types(self):
for t in self.types:
#if not self.lazy_resolve:
self.resolve_type(t)

self.tree.push(t)

def _unpack_arch(self, bs):
return struct.unpack("<" + (("I" if self.bit32 else "H") * int(len(bs) / (4 if self.bit32 else 2))), bs)

def resolve_type(self, t):
if t.resolved:
return t
@@ -187,6 +194,20 @@ def resolve_type(self, t):
t.parent = self.no_parent_type
t.name = self._resolve_string(t.name)
t.desc = self._resolve_string(t.desc)
# try:
# vl = self._unpack_arch(self.data[t.variable_list])
# print(t.path, self.data[t.variable_list], "->", vl)
# skip = False
# for vid in vl:
# if skip:
# skip = False
# continue
# print("*", self._resolve_string(self.variables[vid].name))
# skip = True
# # print([self._resolve_string(i) for i in self._unpack_arch(self.data[t.variable_list])])
# except:
# if t.variable_list != 65535:
# raise
t.resolved = True
return t

@@ -221,6 +242,12 @@ def _ffwdarch(self, seek32, seek16):
def _nbytes(self, n):
return self.reader.read(n)

def _nbytesarch(self, n32, n16):
read = n16
if self.bit32:
read = n32
return self._nbytes(read)

def _float32(self):
b = self.reader.read(4)
if len(b) < 4:
@@ -298,6 +325,10 @@ def _parse_extended_data(self):
self.world.icon_height = self._uint16()
self.world.map_format = self._uint16()

def write(self, dmbname):
writer = DmbWriter(dmbname, self)
writer.write()

def _throttle(self):
if self.throttle:
self.ops += 1
@@ -321,52 +352,59 @@ def _typegen(self, count):
curr.icon_state = self._uarch()
curr.dir = self._uint8()

unknown = self._uint8()
if unknown == 15:
self._ffwd(4)
curr._unknown1 = self._uint8()
if curr._unknown1 == 15:
curr._fdata1 = self._nbytes(4)
curr.text = self._uarch()
self._uarch() # suffix
self._ffwdarch(8, 6)
curr.suffix = self._uarch() # suffix
curr._fdata2 = self._nbytesarch(8, 6)
curr.flags = self._uint32()
self._ffwdarch(16, 8)
curr._fdata3 = self._nbytesarch(16, 8)
curr.variable_list = self._uarch()
curr.layer = self._float32()
if self.world.min_client >= 500:
unknown = self._uint8()
if unknown > 0:
self._ffwd(24)
curr._unknown2 = self._uint8()
if curr._unknown2 > 0:
curr._fdata4 = self._nbytes(24)
curr.builtin_variable_list = self._uarch()
count -= 1
yield curr
self._throttle()

def _mobgen(self, count):
while count > 0:
self._ffwdarch(8, 4)
unknown = self._uint8()
if (unknown & 0x80) > 0:
self._ffwd(6)
mob = Mob()
mob._fdata1 = self._nbytesarch(8, 4)
mob._unknown = self._uint8()
if (mob._unknown & 0x80) > 0:
mob._fdata2 = self._nbytes(6)
count -= 1
yield True
yield mob
self._throttle()

def _crc(self, b):
self.strcrc = byond32(self.strcrc, b, null_terminate=True)

def _stringgen(self, count):
while count > 0:
c = self.reader.seek(0, io.SEEK_CUR)
strlen = (self._uint16() ^ c) & 65535
lb = self._uint16()
strlen = (lb ^ c) & 65535
lastread = strlen
while strlen == 65535:
c += 2
lastread = self._uint16()
nextd = (lastread ^ c) & 65535
strlen += nextd
key = ModularInt(c + 2, mod8)
decoder = StringDecoder(self, key, strlen)
key = c + 2
count -= 1
string = decoder.dump()
if self.lazy_resolve:
yield string
else:
estr = self._nbytes(strlen)
string = RawString(bytearray(estr), key, lazy=True)
if self.check_string_crc:
self._crc(string.decrypt())
if self.string_mode == constants.string_mode_byte_strings:
yield string.decrypt()
elif self.string_mode == constants.string_mode_strings:
yield string.decode()
self._throttle()

@@ -386,10 +424,10 @@ def _procgen(self, count):
ret = Proc()
ret.path = self._uarch()
ret.name = self._uarch()
self._ffwdarch(10, 6)
unknown = self._uint8()
if unknown & 0x80 > 0:
self._ffwd(5)
ret._fdata1 = self._nbytesarch(10, 6)
ret._unknown = self._uint8()
if ret._unknown & 0x80 > 0:
ret._fdata2 = self._nbytes(5)
ret.data = self._uarch()
ret.variable_list = self._uarch()
ret.argument_list = self._uarch()
@@ -475,7 +513,7 @@ def _resolve_string(self, stringid):
return None
ret = self.strings[stringid]
if isinstance(ret, RawString):
val = ret.decode()
val = ret.decrypt()
self.strings[stringid] = val
ret = val
return ret
383 dmb/dmbwriter.py
View file
@@ -0,0 +1,383 @@
import struct
from . import constants
from .dmb import RawString, Type
from .crypt import byond32
import copy
import numpy as np
import io


class DmbWriter:
def __init__(self, dmbname, dmb):
self.dmb = dmb
self.writer = open(dmbname, 'wb')

def __del__(self):
if self.writer is not None:
self.writer.close()
self.writer = None

def _uint8(self, v):
data = struct.pack('<B', v & 0xFF) # interprets the provided integer as 1 byte
self.writer.write(data)
return 1

def _uint16(self, v):
data = struct.pack('<H', v & 0xFFFF) # interprets the provided integer as 2 LE bytes
self.writer.write(data)
return 2

def _uint32(self, v):
data = struct.pack('<I', v & 0xFFFFFFFF) # interprets the provided integer as 4 LE bytes
self.writer.write(data)
return 4

def _float32(self, v):
data = struct.pack('<f', v)
self.writer.write(data)
return 4

def _uarch(self, v):
if self.dmb.bit32:
return self._uint32(v)
else:
return self._uint16(v)

def _nbytes(self, count, bs):
if isinstance(bs, str):
bs = bs.encode('iso-8859-1')
elif isinstance(bs, list):
bs = bytes(bs)
avail = len(bs)
if count > avail:
raise IndexError("Cannot write {0} bytes, provided object only contains {1} bytes.".format(count, avail))
elif count == avail:
self.writer.write(bs)
else:
self.writer.write(bs[0:count])
return count

def _bytes(self, bs):
self.writer.write(bs)
return len(bs)

def _str(self, data):
e = data.encode('iso-8859-1')
self.writer.write(e)
return len(e)

def _byte(self, bs):
if isinstance(bs, int):
self._nbytes(1, [bs])
else:
self._nbytes(1, bs)

def _write_version_data(self):
s1 = "world bin v{0}".format(self.dmb.world.world_version)
self._str(s1)
self._byte(b'\x0A')
s2 = "min compatibility v{0} {1}".format(self.dmb.world.min_server, self.dmb.world.min_client)
self._str(s2)
self._byte(b'\x0A')
return len(s1) + len(s2) + 2

def _write_tiles(self):
length = self.dmb.world.map_x * self.dmb.world.map_y * self.dmb.world.map_z
x = 0
y = 0
z = 0
rle = 0
last_match = (None, None, None)
lxr = len(self.dmb.tiles[z][y])
lyr = len(self.dmb.tiles[z])
lzr = len(self.dmb.tiles)
while length > 0:
tile = self.dmb.tiles[z][y][x]
if isinstance(tile.area, Type):
area = tile.area.id
else:
area = tile.area
if isinstance(tile.turf, Type):
turf = tile.turf.id
else:
turf = tile.turf
unk = tile.unknown

if rle > 0:
if (area, turf, unk) == last_match:
rle += 1
if rle > 255:
raise RuntimeError("RLE overflow.")
elif rle == 255:
self._uarch(area)
self._uarch(turf)
self._uarch(unk)
self._uint8(255)
rle = 0
else:
self._uarch(last_match[0])
self._uarch(last_match[1])
self._uarch(last_match[2])
self._uint8(rle)
rle = 1
last_match = (area, turf, unk)
else:
rle = 1
last_match = (area, turf, unk)
length -= 1
x += 1
if x >= lxr:
x = 0
y += 1
if y >= lyr:
y = 0
z += 1
if z >= lzr and length > 0:
raise ValueError("Length of tile write longer than total length of map.")
if rle > 0:
self._uarch(last_match[0])
self._uarch(last_match[1])
self._uarch(last_match[2])
self._uint8(rle)

def _get_strid(self, strid):
if strid is None:
return 65535
if not isinstance(strid, int):
try: # so pythonic. also comments are hashtags. #pythonic
strid = self.dmb.strings.index(strid)
except:
strid = self.dmb.insert_string(strid)
return strid

def _strid(self, strid):
return self._uarch(self._get_strid(strid))

def _write_types(self):
type_count = len(self.dmb.types)
self._uarch(type_count)
for t in self.dmb.types:
self._strid(t.path)
self._strid(t.parent)
self._strid(t.name)
self._strid(t.desc)
self._strid(t.icon) # TODO: may be wrong?
self._strid(t.icon_state)
self._uint8(t.dir)
self._uint8(t._unknown1)
if t._unknown1 == 15:
self._bytes(t._fdata1)
self._strid(t.text)
self._strid(t.suffix)
self._bytes(t._fdata2)
self._uint32(t.flags)
self._bytes(t._fdata3)
self._uarch(t.variable_list) # TODO: reencode
self._float32(t.layer)
if self.dmb.world.min_client >= 500:
self._uint8(t._unknown2)
if t._unknown2 > 0:
self._bytes(t._fdata4)
self._uarch(t.builtin_variable_list) # TODO: reencode

def _ffwd(self, count):
return self.writer.seek(count, io.SEEK_CUR)

def _write_string_length(self, strlen):
while strlen >= 65535:
s = (strlen ^ self._ffwd(0)) & 65535
self._uint16(s)
strlen -= 65535
s = (strlen ^ self._ffwd(0)) & 65535
self._uint16(s)

def _crc(self, b):
self.strcrc = byond32(self.strcrc, b, null_terminate=True)

def _write_strings(self):
self.strcrc = np.uint32(0xFFFFFFFF)
string_count = len(self.dmb.strings)
self._uarch(string_count)
for s in self.strings:
self._crc(s)
self._write_string_length(len(s))
s = RawString(s, self._ffwd(0), mode=constants.raw_string_mode_decrypted, lazy=True)
data = s.encrypt(True)
self._bytes(data)
self._uint32(self.strcrc)

def _write_mobs(self):
mob_count = len(self.dmb.mobs)
self._uarch(mob_count)
for mob in self.dmb.mobs:
self._bytes(mob._fdata1)
self._uint8(mob._unknown)
if (mob._unknown & 0x80) > 0:
self._bytes(mob._fdata2)

def _write_data(self):
data_count = len(self.dmb.data)
self._uarch(data_count)
for data in self.dmb.data:
dlen = len(data)
if self.dmb.bit32:
dlen /= 4
else:
dlen /= 2
self._uint16(int(dlen))
self._bytes(data)

def _prep_proc_copy(self, proc):
ret = copy.copy(proc)
ret.path = self._get_strid(proc.path)
ret.name = self._get_strid(proc.name)
# TODO: unresolve data, lists
return ret

def _write_procs(self):
proc_count = len(self.written_procs)
self._uarch(proc_count)
for proc in self.written_procs:
self._uarch(proc.path)
self._uarch(proc.name)
self._bytes(proc._fdata1)
self._uint8(proc._unknown)
if (proc._unknown & 0x80) > 0:
self._bytes(proc._fdata2)
self._uarch(proc.data)
self._uarch(proc.variable_list)
self._uarch(proc.argument_list)

def _prep_var_copy(self, var):
ret = copy.copy(var)
ret.name = self._get_strid(var.name)
# TODO: unresolve data, lists
return ret

def _write_vars(self):
var_count = len(self.written_vars)
self._uarch(var_count)
for var in self.written_vars:
self._uint8(var.value._typeid)
self._uint32(var.value._value)
self._uarch(var.name)

def _write_instances(self):
inst_count = len(self.dmb.instances)
self._uarch(inst_count)
for inst in self.dmb.instances:
self._uint8(inst.value._typeid)
self._uint32(inst.value._value)
self._uarch(inst.initializer) # TODO: unresolve

def _write_argprocs(self):
ap_count = len(self.dmb.argprocs)
self._uarch(ap_count)
for a in self.dmb.argprocs:
self._uarch(a)

def _write_mappops(self):
total = 0
tid = 0
last_tid = 0
for zlevel in self.dmb.tiles:
for row in zlevel:
for tile in row:
total += len(tile.instances)
self._uint32(total)
for zlevel in self.dmb.tiles:
for row in zlevel:
for tile in row:
for inst in tile.instances:
self._uint16(tid - last_tid)
last_tid = tid
iid = self.dmb.instances.index(inst)
self._uarch(iid)
tid += 1

def _write_resources(self):
res_count = len(self.dmb.resources)
self._uarch(res_count)
for r in self.dmb.resources:
self._uint32(r.hash)
self._uint8(r.typeid)

def _prep_world_data(self):
written_world = copy.copy(self.dmb.world)
written_world.world_name = self._get_strid(written_world.world_name)
# TODO: there's probably more to deal with
return written_world

def _write_extended_data(self):
self._uarch(self.written_world.default_mob)
self._uarch(self.written_world.default_turf)
self._uarch(self.written_world.default_area)
self._uarch(self.written_world.world_procs)
self._uarch(self.written_world.global_init)
self._uarch(self.written_world.world_domain)
self._uarch(self.written_world.world_name)
self._uint16(self.written_world.tick_lag)
self._uint16(self.written_world.unknown1)
self._uarch(self.written_world.client_type)
self._uarch(self.written_world.image_type)
self._uint8(self.written_world.lazy_eye)
self._uint8(self.written_world.client_dir)
self._uint8(self.written_world.control_freak)
self._uint16(self.written_world.unknown2)
self._uarch(self.written_world.client_script)
self._uint16(self.written_world.unknown3)
self._uint8(self.written_world.view_width)
self._uint8(self.written_world.view_height)
self._uarch(self.written_world.hub_password)
self._uarch(self.written_world.world_status)
self._uint16(self.written_world.unknown4)
self._uint16(self.written_world.unknown5)
self._uint32(self.written_world.version)
self._uint16(self.written_world.cache_lifespan)
self._uarch(self.written_world.default_command_text)
self._uarch(self.written_world.default_command_prompt)
self._uarch(self.written_world.hub_path)
self._uarch(self.written_world.unknown6)
self._uarch(self.written_world.unknown7)
self._uint16(self.written_world.icon_width)
self._uint16(self.written_world.icon_height)
self._uint16(self.written_world.map_format)

def write(self):
f = open('strlist.txt', 'wb')
if self.dmb.string_mode == constants.string_mode_strings:
self.strings = [RawString(s, 0, mode=constants.raw_string_mode_string, lazy=True).encode() for s in self.dmb.strings]
for strb in self.strings:
f.write(strb)
f.write(b'\x0A')
else:
self.strings = self.dmb.strings
f.close()
self.string_mem_len = 0
for s in self.strings:
self.string_mem_len += len(s) + 1

self._write_version_data()
self._uint32(self.dmb.flags)
self._uint16(self.dmb.world.map_x)
self._uint16(self.dmb.world.map_y)
self._uint16(self.dmb.world.map_z)
self._write_tiles()
self._uint32(self.string_mem_len)

self._write_types()
self._write_mobs()

self.written_procs = [self._prep_proc_copy(proc) for proc in self.dmb.procs]
self.written_vars = [self._prep_var_copy(proc) for proc in self.dmb.variables]
self.written_world = self._prep_world_data()

self._write_strings()
self._write_data()
self._write_procs()
self._write_vars()
self._write_argprocs()
self._write_instances()
self._write_mappops()
self._write_extended_data()
self._write_resources()
3 dmb/tree.py
View file
@@ -70,5 +70,8 @@ def complete_path(self, path):
"options": options,
}

def populate_variables(self, dmb):
pass

def __repr__(self):
return self.json()
57 dmb/util.py
View file

This file was deleted.

20 dmb/value.py
View file
@@ -1,74 +1,88 @@
class compiled_value():
pass
def __init__(self, typeid, value):
self._typeid = typeid
self._value = value


class value_null(compiled_value):
def __init__(self, typeid, value):
pass
pass


class value_mob(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
self.value = value


class value_resource(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
self.value = value


class value_type(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
self.value = value


class value_savefile_type(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
pass


class value_file_type(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
pass


class value_list_type(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
pass


class value_client_type(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
pass


class value_string(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
self.value = value


class value_number(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
self.value = value


class value_list(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
self.value = value


class value_proc(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
self.value = value


class value_image(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
self.value = value


class value_unknown(compiled_value):
def __init__(self, typeid, value):
super().__init__(typeid, value)
self.typeid = typeid
self.value = value

2 setup.py
View file
@@ -5,7 +5,7 @@

setup(
name='pydmb',
version='0.1.1',
version='0.1.2',
description=description,
long_description=description,
url='https://github.com/baliame/pydmb',