From bdeef40cc0ac18bfe9aa62125ab381cbc67069ed Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Tue, 10 Mar 2020 15:39:40 +0100 Subject: [PATCH 01/39] Request memory addresses --- lib/city-manager.js | 38 +++++++++++++++++++++++++++++++++++++- test/city-manager-test.js | 19 +++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index cab7312..21d47b0 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -15,6 +15,11 @@ class CityManager { // Sets up the city manager. constructor(dbpf) { + // Pre-initialize all fields. + this.dbpf = null; + this.memRefs = null; + this.$mem = 1; + // If we've received a string, treat it as a path. if (typeof dbpf === 'string') { let file = path.resolve(regions, dbpf); @@ -35,13 +40,44 @@ class CityManager { } // Create the city. - dbpf = new Savegame(fs.readFileSync(file)); + dbpf = this.dbpf = new Savegame(fs.readFileSync(file)); + + } else { + this.dbpf = dbpf; + } + + } + + // ## mem() + // Returns an unused memory address. This is useful if we add new stuff to + // a city - such as buildings etc. - because we need to make sure that the + // memory addresses for every record are unique. + mem() { + + // If we didn't set up the memory references yet, parse them. + if (!this.memRefs) { + let { dbpf } = this; + let set = this.memRefs = new Set(); + for (let { mem } of dbpf.memRefs()) { + set.add(mem); + } + } + // Create a new memory reference, but make sure it doesn't exist yet. + let ref = this.$mem++; + while (this.memRefs.has(ref)) { + ref = this.$mem++; } + this.memRefs.add(ref); + return ref; } // ## loadPlugins(opts) + // Loads a plugins directory and sets an index. This index will be at the + // heart of everything we'll be doing because it allows us to query + // entries by TGI. Note that we have to make sure to index the + // SimCity_1.dat files as well! async loadPlugins(opts) { if (!opts) { opts = { diff --git a/test/city-manager-test.js b/test/city-manager-test.js index 4bba2fd..6c1bab0 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -1,7 +1,6 @@ // # city-manager-test.js "use strict"; -const chai = require('chai'); -const expect = chai.expect; +const { expect } = require('chai'); const CityManager = require('../lib/city-manager'); const path = require('path'); @@ -24,4 +23,20 @@ describe('A city manager', function() { }); + context('#mem()', function() { + + it('returns an unused memory address', async function() { + + let file = path.resolve(__dirname, 'files/City - RCI.sc4'); + let city = new CityManager(file); + + expect(city.mem()).to.equal(1); + city.memRefs.add(2); + expect(city.mem()).to.equal(3); + expect(city.mem()).to.equal(4); + + }); + + }); + }); \ No newline at end of file From b12c65dc892e9d72234f4358daf5858ef9a80ee7 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Wed, 11 Mar 2020 10:00:02 +0100 Subject: [PATCH 02/39] Rename Index to FileIndex --- lib/city-manager.js | 4 +-- lib/{index.js => file-index.js} | 29 +++++++++++----------- test/city-manager-test.js | 2 +- test/{index-test.js => file-index-test.js} | 12 +++++---- test/plop-test.js | 2 +- 5 files changed, 26 insertions(+), 23 deletions(-) rename lib/{index.js => file-index.js} (90%) rename test/{index-test.js => file-index-test.js} (76%) diff --git a/lib/city-manager.js b/lib/city-manager.js index 21d47b0..1c0afb9 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -2,8 +2,8 @@ "use strict"; const path = require('path'); const fs = require('fs'); -const Savegame = require('./savegame'); -const Index = require('./index'); +const Savegame = require('./savegame.js'); +const FileIndex = require('./file-index.js'); const SC4 = path.resolve(process.env.HOMEPATH, 'documents/SimCity 4'); const regions = path.join(SC4, 'regions'); const plugins = path.join(SC4, 'plugins'); diff --git a/lib/index.js b/lib/file-index.js similarity index 90% rename from lib/index.js rename to lib/file-index.js index 40b16c5..7540be1 100644 --- a/lib/index.js +++ b/lib/file-index.js @@ -1,4 +1,4 @@ -// # index.js +// # file-index.js "use strict"; const fs = require('fs'); const path = require('path'); @@ -16,14 +16,14 @@ if (!fs.promises) { }; } -// # Index -// The index is a data structure that scans a list of dbpf files and builds up -// an index of all files in it by their TGI's. This should mimmick how the -// game scans the plugins folder as well. We obivously cannot keep everything -// in memory so we'll keep pointers to where we can find each file **on the -// disk**. Note: we should make use of node's async nature here so that we can -// read in as much files as possible in parallel! -class Index { +// # FileIndex +// The file index is a data structure that scans a list of dbpf files and +// builds up an index of all files in it by their TGI's. This should mimmick +// how the game scans the plugins folder as well. We obivously cannot keep +// everything in memory so we'll keep pointers to where we can find each file +// **on the disk**. Note: we should make use of node's async nature here so +// that we can read in as much files as possible in parallel! +class FileIndex { // ## constructor(opts) constructor(opts) { @@ -127,14 +127,14 @@ class Index { } } -module.exports = Index; +module.exports = FileIndex; // Object that we'll re-use to query so that we don't have to recreate it all // the time. const query = { - "type": 0, - "group": 0, - "instance": 0 + type: 0, + group: 0, + instance: 0 }; // # Record @@ -171,7 +171,8 @@ class Record extends Entry { if (this.file) return this.file; if (this.raw) return this.raw; - // Read from the file, but at the correct offset (and in a syncronous way). + // Read from the file, but at the correct offset (and in a synchronous + // way). let file = String(this.source); let fd = fs.openSync(file, 'r'); let buff = Buffer.allocUnsafe(this.compressedSize); diff --git a/test/city-manager-test.js b/test/city-manager-test.js index 6c1bab0..06768dd 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -1,8 +1,8 @@ // # city-manager-test.js "use strict"; const { expect } = require('chai'); -const CityManager = require('../lib/city-manager'); const path = require('path'); +const CityManager = require('../lib/city-manager.js'); describe('A city manager', function() { diff --git a/test/index-test.js b/test/file-index-test.js similarity index 76% rename from test/index-test.js rename to test/file-index-test.js index 9ada9bb..523aa39 100644 --- a/test/index-test.js +++ b/test/file-index-test.js @@ -1,15 +1,15 @@ -// # index-test.js +// # file-index-test.js "use strict"; const chai = require('chai'); const expect = chai.expect; const path = require('path'); -const Index = require('../lib/index'); -const FileType = require('../lib/file-types'); +const Index = require('../lib/file-index.js'); +const FileType = require('../lib/file-types.js'); describe('The file index', function() { - it.skip('should index all files in a directory', async function() { + it('should index all files in a directory', async function() { let index = new Index({ "dirs": [ @@ -34,7 +34,9 @@ describe('The file index', function() { expect(file.table).to.have.property(0x88EDC900); let building = file.lotObjects.find(x => x.type === 0x00); - console.log(building.x, building.y, building.z); + expect(building.x).to.equal(1); + expect(building.y).to.equal(0); + expect(building.z).to.equal(1.5); }); diff --git a/test/plop-test.js b/test/plop-test.js index 6b4c7dc..c12c313 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -10,7 +10,7 @@ const Stream = require('../lib/stream'); const crc32 = require('../lib/crc'); const { hex, chunk, split } = require('../lib/util'); const { ZoneType, FileType, cClass, SimGrid } = require('../lib/enums'); -const Index = require('../lib/index'); +const Index = require('../lib/file-index.js'); const Savegame = require('../lib/savegame'); const Lot = require('../lib/lot'); const Building = require('../lib/building'); From 88e7758b5f9bbb3452f71a30ae1b1652cbe29353 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Wed, 11 Mar 2020 16:34:38 +0100 Subject: [PATCH 03/39] Try out plopping buildings --- lib/building.js | 7 +- lib/city-manager.js | 305 +++++++++++++++++++++++++++++++++----- lib/exemplar.js | 24 ++- lib/file-index.js | 43 +++++- lib/lot.js | 3 +- test/city-manager-test.js | 35 ++++- 6 files changed, 358 insertions(+), 59 deletions(-) diff --git a/lib/building.js b/lib/building.js index 6d3fe63..d6b523e 100644 --- a/lib/building.js +++ b/lib/building.js @@ -12,8 +12,8 @@ const Type = require('./type'); // Represents a single building from the building file. const Building = class Building extends Type(FileType.BuildingFile) { - // ## constructor() - constructor() { + // ## constructor(opts) + constructor(opts) { super(); this.crc = 0x00000000; this.mem = 0x00000000; @@ -21,7 +21,7 @@ const Building = class Building extends Type(FileType.BuildingFile) { this.minor = 0x0004; this.zotWord = 0x0; this.unknown1 = 0x00; - this.appearance = 0x04; + this.appearance = 0b00000101; this.xMinTract = 0x00; this.zMinTract = 0x00; this.xMaxTract = 0x00; @@ -35,6 +35,7 @@ const Building = class Building extends Type(FileType.BuildingFile) { this.maxZ = this.maxY = this.maxX = 0; this.orientation = 0x00; this.scaffold = 0; + Object.assign(this, opts); } // ## move(dx, dy, dz) diff --git a/lib/city-manager.js b/lib/city-manager.js index 1c0afb9..354c437 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -3,49 +3,76 @@ const path = require('path'); const fs = require('fs'); const Savegame = require('./savegame.js'); -const FileIndex = require('./file-index.js'); +const Lot = require('./lot.js'); +const Building = require('./building.js'); +const BaseTexture = require('./lot-base-texture.js'); +const { FileType } = require('./enums.js'); const SC4 = path.resolve(process.env.HOMEPATH, 'documents/SimCity 4'); const regions = path.join(SC4, 'regions'); -const plugins = path.join(SC4, 'plugins'); + +// Hex constants to make the code more readable. +const ExemplarType = 0x10; +const Buildings = 0x02; +const OccupantSize = 0x27812810; +const LotConfigurations = 0x10; +const LotResourceKey = 0xea260589; +const LotConfigPropertySize = 0x88edc790; // # CityManager +// A class for performing operations on a certain city, such as plopping +// arbitrary lots etc. Have a look at https://sc4devotion.com/forums/ +// index.php?topic=5656.0, contains a lot of relevant info. class CityManager { - // ## constructor(dbpf) + // ## constructor(opts) // Sets up the city manager. - constructor(dbpf) { + constructor(opts = {}) { - // Pre-initialize all fields. - this.dbpf = null; + // Pre-initialize the "private" fields that cannot be modified by the + // options. this.memRefs = null; this.$mem = 1; - // If we've received a string, treat it as a path. - if (typeof dbpf === 'string') { - let file = path.resolve(regions, dbpf); - - // No extension given? Add .sc4 - let ext = path.extname(file); - if (ext !== '.sc4') file += '.sc4'; - - // Check if the file exists. If it doesn't exist, then try again - // with "City - " in front. - if (!fs.existsSync(file)) { - let name = path.basename(file); - let dir = path.dirname(file); - file = path.join(dir, 'City - '+name); - if (!fs.existsSync(file)) { - throw new Error(`City "${dbpf}" could not be found!`); - } - } + // Setup the "public" fields. + this.dbpf = opts.dbpf || null; + this.index = opts.index || null; + + } + + // ## setFileIndex(index) + // Stores the file index to be used for looking up TGI's etc. That's + // required if you want to plop lot's etc. because in that case we need to + // know where to look for the resources! + setFileIndex(index) { + this.index = index; + } - // Create the city. - dbpf = this.dbpf = new Savegame(fs.readFileSync(file)); + // ## load(file) + // Loads the given savegame into the city manager. + load(file) { - } else { - this.dbpf = dbpf; + let full = path.resolve(regions, file); + + // No extension given? Add .sc4 + let ext = path.extname(full); + if (ext !== '.sc4') { + full += '.sc4'; + } + + // Check if the file exists. If it doesn't exist, then try again + // with "City - " in front. + if (!fs.existsSync(full)) { + let name = path.basename(full); + let dir = path.dirname(full); + full = path.join(dir, 'City - '+name); + if (!fs.existsSync(full)) { + throw new Error(`City "${file}" could not be found!`); + } } + // Create the city. + this.dbpf = new Savegame(fs.readFileSync(full)); + } // ## mem() @@ -73,24 +100,220 @@ class CityManager { } - // ## loadPlugins(opts) - // Loads a plugins directory and sets an index. This index will be at the - // heart of everything we'll be doing because it allows us to query - // entries by TGI. Note that we have to make sure to index the - // SimCity_1.dat files as well! - async loadPlugins(opts) { - if (!opts) { - opts = { - "dirs": [plugins] + // ## plop(opts) + // Behold, the mother of all functions. This function allows to plop any + // lot anywhere in the city. Note that this function expects a *building* + // exemplar, which means it only works for *ploppable* buildings. For + // growable buildings the process is different, in that case you have to + // use the "grow" method. + plop(opts = {}) { + + // (1) First of all we need to find the T10 exemplar file with the + // information to plop the lot. Most of the time this resides in an + // .sc4lot file, but it doesn't have to. + let { tgi } = opts; + let record = this.index.find(tgi); + if (!record) { + throw new Error( + `Exemplar ${ JSON.stringify(tgi) } not found!`, + ); + } + + // Check what type of exemplar we're dealing with. As explained by + // RippleJet, there's a fundamental difference between ploppable and + // growable buildings. Apparently ploppable buildings start from a + // building exemplar and then we can look up according + // LotConfiguration exemplar. + let exemplar = record.read(); + if (+exemplar.prop(ExemplarType) !== Buildings) { + throw new Error([ + 'The exemplar is not a building exemplar!', + 'The `.plop()` function expects a ploppable building exemplar!' + ].join(' ')); + } + + // Find the lot resource key, which is the IID where we can find the + // LotResourceKey & then based on that find the appropriate Building + // exemplar. Note that we currently have no other choice than finding + // everything with the same instance ID... + let IID = +exemplar.prop(LotResourceKey); + let exemplars = this.index.findAllTI(FileType.Exemplar, IID); + let lotExemplar = exemplars.find(record => { + let exemplar = record.read(); + return +exemplar.prop(ExemplarType) === LotConfigurations; + }); + + // Create the lot. It will automatically insert it into the zone + // developer file as well. + let lot = this.createLot({ + exemplar: lotExemplar, + x: opts.x, + z: opts.z, + building: IID, + }); + + // Now loop all objects on the lot such as the building, the props + // etc. and insert them. + let { lotObjects } = lotExemplar.read(); + for (let lotObject of lotObjects) { + switch (lotObject.type) { + case 0x00: + this.createBuilding({ + lot, + lotObject, + exemplar: record, + }); + case 0x02: + this.createTexture({ + lot, + lotObject, + }); + } + } + + } + + // ## createLot(opts) + // Creates a new lot object from the given options when plopping a lot. + createLot(opts) { + + // Read in the size of the lot because we'll still need it. + let { exemplar, x, z, building, orientation = 0 } = opts; + let file = exemplar.read(); + let [width, height] = file.prop(LotConfigPropertySize).value; + if (orientation % 2 === 1) { + [width, height] = [height, width]; + } + + // Cool, we can now create a new lot entry. Note that we will need to + // take into account the + let lot = new Lot({ + mem: this.mem(), + IID: building, + buildingIID: building, + zoneType: 0x0f, + + // For now, just put at y = 270. In the future we'll need to read + // in the terrain here. + yPos: 270, + minX: x, + maxX: x+width, + minZ: z, + maxZ: z+height, + commuteX: x, + commuteZ: z, + depth: height, + width: width, + orientation: orientation, + + }); + + // Push the lot in the lotFile. + let { dbpf } = this; + let lots = dbpf.lotFile; + lots.push(lot); + + // Now put the lot in the zone developer file as well. TODO: We should + // actually check first and ensure that no building exists yet here! + let zones = dbpf.zoneDeveloperFile; + for (let x = lot.minX; x <= lot.maxX; x++) { + for (let z = lot.minZ; z <= lot.maxZ; z++) { + zones.cells[x][z] = { + mem: lot.mem, + type: FileType.LotFile, + }; + } + } + + // Don't forget to update the COMSerializer to include the updated + // length! Otherwise the lot won't show up! + let com = dbpf.COMSerializerFile; + com.set(FileType.LotFile, lots.length); + + // Return the lot that we've just created. + return lot; + + } + + // ## createBuilding(opts) + // Creates a new building record and inserts it into the savegame. + createBuilding(opts) { + let { lot, lotObject, exemplar } = opts; + let file = exemplar.read(); + let [width, height, depth] = file.prop(OccupantSize).value; + let { orientation, x, y, z } = lotObject; + + // Create the building. + let building = new Building({ + mem: this.mem(), + + // TODO: we need to rotate the building into place here! + minX: 16*lot.minX + x, + maxX: 16*lot.minX + x + width, + minZ: 16*lot.minZ + z, + maxZ: 16*lot.maxZ + z + depth, + minY: lot.yPos + y, + maxY: lot.yPos + y + height, + orientation: (orientation + lot.orientation) % 4, + + TID: exemplar.type, + GID: exemplar.group, + IID: exemplar.instance, + IID1: exemplar.instance, + + }); + + // Set the correct tract for the building & then put inside the item + // index. + setTract(building); + let { dbpf } = this; + let index = dbpf.itemIndexFile; + for (let x = building.xMinTract; x <= building.xMaxTract; x++) { + for (let z = building.zMinTract; z <= building.zMaxTract; z++) { + index[x][z].push({ + mem: building.mem, + type: FileType.BuildingFile, + }); } } - // Build the index. - let index = this.plugins = new Index(opts); - await index.build(); + // Push in the file with all buildings. + let buildings = dbpf.buildingFile; + buildings.push(); + // Add to the lot developer file as well. + let dev = dbpf.lotDeveloperFile; + dev.buildings.push({ + mem: building.mem, + type: FileType.BuildingFile, + }); + + // At last update the COMSerializer file. + let com = dbpf.COMSerializerFile; + com.set(FileType.BuildingFile, buildings.length); + return building; + + } + + // ## createTexture(opts) + // Creates a base texture. + createTexture(opts) { + // let { lot, lotObject } = opts; + // let { orientation, x, y, z } = lotObject; } } -module.exports = CityManager; \ No newline at end of file +module.exports = CityManager; + +// ## setTract(obj) +// Helper function for setting the correct "Tract" values in the given object +// based on its bounding box. +function setTract(obj) { + const xSize = 2 ** obj.xTractSize; + const ySize = 2 ** obj.yTractSize; + obj.xMinTract = 64 + Math.floor(obj.minX / xSize); + obj.xMaxTract = 64 + Math.floor(obj.maxX / xSize); + obj.yMinTract = 64 + Math.floor(obj.minY / ySize); + obj.yMaxTract = 64 + Math.floor(obj.maxY / ySize); +} diff --git a/lib/exemplar.js b/lib/exemplar.js index 5950848..700e9c9 100644 --- a/lib/exemplar.js +++ b/lib/exemplar.js @@ -68,6 +68,14 @@ class Exemplar { return FileType.Exemplar; } + // ## constructor() + constructor() { + this.id = 'EQZB1###'; + this.parent = [0,0,0]; + this.props = []; + this.table = null; + } + // ## get fileType() get fileType() { return FileType.Exemplar; @@ -88,12 +96,10 @@ class Exemplar { return out; } - // ## constructor() - constructor() { - this.id = 'EQZB1###'; - this.parent = [0,0,0]; - this.props = []; - this.table = null; + // ## prop(key) + // Helper function for accessing a property. + prop(key) { + return this.table[ key ]; } // ## parse(buff) @@ -224,6 +230,12 @@ class Property { this.value = value; } + // ## [Symbol.toPrimitive]() + // Casting the prop to a number will return the numeric value. + [Symbol.toPrimitive](hint) { + return hint === 'number' ? this.value : this.hex; + } + // ## get hex() // Computed property that shows the hex value of the property name. Useful // when comparing this with Reader, because Reader shows everything in hex diff --git a/lib/file-index.js b/lib/file-index.js index 7540be1..0d8355d 100644 --- a/lib/file-index.js +++ b/lib/file-index.js @@ -31,8 +31,13 @@ class FileIndex { // Our array containing all our records. This array will be sorted by // tgi. this.records = null; - this.git = null; - this.igt = null; + this.itg = null; + + // If the options are simply given as a string, consider it to be a + // directory. + if (typeof opts === 'string') { + opts = { dirs: [opts] }; + } let files = this.files = []; if (opts.files) { @@ -81,8 +86,7 @@ class FileIndex { // Allright we now have all records. Time to sort them in their // respective arrays. - this.git = this.records.map(x => x).sort(git); - this.igt = this.records.map(x => x).sort(igt); + this.itg = this.records.filter(Boolean).sort(itg); this.records.sort(tgi); } @@ -121,11 +125,32 @@ class FileIndex { query.instance = instance; let index = bsearch(this.records, query, tgi); - if (index < 0) return null; - return this.records[index]; + if (index < 0) { + return null; + } + let record = this.records[index]; + if ( + record.type !== type || + record.group !== group || + record.instance !== instance + ) { + return null; + } else { + return record; + } } + // ## findAllTI(type, instance) + // Finds all entries with the given Type and Instance ID. TODO: We'll need + // to use a compound index for this. That's for later on though. Brute + // force. + findAllTI(type, instance) { + return this.records.filter(record => { + return record.type === type && record.instance === instance; + }); + } + } module.exports = FileIndex; @@ -209,6 +234,12 @@ function tgi(a, b) { return a.type - b.type || a.group - b.group || a.instance - b.instance; } +// ## itg(a, b) +// TIG comparator that will sort by instance, then type and then group. +function itg(a, b) { + return a.instance - b.instance || a.type - b.type || a.group - b.group; +} + // # git(a, b) // TGI comparator that will sort by group, then instance, then type. function git(a, b) { diff --git a/lib/lot.js b/lib/lot.js index fe87be3..62617a4 100644 --- a/lib/lot.js +++ b/lib/lot.js @@ -22,7 +22,7 @@ const Lot = class Lot extends Type(FileType.LotFile) { // Pre-initialize the lot properties with correct types to produce better // optimized vm code. This is inherent to how V8 optimizes Object // properties. - constructor() { + constructor(opts) { super(); this.crc = 0x00000000; this.mem = 0x00000000; @@ -53,6 +53,7 @@ const Lot = class Lot extends Type(FileType.LotFile) { this.commutes = []; this.commuteBuffer = null; this.debug = 0x00; + Object.assign(this, opts); } // ## get/set historical() diff --git a/test/city-manager-test.js b/test/city-manager-test.js index 06768dd..58e3058 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -2,6 +2,7 @@ "use strict"; const { expect } = require('chai'); const path = require('path'); +const FileIndex = require('../lib/file-index.js'); const CityManager = require('../lib/city-manager.js'); describe('A city manager', function() { @@ -28,7 +29,8 @@ describe('A city manager', function() { it('returns an unused memory address', async function() { let file = path.resolve(__dirname, 'files/City - RCI.sc4'); - let city = new CityManager(file); + let city = new CityManager(); + city.load(file); expect(city.mem()).to.equal(1); city.memRefs.add(2); @@ -39,4 +41,33 @@ describe('A city manager', function() { }); -}); \ No newline at end of file + context('#plop()', function() { + + it.only('plops a ploppable lot', async function() { + + this.slow(1000); + + // First of all we need to build up a file index that the city + // manager can use. + // let dir = path.join(__dirname, 'files/DarkNight_11KingStreetWest'); + let dir = path.join(__dirname,'files/DiegoDL-432ParkAvenue-LM-DN'); + let index = new FileIndex(dir); + await index.build(); + + // Create the city manager. + let game = path.join(__dirname, 'files/City - Plopsaland.sc4'); + let city = new CityManager({ index }); + city.load(game); + + // Plop it baby. + city.plop({ + tgi: [0x6534284a, 0xd60100c4, 0x483248bb], + x: 5, + z: 5, + }); + + }); + + }); + +}); From 93b8f7ad52bb003ca93d9aae4a08d359c6ba0abe Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Wed, 11 Mar 2020 17:59:25 +0100 Subject: [PATCH 04/39] Ensure correct building rotation --- lib/city-manager.js | 93 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 354c437..83d1443 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -142,13 +142,16 @@ class CityManager { let exemplar = record.read(); return +exemplar.prop(ExemplarType) === LotConfigurations; }); + let lotConfig = lotExemplar.read(); // Create the lot. It will automatically insert it into the zone // developer file as well. + let { orientation = 0 } = opts; let lot = this.createLot({ exemplar: lotExemplar, x: opts.x, z: opts.z, + orientation, building: IID, }); @@ -163,11 +166,13 @@ class CityManager { lotObject, exemplar: record, }); + break; case 0x02: this.createTexture({ lot, lotObject, }); + break; } } @@ -243,15 +248,28 @@ class CityManager { let [width, height, depth] = file.prop(OccupantSize).value; let { orientation, x, y, z } = lotObject; + // Find the rectangle the building is occupying on the lot, where the + // origin of the coordinate system is in the top-left corner. Note + // that we already rotate the building into place **in the lot**. We + // still need to rotate the building rectangle later on based on the + // orientation of the lot *itself*. + if (orientation % 2) { + [width, depth] = [depth, width]; + } + let rect = position({ + minX: 16*x - width/2, + maxX: 16*x + width/2, + minZ: 16*z - depth/2, + maxZ: 16*z + depth/2, + }, lot); + // Create the building. let building = new Building({ mem: this.mem(), - // TODO: we need to rotate the building into place here! - minX: 16*lot.minX + x, - maxX: 16*lot.minX + x + width, - minZ: 16*lot.minZ + z, - maxZ: 16*lot.maxZ + z + depth, + // Now use the **rotated** building rectangle and use it to + // position the building appropriately. + ...rect, minY: lot.yPos + y, maxY: lot.yPos + y + height, orientation: (orientation + lot.orientation) % 4, @@ -317,3 +335,68 @@ function setTract(obj) { obj.yMinTract = 64 + Math.floor(obj.minY / ySize); obj.yMaxTract = 64 + Math.floor(obj.maxY / ySize); } + +// ## position(rect, lot) +// Modifies the given rectangle so that it is positioned correctly on the +// given lot. Returns an object { minX, maxX, minZ, maxZ } that can be easily +// assigned to the object. +function position(rect, lot) { + let { minX, maxX, minZ, maxZ } = rect; + function move({ minX, maxX, minZ, maxZ }) { + return { + minX: lot.minX + minX, + maxX: lot.minX + maxX, + minZ: lot.minZ + minZ, + maxZ: lot.maxZ + maxZ, + }; + } + + // First of all, if the lot has default rotation, just return the + // rectangle as is. + if (lot.orientation === 0x00) { + return move({ minX, maxX, minZ, maxZ }); + } + + // In all other cases we need the width an height of the lot **in it's + // local coordinates**. Note that the lot is already rotated, so we need + // to "unrotate" again! + let width = 16*(lot.maxX - lot.minX); + let depth = 16*(lot.maxZ - lot.minZ); + if (lot.orientation % 2 === 1) { + [width, depth] = [depth, width]; + } + + // Now handle the different orientations. + switch (lot.orientation) { + case 0x01: + return move({ + minX: depth-maxZ, + maxX: depth-minZ, + minZ: minX, + maxZ: maxX, + }); + case 0x02: + return move({ + minX: width-maxX, + maxX: width-minX, + minZ: depth-maxZ, + maxZ: depth-minZ, + }); + case 0x03: + return move({ + minX: minZ, + maxX: maxZ, + minZ: width-maxX, + maxZ: width-minX, + }); + default: + return move({ + minX, + maxX, + minZ, + maxZ, + }); + + } + +} From 205a1c9f467aeb8274820cf2cf7c50136e35dd9c Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Wed, 11 Mar 2020 22:06:48 +0100 Subject: [PATCH 05/39] Fix lot plopping It's now possible to plop arbitrary lots anywhere on the map. Still, we're having some troubles with the rotations, so those will still need to be fixed. --- lib/building.js | 10 +++-- lib/city-manager.js | 85 ++++++++++++++++++++++++--------------- lib/savegame.js | 12 ++++++ test/city-manager-test.js | 20 +++++---- 4 files changed, 84 insertions(+), 43 deletions(-) diff --git a/lib/building.js b/lib/building.js index d6b523e..f3edc58 100644 --- a/lib/building.js +++ b/lib/building.js @@ -29,12 +29,12 @@ const Building = class Building extends Type(FileType.BuildingFile) { this.xTractSize = 0x0002; this.zTractSize = 0x0002; this.sgprops = []; - this.unknown2 = 0x01; + this.unknown2 = 0x00; this.IID1 = this.IID = this.TID = this.GID = 0x00000000; this.minZ = this.minY = this.minX = 0; this.maxZ = this.maxY = this.maxX = 0; this.orientation = 0x00; - this.scaffold = 0; + this.scaffold = 0x01; Object.assign(this, opts); } @@ -101,11 +101,13 @@ const Building = class Building extends Type(FileType.BuildingFile) { prop.parse(rs); } - this.unknown2 = rs.byte(); + // There seems to be an error in the Wiki. The unknown byte should + // come **after** the IID1, otherwise they're incorrect. this.GID = rs.dword(); this.TID = rs.dword(); this.IID = rs.dword(); this.IID1 = rs.dword(); + this.unknown2 = rs.byte(); this.minX = rs.float(); this.minY = rs.float(); this.minZ = rs.float(); @@ -159,11 +161,11 @@ const Building = class Building extends Type(FileType.BuildingFile) { // Serialize the fixed remainder. let two = Buffer.allocUnsafe(46); ws = new WriteStream(two); - ws.byte(this.unknown2); ws.dword(this.GID); ws.dword(this.TID); ws.dword(this.IID); ws.dword(this.IID1); + ws.byte(this.unknown2); ws.float(this.minX); ws.float(this.minY); ws.float(this.minZ); diff --git a/lib/city-manager.js b/lib/city-manager.js index 83d1443..67f3143 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -75,6 +75,12 @@ class CityManager { } + // ## save(opts) + // Saves the city to the given file. + save(opts) { + return this.dbpf.save(opts); + } + // ## mem() // Returns an unused memory address. This is useful if we add new stuff to // a city - such as buildings etc. - because we need to make sure that the @@ -183,12 +189,11 @@ class CityManager { createLot(opts) { // Read in the size of the lot because we'll still need it. + let { dbpf } = this; + let lots = dbpf.lotFile; let { exemplar, x, z, building, orientation = 0 } = opts; let file = exemplar.read(); - let [width, height] = file.prop(LotConfigPropertySize).value; - if (orientation % 2 === 1) { - [width, height] = [height, width]; - } + let [width, depth] = file.prop(LotConfigPropertySize).value; // Cool, we can now create a new lot entry. Note that we will need to // take into account the @@ -196,26 +201,35 @@ class CityManager { mem: this.mem(), IID: building, buildingIID: building, - zoneType: 0x0f, + zoneType: 0x01, // For now, just put at y = 270. In the future we'll need to read // in the terrain here. yPos: 270, minX: x, - maxX: x+width, + maxX: x+(orientation % 2 === 1 ? depth : width)-1, minZ: z, - maxZ: z+height, + maxZ: z+(orientation % 2 === 1 ? width : depth)-1, commuteX: x, commuteZ: z, - depth: height, - width: width, + width, + depth, orientation: orientation, + // Important! ZoneWealth cannot be set to 0, otherwise CTD! + zoneWealth: 0x03, + + // Apparently jobCapacities is also required, otherwise CTD! The + // capacity is stored I guess in the LotConfig exemplar, or + // perhaps in the building exemplar. + jobCapacities: [{ + demandSourceIndex: 0x00003320, + capacity: 0, + }], + }); // Push the lot in the lotFile. - let { dbpf } = this; - let lots = dbpf.lotFile; lots.push(lot); // Now put the lot in the zone developer file as well. TODO: We should @@ -253,7 +267,7 @@ class CityManager { // that we already rotate the building into place **in the lot**. We // still need to rotate the building rectangle later on based on the // orientation of the lot *itself*. - if (orientation % 2) { + if (orientation % 2 === 1) { [width, depth] = [depth, width]; } let rect = position({ @@ -262,6 +276,7 @@ class CityManager { minZ: 16*z - depth/2, maxZ: 16*z + depth/2, }, lot); + console.log(rect); // Create the building. let building = new Building({ @@ -274,16 +289,16 @@ class CityManager { maxY: lot.yPos + y + height, orientation: (orientation + lot.orientation) % 4, + // Store the TGI of the building exemplar. TID: exemplar.type, GID: exemplar.group, IID: exemplar.instance, IID1: exemplar.instance, }); - - // Set the correct tract for the building & then put inside the item - // index. setTract(building); + + // Put the building in the index at the correct spot. let { dbpf } = this; let index = dbpf.itemIndexFile; for (let x = building.xMinTract; x <= building.xMaxTract; x++) { @@ -297,7 +312,7 @@ class CityManager { // Push in the file with all buildings. let buildings = dbpf.buildingFile; - buildings.push(); + buildings.push(building); // Add to the lot developer file as well. let dev = dbpf.lotDeveloperFile; @@ -328,12 +343,12 @@ module.exports = CityManager; // Helper function for setting the correct "Tract" values in the given object // based on its bounding box. function setTract(obj) { - const xSize = 2 ** obj.xTractSize; - const ySize = 2 ** obj.yTractSize; + const xSize = 16 * 2**obj.xTractSize; + const zSize = 16 * 2**obj.zTractSize; obj.xMinTract = 64 + Math.floor(obj.minX / xSize); obj.xMaxTract = 64 + Math.floor(obj.maxX / xSize); - obj.yMinTract = 64 + Math.floor(obj.minY / ySize); - obj.yMaxTract = 64 + Math.floor(obj.maxY / ySize); + obj.zMinTract = 64 + Math.floor(obj.minZ / zSize); + obj.zMaxTract = 64 + Math.floor(obj.maxZ / zSize); } // ## position(rect, lot) @@ -344,10 +359,10 @@ function position(rect, lot) { let { minX, maxX, minZ, maxZ } = rect; function move({ minX, maxX, minZ, maxZ }) { return { - minX: lot.minX + minX, - maxX: lot.minX + maxX, - minZ: lot.minZ + minZ, - maxZ: lot.maxZ + maxZ, + minX: 16*lot.minX + minX, + maxX: 16*lot.minX + maxX, + minZ: 16*lot.minZ + minZ, + maxZ: 16*lot.minZ + maxZ, }; } @@ -358,22 +373,24 @@ function position(rect, lot) { } // In all other cases we need the width an height of the lot **in it's - // local coordinates**. Note that the lot is already rotated, so we need - // to "unrotate" again! - let width = 16*(lot.maxX - lot.minX); - let depth = 16*(lot.maxZ - lot.minZ); + // local coordinates**. Note: the lot's width & depth properties are given + // in *local coordinates*. This means that if the orientation is east or + // west, we should still swap width & depth! + let { width, depth } = lot; if (lot.orientation % 2 === 1) { [width, depth] = [depth, width]; } + depth *= 16; + width *= 16; // Now handle the different orientations. switch (lot.orientation) { case 0x01: return move({ - minX: depth-maxZ, - maxX: depth-minZ, - minZ: minX, - maxZ: maxX, + minX: minX, + maxX: maxX, + minZ: minZ, + maxZ: maxZ, }); case 0x02: return move({ @@ -400,3 +417,7 @@ function position(rect, lot) { } } + +function clone(obj) { + return Object.create(Object.getPrototypeOf(obj), Object.getOwnPropertyDescriptors(obj)); +} \ No newline at end of file diff --git a/lib/savegame.js b/lib/savegame.js index e013120..42f4d8d 100644 --- a/lib/savegame.js +++ b/lib/savegame.js @@ -13,18 +13,27 @@ module.exports = class Savegame extends DBPF { let entry = this.entries.find(x => x.type === FileType.LotFile); return entry ? entry.read() : null; } + get lots() { + return this.lotFile; + } // ## get buildingFile() get buildingFile() { let entry = this.entries.find(x => x.type === FileType.BuildingFile); return entry ? entry.read() : null; } + get buildings() { + return this.buildingFile; + } // ## get propFile() get propFile() { let entry = this.entries.find(x => x.type === FileType.PropFile); return entry ? entry.read() : null; } + get props() { + return this.propFile; + } // ## get baseTextureFile() get baseTextureFile() { @@ -37,6 +46,9 @@ module.exports = class Savegame extends DBPF { let entry = this.getByType(FileType.ItemIndexFile); return entry ? entry.read() : null; } + get itemIndex() { + return this.itemIndexFile; + } // ## get zoneDeveloperFile() get zoneDeveloperFile() { diff --git a/test/city-manager-test.js b/test/city-manager-test.js index 58e3058..bbde52a 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -50,21 +50,27 @@ describe('A city manager', function() { // First of all we need to build up a file index that the city // manager can use. // let dir = path.join(__dirname, 'files/DarkNight_11KingStreetWest'); - let dir = path.join(__dirname,'files/DiegoDL-432ParkAvenue-LM-DN'); + // let dir = path.join(__dirname,'files/DiegoDL-432ParkAvenue-LM-DN'); + let dir = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Plugins'); let index = new FileIndex(dir); await index.build(); // Create the city manager. - let game = path.join(__dirname, 'files/City - Plopsaland.sc4'); + let game = path.join(__dirname, 'files/City - 432.sc4'); let city = new CityManager({ index }); city.load(game); // Plop it baby. - city.plop({ - tgi: [0x6534284a, 0xd60100c4, 0x483248bb], - x: 5, - z: 5, - }); + for (let i = 0; i < 1; i++) { + city.plop({ + tgi: [0x6534284a, 0xd60100c4, 0x483248bb], + x: 5+5*i, + z: 5, + }); + } + let regions = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments'); + let file = path.join(regions, 'City - Plopsaland.sc4'); + await city.save({ file }); }); From 2987726ce6631dbe97d3d25f1f3e26ead31e4e94 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Wed, 11 Mar 2020 22:53:47 +0100 Subject: [PATCH 06/39] Try to fix rotations --- lib/city-manager.js | 56 +++++++++++---------------------------- test/city-manager-test.js | 3 ++- 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 67f3143..9cef471 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -214,7 +214,7 @@ class CityManager { commuteZ: z, width, depth, - orientation: orientation, + orientation, // Important! ZoneWealth cannot be set to 0, otherwise CTD! zoneWealth: 0x03, @@ -276,7 +276,6 @@ class CityManager { minZ: 16*z - depth/2, maxZ: 16*z + depth/2, }, lot); - console.log(rect); // Create the building. let building = new Building({ @@ -358,66 +357,43 @@ function setTract(obj) { function position(rect, lot) { let { minX, maxX, minZ, maxZ } = rect; function move({ minX, maxX, minZ, maxZ }) { + let x = 16*lot.minX; + let z = 16*lot.minZ; return { - minX: 16*lot.minX + minX, - maxX: 16*lot.minX + maxX, - minZ: 16*lot.minZ + minZ, - maxZ: 16*lot.minZ + maxZ, + minX: x + minX, + maxX: x + maxX, + minZ: z + minZ, + maxZ: z + maxZ, }; } - // First of all, if the lot has default rotation, just return the - // rectangle as is. - if (lot.orientation === 0x00) { - return move({ minX, maxX, minZ, maxZ }); - } - - // In all other cases we need the width an height of the lot **in it's - // local coordinates**. Note: the lot's width & depth properties are given - // in *local coordinates*. This means that if the orientation is east or - // west, we should still swap width & depth! + // Find the width & the depth of the lot. We can get this from the lot + // itself, but this doesn't take into account yet that the lot is rotated, + // so that's something we still need to do ourselves. let { width, depth } = lot; if (lot.orientation % 2 === 1) { [width, depth] = [depth, width]; } - depth *= 16; - width *= 16; - // Now handle the different orientations. + // TODO: Don't think this works, but we need a lot that's a bit more + // "cornered", otherwise we won't be able to see the effects of it. switch (lot.orientation) { case 0x01: - return move({ - minX: minX, - maxX: maxX, - minZ: minZ, - maxZ: maxZ, - }); - case 0x02: - return move({ - minX: width-maxX, - maxX: width-minX, - minZ: depth-maxZ, - maxZ: depth-minZ, - }); case 0x03: return move({ minX: minZ, maxX: maxZ, - minZ: width-maxX, - maxZ: width-minX, + minZ: minX, + maxZ: maxX, }); - default: + default: { return move({ minX, maxX, minZ, maxZ, }); - + } } } - -function clone(obj) { - return Object.create(Object.getPrototypeOf(obj), Object.getOwnPropertyDescriptors(obj)); -} \ No newline at end of file diff --git a/test/city-manager-test.js b/test/city-manager-test.js index bbde52a..2184cec 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -61,11 +61,12 @@ describe('A city manager', function() { city.load(game); // Plop it baby. - for (let i = 0; i < 1; i++) { + for (let i = 0; i < 4; i++) { city.plop({ tgi: [0x6534284a, 0xd60100c4, 0x483248bb], x: 5+5*i, z: 5, + orientation: i % 4, }); } let regions = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments'); From b73ef08cd480cf7b531a2d282424d795df42fe72 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 00:05:56 +0100 Subject: [PATCH 07/39] Fix orientations --- lib/city-manager.js | 24 ++++++++++++++++++++---- test/city-manager-test.js | 7 ++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 9cef471..08906cb 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -201,7 +201,7 @@ class CityManager { mem: this.mem(), IID: building, buildingIID: building, - zoneType: 0x01, + zoneType: 0x0f, // For now, just put at y = 270. In the future we'll need to read // in the terrain here. @@ -372,19 +372,35 @@ function position(rect, lot) { // so that's something we still need to do ourselves. let { width, depth } = lot; if (lot.orientation % 2 === 1) { - [width, depth] = [depth, width]; + [width, depth] = [16*depth, 16*width]; + } else { + width *= 16; + depth *= 16; } // TODO: Don't think this works, but we need a lot that's a bit more // "cornered", otherwise we won't be able to see the effects of it. switch (lot.orientation) { case 0x01: + return move({ + minX: width-maxZ, + maxX: width-minZ, + minZ: minX, + maxZ: maxX, + }); + case 0x02: + return move({ + minX: width-maxX, + maxX: width-minX, + minZ: depth-maxZ, + maxZ: depth-minZ, + }); case 0x03: return move({ minX: minZ, maxX: maxZ, - minZ: minX, - maxZ: maxX, + minZ: depth-maxX, + maxZ: depth-minX, }); default: { return move({ diff --git a/test/city-manager-test.js b/test/city-manager-test.js index 2184cec..f472ba3 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -63,9 +63,10 @@ describe('A city manager', function() { // Plop it baby. for (let i = 0; i < 4; i++) { city.plop({ - tgi: [0x6534284a, 0xd60100c4, 0x483248bb], - x: 5+5*i, - z: 5, + // tgi: [0x6534284a, 0xd60100c4, 0x483248bb], + tgi: [0x6534284a,0x76fbb03a,0x290dc058], + x: (1+i)*8, + z: 8, orientation: i % 4, }); } From 0a00ced90f70779317fcba69349b704ed7547871 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 09:49:26 +0100 Subject: [PATCH 08/39] Use binary-search-bounds We now use the binary-search-bounds module instead of the binary-search module as it exposes more useful methods such as le and lt etc. --- lib/com-serializer-file.js | 1 - lib/file-index.js | 106 +++++++++++++++++-------------------- lib/sim-grid-file.js | 4 +- package.json | 2 +- 4 files changed, 53 insertions(+), 60 deletions(-) diff --git a/lib/com-serializer-file.js b/lib/com-serializer-file.js index 4d91777..521d957 100644 --- a/lib/com-serializer-file.js +++ b/lib/com-serializer-file.js @@ -1,6 +1,5 @@ // # com-serializer-file.js "use strict"; -const bsearch = require('binary-search'); const Stream = require('./stream'); const WriteStream = require('./write-stream'); const { FileType } = require('./enums'); diff --git a/lib/file-index.js b/lib/file-index.js index 0d8355d..84e3d01 100644 --- a/lib/file-index.js +++ b/lib/file-index.js @@ -3,7 +3,7 @@ const fs = require('fs'); const path = require('path'); const os = require('os'); -const bsearch = require('binary-search'); +const bsearch = require('binary-search-bounds'); const DBPF = require('./dbpf'); const { Entry } = DBPF; const {default:PQueue} = require('p-queue'); @@ -11,9 +11,7 @@ const {default:PQueue} = require('p-queue'); // Patch fs promises. if (!fs.promises) { const util = require('util'); - fs.promises = { - "readFile": util.promisify(fs.readFile) - }; + fs.promises = { readFile: util.promisify(fs.readFile) }; } // # FileIndex @@ -31,7 +29,18 @@ class FileIndex { // Our array containing all our records. This array will be sorted by // tgi. this.records = null; - this.itg = null; + + // No options specified? Use the default plugins directory. Note that + // we should look for the SimCity_1.dat core files as well of course. + if (!opts) { + let plugins = path.join( + process.env.HOMEPATH, + 'Documents/SimCity 4/Plugins', + ); + opts = { + dirs: [plugins], + }; + } // If the options are simply given as a string, consider it to be a // directory. @@ -68,7 +77,7 @@ class FileIndex { // Limit the amount of reads that we carry out. let cpus = os.cpus(); let max = Math.max(2, cpus.length, opts.concurrency || 0); - let Q = new PQueue({"concurrency": max}); + let Q = new PQueue({ concurrency: max }); let all = []; for (let file of this.files) { @@ -86,13 +95,13 @@ class FileIndex { // Allright we now have all records. Time to sort them in their // respective arrays. - this.itg = this.records.filter(Boolean).sort(itg); - this.records.sort(tgi); + this.records.sort(compare); } // ## async addToIndex(file) - // Asynchronously adds the given file to the index. + // Asynchronously adds the given file to the index. Note that this is not + // meant for external use as this doesn't sort the records! async addToIndex(file) { // Read in the file. @@ -100,7 +109,9 @@ class FileIndex { const source = new Source(file); // Ensure that the file is a dbpf file, ignore otherwise. - if (buff.toString('utf8', 0, 4) !== 'DBPF') return; + if (buff.toString('utf8', 0, 4) !== 'DBPF') { + return; + } // Parse. let dbpf = new DBPF(buff); @@ -120,35 +131,32 @@ class FileIndex { } else if (typeof type === 'object') { ({type, group, instance} = type); } - query.type = type; - query.group = group; - query.instance = instance; + let query = { + type, + group, + instance, + }; - let index = bsearch(this.records, query, tgi); - if (index < 0) { - return null; - } - let record = this.records[index]; - if ( - record.type !== type || - record.group !== group || - record.instance !== instance - ) { - return null; - } else { - return record; - } + let index = bsearch.eq(this.records, query, compare); + return index > -1 ? this.records[index] : null; } // ## findAllTI(type, instance) - // Finds all entries with the given Type and Instance ID. TODO: We'll need - // to use a compound index for this. That's for later on though. Brute - // force. + // Finds all entries with the given Type and Instance ID. findAllTI(type, instance) { - return this.records.filter(record => { - return record.type === type && record.instance === instance; - }); + let query = { type, instance, group: 0 }; + let index = bsearch.lt(this.records, query, compare); + let out = []; + let record; + while ( + (record = this.records[++index]) && + record.type === type && + record.instance === instance + ) { + out.push(record); + } + return out; } } @@ -225,32 +233,18 @@ class Source { } } -// # extRegex -const extRegex = /^(sc4desc)|(sc4lot)|(sc4model)|(dat)$/; - -// # tgi(a, b) -// TGI comparator that will sort by type, then group and then instance. -function tgi(a, b) { - return a.type - b.type || a.group - b.group || a.instance - b.instance; -} - -// ## itg(a, b) -// TIG comparator that will sort by instance, then type and then group. -function itg(a, b) { +// # compare(a, b) +// The function that we use for sorting all files in our index. This +// effectively creates an *index* on all the records so that we can use a +// binary search algorithm for finding them. Given that the instance id (IID) +// is what we'll use the most, this will be the main sorting key, followed by +// type id (TID) and then by group id (GID). +function compare(a, b) { return a.instance - b.instance || a.type - b.type || a.group - b.group; } -// # git(a, b) -// TGI comparator that will sort by group, then instance, then type. -function git(a, b) { - return a.group - b.group || a.instance - b.instance || a.type - b.type; -} - -// # igt(a, b) -// TGI comperator that will sort by instance, then group, then type. -function igt(a, b) { - return a.instance - b.instance || a.group - b.group || a.type - b.type; -} +// # extRegex +const extRegex = /^(sc4desc)|(sc4lot)|(sc4model)|(dat)$/; // # collect(dir, all) // Recursively crawls the given directory and collects all files within it. diff --git a/lib/sim-grid-file.js b/lib/sim-grid-file.js index 571254b..c6cab98 100644 --- a/lib/sim-grid-file.js +++ b/lib/sim-grid-file.js @@ -1,6 +1,6 @@ // # sim-grid.js "use strict"; -const bsearch = require('binary-search'); +const bsearch = require('binary-search-bounds'); const Stream = require('./stream'); const WriteStream = require('./write-stream'); const crc32 = require('./crc'); @@ -71,7 +71,7 @@ class SimGridFile { // ## get(dataId) // Returns the grid for the given data Id. get(dataId) { - let index = bsearch(this.sorted, {"dataId": dataId}, compare); + let index = bsearch.eq(this.sorted, { dataId }, compare); return index > -1 ? this.sorted[index] : null; } diff --git a/package.json b/package.json index e82bfcb..ba0af7c 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "three": "^0.106.2" }, "dependencies": { - "binary-search": "^1.3.6", + "binary-search-bounds": "^2.0.4", "chalk": "^2.4.2", "commander": "^2.20.3", "ini": "^1.3.5", From b36dbc3b01e0e2e6f56d1b1e06fb5c2812f64a6f Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 11:39:30 +0100 Subject: [PATCH 09/39] Allow DBPF files to be read in lazily --- lib/dbpf.js | 325 ++++++++++++++++++++++++++++++---------- test/dbpf-test.js | 35 ++++- test/file-index-test.js | 15 +- 3 files changed, 292 insertions(+), 83 deletions(-) diff --git a/lib/dbpf.js b/lib/dbpf.js index df791c2..69ebd74 100644 --- a/lib/dbpf.js +++ b/lib/dbpf.js @@ -1,5 +1,6 @@ // # dbpf.js "use strict"; +const util = require('util'); const fs = require('fs'); const TYPES = require('./file-types'); const Stream = require('./stream'); @@ -14,12 +15,26 @@ const { hex, tgi } = require('./util'); // might be compressed etc. const DBPF = module.exports = class DBPF { - // ## constructor(buff) - constructor(buff) { + // ## constructor(file) + // Constructs the dbpf for the given file. For backwards compatibility and + // ease of use, a DBPF is synchronous by default, but we should support + // async parsing as well: that's important if we're reading in an entire + // plugins directory for example. Note that DBPF's are always constructed + // from files, **not** from buffers. As such we don't have to keep the + // entire buffer in memory and we can read the required parts of the file + // "on the fly". That's what the DBPF format was designed for! + constructor(file = null, opts = {}) { // DBPF Magic number. By default this is DBPF, but we'll allow the // user to change this so that a dbpf can be de-activated. this.id = 'DBPF'; + this.file = null; + this.buffer = null; + if (Buffer.isBuffer(file)) { + this.buffer = file; + } else { + this.file = file; + } // Versioning of DBPF. In SC4 we always use DBPF 1.0. this.major = 1; @@ -34,14 +49,35 @@ const DBPF = module.exports = class DBPF { // TGI - is found where. this.entries = []; this.index = new Index(this); - - // If a buffer was specified, parse the dbpf from it. - if (buff) { - if (typeof buff === 'string') { - buff = fs.readFileSync(buff); + this.indexCount = 0; + this.indexOffset = 0; + this.indexSize = 0; + + // If a buffer or file was specified, parse the dbpf from it. Note: + // we've specified that it's *way* faster to read the file in at once. + // The problem is that we can't load an entire plugins directory into + // memory and keep it there. So if we don't want to load the file + // right away, it should be *explicitly* disabled, which is what we + // should do in that case... + if (!opts.async) { + if (this.file || this.buffer) { + if (this.file && !opts.lazy) { + this.buffer = fs.readFileSync(this.file); + } + this.readSync(); } - this.parse(buff); + } else { + throw new Error( + 'Async reading of DBPF files is not supported yet!', + ); } + + } + + // ## find(...args) + // Proxies to entries.find() + find(...args) { + return this.entries.find(...args); } // ## add(tfi, file) @@ -58,6 +94,87 @@ const DBPF = module.exports = class DBPF { return entry; } + // ## readBytesSync(offset, length) + // Returns a buffer contain the bytes starting at offset and with the + // given length. We use this method so that we can use a buffer or a file + // as underlying source interchangeably. + readBytesSync(offset, length) { + if (this.buffer) { + let { buffer } = this; + return Buffer.from(buffer.buffer, buffer.offset+offset, length); + } else { + let fd = fs.openSync(this.file); + let buffer = Buffer.allocUnsafe(length); + fs.readSync(fd, buffer, 0, length, offset); + fs.closeSync(fd); + return buffer; + } + } + + // ## readSync() + // Reads in the DBPF in a *synchronous* way. That's useful if you're + // testing stuff out, but for bulk reading you should use the async + // reading. + readSync() { + + // First of all we need to read the header, and only the header. From + // this we can derive where to find the index so that we can parse the + // entries from it. + this.parseHeader(this.readBytesSync(0, 96)); + + // Header is parsed which means we now know the offset of the index. + // Let's parse the index as well then. + let buffer = this.readBytesSync(this.indexOffset, this.indexSize); + let rs = new Stream(buffer); + this.index.parse(rs); + + } + + // ## parseHeader(buff) + // Parses the header of the DBPF file from the given buffer. + parseHeader(buff) { + + let rs = new Stream(buff); + this.id = rs.string(4); + if (this.id !== 'DBPF') { + throw new Error(`${ this.file } is not a valid DBPF file!`); + } + this.major = rs.uint32(); + this.minor = rs.uint32(); + + // 12 unknown bytes. + rs.skip(12); + + // Read in creation & modification date. + this.created = new Date(1000*rs.uint32()); + this.modified = new Date(1000*rs.uint32()); + + // Update the major version of the index used. + const index = this.index; + index.major = rs.uint32(); + + // Read in where we can find the file index and the holes. Note that + // this is specific to the serialization of a dbpf, we only need this + // when parsing so we won't store these values on the dbpf itself. + const indexCount = this.indexCount = rs.uint32(); + const indexOffset = this.indexOffset = rs.uint32(); + const indexSize = this.indexSize = rs.uint32(); + const holesCount = rs.uint32(); + const holesOffset = rs.uint32(); + const holesSize = rs.uint32(); + + // Read in the minor version of the index, for some weird reason this + // comes here in the header. + index.minor = rs.uint32(); + + // Read in all entries from the file index. It's very important that + // we update the length of all entries first so that the index knows + // how much entries it should parse! + const entries = this.entries = []; + entries.length = this.indexCount; + + } + // ## parse(buff) // Decodes the DBPF file from the given buffer. parse(buff) { @@ -111,7 +228,7 @@ const DBPF = module.exports = class DBPF { // it's just easier. save(opts) { if (typeof opts === 'string') { - opts = {"file": opts}; + opts = { file: opts }; } this.modified = new Date(); let buff = this.toBuffer(opts); @@ -171,31 +288,37 @@ const DBPF = module.exports = class DBPF { dir.push(list.length); } list.push({ - "type": entry.type, - "group": entry.group, - "instance": entry.instance, - "buffer": buffer, - "compressed": entry.compressed, - "fileSize": fileSize, - "compressedSize": buffer.byteLength + type: entry.type, + group: entry.group, + instance: entry.instance, + buffer: buffer, + compressed: entry.compressed, + fileSize: fileSize, + compressedSize: buffer.byteLength }); - } else if (entry.raw) { + } else { + + // Allright, the entry has not been decoded into a file yet. + // Check if has even been read. If not the case we will need + // to do this first. + if (!entry.raw) { + entry.readRaw(); + } + if (entry.compressed) { dir.push(list.length); } list.push({ - "type": entry.type, - "group": entry.group, - "instance": entry.instance, - "buffer": entry.raw, - "compressed": entry.compressed, - "fileSize": entry.fileSize, - "compressedSize": entry.compressedSize + type: entry.type, + group: entry.group, + instance: entry.instance, + buffer: entry.raw, + compressed: entry.compressed, + fileSize: entry.fileSize, + compressedSize: entry.compressedSize }); - } else { - throw new Error('Entry has no buffer and no file! Can\'t serialize it!'); } } @@ -215,13 +338,13 @@ const DBPF = module.exports = class DBPF { } list.push({ - "type": 0xE86B1EEF, - "group": 0xE86B1EEF, - "instance": 0x286B1F03, - "buffer": buff, - "compressed": false, - "fileSize": buff.byteLength, - "compressedSize": buff.byteLength + type: 0xE86B1EEF, + group: 0xE86B1EEF, + instance: 0x286B1F03, + buffer: buff, + compressed: false, + fileSize: buff.byteLength, + compressedSize: buff.byteLength }); } @@ -327,10 +450,10 @@ const DBPF = module.exports = class DBPF { // Allright, first entry is of type "SIZE MEM CRC", we assume that // all following entries are as well. all.push({ - "mem": slice.readUInt32LE(8), - "type": entry.type, - "entry": entry, - "index": 0 + mem: slice.readUInt32LE(8), + type: entry.type, + entry: entry, + index: 0 }); let index = size; buff = buff.slice(size); @@ -339,10 +462,10 @@ const DBPF = module.exports = class DBPF { let slice = buff.slice(0, size); let mem = slice.readUInt32LE(8); all.push({ - "mem": slice.readUInt32LE(8), - "type": entry.type, - "entry": entry, - "index": index + mem: slice.readUInt32LE(8), + type: entry.type, + entry, + index, }); index += size; buff = buff.slice(size); @@ -380,9 +503,9 @@ const DBPF = module.exports = class DBPF { let index = raw.indexOf(hex); if (index > -1) { out[i].push({ - "class": cClass[entry.type], - "entry": entry, - "index": index + class: cClass[entry.type], + entry, + index, }); } } @@ -463,11 +586,14 @@ class Index { // An index is always tied to a DBPF file, so we'll have mutual // references. - Object.defineProperty(this, 'dbpf', {"value": dbpf}); + Object.defineProperty(this, 'dbpf', { + value: dbpf, + enumerable: false, + writable: false, + }); // Store the raw array of all entries. This array should always be // treated by reference because it is shared with the DBPF class. - this.entries = dbpf.entries; this.entriesById = Object.create(null); // Versioning of the index. By default we use 7.0 in SC4. @@ -476,6 +602,11 @@ class Index { } + // ## get entries() + get entries() { + return this.dbpf.entries; + } + // ## get(tgi) // Returns an entry by tgi get(tgi) { @@ -488,16 +619,15 @@ class Index { // buffer! This is because an index is inherently intertwined with a DBPF. parse(rs, opts = {}) { - let entries = this.entries; + let { dbpf, entries } = this; let byId = this.entriesById; let length = entries.length; let dir = null; for (let i = 0; i < length; i++) { - let entry = entries[i] = new Entry(); + let entry = entries[i] = new Entry(dbpf); entry.parse(rs, { - "major": this.major, - "minor": this.minor, - "buffer": rs.buffer + major: this.major, + minor: this.minor, }); byId[ entry.id ] = entry; @@ -513,14 +643,14 @@ class Index { // accordingly. if (dir) { let parsed = parseDir(dir, { - "major": this.major, - "minor": this.minor + major: this.major, + minor: this.minor }); // Now find all compressed entries and update their compressed // sizes so that we can store the raw buffers correctly afterwards. for (let id in parsed) { - let {size} = parsed[id]; + let { size } = parsed[id]; let entry = this.get(id); entry.compressed = true; entry.fileSize = size; @@ -541,8 +671,13 @@ class Index { // it will be parsed appropriately. class Entry { - // ## constructor() - constructor() { + // ## constructor(dbpf) + constructor(dbpf) { + Object.defineProperty(this, 'dbpf',{ + value: dbpf, + enumerable: false, + witable: false, + }); this.type = 0; this.group = 0; this.instance = 0; @@ -617,6 +752,9 @@ class Entry { // Returns the decompressed raw entry buffer. If the entry is not // compressed, then the buffer is returned as is. decompress() { + if (!this.raw) { + this.readRaw(); + } let buff = this.raw; if (this.compressed) { buff = decompress(buff); @@ -624,25 +762,31 @@ class Entry { return buff; } + // ## readRaw() + // **Synchronously** reads the entry's raw buffer and stores it in the + // "raw" property. + readRaw() { + + // Find the raw entry buffer inside the DBPF file. We'll do this in a + // sync way, but we should support doing this asynchronously as well. + this.raw = this.dbpf.readBytesSync(this.offset, this.compressedSize); + + } + // ## read() // Tries to convert the raw buffer of the entry into a known file type. If // this fails, we'll simply return the raw buffer, but decompressed if the // entry was compressed. read() { - // If the entry was already read, don't read it again. + // If the entry was already read, don't read it again. Note that it's + // possible to dispose the entry to free up some memory if required. if (this.file) { return this.file; } - // No raw buffer stored? Unable to read the entry then. - if (!this.raw) { - throw new Error( - 'No raw buffer set for the entry! Hence cannot read it!' - ); - } - // If the entry is compressed, decompress it. + this.readRaw(); let buff = this.decompress(); // Now check for known file types. @@ -653,7 +797,7 @@ class Entry { return buff; } else { let file = this.file = new Klass(); - file.parse(buff, {"entry": this}); + file.parse(buff, { entry: this }); return file; } @@ -664,6 +808,21 @@ class Entry { return tgi(this.type, this.group, this.instance); } + // ## [util.inspect.custom]() + // If we console.log an entry in node we want to convert the TGI to their + // hex equivalents so that it's easier to debug. + [util.inspect.custom]() { + return { + type: hex(this.type), + group: hex(this.group), + instance: hex(this.instance), + fileSize: this.fileSize, + compressedSize: this.compressedSize, + offset: this.offset, + compressed: this.compressed, + }; + } + } // Export on the DBPF class. @@ -672,25 +831,39 @@ DBPF.Entry = Entry; // # parseDir(dir, opts) // Helper function for parsing the "dir" entry. Returns a json object. function parseDir(dir, opts = {}) { - const { major = 7, minor = 0} = opts; - const rs = new Stream(dir.raw); - const byteLength = major === 7 && minor === 1 ? 20 : 16; - const n = dir.fileSize / byteLength; + const { + major = 7, + minor = 0, + } = opts; + let { dbpf } = dir; + + // Read in the bytes for the dir. + let size = dir.fileSize; + let offset = dir.offset; + let byteLength = major === 7 && minor === 1 ? 20 : 16; + let n = size / byteLength; + let buffer = dbpf.readBytesSync(offset, size); + let rs = new Stream(buffer); + + // Now create the dir entry. let out = Object.create(null); for (let i = 0; i < n; i++) { let type = rs.uint32(); let group = rs.uint32(); let instance = rs.uint32(); let size = rs.uint32(); - if (byteLength > 16) this.skip(4); + if (byteLength > 16) { + rs.skip(4); + } let id = tgi(type, group, instance); out[id] = { - "type": type, - "group": group, - "instance": instance, - "id": id, - "size": size + type, + group, + instance, + id, + size, }; } return out; -} \ No newline at end of file + +} diff --git a/test/dbpf-test.js b/test/dbpf-test.js index aea8f41..71d78f2 100644 --- a/test/dbpf-test.js +++ b/test/dbpf-test.js @@ -15,19 +15,48 @@ const { ZoneType, cClass } = require('../lib/enums'); describe('A DBPF file', function() { - it('should be parsed', function() { + it('parses from a file', function() { let file = path.resolve(__dirname, 'files/cement.sc4lot'); - let buff = fs.readFileSync(file); // Parse the dbpf. + let dbpf = new DBPF(file); + + // Find an entry and verify that it gets read correctly. + let entry = dbpf.find(entry => { + return ( + entry.type === 0x6534284a && + entry.group === 0xa8fbd372 && + entry.instance === 0x8a73e853 + ); + }); + let exemplar = entry.read(); + expect(+exemplar.prop(0x10)).to.equal(0x10); + + }); + + it('parses from a buffer', function() { + + let file = path.resolve(__dirname, 'files/cement.sc4lot'); + let buff = fs.readFileSync(file); let dbpf = new DBPF(buff); + // Find an entry and verify that it gets read correctly. + let entry = dbpf.find(entry => { + return ( + entry.type === 0x6534284a && + entry.group === 0xa8fbd372 && + entry.instance === 0x8a73e853 + ); + }); + let exemplar = entry.read(); + expect(+exemplar.prop(0x10)).to.equal(0x10); + }); it('should be serialized to a buffer', function() { let file = path.resolve(__dirname, 'files/cement.sc4lot'); - let dbpf = new DBPF(fs.readFileSync(file)); + let dbpf = new DBPF(file); // Serialize the DBPF into a buffer and immediately parse again so // that we can compare. diff --git a/test/file-index-test.js b/test/file-index-test.js index 523aa39..febee49 100644 --- a/test/file-index-test.js +++ b/test/file-index-test.js @@ -1,11 +1,10 @@ // # file-index-test.js "use strict"; -const chai = require('chai'); -const expect = chai.expect; const path = require('path'); - +const { expect } = require('chai'); const Index = require('../lib/file-index.js'); const FileType = require('../lib/file-types.js'); +const dir = path.join(__dirname, 'files'); describe('The file index', function() { @@ -40,4 +39,12 @@ describe('The file index', function() { }); -}); \ No newline at end of file + it.only('indexes all building and prop families', async function() { + + let nybt = path.join(dir, 'NYBT Gracie Manor'); + let index = new Index(nybt); + await index.build(); + + }); + +}); From df451090fd583b51695c1ca2475e526b57c57bab Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 12:14:48 +0100 Subject: [PATCH 10/39] Free dbpf memory A method has been added to free up the memory a DBPF is taking up. Useful for indexing a large amount of DBPF files: we can simply unload the ones we no longer need. --- lib/dbpf.js | 148 ++++++++++++++++++++-------------------------- test/dbpf-test.js | 33 +++++++++++ 2 files changed, 97 insertions(+), 84 deletions(-) diff --git a/lib/dbpf.js b/lib/dbpf.js index 69ebd74..5fead45 100644 --- a/lib/dbpf.js +++ b/lib/dbpf.js @@ -28,12 +28,17 @@ const DBPF = module.exports = class DBPF { // DBPF Magic number. By default this is DBPF, but we'll allow the // user to change this so that a dbpf can be de-activated. this.id = 'DBPF'; - this.file = null; - this.buffer = null; + + // If the file specified is actually a buffer, store that we don't + // have a file. Note that this is not recommended: we need to be able + // to load and unload DBPFs on the fly because we simply cannot load + // them all into memory! if (Buffer.isBuffer(file)) { + this.file = null; this.buffer = file; } else { this.file = file; + this.buffer = null; } // Versioning of DBPF. In SC4 we always use DBPF 1.0. @@ -53,23 +58,9 @@ const DBPF = module.exports = class DBPF { this.indexOffset = 0; this.indexSize = 0; - // If a buffer or file was specified, parse the dbpf from it. Note: - // we've specified that it's *way* faster to read the file in at once. - // The problem is that we can't load an entire plugins directory into - // memory and keep it there. So if we don't want to load the file - // right away, it should be *explicitly* disabled, which is what we - // should do in that case... - if (!opts.async) { - if (this.file || this.buffer) { - if (this.file && !opts.lazy) { - this.buffer = fs.readFileSync(this.file); - } - this.readSync(); - } - } else { - throw new Error( - 'Async reading of DBPF files is not supported yet!', - ); + // If the user specified a file, parse the DBPF right away. + if (file) { + this.parse(); } } @@ -94,37 +85,63 @@ const DBPF = module.exports = class DBPF { return entry; } - // ## readBytesSync(offset, length) + // ## free() + // This method unloads the underlying buffer of the dbpf so that it can be + // garbage collected to free up some memory. This is useful if we just + // needed the DBPF for indexing but are not planning to use it soon. In + // that case the cache can decide to free up the dbpf and only read it in + // again upon the next read. + free() { + if (!this.file) { + console.warn([ + 'No file is set.', + 'This means you will no longer be able to use this DBPF!' + ].join(' ')); + } + + // Delete the buffer & loop all our entries so that we unload those as + // well. + this.buffer = null; + for (let entry of this) { + entry.free(); + } + return this; + + } + + // ## readBytes(offset, length) // Returns a buffer contain the bytes starting at offset and with the // given length. We use this method so that we can use a buffer or a file // as underlying source interchangeably. - readBytesSync(offset, length) { - if (this.buffer) { - let { buffer } = this; - return Buffer.from(buffer.buffer, buffer.offset+offset, length); - } else { - let fd = fs.openSync(this.file); - let buffer = Buffer.allocUnsafe(length); - fs.readSync(fd, buffer, 0, length, offset); - fs.closeSync(fd); - return buffer; + readBytes(offset, length) { + + // If we don't have a buffer, but we do have a file - which can happen + // if the DBPF was "unloaded" to free up memory - we'll have to load + // it in memory again. Note that it's your responsibility to free up + // memory the DBPF is taking up. You can use the `.free()` method for + // that. + let { buffer } = this; + if (!buffer) { + buffer = this.buffer = fs.readFileSync(this.file); } + return Buffer.from(buffer.buffer, buffer.offset+offset, length); + } - // ## readSync() + // ## parse() // Reads in the DBPF in a *synchronous* way. That's useful if you're // testing stuff out, but for bulk reading you should use the async // reading. - readSync() { + parse() { // First of all we need to read the header, and only the header. From // this we can derive where to find the index so that we can parse the // entries from it. - this.parseHeader(this.readBytesSync(0, 96)); + this.parseHeader(this.readBytes(0, 96)); // Header is parsed which means we now know the offset of the index. // Let's parse the index as well then. - let buffer = this.readBytesSync(this.indexOffset, this.indexSize); + let buffer = this.readBytes(this.indexOffset, this.indexSize); let rs = new Stream(buffer); this.index.parse(rs); @@ -175,54 +192,6 @@ const DBPF = module.exports = class DBPF { } - // ## parse(buff) - // Decodes the DBPF file from the given buffer. - parse(buff) { - - let rs = new Stream(buff); - this.id = rs.string(4); - this.major = rs.uint32(); - this.minor = rs.uint32(); - - // 12 unknown bytes. - rs.skip(12); - - // Read in creation & modification date. - this.created = new Date(1000*rs.uint32()); - this.modified = new Date(1000*rs.uint32()); - - // Update the major version of the index used. - const index = this.index; - index.major = rs.uint32(); - - // Read in where we can find the file index and the holes. Note that - // this is specific to the serialization of a dbpf, we only need this - // when parsing so we won't store these values on the dbpf itself. - const indexCount = rs.uint32(); - const indexOffset = rs.uint32(); - const indexSize = rs.uint32(); - const holesCount = rs.uint32(); - const holesOffset = rs.uint32(); - const holesSize = rs.uint32(); - - // Read in the minor version of the index, for some weird reason this - // comes here in the header. - index.minor = rs.uint32(); - rs.skip(4); - - // Read in all entries from the file index. It's very important that - // we update the length of all entries first so that the index knows - // how much entries it should parse! - const entries = this.entries; - entries.length = 0; - entries.length = indexCount; - rs.jump(indexOffset); - index.parse(rs); - - return this; - - } - // ## save(opts) // Saves the DBPF to a file. Note: we're going to do this in a sync way, // it's just easier. @@ -696,6 +665,15 @@ class Entry { } + // ## free() + // Frees up the memory the entry is taking up. Useful when DBPFs get + // unloaded to not take up too much memory. + free() { + this.raw = null; + this.file = null; + return this; + } + // ## get tgi() get tgi() { return [ this.type, this.group, this.instance ]; } set tgi(tgi) { @@ -769,7 +747,7 @@ class Entry { // Find the raw entry buffer inside the DBPF file. We'll do this in a // sync way, but we should support doing this asynchronously as well. - this.raw = this.dbpf.readBytesSync(this.offset, this.compressedSize); + this.raw = this.dbpf.readBytes(this.offset, this.compressedSize); } @@ -820,6 +798,8 @@ class Entry { compressedSize: this.compressedSize, offset: this.offset, compressed: this.compressed, + file: String(this.file), + raw: this.raw ? '[Object Buffer]' : null, }; } @@ -842,7 +822,7 @@ function parseDir(dir, opts = {}) { let offset = dir.offset; let byteLength = major === 7 && minor === 1 ? 20 : 16; let n = size / byteLength; - let buffer = dbpf.readBytesSync(offset, size); + let buffer = dbpf.readBytes(offset, size); let rs = new Stream(buffer); // Now create the dir entry. diff --git a/test/dbpf-test.js b/test/dbpf-test.js index 71d78f2..cf7a88c 100644 --- a/test/dbpf-test.js +++ b/test/dbpf-test.js @@ -54,6 +54,39 @@ describe('A DBPF file', function() { }); + it('frees memory the DBPF is taking up', function() { + + // Read in the DBPF and make sure all entries are properly read. + let file = path.resolve(__dirname, 'files/cement.sc4lot'); + let dbpf = new DBPF(file); + expect(dbpf.buffer).to.be.ok; + for (let entry of dbpf) { + entry.read(); + expect(entry.raw).to.be.ok; + } + let entry = dbpf.find(entry => { + return ( + entry.type === 0x6534284a && + entry.group === 0xa8fbd372 && + entry.instance === 0x8a73e853 + ); + }); + + // Free up the DBPF memory. + dbpf.free(); + expect(dbpf.buffer).to.be.null; + for (let entry of dbpf) { + expect(entry.raw).to.be.null; + expect(entry.file).to.be.null; + } + + // Check that the DBPF gets automatically reloaded if we request to + // read an entry. + let exemplar = entry.read(); + expect(+exemplar.prop(0x10)).to.equal(0x10); + + }); + it('should be serialized to a buffer', function() { let file = path.resolve(__dirname, 'files/cement.sc4lot'); let dbpf = new DBPF(file); From dfa97aa499a092f59118b3f890efd8307ec3dbab Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 12:47:17 +0100 Subject: [PATCH 11/39] Use a LRU cache for the file index The FileIndex now uses a LRU cache which properly frees up memory for DBPF files depending on how often the entries in it are accessed. Note that you should make sure to provide sufficient memory, otherwise it will slow you down tremendously. Let's hope the cache here doesn't get in the way and provides proper memory management. --- lib/dbpf.js | 23 +++++-- lib/file-index.js | 129 ++++++++++++++-------------------------- package.json | 1 + test/file-index-test.js | 26 ++++++-- 4 files changed, 84 insertions(+), 95 deletions(-) diff --git a/lib/dbpf.js b/lib/dbpf.js index 5fead45..12b240b 100644 --- a/lib/dbpf.js +++ b/lib/dbpf.js @@ -2,6 +2,7 @@ "use strict"; const util = require('util'); const fs = require('fs'); +const { EventEmitter } = require('events'); const TYPES = require('./file-types'); const Stream = require('./stream'); const crc32 = require('./crc'); @@ -13,7 +14,7 @@ const { hex, tgi } = require('./util'); // A class that represents a DBPF file. A DBPF file is basically just a custom // file archive format, a bit like .zip etc. as it contains other files that // might be compressed etc. -const DBPF = module.exports = class DBPF { +const DBPF = module.exports = class DBPF extends EventEmitter { // ## constructor(file) // Constructs the dbpf for the given file. For backwards compatibility and @@ -27,6 +28,7 @@ const DBPF = module.exports = class DBPF { // DBPF Magic number. By default this is DBPF, but we'll allow the // user to change this so that a dbpf can be de-activated. + super(); this.id = 'DBPF'; // If the file specified is actually a buffer, store that we don't @@ -85,6 +87,15 @@ const DBPF = module.exports = class DBPF { return entry; } + // ## load() + // If the buffer is not loaded yet, load it. + load() { + if (!this.buffer) { + this.buffer = fs.readFileSync(this.file); + } + return this.buffer; + } + // ## free() // This method unloads the underlying buffer of the dbpf so that it can be // garbage collected to free up some memory. This is useful if we just @@ -120,10 +131,7 @@ const DBPF = module.exports = class DBPF { // it in memory again. Note that it's your responsibility to free up // memory the DBPF is taking up. You can use the `.free()` method for // that. - let { buffer } = this; - if (!buffer) { - buffer = this.buffer = fs.readFileSync(this.file); - } + let buffer = this.load(); return Buffer.from(buffer.buffer, buffer.offset+offset, length); } @@ -763,6 +771,11 @@ class Entry { return this.file; } + // Before reading the entry, we'll emit an event on the owning DBPF. + // As such a cache can keep track of which DBPF files are read from + // most often. + this.dbpf.emit('read'); + // If the entry is compressed, decompress it. this.readRaw(); let buff = this.decompress(); diff --git a/lib/file-index.js b/lib/file-index.js index 84e3d01..26060de 100644 --- a/lib/file-index.js +++ b/lib/file-index.js @@ -4,6 +4,7 @@ const fs = require('fs'); const path = require('path'); const os = require('os'); const bsearch = require('binary-search-bounds'); +const LRUCache = require('lru-cache'); const DBPF = require('./dbpf'); const { Entry } = DBPF; const {default:PQueue} = require('p-queue'); @@ -24,22 +25,36 @@ if (!fs.promises) { class FileIndex { // ## constructor(opts) - constructor(opts) { + constructor(opts = {}) { + + // Set up the cache that we'll use to free up memory of DBPF files + // that are not read often. + let { mem = Number(os.totalmem) } = opts; + this.cache = new LRUCache({ + max: 0.5*mem, + length(n, dbpf) { + return dbpf.buffer.byteLength; + }, + dispose(dbpf, n) { + dbpf.free(); + }, + }); // Our array containing all our records. This array will be sorted by // tgi. this.records = null; - // No options specified? Use the default plugins directory. Note that - // we should look for the SimCity_1.dat core files as well of course. - if (!opts) { + // No directory or files specified? Use the default plugins directory. + // Note that we should look for the SimCity_1.dat core files as well + // of course. + if (!opts.files && !opts.dirs) { let plugins = path.join( process.env.HOMEPATH, 'Documents/SimCity 4/Plugins', ); - opts = { + opts = Object.assign({ dirs: [plugins], - }; + }, opts); } // If the options are simply given as a string, consider it to be a @@ -104,23 +119,40 @@ class FileIndex { // meant for external use as this doesn't sort the records! async addToIndex(file) { - // Read in the file. + // Asynchronously load read in the file to add. As such we can make + // use of the OS's multithreading for reading files, even though JS is + // single threaded. let buff = await fs.promises.readFile(file); - const source = new Source(file); // Ensure that the file is a dbpf file, ignore otherwise. if (buff.toString('utf8', 0, 4) !== 'DBPF') { return; } - // Parse. + // Parse the DBPF. let dbpf = new DBPF(buff); + dbpf.file = file; for (let entry of dbpf.entries) { - let record = new Record(entry); - record.source = source; - this.records.push(record); + this.records.push(entry); } + // Important! Make sure to free the buffer, otherwise the `read` + // handler still has access to it and won't ever be cleared from + // memory! + buff = null; + + // Note done yet. Listen to how many times the entries of the DBPF are + // read so that we can update the cache with it. + this.cache.set(dbpf); + dbpf.on('read', () => { + if (!this.cache.has(dbpf)) { + if (!dbpf.buffer) { + dbpf.load(); + } + this.cache.set(dbpf); + } + }); + } // ## find(type, group, instance) @@ -162,77 +194,6 @@ class FileIndex { } module.exports = FileIndex; -// Object that we'll re-use to query so that we don't have to recreate it all -// the time. -const query = { - type: 0, - group: 0, - instance: 0 -}; - -// # Record -// Represents an record in the index. -class Record extends Entry { - - // ## constructor(entry) - // Creates the record from the underlying dbpf entry. The difference here - // is that we no longer store the raw buffer because we don't want to keep - // everything in memory. We'll provide functionality to read the file by - // ourselves. - constructor(entry) { - super(); - this.type = entry.type; - this.group = entry.group; - this.instance = entry.instance; - this.fileSize = entry.fileSize; - this.compressedSize = entry.compressedSize; - this.offset = entry.offset; - this.compressed = entry.compressed; - - // Additionally store a reference to the source file where we can find - // the entry as well. - this.source = null; - - } - - // ## read() - // Overrides the DBPF's read method because we no longer have a raw buffer - // set. We need to read that one in first. - read() { - - // Entry already read? Don't read it again. - if (this.file) return this.file; - if (this.raw) return this.raw; - - // Read from the file, but at the correct offset (and in a synchronous - // way). - let file = String(this.source); - let fd = fs.openSync(file, 'r'); - let buff = Buffer.allocUnsafe(this.compressedSize); - fs.readSync(fd, buff, 0, buff.byteLength, this.offset); - fs.closeSync(fd); - this.raw = buff; - - // Call the super method. - return super.read(); - - } - -} - -// # Source -// Represents a source file. We use this so that we don't have to include the -// long filename everywhere. Saves on memory because we can work with -// pointers, yeah well references. -class Source { - constructor(file) { - this.file = file; - } - toString() { - return this.file; - } -} - // # compare(a, b) // The function that we use for sorting all files in our index. This // effectively creates an *index* on all the records so that we can use a @@ -263,4 +224,4 @@ function collect(dir, all) { } } return all; -} \ No newline at end of file +} diff --git a/package.json b/package.json index ba0af7c..3c0acfb 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "commander": "^2.20.3", "ini": "^1.3.5", "inquirer": "^6.5.2", + "lru-cache": "^5.1.1", "node-addon-api": "^1.7.1", "node-gyp-build": "^4.2.1", "ora": "^3.4.0", diff --git a/test/file-index-test.js b/test/file-index-test.js index febee49..e5e109a 100644 --- a/test/file-index-test.js +++ b/test/file-index-test.js @@ -10,11 +10,9 @@ describe('The file index', function() { it('should index all files in a directory', async function() { - let index = new Index({ - "dirs": [ - path.resolve(__dirname, 'files/DarkNight_11KingStreetWest') - ] - }); + let index = new Index( + path.resolve(__dirname, 'files/DarkNight_11KingStreetWest') + ); // Build up the index. This is done asynchronously so that files can // be read in parallel while parsing. @@ -39,12 +37,28 @@ describe('The file index', function() { }); - it.only('indexes all building and prop families', async function() { + it('uses a memory limit for the cache', async function() { + + let nybt = path.join(dir, 'NYBT Gracie Manor'); + let index = new Index({ + dirs: [nybt], + mem: 1500000, + }); + await index.build(); + for (let entry of index.records) { + entry.read(); + } + + }); + + it.skip('indexes all building and prop families', async function() { let nybt = path.join(dir, 'NYBT Gracie Manor'); let index = new Index(nybt); await index.build(); + console.log(index); + }); }); From a89571785764d39ae5c7ebdfce806c8a1a7be43f Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 14:12:50 +0100 Subject: [PATCH 12/39] Index all building & prop families --- lib/file-index.js | 47 ++++++++++++++++++++++++++++++++++------- test/file-index-test.js | 8 ++++--- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/lib/file-index.js b/lib/file-index.js index 26060de..b51e488 100644 --- a/lib/file-index.js +++ b/lib/file-index.js @@ -5,9 +5,11 @@ const path = require('path'); const os = require('os'); const bsearch = require('binary-search-bounds'); const LRUCache = require('lru-cache'); -const DBPF = require('./dbpf'); -const { Entry } = DBPF; const {default:PQueue} = require('p-queue'); +const DBPF = require('./dbpf.js'); +const { FileType } = require('./enums.js'); + +const Family = 0x27812870; // Patch fs promises. if (!fs.promises) { @@ -44,6 +46,17 @@ class FileIndex { // tgi. this.records = null; + // A map that contains references to all building & prop families. + // Note that we cannot use a separate data structure for this because + // the order in which files get loaded is important here! + this.families = new Map(); + + // If the options are simply given as a string, consider it to be a + // directory. + if (typeof opts === 'string') { + opts = { dirs: [opts] }; + } + // No directory or files specified? Use the default plugins directory. // Note that we should look for the SimCity_1.dat core files as well // of course. @@ -57,12 +70,6 @@ class FileIndex { }, opts); } - // If the options are simply given as a string, consider it to be a - // directory. - if (typeof opts === 'string') { - opts = { dirs: [opts] }; - } - let files = this.files = []; if (opts.files) { files.push(...opts.files); @@ -134,6 +141,22 @@ class FileIndex { dbpf.file = file; for (let entry of dbpf.entries) { this.records.push(entry); + + // If the entry is an exemplar, we'll parse it right away to check + // if it contains a building or prop family. + if (entry.type === FileType.Exemplar) { + let exemplar = entry.read(); + let family = exemplar.prop(Family); + if (family) { + let [IID] = family.value; + let arr = this.families.get(IID); + if (!arr) { + this.families.set(IID, arr = []); + } + arr.push(entry); + } + } + } // Important! Make sure to free the buffer, otherwise the `read` @@ -191,6 +214,14 @@ class FileIndex { return out; } + // ## family(id) + // Checks if the a prop or building family exists with the given IID and + // if so returns the family array. + family(id) { + let arr = this.families.get(id); + return arr || null; + } + } module.exports = FileIndex; diff --git a/test/file-index-test.js b/test/file-index-test.js index e5e109a..4e4c953 100644 --- a/test/file-index-test.js +++ b/test/file-index-test.js @@ -51,13 +51,15 @@ describe('The file index', function() { }); - it.skip('indexes all building and prop families', async function() { + it('indexes all building and prop families', async function() { let nybt = path.join(dir, 'NYBT Gracie Manor'); let index = new Index(nybt); await index.build(); - - console.log(index); + let { families } = index; + expect(families).to.have.length(2); + expect(index.family(0x5484CA20)).to.have.length(4); + expect(index.family(0x5484CA1F)).to.have.length(4); }); From 5b0077ca4c11e58557ec626d4a1f705748c96d42 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 15:28:28 +0100 Subject: [PATCH 13/39] Write outline of a skyline function --- lib/dbpf.js | 4 +- lib/exemplar.js | 7 ++ lib/file-index.js | 5 ++ lib/file-types.js | 1 + lib/lot-object.js | 5 ++ lib/savegame.js | 3 + lib/skyline.js | 164 ++++++++++++++++++++++++++++++++++++++++++++++ test/plop-test.js | 33 ++++++++-- 8 files changed, 215 insertions(+), 7 deletions(-) create mode 100644 lib/skyline.js diff --git a/lib/dbpf.js b/lib/dbpf.js index 12b240b..32f788f 100644 --- a/lib/dbpf.js +++ b/lib/dbpf.js @@ -528,8 +528,9 @@ DBPF.FileTypes = Object.create(null); // Register our known filetypes. For now only Exemplars and LotFiles. More // might be added. +const Exemplar = require('./exemplar.js'); DBPF.register([ - require('./exemplar'), + Exemplar, require('./lot').Array, require('./building').Array, require('./prop').Array, @@ -542,6 +543,7 @@ DBPF.register([ require('./lot-developer-file'), require('./com-serializer-file'), ]); +DBPF.register(FileType.Cohort, Exemplar); // Register the different sim grids. We use the same class for multiple type // ids, so we need to register under id manually. diff --git a/lib/exemplar.js b/lib/exemplar.js index 700e9c9..fc3b1ac 100644 --- a/lib/exemplar.js +++ b/lib/exemplar.js @@ -102,6 +102,13 @@ class Exemplar { return this.table[ key ]; } + // ## value(key) + // Helper function for directly accessing the value of a property. + value(key) { + let prop = this.prop(key); + return prop ? prop.value : undefined; + } + // ## parse(buff) // Parses an exemplar file from a buffer. parse(buff) { diff --git a/lib/file-index.js b/lib/file-index.js index b51e488..be608cf 100644 --- a/lib/file-index.js +++ b/lib/file-index.js @@ -46,6 +46,10 @@ class FileIndex { // tgi. this.records = null; + // We'll also keep track of all exemplar records because we'll often + // need them and it can be cumbersome to loop just about everything. + this.exemplars = []; + // A map that contains references to all building & prop families. // Note that we cannot use a separate data structure for this because // the order in which files get loaded is important here! @@ -145,6 +149,7 @@ class FileIndex { // If the entry is an exemplar, we'll parse it right away to check // if it contains a building or prop family. if (entry.type === FileType.Exemplar) { + this.exemplars.push(entry); let exemplar = entry.read(); let family = exemplar.prop(Family); if (family) { diff --git a/lib/file-types.js b/lib/file-types.js index 176e3f1..e356e1a 100644 --- a/lib/file-types.js +++ b/lib/file-types.js @@ -30,6 +30,7 @@ // Cohort TypeID=05342861 const TYPES = module.exports = { "Exemplar": 0x6534284A, + "Cohort": 0x05342861, "DIR": 0xE86B1EEF, "PNG": 0x856DDBAC, diff --git a/lib/lot-object.js b/lib/lot-object.js index 022243d..8cef5c6 100644 --- a/lib/lot-object.js +++ b/lib/lot-object.js @@ -33,6 +33,11 @@ class LotObject { get z() { return this.values[5]/scale; } set z(value) { this.values[5] = Math.round(scale*value); } + // ## get IID() + get IID() { + return this.values[12]; + } + } module.exports = LotObject; \ No newline at end of file diff --git a/lib/savegame.js b/lib/savegame.js index 42f4d8d..f583cf1 100644 --- a/lib/savegame.js +++ b/lib/savegame.js @@ -55,6 +55,9 @@ module.exports = class Savegame extends DBPF { let entry = this.getByType(FileType.ZoneDeveloperFile); return entry ? entry.read() : null; } + get zones() { + return this.zoneDeveloperFile; + } // ## get lotDeveloperFile() get lotDeveloperFile() { diff --git a/lib/skyline.js b/lib/skyline.js new file mode 100644 index 0000000..72cc3fa --- /dev/null +++ b/lib/skyline.js @@ -0,0 +1,164 @@ +// # skyline.js +"use strict"; +const bsearch = require('binary-search-bounds'); +const LotConfigPropertySize = 0x88edc790; +const OccupantSize = 0x27812810; + +// # skyline(opts) +// Just for fun: plop a random skyline. +function skyline(opts) { + let { + city, + center = [], + radius + } = opts; + + // Make sure to index all lots by height. + let lots = indexLots(city); + + // Create our skyline function that will determine the maximum height + // based on the distance from the center. + let fn = makeSkylineFunction({ + center, + radius, + }); + + // Now loop all city tiles & plop away. + let zones = city.dbpf.zones; + const { xSize, zSize } = zones; + for (let x = 0; x < xSize; x++) { + outer: + for (let z = 0; z < zSize; z++) { + + // First of all make sure there's no lot yet on this tile. Not + // going to overplop lots. + if (zones.cells[x][z]) { + continue; + } + + // Calculate the maximum height for this tile & select the + // appropriate lot range. + let max = fn(x, z); + let last = bsearch.ge(lots, { height: max }, compare); + + // No suitable lots found to plop? Pity, go on. + if (last === 0) { + continue; + } + + // Now pick a random lot from the suitable lots. We will favor + // higher lots more to create a nicer effect. + let index = Math.floor(last * Math.random()**0.75); + let lot = lots[index].entry; + + // Cool, an appropriate lot was found, but we're not done yet. + // It's possible if there's no space to plop the lot, we're not + // going to bother. Perhaps that we can retry later, but ok. + // Note: we'll still have to take into account the rotation as + // well here! + let [width, depth] = lot.read().value(LotConfigPropertySize); + for (let i = 0; i < width; i++) { + for (let j = 0; j < depth; j++) { + let xx = x + i; + let zz = z + j; + let row = zones.cells[xx]; + if (row && row[zz]) { + continue outer; + } + } + } + + // Cool, we got space left to plop the lot. Just do it baby. + city.grow({ + x, + z, + }); + + } + } + +} +module.exports = skyline; + +// ## makeSkylineFunction(opts) +// Factory for the skyline function that returns an appropriate height for the +// given (x, z) tile. +function makeSkylineFunction(opts) { + let { + center: [ + cx = 32, + cz = 32, + ], + min = 10, + max = 400, + radius = 16, + } = opts; + let diff = (max-min); + return function(x, z) { + let t = Math.sqrt((cx-x)**2 + (cz-z)**2) / radius; + if (t > radius) { + return -1; + } else { + return min+diff*Math.exp(-((2*t)**2)); + } + }; +} + +// ## indexLots(city) +// Creates an index of all lots by height that are available to the city. +function indexLots(city) { + let { index } = city; + let lots = []; + + // Loop every exemplar that we have indexed. If its a lot configurations + // exemplar, then read it so that we can find the building that appears on + // the lot. + for (let entry of index.exemplars) { + let file = entry.read(); + + // Not a Lot Configurations exemplar? Don't bother. + if (+file.prop(0x10) !== 0x10) { + continue; + } + + // Find the building on the lot. + let lotObjects = file.lotObjects; + let rep = lotObjects.find(({ type }) => type === 0x00); + let IID = rep.IID; + + // Check if the building belongs to a family. + let family = index.family(IID); + if (family) { + let pivot = family[0].read(); + let prop = pivot.prop(OccupantSize); + if (!prop) { + let { parent } = pivot; + pivot = index.find(parent).read(); + } + if (!pivot) { + console.warn('Could not find the size for a lot!'); + continue; + } + let [width, height, depth] = pivot.prop(OccupantSize).value; + lots.push({ + entry, + height, + }); + } else { + + // TODO... + + } + + } + + lots.sort(compare); + return lots; + +} + +// ## compare(a, b) +// The function we use to sort all lots. +function compare(a, b) { + return a.height - b.height; +} diff --git a/test/plop-test.js b/test/plop-test.js index c12c313..b2841bd 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -10,14 +10,17 @@ const Stream = require('../lib/stream'); const crc32 = require('../lib/crc'); const { hex, chunk, split } = require('../lib/util'); const { ZoneType, FileType, cClass, SimGrid } = require('../lib/enums'); -const Index = require('../lib/file-index.js'); +const CityManager = require('../lib/city-manager.js'); +const FileIndex = require('../lib/file-index.js'); const Savegame = require('../lib/savegame'); const Lot = require('../lib/lot'); const Building = require('../lib/building'); +const skyline = require('../lib/skyline.js'); const HOME = process.env.HOMEPATH; +const PLUGINS = path.resolve(HOME, 'documents/SimCity 4/plugins'); const REGION = path.resolve(HOME, 'documents/SimCity 4/regions/experiments'); -describe.skip('A city manager', function() { +describe('A city manager', function() { it.skip('should decode the cSC4Occupant class', function() { @@ -145,7 +148,7 @@ describe.skip('A city manager', function() { }); - it('should play with the grids in an established city', async function() { + it.skip('should play with the grids in an established city', async function() { let buff = fs.readFileSync(path.resolve(__dirname, 'files/City - Established.sc4')); let dbpf = new Savegame(buff); @@ -216,7 +219,7 @@ describe.skip('A city manager', function() { // Beware!! If the tracts are not set correctly we've created immortal // flora. Probably when deleting within a tract the game only looks for // stuff in that tract. That's quite logical actually. - it('should create forested streets', async function() { + it.skip('should create forested streets', async function() { let buff = fs.readFileSync(path.resolve(__dirname, 'files/City - Million Trees.sc4')); let dbpf = new Savegame(buff); @@ -257,7 +260,7 @@ describe.skip('A city manager', function() { }); - it('should move a building', async function() { + it.skip('should move a building', async function() { let buff = fs.readFileSync(path.resolve(__dirname, 'files/City - Move bitch.sc4')); let dbpf = new Savegame(buff); @@ -291,4 +294,22 @@ describe.skip('A city manager', function() { }); -}); \ No newline at end of file + it.only('builds a skyline', async function() { + + let dir = path.join(__dirname, 'files'); + let file = path.join(dir, 'City - Plopsaland.sc4'); + let nybt = path.join(PLUGINS, 'NYBT'); + let dbpf = new Savegame(file); + let index = new FileIndex(nybt); + await index.build(); + let city = new CityManager({ + dbpf, + index, + }); + + // Create the skyline in the city. + skyline({ city }); + + }); + +}); From d7c0614b8ad63ce28798cbfbbdfbd1830c351cc3 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 16:05:14 +0100 Subject: [PATCH 14/39] Create city with skyline function --- lib/city-manager.js | 97 ++++++++++++++++++++++++++++++++++++++- lib/skyline.js | 11 +++-- test/city-manager-test.js | 35 ++++++++++++-- test/file-index-test.js | 4 +- test/plop-test.js | 10 ++-- 5 files changed, 140 insertions(+), 17 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 08906cb..aea44f1 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -106,6 +106,30 @@ class CityManager { } + // ## getProperty(file, key) + // Helper function for getting a property from an exemplar, taking into + // account that the exemplar may possibly inherit from a parent cohort. + // Perhaps that we should put this in the index though... + getProperty(file, key) { + let prop = file.prop(key); + if (!prop) { + let { parent } = file; + let entry = this.index.find(parent); + if (entry) { + let file = entry.read(); + prop = file.prop(key); + } + } + return prop; + } + + // ## getPropertyValue(file, prop) + // Returns the direct value for the given property. + getPropertyValue(file, key) { + let prop = this.getProperty(file, key); + return prop ? prop.value : undefined; + } + // ## plop(opts) // Behold, the mother of all functions. This function allows to plop any // lot anywhere in the city. Note that this function expects a *building* @@ -184,6 +208,77 @@ class CityManager { } + // ## grow(opts) + // This method is similar to the `plop()` method, but this time it starts + // from a *Lot Configurations* exemplar, not a ploppable building exemplar + // - which is how the game does it. From then on the logic is pretty much + // the same. + grow(opts) { + + let { tgi, exemplar } = opts; + let record = exemplar ? exemplar : this.index.find(tgi); + if (!record) { + throw new Error( + `Exemplar ${ JSON.stringify(tgi) } not found!`, + ); + } + + // Ensure that the exemplar that was specified. + let props = record.read(); + if (+props.value(ExemplarType) !== LotConfigurations) { + throw new Error([ + 'The exemplar is not a lot configurations exemplar!', + 'The `.grow()` function expects a lot exemplar!' + ].join(' ')); + } + + // Find the appropriate building exemplar. Note that it's possible + // that the building belongs to a family. In that case we'll pick a + // random building from the family. + let IID = props.lotObjects.find(({ type }) => type === 0x00).IID; + let family = this.index.family(IID); + let buildingExemplar; + if (family) { + let buildings = family.filter(entry => { + let file = entry.read(); + return file.value(0x10) === 0x02; + }); + buildingExemplar = buildings[ Math.round()*buildings.length | 0]; + } else { + let exemplars = this.index.findAllTI(FileType.Exemplar, IID) + .filter(entry => { + let file = entry.read(); + return file.value(0x10) === 0x02; + }); + buildingExemplar = exemplars[ exemplars.length-1 ]; + } + + // Same logic now as for plopping buildings. + let { orientation = 0 } = opts; + let lot = this.createLot({ + exemplar: record, + x: opts.x, + z: opts.z, + orientation, + building: buildingExemplar.instance, + }); + + // Loop all objects on the lot such and insert them. + let { lotObjects } = props; + for (let lotObject of lotObjects) { + switch (lotObject.type) { + case 0x00: + this.createBuilding({ + lot, + lotObject, + exemplar: buildingExemplar, + }); + break; + } + } + + } + // ## createLot(opts) // Creates a new lot object from the given options when plopping a lot. createLot(opts) { @@ -259,7 +354,7 @@ class CityManager { createBuilding(opts) { let { lot, lotObject, exemplar } = opts; let file = exemplar.read(); - let [width, height, depth] = file.prop(OccupantSize).value; + let [width, height, depth] = this.getPropertyValue(file, OccupantSize); let { orientation, x, y, z } = lotObject; // Find the rectangle the building is occupying on the lot, where the diff --git a/lib/skyline.js b/lib/skyline.js index 72cc3fa..dbbecdd 100644 --- a/lib/skyline.js +++ b/lib/skyline.js @@ -62,7 +62,7 @@ function skyline(opts) { let xx = x + i; let zz = z + j; let row = zones.cells[xx]; - if (row && row[zz]) { + if (!row || (row && row[zz])) { continue outer; } } @@ -70,6 +70,7 @@ function skyline(opts) { // Cool, we got space left to plop the lot. Just do it baby. city.grow({ + exemplar: lot, x, z, }); @@ -89,14 +90,14 @@ function makeSkylineFunction(opts) { cx = 32, cz = 32, ], - min = 10, - max = 400, - radius = 16, + min = 30, + max = 200, + radius = 32, } = opts; let diff = (max-min); return function(x, z) { let t = Math.sqrt((cx-x)**2 + (cz-z)**2) / radius; - if (t > radius) { + if (t > 1) { return -1; } else { return min+diff*Math.exp(-((2*t)**2)); diff --git a/test/city-manager-test.js b/test/city-manager-test.js index f472ba3..11ffaa0 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -41,9 +41,36 @@ describe('A city manager', function() { }); + context('#grow()', function() { + + it.only('grows a lot', async function() { + + this.slow(1000); + + let dir = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Plugins'); + let index = new FileIndex(dir); + await index.build(); + + // Create the city manager. + let game = path.join(__dirname, 'files/City - 432.sc4'); + let city = new CityManager({ index }); + city.load(game); + + // Grow a lot. + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0x8fcc0f62], + x: 10, + z: 10, + orientation: 0, + }); + + }); + + }); + context('#plop()', function() { - it.only('plops a ploppable lot', async function() { + it('plops a ploppable lot', async function() { this.slow(1000); @@ -61,10 +88,10 @@ describe('A city manager', function() { city.load(game); // Plop it baby. - for (let i = 0; i < 4; i++) { + for (let i = 0; i < 1; i++) { city.plop({ - // tgi: [0x6534284a, 0xd60100c4, 0x483248bb], - tgi: [0x6534284a,0x76fbb03a,0x290dc058], + tgi: [0x6534284a, 0xd60100c4, 0x483248bb], + // tgi: [0x6534284a,0x76fbb03a,0x290dc058], x: (1+i)*8, z: 8, orientation: i % 4, diff --git a/test/file-index-test.js b/test/file-index-test.js index 4e4c953..6ef95b2 100644 --- a/test/file-index-test.js +++ b/test/file-index-test.js @@ -39,7 +39,7 @@ describe('The file index', function() { it('uses a memory limit for the cache', async function() { - let nybt = path.join(dir, 'NYBT Gracie Manor'); + let nybt = path.join(dir, 'NYBT/Aaron Graham/NYBT Gracie Manor'); let index = new Index({ dirs: [nybt], mem: 1500000, @@ -53,7 +53,7 @@ describe('The file index', function() { it('indexes all building and prop families', async function() { - let nybt = path.join(dir, 'NYBT Gracie Manor'); + let nybt = path.join(dir, 'NYBT/Aaron Graham/NYBT Gracie Manor'); let index = new Index(nybt); await index.build(); let { families } = index; diff --git a/test/plop-test.js b/test/plop-test.js index b2841bd..84ef661 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -299,17 +299,17 @@ describe('A city manager', function() { let dir = path.join(__dirname, 'files'); let file = path.join(dir, 'City - Plopsaland.sc4'); let nybt = path.join(PLUGINS, 'NYBT'); - let dbpf = new Savegame(file); let index = new FileIndex(nybt); await index.build(); - let city = new CityManager({ - dbpf, - index, - }); + let city = new CityManager({ index }); + city.load(file); // Create the skyline in the city. skyline({ city }); + // Save the city. + city.save({ file: path.join(REGION, 'City - Skyline.sc4') }); + }); }); From 57554e36df3f4a2f4c9e827429878e10cbb16a81 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 16:41:46 +0100 Subject: [PATCH 15/39] Enlarge gui canvas --- lib/gui/gui.js | 7 ++++--- lib/gui/index.html | 6 ++++++ lib/skyline.js | 15 ++++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/gui/gui.js b/lib/gui/gui.js index b7f01d6..3f61124 100644 --- a/lib/gui/gui.js +++ b/lib/gui/gui.js @@ -7,8 +7,8 @@ const Savegame = require('../savegame'); // Do some stuff. let renderer = window.r = new Renderer({ - "width": 1000, - "height": 750, + "width": window.innerWidth-2, + "height": window.innerHeight-4, "position": [16*64, 1000, 16*64], "target": [16*64,0,16*64], "perspective": true, @@ -19,7 +19,8 @@ document.body.append(renderer.el); // Append a box. -let file = path.resolve(__dirname, '../../test/files/city.sc4'); +// let file = path.resolve(__dirname, '../../test/files/city.sc4'); +let file = path.resolve(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments/City - Skyline.sc4'); let dbpf = new Savegame(fs.readFileSync(file)); // Read all buildings. diff --git a/lib/gui/index.html b/lib/gui/index.html index 682b6b6..9c3c544 100644 --- a/lib/gui/index.html +++ b/lib/gui/index.html @@ -3,6 +3,12 @@ + diff --git a/lib/skyline.js b/lib/skyline.js index dbbecdd..216bfdb 100644 --- a/lib/skyline.js +++ b/lib/skyline.js @@ -36,6 +36,11 @@ function skyline(opts) { continue; } + // Random voids. + if (Math.random() < 0.2) { + continue; + } + // Calculate the maximum height for this tile & select the // appropriate lot range. let max = fn(x, z); @@ -90,18 +95,14 @@ function makeSkylineFunction(opts) { cx = 32, cz = 32, ], - min = 30, + min = 15, max = 200, radius = 32, } = opts; let diff = (max-min); return function(x, z) { let t = Math.sqrt((cx-x)**2 + (cz-z)**2) / radius; - if (t > 1) { - return -1; - } else { - return min+diff*Math.exp(-((2*t)**2)); - } + return min+diff*Math.exp(-((2*t)**2)); }; } @@ -118,7 +119,7 @@ function indexLots(city) { let file = entry.read(); // Not a Lot Configurations exemplar? Don't bother. - if (+file.prop(0x10) !== 0x10) { + if (file.value(0x10) !== 0x10) { continue; } From 65a267411b05de2f703f9ad23d66a81efa62471b Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 16:46:38 +0100 Subject: [PATCH 16/39] Limit geometries created in gui --- lib/gui/gui.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/gui/gui.js b/lib/gui/gui.js index 3f61124..eec8bcd 100644 --- a/lib/gui/gui.js +++ b/lib/gui/gui.js @@ -43,8 +43,9 @@ for (let building of buildings) { let z = building.minZ + depth/2; // Create a box for it. - let box = new three.BoxBufferGeometry(width, height, depth); + let box = new three.BoxBufferGeometry(1, 1, 1); let mesh = new three.Mesh(box, material); + mesh.scale.set(width, height, depth); renderer.add(mesh); mesh.position.set(x, y, z); } From a907fd93804e3a7671075bd62ce5fb12635a2134 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 12 Mar 2020 17:08:48 +0100 Subject: [PATCH 17/39] Allow random lot orientations --- lib/gui/gui.js | 2 +- lib/skyline.js | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/gui/gui.js b/lib/gui/gui.js index eec8bcd..2751bf0 100644 --- a/lib/gui/gui.js +++ b/lib/gui/gui.js @@ -31,6 +31,7 @@ let material = new three.MeshStandardMaterial({ }); let buildings = dbpf.buildingFile; +let box = new three.BoxBufferGeometry(1, 1, 1); for (let building of buildings) { let width = building.maxX - building.minX; @@ -43,7 +44,6 @@ for (let building of buildings) { let z = building.minZ + depth/2; // Create a box for it. - let box = new three.BoxBufferGeometry(1, 1, 1); let mesh = new three.Mesh(box, material); mesh.scale.set(width, height, depth); renderer.add(mesh); diff --git a/lib/skyline.js b/lib/skyline.js index 216bfdb..a59ba67 100644 --- a/lib/skyline.js +++ b/lib/skyline.js @@ -37,7 +37,7 @@ function skyline(opts) { } // Random voids. - if (Math.random() < 0.2) { + if (Math.random() < 0.1) { continue; } @@ -61,7 +61,11 @@ function skyline(opts) { // going to bother. Perhaps that we can retry later, but ok. // Note: we'll still have to take into account the rotation as // well here! + let orientation = Math.random()*4 | 0; let [width, depth] = lot.read().value(LotConfigPropertySize); + if (orientation % 2 === 1) { + [width, depth] = [depth, width]; + } for (let i = 0; i < width; i++) { for (let j = 0; j < depth; j++) { let xx = x + i; @@ -78,6 +82,7 @@ function skyline(opts) { exemplar: lot, x, z, + orientation, }); } From aa8e0116d04c45847d621befb0ad687eac6c9399 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Fri, 13 Mar 2020 10:08:38 +0100 Subject: [PATCH 18/39] Use inheritance chain for families --- lib/city-manager.js | 26 +++++++-------------- lib/file-index.js | 57 +++++++++++++++++++++++++++++++++++---------- lib/gui/gui.js | 3 ++- lib/skyline.js | 43 +++++++++++++++++++++++----------- test/plop-test.js | 19 +++++++++++++-- 5 files changed, 102 insertions(+), 46 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index aea44f1..96ca32e 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -107,27 +107,17 @@ class CityManager { } // ## getProperty(file, key) - // Helper function for getting a property from an exemplar, taking into - // account that the exemplar may possibly inherit from a parent cohort. - // Perhaps that we should put this in the index though... + // Helper function for getting a property from an exemplar, taking into + // account the inheritance chain. It's the index that is actually + // responsible for this though. getProperty(file, key) { - let prop = file.prop(key); - if (!prop) { - let { parent } = file; - let entry = this.index.find(parent); - if (entry) { - let file = entry.read(); - prop = file.prop(key); - } - } - return prop; + return this.index.getProperty(file, key); } // ## getPropertyValue(file, prop) // Returns the direct value for the given property. getPropertyValue(file, key) { - let prop = this.getProperty(file, key); - return prop ? prop.value : undefined; + return this.index.getPropertyValue(file, key); } // ## plop(opts) @@ -241,14 +231,16 @@ class CityManager { if (family) { let buildings = family.filter(entry => { let file = entry.read(); - return file.value(0x10) === 0x02; + let type = this.index.getPropertyValue(file, 0x10); + return type === 0x02; }); buildingExemplar = buildings[ Math.round()*buildings.length | 0]; } else { let exemplars = this.index.findAllTI(FileType.Exemplar, IID) .filter(entry => { let file = entry.read(); - return file.value(0x10) === 0x02; + let type = this.index.getPropertyValue(file, 0x10); + return type === 0x02; }); buildingExemplar = exemplars[ exemplars.length-1 ]; } diff --git a/lib/file-index.js b/lib/file-index.js index be608cf..15db5b5 100644 --- a/lib/file-index.js +++ b/lib/file-index.js @@ -123,6 +123,21 @@ class FileIndex { // respective arrays. this.records.sort(compare); + // Loop all exemplars so that we can find the building and prop + // families. + for (let entry of this.exemplars) { + let file = entry.read(); + let family = this.getPropertyValue(file, Family); + if (family) { + let [IID] = family; + let arr = this.families.get(IID); + if (!arr) { + this.families.set(IID, arr = []); + } + arr.push(entry); + } + } + } // ## async addToIndex(file) @@ -146,20 +161,11 @@ class FileIndex { for (let entry of dbpf.entries) { this.records.push(entry); - // If the entry is an exemplar, we'll parse it right away to check - // if it contains a building or prop family. + // If the entry is an exemplar, push it in all our exemplars. Note + // that we don't read it yet and check for a family: we need to + // have access to **all** exemplars first! if (entry.type === FileType.Exemplar) { this.exemplars.push(entry); - let exemplar = entry.read(); - let family = exemplar.prop(Family); - if (family) { - let [IID] = family.value; - let arr = this.families.get(IID); - if (!arr) { - this.families.set(IID, arr = []); - } - arr.push(entry); - } } } @@ -227,6 +233,33 @@ class FileIndex { return arr || null; } + // ## getProperty(exemplar, key) + // This function accepts a parsed exemplar file and looks up the property + // with the given key. If the property doesn't exist, then tries to look + // it up in the parent cohort and so on all the way up. + getProperty(exemplar, key) { + let original = exemplar; + let prop = exemplar.prop(key); + while (!prop && exemplar.parent[0]) { + let { parent } = exemplar; + let entry = this.find(parent); + if (!entry) { + break; + } + exemplar = entry.read(); + prop = exemplar.prop(key); + } + return prop; + } + + // ## getPropertyValue(exemplar, key) + // Directly returns the value for the given property in the exemplar. If + // it doesn't exist, looks it up in the parent cohort. + getPropertyValue(exemplar, key) { + let prop = this.getProperty(exemplar, key); + return prop ? prop.value : undefined; + } + } module.exports = FileIndex; diff --git a/lib/gui/gui.js b/lib/gui/gui.js index 2751bf0..e9a8dc3 100644 --- a/lib/gui/gui.js +++ b/lib/gui/gui.js @@ -20,7 +20,8 @@ document.body.append(renderer.el); // Append a box. // let file = path.resolve(__dirname, '../../test/files/city.sc4'); -let file = path.resolve(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments/City - Skyline.sc4'); +// let file = path.resolve(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments/City - Skyline.sc4'); +let file = path.resolve(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments/City - Plopsaland.sc4'); let dbpf = new Savegame(fs.readFileSync(file)); // Read all buildings. diff --git a/lib/skyline.js b/lib/skyline.js index a59ba67..11ddf52 100644 --- a/lib/skyline.js +++ b/lib/skyline.js @@ -1,6 +1,8 @@ // # skyline.js "use strict"; const bsearch = require('binary-search-bounds'); +const { FileType } = require('./enums.js'); +const { hex } = require('./util.js'); const LotConfigPropertySize = 0x88edc790; const OccupantSize = 0x27812810; @@ -37,9 +39,9 @@ function skyline(opts) { } // Random voids. - if (Math.random() < 0.1) { - continue; - } + // if (Math.random() < 0.1) { + // continue; + // } // Calculate the maximum height for this tile & select the // appropriate lot range. @@ -135,28 +137,41 @@ function indexLots(city) { // Check if the building belongs to a family. let family = index.family(IID); + let building; if (family) { let pivot = family[0].read(); - let prop = pivot.prop(OccupantSize); - if (!prop) { - let { parent } = pivot; - pivot = index.find(parent).read(); - } if (!pivot) { console.warn('Could not find the size for a lot!'); continue; } - let [width, height, depth] = pivot.prop(OccupantSize).value; - lots.push({ - entry, - height, - }); + building = pivot; } else { - // TODO... + // Cool, the building on the lot does not belong to a family. In + // that case just find it. + let buildings = index.findAllTI(FileType.Exemplar, IID) + .filter(entry => { + let file = entry.read(); + let type = index.getPropertyValue(file, 0x10); + return type === 0x02; + }); + if (buildings.length === 0) { + console.warn([ + `No building found with IID ${ hex(IID) }, but referenced on lot ${ hex(entry.instance) }!` + ].join(' ')); + continue; + } + building = buildings[0].read(); } + let prop = index.getProperty(building, OccupantSize); + let [width, height, depth] = prop.value; + lots.push({ + entry, + height, + }); + } lots.sort(compare); diff --git a/test/plop-test.js b/test/plop-test.js index 84ef661..eaa4f1e 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -296,11 +296,25 @@ describe('A city manager', function() { it.only('builds a skyline', async function() { + this.timeout(0); + let dir = path.join(__dirname, 'files'); let file = path.join(dir, 'City - Plopsaland.sc4'); let nybt = path.join(PLUGINS, 'NYBT'); - let index = new FileIndex(nybt); + + let c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; + // let index = new FileIndex(nybt); + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + // path.join(c, 'SimCity_2.dat'), + // path.join(c, 'SimCity_3.dat'), + // path.join(c, 'SimCity_4.dat'), + // path.join(c, 'SimCity_5.dat'), + ] + }); await index.build(); + let city = new CityManager({ index }); city.load(file); @@ -308,7 +322,8 @@ describe('A city manager', function() { skyline({ city }); // Save the city. - city.save({ file: path.join(REGION, 'City - Skyline.sc4') }); + let out = path.join(REGION, 'City - Plopsaland.sc4'); + await city.save({ file: out }); }); From 47b92d918406c432069599462290ae8411bb6ecc Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Fri, 13 Mar 2020 10:47:42 +0100 Subject: [PATCH 19/39] Filter out residential & commercial --- lib/skyline.js | 65 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/lib/skyline.js b/lib/skyline.js index 11ddf52..ff25508 100644 --- a/lib/skyline.js +++ b/lib/skyline.js @@ -5,6 +5,7 @@ const { FileType } = require('./enums.js'); const { hex } = require('./util.js'); const LotConfigPropertySize = 0x88edc790; const OccupantSize = 0x27812810; +const OccupantGroups = 0xaa1dd396; // # skyline(opts) // Just for fun: plop a random skyline. @@ -72,10 +73,21 @@ function skyline(opts) { for (let j = 0; j < depth; j++) { let xx = x + i; let zz = z + j; - let row = zones.cells[xx]; - if (!row || (row && row[zz])) { + if (xx >= xSize || zz >= zSize) { continue outer; } + let row = zones.cells[xx][zz]; + if (row) { + continue outer; + } + + // Mock a 8x8 city grid. + let grid = 16; + let half = Math.floor(grid/2); + if (xx % grid === half || zz % grid === half) { + continue outer; + } + } } @@ -103,7 +115,7 @@ function makeSkylineFunction(opts) { cz = 32, ], min = 15, - max = 200, + max = 400, radius = 32, } = opts; let diff = (max-min); @@ -126,7 +138,14 @@ function indexLots(city) { let file = entry.read(); // Not a Lot Configurations exemplar? Don't bother. - if (file.value(0x10) !== 0x10) { + if (index.getPropertyValue(file, 0x10) !== 0x10) { + continue; + } + + // Check the lot size. We're not going to include lots that are too + // big. + let [xSize, zSize] = index.getPropertyValue(file, LotConfigPropertySize); + if (Math.max(xSize, zSize) > 5) { continue; } @@ -165,8 +184,46 @@ function indexLots(city) { } + // Now check the occupant groups of the building. We'll only want + // residential and commerical for now. + let groups = index.getPropertyValue(building, OccupantGroups); + if (!groups) { + continue; + } + let required = [ + // 0x00011010, + 0x00011020, + 0x00011030, + // 0x00013110, + 0x00013120, + 0x00013130, + 0x00013320, + 0x00013330, + ]; + let filter = groups.filter(group => { + return required.includes(group); + }); + + // Filter out the New York style. + let filter2 = groups.filter(group => { + return true; + return group === 0x2001 || group === 0x2002; + }); + + if (filter.length === 0 || filter2.length === 0) { + continue; + } + let prop = index.getProperty(building, OccupantSize); let [width, height, depth] = prop.value; + + // Check the filling degree of the building. If that's not + // sufficiently high, don't include the building. + let fill = (width*depth) / (xSize*zSize*16*16); + if (fill < 0.5 + 0.5*height/400) { + continue; + } + lots.push({ entry, height, From 02cd677bf11fd5f392b4a17b899a3fa995838f78 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Fri, 13 Mar 2020 15:59:26 +0100 Subject: [PATCH 20/39] Create nicer skylines --- lib/city-manager.js | 2 +- lib/skyline.js | 25 +++++++++++++++---------- test/plop-test.js | 8 ++++---- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 96ca32e..2c6c9aa 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -234,7 +234,7 @@ class CityManager { let type = this.index.getPropertyValue(file, 0x10); return type === 0x02; }); - buildingExemplar = buildings[ Math.round()*buildings.length | 0]; + buildingExemplar = buildings[ Math.random()*buildings.length | 0]; } else { let exemplars = this.index.findAllTI(FileType.Exemplar, IID) .filter(entry => { diff --git a/lib/skyline.js b/lib/skyline.js index ff25508..98c7fce 100644 --- a/lib/skyline.js +++ b/lib/skyline.js @@ -48,16 +48,18 @@ function skyline(opts) { // appropriate lot range. let max = fn(x, z); let last = bsearch.ge(lots, { height: max }, compare); - + let first = bsearch.le(lots, { height: 0.25*max }, compare)+1; + let diff = last - first; + // No suitable lots found to plop? Pity, go on. - if (last === 0) { + if (diff === 0) { continue; } // Now pick a random lot from the suitable lots. We will favor // higher lots more to create a nicer effect. - let index = Math.floor(last * Math.random()**0.75); - let lot = lots[index].entry; + let index = Math.floor(diff * Math.random()**0.75); + let lot = lots[first + index].entry; // Cool, an appropriate lot was found, but we're not done yet. // It's possible if there's no space to plop the lot, we're not @@ -195,9 +197,9 @@ function indexLots(city) { 0x00011020, 0x00011030, // 0x00013110, - 0x00013120, + // 0x00013120, 0x00013130, - 0x00013320, + // 0x00013320, 0x00013330, ]; let filter = groups.filter(group => { @@ -207,7 +209,8 @@ function indexLots(city) { // Filter out the New York style. let filter2 = groups.filter(group => { return true; - return group === 0x2001 || group === 0x2002; + // return group === 0x2000 || group === 0x2001; + // return group === 0x2001 || group === 0x2002; }); if (filter.length === 0 || filter2.length === 0) { @@ -219,9 +222,11 @@ function indexLots(city) { // Check the filling degree of the building. If that's not // sufficiently high, don't include the building. - let fill = (width*depth) / (xSize*zSize*16*16); - if (fill < 0.5 + 0.5*height/400) { - continue; + if (xSize * zSize > 2) { + let fill = (width*depth) / (xSize*zSize*16*16); + if (fill < 0.5 + 0.5*height/400) { + continue; + } } lots.push({ diff --git a/test/plop-test.js b/test/plop-test.js index eaa4f1e..de28c25 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -307,10 +307,10 @@ describe('A city manager', function() { let index = new FileIndex({ files: [ path.join(c, 'SimCity_1.dat'), - // path.join(c, 'SimCity_2.dat'), - // path.join(c, 'SimCity_3.dat'), - // path.join(c, 'SimCity_4.dat'), - // path.join(c, 'SimCity_5.dat'), + path.join(c, 'SimCity_2.dat'), + path.join(c, 'SimCity_3.dat'), + path.join(c, 'SimCity_4.dat'), + path.join(c, 'SimCity_5.dat'), ] }); await index.build(); From dec6f7458838538b86134f5292a701e0cc7977db Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Fri, 13 Mar 2020 17:58:27 +0100 Subject: [PATCH 21/39] Add build method to share common functionality --- lib/city-manager.js | 192 ++++++++++++++++++++++++++------------ lib/lot-base-texture.js | 18 +++- lib/savegame.js | 3 + test/city-manager-test.js | 8 +- 4 files changed, 158 insertions(+), 63 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 2c6c9aa..1a6f293 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -131,12 +131,14 @@ class CityManager { // (1) First of all we need to find the T10 exemplar file with the // information to plop the lot. Most of the time this resides in an // .sc4lot file, but it doesn't have to. - let { tgi } = opts; - let record = this.index.find(tgi); - if (!record) { - throw new Error( - `Exemplar ${ JSON.stringify(tgi) } not found!`, - ); + let { tgi, building } = opts; + if (!building && tgi) { + building = this.index.find(tgi); + if (!building) { + throw new Error( + `Exemplar ${ JSON.stringify(tgi) } not found!`, + ); + } } // Check what type of exemplar we're dealing with. As explained by @@ -144,8 +146,8 @@ class CityManager { // growable buildings. Apparently ploppable buildings start from a // building exemplar and then we can look up according // LotConfiguration exemplar. - let exemplar = record.read(); - if (+exemplar.prop(ExemplarType) !== Buildings) { + let file = building.read(); + if (this.getPropertyValue(file, ExemplarType) !== Buildings) { throw new Error([ 'The exemplar is not a building exemplar!', 'The `.plop()` function expects a ploppable building exemplar!' @@ -156,46 +158,23 @@ class CityManager { // LotResourceKey & then based on that find the appropriate Building // exemplar. Note that we currently have no other choice than finding // everything with the same instance ID... - let IID = +exemplar.prop(LotResourceKey); + let IID = this.getPropertyValue(file, LotResourceKey); let exemplars = this.index.findAllTI(FileType.Exemplar, IID); let lotExemplar = exemplars.find(record => { let exemplar = record.read(); - return +exemplar.prop(ExemplarType) === LotConfigurations; + let type = this.getPropertyValue(exemplar, ExemplarType); + return type === LotConfigurations; }); - let lotConfig = lotExemplar.read(); - // Create the lot. It will automatically insert it into the zone - // developer file as well. - let { orientation = 0 } = opts; - let lot = this.createLot({ - exemplar: lotExemplar, + // Cool, we have both the building & the lot exemplar. Create the lot. + let lot = this.build({ + lot: lotExemplar, + building, x: opts.x, z: opts.z, - orientation, - building: IID, + orientation: opts.orientation, }); - // Now loop all objects on the lot such as the building, the props - // etc. and insert them. - let { lotObjects } = lotExemplar.read(); - for (let lotObject of lotObjects) { - switch (lotObject.type) { - case 0x00: - this.createBuilding({ - lot, - lotObject, - exemplar: record, - }); - break; - case 0x02: - this.createTexture({ - lot, - lotObject, - }); - break; - } - } - } // ## grow(opts) @@ -245,30 +224,71 @@ class CityManager { buildingExemplar = exemplars[ exemplars.length-1 ]; } - // Same logic now as for plopping buildings. - let { orientation = 0 } = opts; + // Now that we have both the building exemplar and as well as the lot + // exemplar we can create the lot and insert everything on it into the + // city. + let { x, z, orientation } = opts; + let lot = this.build({ + building: buildingExemplar, + lot: record, + x, + z, + orientation, + }); + + } + + // ## build(opts) + // This method is responsible for inserting all *physical* entities into + // the city such as a lot, a building, the props on the lot, the textures + // etc. It's not really meant for public use, you should use the `.plop()` + // or `.grow()` methods instead. It requires a lot exemplar and a building + // exemplar to be specified. It's the `.plop()` and `.grow()` methods that + // are responsible for deciding what building will be inserted. + build(opts) { + + // First of all create the lot record & insert it into the city. + let { + lot: lotExemplar, + building, + x = 0, + z = 0, + orientation = 0, + } = opts; let lot = this.createLot({ - exemplar: record, + exemplar: lotExemplar, + building: building.instance, x: opts.x, z: opts.z, orientation, - building: buildingExemplar.instance, }); // Loop all objects on the lot such and insert them. - let { lotObjects } = props; + let { lotObjects } = lotExemplar.read(); + let textures = []; for (let lotObject of lotObjects) { switch (lotObject.type) { case 0x00: this.createBuilding({ lot, lotObject, - exemplar: buildingExemplar, + exemplar: building, }); break; + case 0x02: + + // Note: We can't handle textures right away because they + // need to be put in a *single* BaseTexture entry. As such + // we'll simply collect them for now. + textures.push(lotObject); + break; } } + // At last return the created lot so that the calling function can + // modify the properties such as capcity, zoneWealth, zoneDensity etc. + return lot; + } // ## createLot(opts) @@ -280,7 +300,10 @@ class CityManager { let lots = dbpf.lotFile; let { exemplar, x, z, building, orientation = 0 } = opts; let file = exemplar.read(); - let [width, depth] = file.prop(LotConfigPropertySize).value; + let [width, depth] = this.getPropertyValue( + file, + LotConfigPropertySize + ); // Cool, we can now create a new lot entry. Note that we will need to // take into account the @@ -386,15 +409,7 @@ class CityManager { // Put the building in the index at the correct spot. let { dbpf } = this; - let index = dbpf.itemIndexFile; - for (let x = building.xMinTract; x <= building.xMaxTract; x++) { - for (let z = building.zMinTract; z <= building.zMaxTract; z++) { - index[x][z].push({ - mem: building.mem, - type: FileType.BuildingFile, - }); - } - } + this.addToItemIndex(building, FileType.BuildingFile); // Push in the file with all buildings. let buildings = dbpf.buildingFile; @@ -415,10 +430,71 @@ class CityManager { } // ## createTexture(opts) - // Creates a base texture. + // Creates a texture entry in the BaseTexture file of the city for the + // given lot. createTexture(opts) { - // let { lot, lotObject } = opts; - // let { orientation, x, y, z } = lotObject; + + // Create a new texture instance and copy some lot properties in it + // such as the tracts etc. + let { lot, textures } = opts; + let texture = new BaseTexture({ + mem: this.mem(), + xMinTract: lot.xMinTract, + zMinTract: lot.zMinTract, + xMaxTract: lot.xMaxTract, + zMaxTract: lot.zMaxTract, + minX: 16*lot.minX+0.1, + maxX: 16*lot.maxX-0.1, + minZ: 16*lot.minZ+0.1, + maxZ: 16*lot.maxZ-0.1, + + // TODO: This is only for flat cities, should use terrain queries + // later on! + minY: 270, + maxY: 270.1000061035156, + + }); + + // Add all required textures. + for (let def of textures) { + let { orientation, x, z, IID } = def; + texture.add({ + IID, + x: lot.minX + Math.floor(x), + z: lot.maxX + Math.floor(z), + orientation: (lot.orientation + orientation) % 4, + }); + } + + // Cool, now push the base texture in the city & update the + // COMSerializer as well. + let { dbpf } = this; + dbpf.textures.push(texture); + let com = dbpf.COMSerializerFile; + com.set(FileType.BaseTextureFile, dbpf.textures.length); + + // Update the item index as well. + this.addToItemIndex(texture, FileType.BaseTextureFile); + + // Return the base texture that we've created. + return texture; + + } + + // ## addToItemIndex(obj, type) + // Helper function for adding the given object - that exposes the tract + // coordinates - to the item index. + addToItemIndex(obj, type) { + let { dbpf } = this; + let index = dbpf.itemIndexFile; + for (let x = obj.xMinTract; x <= obj.xMaxTract; x++) { + for (let z = obj.zMinTract; z <= obj.zMaxTract; z++) { + index[x][z].push({ + mem: obj.mem, + type: type, + }); + } + } } } diff --git a/lib/lot-base-texture.js b/lib/lot-base-texture.js index b141cad..43204ae 100644 --- a/lib/lot-base-texture.js +++ b/lib/lot-base-texture.js @@ -10,8 +10,8 @@ const Type = require('./type'); // # LotBaseTexture class LotBaseTexture extends Type(FileType.BaseTextureFile) { - // ## constructor() - constructor() { + // ## constructor(opts) + constructor(opts) { super(); this.crc = 0x00000000; this.mem = 0x00000000; @@ -36,6 +36,7 @@ class LotBaseTexture extends Type(FileType.BaseTextureFile) { this.maxZ = this.maxY = this.maxX = 0; this.u10 = 0x02; this.textures = []; + Object.assign(this, opts); } // ## move(dx, dz) @@ -162,6 +163,14 @@ class LotBaseTexture extends Type(FileType.BaseTextureFile) { } + // ## add(opts) + // Adds a single texture into the array of all textures. + add(opts) { + let texture = new Texture(opts); + this.textures.push(texture); + return texture; + } + } module.exports = LotBaseTexture; @@ -175,8 +184,8 @@ Object.defineProperty(LotBaseTexture.Array.prototype, 'textures', { // # Texture class Texture { - // ## constructor() - constructor() { + // ## constructor(opts) + constructor(opts) { this.IID = 0x00000000; this.z = this.x = 0; this.orientation = 0; @@ -187,6 +196,7 @@ class Texture { this.u5 = 0x00; this.u6 = 0x00; this.u7 = 0x00; + Object.assign(this, opts); } // ## parse(rs) diff --git a/lib/savegame.js b/lib/savegame.js index f583cf1..804a597 100644 --- a/lib/savegame.js +++ b/lib/savegame.js @@ -40,6 +40,9 @@ module.exports = class Savegame extends DBPF { let entry = this.getByType(FileType.BaseTextureFile); return entry ? entry.read() : null; } + get textures() { + return this.baseTextureFile; + } // ## get itemIndexFile() get itemIndexFile() { diff --git a/test/city-manager-test.js b/test/city-manager-test.js index 11ffaa0..a7f7f1e 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -43,7 +43,7 @@ describe('A city manager', function() { context('#grow()', function() { - it.only('grows a lot', async function() { + it('grows a lot', async function() { this.slow(1000); @@ -56,6 +56,8 @@ describe('A city manager', function() { let city = new CityManager({ index }); city.load(game); + console.log(city.dbpf.textures[0]); + // Grow a lot. city.grow({ tgi: [0x6534284a,0xa8fbd372,0x8fcc0f62], @@ -64,6 +66,10 @@ describe('A city manager', function() { orientation: 0, }); + let regions = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments'); + let file = path.join(regions, 'City - Plopsaland.sc4'); + await city.save({ file }); + }); }); From ab8fb180dc91c9fd3c086ea7af49065a05de16a6 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Fri, 13 Mar 2020 18:12:15 +0100 Subject: [PATCH 22/39] Find exemplars for given IID, including families. --- lib/city-manager.js | 63 ++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 1a6f293..95c5a94 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -159,12 +159,7 @@ class CityManager { // exemplar. Note that we currently have no other choice than finding // everything with the same instance ID... let IID = this.getPropertyValue(file, LotResourceKey); - let exemplars = this.index.findAllTI(FileType.Exemplar, IID); - let lotExemplar = exemplars.find(record => { - let exemplar = record.read(); - let type = this.getPropertyValue(exemplar, ExemplarType); - return type === LotConfigurations; - }); + let lotExemplar = this.findExemplarOfType(IID, LotConfigurations); // Cool, we have both the building & the lot exemplar. Create the lot. let lot = this.build({ @@ -205,24 +200,7 @@ class CityManager { // that the building belongs to a family. In that case we'll pick a // random building from the family. let IID = props.lotObjects.find(({ type }) => type === 0x00).IID; - let family = this.index.family(IID); - let buildingExemplar; - if (family) { - let buildings = family.filter(entry => { - let file = entry.read(); - let type = this.index.getPropertyValue(file, 0x10); - return type === 0x02; - }); - buildingExemplar = buildings[ Math.random()*buildings.length | 0]; - } else { - let exemplars = this.index.findAllTI(FileType.Exemplar, IID) - .filter(entry => { - let file = entry.read(); - let type = this.index.getPropertyValue(file, 0x10); - return type === 0x02; - }); - buildingExemplar = exemplars[ exemplars.length-1 ]; - } + let buildingExemplar = this.findExemplarOfType(IID, 0x02); // Now that we have both the building exemplar and as well as the lot // exemplar we can create the lot and insert everything on it into the @@ -275,6 +253,11 @@ class CityManager { exemplar: building, }); break; + case 0x01: + this.createProp({ + lot, + lotObject, + }); case 0x02: // Note: We can't handle textures right away because they @@ -429,6 +412,17 @@ class CityManager { } + // ## createProp(opts) + // Creates a new prop record in and inserts it into the save game. Takes + // into account the position it should take up in a lot. + createProp({ lot, lotObject }) { + + // Note: in contrast to the building, we don't know yet what prop + // we're going to insert if we're dealing with a prop family. As such + // we'll check the families first. + + } + // ## createTexture(opts) // Creates a texture entry in the BaseTexture file of the city for the // given lot. @@ -497,6 +491,27 @@ class CityManager { } } + // ## findExemplarOfType(IID, type) + // Helper function that can find an exemplar with the given instance of + // the given type. It will make use of the families we have as well. + findExemplarOfType(IID, type) { + let { index } = this; + let family = index.family(IID); + const filter = entry => { + let file = entry.read(); + return index.getPropertyValue(file, 0x10) === type; + }; + if (family) { + let exemplars = family.filter(filter); + return exemplars[ Math.random()*exemplars.length | 0 ]; + } else { + let exemplars = index + .findAllTI(FileType.Exemplar, IID) + .filter(filter); + return exemplars[ exemplars.length-1 ]; + } + } + } module.exports = CityManager; From 539dd3accbd02de202cd54bf7d459acafda825c4 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Fri, 13 Mar 2020 19:09:33 +0100 Subject: [PATCH 23/39] Add basic functionality to show props --- lib/city-manager.js | 56 +++++++++++++++++++++++++++++++++++++++ lib/prop.js | 7 ++--- lib/savegame.js | 8 +++++- test/city-manager-test.js | 39 +++++++++++++++++++++++++-- 4 files changed, 104 insertions(+), 6 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 95c5a94..6394aba 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -5,6 +5,7 @@ const fs = require('fs'); const Savegame = require('./savegame.js'); const Lot = require('./lot.js'); const Building = require('./building.js'); +const Prop = require('./prop.js'); const BaseTexture = require('./lot-base-texture.js'); const { FileType } = require('./enums.js'); const SC4 = path.resolve(process.env.HOMEPATH, 'documents/SimCity 4'); @@ -420,6 +421,61 @@ class CityManager { // Note: in contrast to the building, we don't know yet what prop // we're going to insert if we're dealing with a prop family. As such // we'll check the families first. + let { IID, orientation, x, y, z } = lotObject; + let exemplar = this.findExemplarOfType(IID, 0x1e); + if (!exemplar) { + // console.warn(`Missing prop ${ IID }!`); + return; + } + let file = exemplar.read(); + let [width, height, depth] = this.getPropertyValue(file, OccupantSize); + + // Find the rectangle the prop is occupying & then position it + // correctly, taking into account the lot dimensions. + if (orientation % 2 === 1) { + [width, depth] = [depth, width]; + } + let rect = position({ + minX: 16*x - width/2, + maxX: 16*x + width/2, + minZ: 16*z - depth/2, + maxZ: 16*z + depth/2, + }, lot); + + // Create the prop. + let prop = new Prop({ + mem: this.mem(), + + ...rect, + minY: lot.yPos + y, + maxY: lot.yPos + y + height, + orientation: (orientation + lot.orientation) % 4, + + // Store the TGI of the prop. + TID: exemplar.type, + GID: exemplar.group, + IID: exemplar.instance, + IID1: exemplar.instance, + OID: this.mem(), + + appearance: 5, + state: 1, + + }); + setTract(prop); + + // Push in the file with all props. + let { dbpf } = this; + let { props } = dbpf; + props.push(prop); + + // Put the prop in the index. + this.addToItemIndex(prop, FileType.PropFile); + + // Update the COM serializer and we're done. + let com = dbpf.COMSerializerFile; + com.set(FileType.PropFile, props.length); + return props; } diff --git a/lib/prop.js b/lib/prop.js index 9fa4e73..93ec8c7 100644 --- a/lib/prop.js +++ b/lib/prop.js @@ -11,8 +11,8 @@ const Type = require('./type'); // Represents a single prop from the prop file. class Prop extends Type(FileType.PropFile) { - // ## constructor() - constructor() { + // ## constructor(opts) + constructor(opts) { super(); this.crc = 0x00000000; this.mem = 0x00000000; @@ -20,7 +20,7 @@ class Prop extends Type(FileType.PropFile) { this.minor = 0x0004; this.zot = 0x0000; this.unknown1 = 0x00; - this.appearance = 0x00; + this.appearance = 0b00000101; this.unknown2 = 0xA823821E; this.zMinTract = this.xMinTract = 0x00; this.zMaxTract = this.xMaxTract = 0x00; @@ -39,6 +39,7 @@ class Prop extends Type(FileType.PropFile) { this.lotType = 0x02; this.OID = 0x00000000; this.condition = 0x00; + Object.assign(this, opts); } // ## parse(rs) diff --git a/lib/savegame.js b/lib/savegame.js index 804a597..e0e8677 100644 --- a/lib/savegame.js +++ b/lib/savegame.js @@ -2,6 +2,7 @@ "use strict"; const DBPF = require('./dbpf'); const FileType = require('./file-types'); +const Prop = require('./prop.js'); // # Savegame() // A class specifically designed for some Savegame functionality. Obviously @@ -27,9 +28,14 @@ module.exports = class Savegame extends DBPF { } // ## get propFile() + // Getter for the propfile. If it doesn't exist yet, we'll create it. get propFile() { let entry = this.entries.find(x => x.type === FileType.PropFile); - return entry ? entry.read() : null; + if (!entry) { + let tgi = [FileType.PropFile, 698035483, 0]; + entry = this.add(tgi, new Prop.Array()); + } + return entry.read(); } get props() { return this.propFile; diff --git a/test/city-manager-test.js b/test/city-manager-test.js index a7f7f1e..0c02294 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -43,6 +43,8 @@ describe('A city manager', function() { context('#grow()', function() { + const regions = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments'); + it('grows a lot', async function() { this.slow(1000); @@ -56,7 +58,7 @@ describe('A city manager', function() { let city = new CityManager({ index }); city.load(game); - console.log(city.dbpf.textures[0]); + // console.log(city.dbpf.textures[0]); // Grow a lot. city.grow({ @@ -66,7 +68,40 @@ describe('A city manager', function() { orientation: 0, }); - let regions = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments'); + let file = path.join(regions, 'City - Plopsaland.sc4'); + await city.save({ file }); + + }); + + it.only('grows a lot with props', async function() { + + this.slow(1000); + + const dir = 'c:/GOG Games/Simcity 4 Deluxe Edition'; + let index = new FileIndex({ + files: [ + path.join(dir, 'Simcity_1.dat'), + path.join(dir, 'Simcity_2.dat'), + path.join(dir, 'Simcity_3.dat'), + path.join(dir, 'Simcity_4.dat'), + path.join(dir, 'Simcity_5.dat'), + ], + }); + await index.build(); + + // Create the city manager. + let game = path.join(__dirname, 'files/City - 432.sc4'); + let city = new CityManager({ index }); + city.load(game); + + // Grow a lot. + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0x600040d0], + x: 10, + z: 10, + orientation: 0, + }); + let file = path.join(regions, 'City - Plopsaland.sc4'); await city.save({ file }); From 5b75d7f2429d18bacb0affaa4736a07f16b9a388 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Sat, 14 Mar 2020 00:07:17 +0100 Subject: [PATCH 24/39] Try to get the textures to appear --- lib/city-manager.js | 24 ++++++++++++------ lib/lot-base-texture.js | 14 +++++------ test/plop-test.js | 55 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 6394aba..13c343a 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -269,6 +269,12 @@ class CityManager { } } + // Create the textures. + this.createTexture({ + lot, + textures, + }); + // At last return the created lot so that the calling function can // modify the properties such as capcity, zoneWealth, zoneDensity etc. return lot; @@ -484,15 +490,13 @@ class CityManager { // given lot. createTexture(opts) { - // Create a new texture instance and copy some lot properties in it - // such as the tracts etc. + // Create a new texture instance and copy some lot properties in it. let { lot, textures } = opts; let texture = new BaseTexture({ mem: this.mem(), - xMinTract: lot.xMinTract, - zMinTract: lot.zMinTract, - xMaxTract: lot.xMaxTract, - zMaxTract: lot.zMaxTract, + + // Apparently the "insets" the texture with 0.1, which gets + // rounded of when transformed into a Float32 by the way. minX: 16*lot.minX+0.1, maxX: 16*lot.maxX-0.1, minZ: 16*lot.minZ+0.1, @@ -504,16 +508,22 @@ class CityManager { maxY: 270.1000061035156, }); + setTract(texture); // Add all required textures. + let i = 0; for (let def of textures) { let { orientation, x, z, IID } = def; texture.add({ IID, x: lot.minX + Math.floor(x), - z: lot.maxX + Math.floor(z), + z: lot.minZ + Math.floor(z), orientation: (lot.orientation + orientation) % 4, + // priority: (1-i), + // u7: i, + }); + i++; } // Cool, now push the base texture in the city & update the diff --git a/lib/lot-base-texture.js b/lib/lot-base-texture.js index 43204ae..4370fc7 100644 --- a/lib/lot-base-texture.js +++ b/lib/lot-base-texture.js @@ -20,8 +20,8 @@ class LotBaseTexture extends Type(FileType.BaseTextureFile) { this.u1 = 0x00; this.u2 = 0x00; this.u3 = 0x00; - this.u4 = 0x00; - this.u5 = 0x00000000; + this.u4 = 0x05; + this.u5 = 0x497f6d9d; this.xMinTract = 0x40; this.zMinTract = 0x40; this.xMaxTract = 0x40; @@ -190,11 +190,11 @@ class Texture { this.z = this.x = 0; this.orientation = 0; this.priority = 0x00; - this.u2 = 0x00; - this.u3 = 0x00; - this.u4 = 0x00; - this.u5 = 0x00; - this.u6 = 0x00; + this.u2 = 0xff; + this.u3 = 0xff; + this.u4 = 0xff; + this.u5 = 0xff; + this.u6 = 0xff; this.u7 = 0x00; Object.assign(this, opts); } diff --git a/test/plop-test.js b/test/plop-test.js index de28c25..d0c6b7a 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -327,4 +327,59 @@ describe('A city manager', function() { }); + it.only('includes the textures when plopping', async function() { + + this.timeout(0); + + let dir = path.join(__dirname, 'files'); + let c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; + let file = path.join(dir, 'City - Textures.sc4'); + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + ], + dirs: [ + path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), + ], + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(file); + // console.log(city.dbpf.textures[0]); + + for (let i = 0; i < 9; i++) { + for (let j = 0; j < 9; j++) { + if (i === 1 && j === 1) { + continue; + } + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x: i, + z: j, + // orientation: i, + }); + } + } + + // console.table(city.dbpf.textures, [ + // 'u1', 'u2', 'u3', 'u4', 'u5', 'u6', 'u7', 'u8', 'u9', + // ]); + + // console.table([ + // city.dbpf.textures[0].textures[0], + // city.dbpf.textures[1].textures[0], + // ]); + + // console.table([ + // city.dbpf.textures[0].textures[1], + // city.dbpf.textures[1].textures[1], + // ]); + + // Save + let out = path.join(REGION, 'City - Textures.sc4'); + await city.save({ file: out }); + + }); + }); From 0f6bafdf693637cc33d8b7bbd328cae6ea76461b Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Sun, 15 Mar 2020 17:24:39 +0100 Subject: [PATCH 25/39] Insert base textures We're finally able to insert the base textures as well. Nice. --- lib/city-manager.js | 32 +++++++++++++++++++---------- lib/lot-object.js | 17 +++++++++++++++ test/plop-test.js | 50 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 86 insertions(+), 13 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 13c343a..8f958b2 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -18,6 +18,8 @@ const OccupantSize = 0x27812810; const LotConfigurations = 0x10; const LotResourceKey = 0xea260589; const LotConfigPropertySize = 0x88edc790; +const Wealth = 0x27812832; +const INSET = 0.1; // # CityManager // A class for performing operations on a certain city, such as plopping @@ -234,12 +236,19 @@ class CityManager { z = 0, orientation = 0, } = opts; + + // It's important to be able to know building properties to pass to + // the lot, such as the wealth etc. Therefore, read in the building. + let file = building.read(); + let zoneWealth = this.getPropertyValue(file, Wealth) || 0x00; + let lot = this.createLot({ exemplar: lotExemplar, building: building.instance, x: opts.x, z: opts.z, orientation, + zoneWealth, }); // Loop all objects on the lot such and insert them. @@ -315,9 +324,7 @@ class CityManager { width, depth, orientation, - - // Important! ZoneWealth cannot be set to 0, otherwise CTD! - zoneWealth: 0x03, + zoneWealth: opts.zoneWealth || 0x00, // Apparently jobCapacities is also required, otherwise CTD! The // capacity is stored I guess in the LotConfig exemplar, or @@ -495,17 +502,17 @@ class CityManager { let texture = new BaseTexture({ mem: this.mem(), - // Apparently the "insets" the texture with 0.1, which gets - // rounded of when transformed into a Float32 by the way. - minX: 16*lot.minX+0.1, - maxX: 16*lot.maxX-0.1, - minZ: 16*lot.minZ+0.1, - maxZ: 16*lot.maxZ-0.1, + // Apparently the game requires "insets" on the texture - which it + // sets to 0.1, which get rounded to Float32's by the way. + minX: 16*lot.minX + INSET, + maxX: 16*(lot.maxX+1) - INSET, + minZ: 16*lot.minZ + INSET, + maxZ: 16*(lot.maxZ+1) - INSET, // TODO: This is only for flat cities, should use terrain queries // later on! minY: 270, - maxY: 270.1000061035156, + maxY: 270 + INSET, }); setTract(texture); @@ -519,8 +526,11 @@ class CityManager { x: lot.minX + Math.floor(x), z: lot.minZ + Math.floor(z), orientation: (lot.orientation + orientation) % 4, + + // priority: 0, + + // priority: // priority: (1-i), - // u7: i, }); i++; diff --git a/lib/lot-object.js b/lib/lot-object.js index 8cef5c6..a305f13 100644 --- a/lib/lot-object.js +++ b/lib/lot-object.js @@ -33,6 +33,23 @@ class LotObject { get z() { return this.values[5]/scale; } set z(value) { this.values[5] = Math.round(scale*value); } + get minX() { return this.values[6]/scale; } + get minZ() { return this.values[7]/scale; } + get maxX() { return this.values[8]/scale; } + get maxZ() { return this.values[9]/scale; } + get usage() { return this.values[10]; } + + // ## get OID() + // The Object id is rep 12. The wiki says about this: + // 0xA0000000 = 0,2,4,6,8,a,c,e - Random one of these characters in case + // the other ID's are the same. + // 0x0BBBB000 = Object Family + // 0x00000CCC = Unique Object ID for this family. Incremental for similar + // objects. + get OID() { + return this.values[11]; + } + // ## get IID() get IID() { return this.values[12]; diff --git a/test/plop-test.js b/test/plop-test.js index d0c6b7a..29f630a 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -19,6 +19,7 @@ const skyline = require('../lib/skyline.js'); const HOME = process.env.HOMEPATH; const PLUGINS = path.resolve(HOME, 'documents/SimCity 4/plugins'); const REGION = path.resolve(HOME, 'documents/SimCity 4/regions/experiments'); +const c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; describe('A city manager', function() { @@ -327,12 +328,11 @@ describe('A city manager', function() { }); - it.only('includes the textures when plopping', async function() { + it.skip('includes the textures when plopping', async function() { this.timeout(0); let dir = path.join(__dirname, 'files'); - let c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; let file = path.join(dir, 'City - Textures.sc4'); let index = new FileIndex({ files: [ @@ -382,4 +382,50 @@ describe('A city manager', function() { }); + it.only('includes the base texture when plopping', async function() { + + this.timeout(0); + let dir = path.join(__dirname, 'files'); + let out = path.join(REGION, 'City - Base Textures.sc4'); + // let source = path.join(dir, 'City - Base Textures.sc4'); + let source = out; + + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + ], + dirs: [ + path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), + ], + }); + await index.build(); + + let float = x => { + return new Float32Array([x])[0]; + }; + + let city = new CityManager({ index }); + city.load(source); + + // for (let i = 0; i < 64; i++) { + // for (let j = 0; j < 64; j++) { + // if (i === 1 && j === 1) { + // continue; + // } + // city.grow({ + // tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + // x: i, + // z: j, + // // orientation: (i+j) % 4, + // }); + // } + // } + + let { textures } = city.dbpf; + console.log('Texture entries in the city:', textures.length); + + // await city.save({ file: out }); + + }); + }); From b3c1977b5f48a478ad59b6f88c6e6fa3ebb1bee2 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Sun, 15 Mar 2020 19:19:57 +0100 Subject: [PATCH 26/39] Correctly position textures in city --- lib/city-manager.js | 122 ++++++++++++++++++-------------------------- test/plop-test.js | 75 ++++++++++++++++++--------- 2 files changed, 100 insertions(+), 97 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 8f958b2..fba7af8 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -369,6 +369,11 @@ class CityManager { let [width, height, depth] = this.getPropertyValue(file, OccupantSize); let { orientation, x, y, z } = lotObject; + // First of all we need to find the *oriented* position of the + // building on the lot. This means that we need to take into account + // that the lot may have been rotated. + let [xx, zz] = orient([x, z], lot); + // Find the rectangle the building is occupying on the lot, where the // origin of the coordinate system is in the top-left corner. Note // that we already rotate the building into place **in the lot**. We @@ -377,12 +382,6 @@ class CityManager { if (orientation % 2 === 1) { [width, depth] = [depth, width]; } - let rect = position({ - minX: 16*x - width/2, - maxX: 16*x + width/2, - minZ: 16*z - depth/2, - maxZ: 16*z + depth/2, - }, lot); // Create the building. let building = new Building({ @@ -390,7 +389,10 @@ class CityManager { // Now use the **rotated** building rectangle and use it to // position the building appropriately. - ...rect, + minX: xx - width/2, + maxX: xx + width/2, + minZ: zz - depth/2, + maxZ: zz + depth/2, minY: lot.yPos + y, maxY: lot.yPos + y + height, orientation: (orientation + lot.orientation) % 4, @@ -448,18 +450,16 @@ class CityManager { if (orientation % 2 === 1) { [width, depth] = [depth, width]; } - let rect = position({ - minX: 16*x - width/2, - maxX: 16*x + width/2, - minZ: 16*z - depth/2, - maxZ: 16*z + depth/2, - }, lot); + let [xx, zz] = orient([x, z], lot); // Create the prop. let prop = new Prop({ mem: this.mem(), - ...rect, + minX: xx - width/2, + maxX: xx + width/2, + minZ: zz - depth/2, + maxZ: zz + depth/2, minY: lot.yPos + y, maxY: lot.yPos + y + height, orientation: (orientation + lot.orientation) % 4, @@ -518,22 +518,24 @@ class CityManager { setTract(texture); // Add all required textures. - let i = 0; + // let i = 0; for (let def of textures) { let { orientation, x, z, IID } = def; + let [xx, zz] = orient([x, z], lot, { bare: true }); texture.add({ IID, - x: lot.minX + Math.floor(x), - z: lot.minZ + Math.floor(z), + x: lot.minX + Math.floor(xx), + z: lot.minZ + Math.floor(zz), orientation: (lot.orientation + orientation) % 4, // priority: 0, // priority: // priority: (1-i), + // priority: Math.min(i, 1), }); - i++; + // i++; } // Cool, now push the base texture in the city & update the @@ -604,66 +606,42 @@ function setTract(obj) { obj.zMaxTract = 64 + Math.floor(obj.maxZ / zSize); } -// ## position(rect, lot) -// Modifies the given rectangle so that it is positioned correctly on the -// given lot. Returns an object { minX, maxX, minZ, maxZ } that can be easily -// assigned to the object. -function position(rect, lot) { - let { minX, maxX, minZ, maxZ } = rect; - function move({ minX, maxX, minZ, maxZ }) { - let x = 16*lot.minX; - let z = 16*lot.minZ; - return { - minX: x + minX, - maxX: x + maxX, - minZ: z + minZ, - maxZ: z + maxZ, - }; - } - - // Find the width & the depth of the lot. We can get this from the lot - // itself, but this doesn't take into account yet that the lot is rotated, - // so that's something we still need to do ourselves. +// ## orient([x, y], lot, opts) +// Helper function for transforming the point [x, y] that is given in +// **local** lot coordinates into global **city** coordinates. Note that local +// lot coordinates use an origin in the bottom-left corner of the lot with an +// y axis that is going up. This means that we'll need to invert properly! +function orient([x, y], lot, opts = {}) { let { width, depth } = lot; - if (lot.orientation % 2 === 1) { - [width, depth] = [16*depth, 16*width]; - } else { - width *= 16; - depth *= 16; - } - // TODO: Don't think this works, but we need a lot that's a bit more - // "cornered", otherwise we won't be able to see the effects of it. + // First of all we need to swap because orientation 0 in the city is "up", + // while orientation 0 in the is "down", and that's also how the + // coordinates are expressed! + [x, y] = [width-x, depth-y]; + + // Based on the lot orientation, position correctly. switch (lot.orientation) { case 0x01: - return move({ - minX: width-maxZ, - maxX: width-minZ, - minZ: minX, - maxZ: maxX, - }); + [x, y] = [depth-y, x]; + break; case 0x02: - return move({ - minX: width-maxX, - maxX: width-minX, - minZ: depth-maxZ, - maxZ: depth-minZ, - }); + [x, y] = [width-x, depth-y]; + break; case 0x03: - return move({ - minX: minZ, - maxX: maxZ, - minZ: depth-maxX, - maxZ: depth-minX, - }); - default: { - return move({ - minX, - maxX, - minZ, - maxZ, - }); - } + [x, y] = [y, width-x]; + break; + + } + + // If we didn't request bare coordinates explicitly, transform to city + // coordinates. + if (opts.bare) { + return [x, y]; + } else { + return [ + 16*(lot.minX + x), + 16*(lot.minZ + y), + ]; } } diff --git a/test/plop-test.js b/test/plop-test.js index 29f630a..c4c09fc 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -357,7 +357,7 @@ describe('A city manager', function() { tgi: [0x6534284a,0xa8fbd372,0xa706ed25], x: i, z: j, - // orientation: i, + orientation: (i+j) % 4, }); } } @@ -382,13 +382,13 @@ describe('A city manager', function() { }); - it.only('includes the base texture when plopping', async function() { + it.skip('includes the base texture when plopping', async function() { this.timeout(0); let dir = path.join(__dirname, 'files'); let out = path.join(REGION, 'City - Base Textures.sc4'); - // let source = path.join(dir, 'City - Base Textures.sc4'); - let source = out; + let source = path.join(dir, 'City - Base Textures.sc4'); + // let source = out; let index = new FileIndex({ files: [ @@ -400,31 +400,56 @@ describe('A city manager', function() { }); await index.build(); - let float = x => { - return new Float32Array([x])[0]; - }; + let city = new CityManager({ index }); + city.load(source); + + console.log(city.dbpf.lots[0].orientation); + + for (let i = 0; i < 5; i++) { + for (let j = 0; j < 5; j++) { + if (i === 1 && j === 1) { + continue; + } + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x: i, + z: j, + orientation: (i+j) % 4, + }); + // break; + } + // break; + } + + await city.save({ file: out }); + + }); + + it.only('positions a 1x2 lot', async function() { + + this.timeout(); + let dir = path.join(__dirname, 'files'); + let out = path.join(REGION, 'City - Base Textures.sc4'); + let source = path.join(dir, 'City - Base Textures.sc4'); + + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + ], + }); + await index.build(); let city = new CityManager({ index }); city.load(source); - // for (let i = 0; i < 64; i++) { - // for (let j = 0; j < 64; j++) { - // if (i === 1 && j === 1) { - // continue; - // } - // city.grow({ - // tgi: [0x6534284a,0xa8fbd372,0xa706ed25], - // x: i, - // z: j, - // // orientation: (i+j) % 4, - // }); - // } - // } - - let { textures } = city.dbpf; - console.log('Texture entries in the city:', textures.length); - - // await city.save({ file: out }); + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0x60000b70], + x: 2, + z: 2, + orientation: 0, + }); + + await city.save({ file: out }); }); From 13f3befbfadf1555ff87090d566d7d7eee21a8f4 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Mon, 16 Mar 2020 11:33:22 +0100 Subject: [PATCH 27/39] Create required entries on empty cities We now create the required entries in the city dbpf when they don't exist yet if we're reading in the city. We've verified that it works properly on a large city tile. --- lib/city-manager.js | 42 +++++++++++++++-------------- lib/dbpf.js | 4 +-- lib/savegame.js | 64 +++++++++++++++++++++++++++------------------ 3 files changed, 63 insertions(+), 47 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index fba7af8..ed6042c 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -76,6 +76,14 @@ class CityManager { // Create the city. this.dbpf = new Savegame(fs.readFileSync(full)); + // Index the *initial* mem refs as well. We need to make sure we do + // this before creating any new files in the dbpf, otherwise we're + // indexing mem refs we might have created ourselves. + let set = this.memRefs = new Set(); + for (let { mem } of this.dbpf.memRefs()) { + set.add(mem); + } + } // ## save(opts) @@ -90,15 +98,6 @@ class CityManager { // memory addresses for every record are unique. mem() { - // If we didn't set up the memory references yet, parse them. - if (!this.memRefs) { - let { dbpf } = this; - let set = this.memRefs = new Set(); - for (let { mem } of dbpf.memRefs()) { - set.add(mem); - } - } - // Create a new memory reference, but make sure it doesn't exist yet. let ref = this.$mem++; while (this.memRefs.has(ref)) { @@ -518,24 +517,29 @@ class CityManager { setTract(texture); // Add all required textures. - // let i = 0; for (let def of textures) { let { orientation, x, z, IID } = def; let [xx, zz] = orient([x, z], lot, { bare: true }); + + // Note: the orientation is given in **lot** coordinates, + // but orientation 0 in the city is 2 in the lot, so add 2 to it. + // Additionally we'll also need to handle mirroring. + let mirrored = orientation >= 0x80000000; + orientation %= 0x80000000; + orientation = (lot.orientation + orientation) % 4; + if (mirrored) { + orientation += 4; + } + + // Create the texture at last. texture.add({ IID, x: lot.minX + Math.floor(xx), z: lot.minZ + Math.floor(zz), - orientation: (lot.orientation + orientation) % 4, - - // priority: 0, - - // priority: - // priority: (1-i), - // priority: Math.min(i, 1), - + orientation, + priority: 0, }); - // i++; + } // Cool, now push the base texture in the city & update the diff --git a/lib/dbpf.js b/lib/dbpf.js index 32f788f..f92e0ac 100644 --- a/lib/dbpf.js +++ b/lib/dbpf.js @@ -73,9 +73,9 @@ const DBPF = module.exports = class DBPF extends EventEmitter { return this.entries.find(...args); } - // ## add(tfi, file) + // ## add(tgi, file) // Adds a new entry to the dbpf file. - add(tfi, file) { + add(tgi, file) { let entry = new Entry(); entry.tgi = tgi; this.entries.push(entry); diff --git a/lib/savegame.js b/lib/savegame.js index e0e8677..9d1387e 100644 --- a/lib/savegame.js +++ b/lib/savegame.js @@ -2,17 +2,20 @@ "use strict"; const DBPF = require('./dbpf'); const FileType = require('./file-types'); -const Prop = require('./prop.js'); // # Savegame() // A class specifically designed for some Savegame functionality. Obviously // extends the DBPF class because savegames are dbpf files. -module.exports = class Savegame extends DBPF { +class Savegame extends DBPF { + + // ## get GID() + get GID() { + return this.entries[0].group; + } // ## get lotFile() get lotFile() { - let entry = this.entries.find(x => x.type === FileType.LotFile); - return entry ? entry.read() : null; + return this.readByType(FileType.LotFile); } get lots() { return this.lotFile; @@ -20,8 +23,7 @@ module.exports = class Savegame extends DBPF { // ## get buildingFile() get buildingFile() { - let entry = this.entries.find(x => x.type === FileType.BuildingFile); - return entry ? entry.read() : null; + return this.readByType(FileType.BuildingFile); } get buildings() { return this.buildingFile; @@ -30,12 +32,7 @@ module.exports = class Savegame extends DBPF { // ## get propFile() // Getter for the propfile. If it doesn't exist yet, we'll create it. get propFile() { - let entry = this.entries.find(x => x.type === FileType.PropFile); - if (!entry) { - let tgi = [FileType.PropFile, 698035483, 0]; - entry = this.add(tgi, new Prop.Array()); - } - return entry.read(); + return this.readByType(FileType.PropFile); } get props() { return this.propFile; @@ -43,8 +40,7 @@ module.exports = class Savegame extends DBPF { // ## get baseTextureFile() get baseTextureFile() { - let entry = this.getByType(FileType.BaseTextureFile); - return entry ? entry.read() : null; + return this.readByType(FileType.BaseTextureFile); } get textures() { return this.baseTextureFile; @@ -52,8 +48,7 @@ module.exports = class Savegame extends DBPF { // ## get itemIndexFile() get itemIndexFile() { - let entry = this.getByType(FileType.ItemIndexFile); - return entry ? entry.read() : null; + return this.readByType(FileType.ItemIndexFile); } get itemIndex() { return this.itemIndexFile; @@ -61,8 +56,7 @@ module.exports = class Savegame extends DBPF { // ## get zoneDeveloperFile() get zoneDeveloperFile() { - let entry = this.getByType(FileType.ZoneDeveloperFile); - return entry ? entry.read() : null; + return this.readByType(FileType.ZoneDeveloperFile); } get zones() { return this.zoneDeveloperFile; @@ -70,25 +64,43 @@ module.exports = class Savegame extends DBPF { // ## get lotDeveloperFile() get lotDeveloperFile() { - let entry = this.getByType(FileType.LotDeveloperFile); - return entry ? entry.read() : null; + return this.readByType(FileType.LotDeveloperFile); } // ## get floraFile() get floraFile() { - let entry = this.getByType(FileType.FloraFile); - return entry ? entry.read() : null; + return this.readByType(FileType.FloraFile); } // ## get COMSerializerFile() get COMSerializerFile() { - let entry = this.getByType(FileType.COMSerializerFile); - return entry ? entry.read() : null; + return this.readByType(FileType.COMSerializerFile); } // # getByType(type) + // This method returns an entry in the savegame by type. If it doesn't + // exist yet, it is created. getByType(type) { - return this.entries.find(x => x.type === type); + let entry = this.entries.find(x => x.type === type); + if (!entry) { + const Klass = DBPF.FileTypes[type]; + if (Klass) { + if (Klass.Array) { + Klass = Klass.Array; + } + let tgi = [type, this.GID, 0]; + entry = this.add(tgi, new Klass()); + } + } + return entry || null; + } + + // ## readByType(type) + // Helper function that reads an entry when it can be returned. + readByType(type) { + let entry = this.getByType(type); + return entry ? entry.read() : null; } -}; \ No newline at end of file +}; +module.exports = Savegame; From 5c6e7a51643a9d5c6a597db17b6e398ea64ea49d Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Mon, 16 Mar 2020 13:13:34 +0100 Subject: [PATCH 28/39] No longer set priority for base textures --- lib/city-manager.js | 5 ++--- test/plop-test.js | 41 +++++++++++++++++++++++++++++++++++------ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index ed6042c..20b64c3 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -510,8 +510,8 @@ class CityManager { // TODO: This is only for flat cities, should use terrain queries // later on! - minY: 270, - maxY: 270 + INSET, + minY: lot.minY, + maxY: lot.maxY + INSET, }); setTract(texture); @@ -537,7 +537,6 @@ class CityManager { x: lot.minX + Math.floor(xx), z: lot.minZ + Math.floor(zz), orientation, - priority: 0, }); } diff --git a/test/plop-test.js b/test/plop-test.js index c4c09fc..92257a1 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -295,7 +295,7 @@ describe('A city manager', function() { }); - it.only('builds a skyline', async function() { + it.skip('builds a skyline', async function() { this.timeout(0); @@ -328,6 +328,37 @@ describe('A city manager', function() { }); + it.skip('builds a skyline on a medium tile', async function() { + + this.timeout(0); + + let dir = path.join(__dirname, 'files'); + let file = path.join(dir, 'City - Medium tile.sc4'); + + let c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + path.join(c, 'SimCity_2.dat'), + path.join(c, 'SimCity_3.dat'), + path.join(c, 'SimCity_4.dat'), + path.join(c, 'SimCity_5.dat'), + ] + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(file); + + // Create the skyline in the city. + skyline({ city }); + + // Save the city. + let out = path.join(REGION, 'City - Medium tile.sc4'); + await city.save({ file: out }); + + }); + it.skip('includes the textures when plopping', async function() { this.timeout(0); @@ -382,7 +413,7 @@ describe('A city manager', function() { }); - it.skip('includes the base texture when plopping', async function() { + it.only('includes the base texture when plopping', async function() { this.timeout(0); let dir = path.join(__dirname, 'files'); @@ -403,8 +434,6 @@ describe('A city manager', function() { let city = new CityManager({ index }); city.load(source); - console.log(city.dbpf.lots[0].orientation); - for (let i = 0; i < 5; i++) { for (let j = 0; j < 5; j++) { if (i === 1 && j === 1) { @@ -425,7 +454,7 @@ describe('A city manager', function() { }); - it.only('positions a 1x2 lot', async function() { + it.skip('positions a 1x2 lot', async function() { this.timeout(); let dir = path.join(__dirname, 'files'); @@ -443,7 +472,7 @@ describe('A city manager', function() { city.load(source); city.grow({ - tgi: [0x6534284a,0xa8fbd372,0x60000b70], + tgi: [0x6534284a,0xa8fbd372,0x600045d0], x: 2, z: 2, orientation: 0, From b0c85e3f5d038e193f6377e8f4f699916b37ff1c Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Mon, 16 Mar 2020 15:57:00 +0100 Subject: [PATCH 29/39] Fix brown boxes for props --- lib/city-manager.js | 24 +++++++++++--- lib/lot-object.js | 7 +++++ test/plop-test.js | 77 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 20b64c3..cd7e52a 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -18,6 +18,7 @@ const OccupantSize = 0x27812810; const LotConfigurations = 0x10; const LotResourceKey = 0xea260589; const LotConfigPropertySize = 0x88edc790; +const LotConfigPropertyZoneTypes = 0x88edc793; const Wealth = 0x27812832; const INSET = 0.1; @@ -267,6 +268,7 @@ class CityManager { lot, lotObject, }); + break; case 0x02: // Note: We can't handle textures right away because they @@ -303,6 +305,12 @@ class CityManager { LotConfigPropertySize ); + // Determine the zone type. + let zoneTypes = this.getPropertyValue( + file, + LotConfigPropertyZoneTypes, + ); + // Cool, we can now create a new lot entry. Note that we will need to // take into account the let lot = new Lot({ @@ -324,6 +332,7 @@ class CityManager { depth, orientation, zoneWealth: opts.zoneWealth || 0x00, + zoneType: zoneTypes[0] || 0x0f, // Apparently jobCapacities is also required, otherwise CTD! The // capacity is stored I guess in the LotConfig exemplar, or @@ -435,10 +444,11 @@ class CityManager { // Note: in contrast to the building, we don't know yet what prop // we're going to insert if we're dealing with a prop family. As such // we'll check the families first. - let { IID, orientation, x, y, z } = lotObject; + let { OID, IIDs, orientation, x, y, z } = lotObject; + let IID = rand(IIDs); let exemplar = this.findExemplarOfType(IID, 0x1e); + if (!exemplar) { - // console.warn(`Missing prop ${ IID }!`); return; } let file = exemplar.read(); @@ -468,10 +478,10 @@ class CityManager { GID: exemplar.group, IID: exemplar.instance, IID1: exemplar.instance, - OID: this.mem(), + OID, appearance: 5, - state: 1, + state: 0, }); setTract(prop); @@ -648,3 +658,9 @@ function orient([x, y], lot, opts = {}) { } } + +// ## rand(arr) +// Helper function that randomly selects a value from a given array. +function rand(arr) { + return arr[Math.random()*arr.length | 0]; +} diff --git a/lib/lot-object.js b/lib/lot-object.js index a305f13..f43032b 100644 --- a/lib/lot-object.js +++ b/lib/lot-object.js @@ -55,6 +55,13 @@ class LotObject { return this.values[12]; } + // ## get IIDs() + // Note: every rep starting from 13 can be an IID, that's another way to + // create families. We should take this into account as well hence. + get IIDs() { + return this.values.slice(12); + } + } module.exports = LotObject; \ No newline at end of file diff --git a/test/plop-test.js b/test/plop-test.js index 92257a1..697fc35 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -295,7 +295,7 @@ describe('A city manager', function() { }); - it.skip('builds a skyline', async function() { + it.only('builds a skyline', async function() { this.timeout(0); @@ -413,7 +413,7 @@ describe('A city manager', function() { }); - it.only('includes the base texture when plopping', async function() { + it.skip('includes the base texture when plopping', async function() { this.timeout(0); let dir = path.join(__dirname, 'files'); @@ -482,4 +482,77 @@ describe('A city manager', function() { }); + it.skip('fixes the brown boxes', async function() { + + this.timeout(); + + let dir = path.join(__dirname, 'files'); + let out = path.join(REGION, 'City - Base Textures.sc4'); + let source = path.join(dir, 'City - Base Textures.sc4'); + + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + ], + }); + await index.build(); + + let entry = index.find([0x29A5D1EC,0x2A2458F9,0x213B0000]); + console.log(entry.dbpf.file); + + return; + + let city = new CityManager({ index }); + city.load(source); + + for (let i = 2; i < 64; i++) { + for (let j = 2; j < 32; j++) { + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0x600029b0], + x: i, + z: 2*j, + orientation: 0, + }); + break; + } + break; + } + + await city.save({ file: out }); + + }); + + it.only('plops a lot with an ATC prop', async function() { + + this.timeout(); + let dir = path.join(__dirname, 'files'); + let source = path.join(dir, 'City - ATC.sc4'); + let out = path.join(REGION, 'City - ATC.sc4'); + + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + ], + dirs: [ + path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), + ], + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(source); + + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x: 2, + z: 1, + orientation: 2, + }); + + console.table(city.dbpf.props); + + await city.save({ file: out }); + + }); + }); From 80bf562db7564252d8ebb5419fefbd1a0c8bba08 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Mon, 16 Mar 2020 16:43:26 +0100 Subject: [PATCH 30/39] Position asymmetric props --- lib/city-manager.js | 70 +++++++++++++++++++++------------------------ lib/lot-object.js | 19 ++++++++---- test/plop-test.js | 2 -- 3 files changed, 47 insertions(+), 44 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index cd7e52a..1287667 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -202,7 +202,8 @@ class CityManager { // Find the appropriate building exemplar. Note that it's possible // that the building belongs to a family. In that case we'll pick a // random building from the family. - let IID = props.lotObjects.find(({ type }) => type === 0x00).IID; + let IIDs = props.lotObjects.find(({ type }) => type === 0x00).IIDs; + let IID = rand(IIDs); let buildingExemplar = this.findExemplarOfType(IID, 0x02); // Now that we have both the building exemplar and as well as the lot @@ -375,21 +376,7 @@ class CityManager { let { lot, lotObject, exemplar } = opts; let file = exemplar.read(); let [width, height, depth] = this.getPropertyValue(file, OccupantSize); - let { orientation, x, y, z } = lotObject; - - // First of all we need to find the *oriented* position of the - // building on the lot. This means that we need to take into account - // that the lot may have been rotated. - let [xx, zz] = orient([x, z], lot); - - // Find the rectangle the building is occupying on the lot, where the - // origin of the coordinate system is in the top-left corner. Note - // that we already rotate the building into place **in the lot**. We - // still need to rotate the building rectangle later on based on the - // orientation of the lot *itself*. - if (orientation % 2 === 1) { - [width, depth] = [depth, width]; - } + let { orientation, y } = lotObject; // Create the building. let building = new Building({ @@ -397,10 +384,7 @@ class CityManager { // Now use the **rotated** building rectangle and use it to // position the building appropriately. - minX: xx - width/2, - maxX: xx + width/2, - minZ: zz - depth/2, - maxZ: zz + depth/2, + ...position(lotObject, lot), minY: lot.yPos + y, maxY: lot.yPos + y + height, orientation: (orientation + lot.orientation) % 4, @@ -444,31 +428,24 @@ class CityManager { // Note: in contrast to the building, we don't know yet what prop // we're going to insert if we're dealing with a prop family. As such // we'll check the families first. - let { OID, IIDs, orientation, x, y, z } = lotObject; + let { OID, IIDs, orientation, y } = lotObject; let IID = rand(IIDs); let exemplar = this.findExemplarOfType(IID, 0x1e); + // Missing props? Just ignore them. if (!exemplar) { return; } + + // Get the dimensions of the prop bounding box. let file = exemplar.read(); let [width, height, depth] = this.getPropertyValue(file, OccupantSize); - // Find the rectangle the prop is occupying & then position it - // correctly, taking into account the lot dimensions. - if (orientation % 2 === 1) { - [width, depth] = [depth, width]; - } - let [xx, zz] = orient([x, z], lot); - - // Create the prop. + // Create the prop & position correctly. let prop = new Prop({ mem: this.mem(), - minX: xx - width/2, - maxX: xx + width/2, - minZ: zz - depth/2, - maxZ: zz + depth/2, + ...position(lotObject, lot), minY: lot.yPos + y, maxY: lot.yPos + y + height, orientation: (orientation + lot.orientation) % 4, @@ -492,7 +469,7 @@ class CityManager { props.push(prop); // Put the prop in the index. - this.addToItemIndex(prop, FileType.PropFile); + this.addToItemIndex(prop, FileType.PropFile, lotObject); // Update the COM serializer and we're done. let com = dbpf.COMSerializerFile; @@ -569,11 +546,14 @@ class CityManager { // ## addToItemIndex(obj, type) // Helper function for adding the given object - that exposes the tract // coordinates - to the item index. - addToItemIndex(obj, type) { + addToItemIndex(obj, type, euh) { let { dbpf } = this; let index = dbpf.itemIndexFile; for (let x = obj.xMinTract; x <= obj.xMaxTract; x++) { for (let z = obj.zMinTract; z <= obj.zMaxTract; z++) { + if (!index[x][z]) { + console.log(obj, euh.minX, euh.maxX, euh.minZ, euh.maxZ); + } index[x][z].push({ mem: obj.mem, type: type, @@ -613,9 +593,9 @@ module.exports = CityManager; function setTract(obj) { const xSize = 16 * 2**obj.xTractSize; const zSize = 16 * 2**obj.zTractSize; - obj.xMinTract = 64 + Math.floor(obj.minX / xSize); + obj.xMinTract = Math.max(64, 64 + Math.floor(obj.minX / xSize)); obj.xMaxTract = 64 + Math.floor(obj.maxX / xSize); - obj.zMinTract = 64 + Math.floor(obj.minZ / zSize); + obj.zMinTract = Math.max(64, 64 + Math.floor(obj.minZ / zSize)); obj.zMaxTract = 64 + Math.floor(obj.maxZ / zSize); } @@ -659,6 +639,22 @@ function orient([x, y], lot, opts = {}) { } +// ## position(lotObject, lot) +// Returns the rectangle we need to position the given lotObject on, taken +// into account it's positioned on the given lot. +function position(lotObject, lot) { + let { minX, maxX, minZ, maxZ } = lotObject; + [minX, minZ] = orient([minX, minZ], lot); + [maxX, maxZ] = orient([maxX, maxZ], lot); + if (minX > maxX) { + [minX, maxX] = [maxX, minX]; + } + if (minZ > maxZ) { + [minZ, maxZ] = [maxZ, minZ]; + } + return { minX, maxX, minZ, maxZ }; +} + // ## rand(arr) // Helper function that randomly selects a value from a given array. function rand(arr) { diff --git a/lib/lot-object.js b/lib/lot-object.js index f43032b..1032ae0 100644 --- a/lib/lot-object.js +++ b/lib/lot-object.js @@ -33,10 +33,10 @@ class LotObject { get z() { return this.values[5]/scale; } set z(value) { this.values[5] = Math.round(scale*value); } - get minX() { return this.values[6]/scale; } - get minZ() { return this.values[7]/scale; } - get maxX() { return this.values[8]/scale; } - get maxZ() { return this.values[9]/scale; } + get minX() { return signed(this.values[6])/scale; } + get minZ() { return signed(this.values[7])/scale; } + get maxX() { return signed(this.values[8])/scale; } + get maxZ() { return signed(this.values[9])/scale; } get usage() { return this.values[10]; } // ## get OID() @@ -64,4 +64,13 @@ class LotObject { } -module.exports = LotObject; \ No newline at end of file +module.exports = LotObject; + +// ## signed(x) +// Helper function that ensures certain 32 bit integers are considered as +// signed. +let arr = new Int32Array([0]); +function signed(x) { + arr[0] = x; + return arr[0]; +} diff --git a/test/plop-test.js b/test/plop-test.js index 697fc35..14dcf10 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -549,8 +549,6 @@ describe('A city manager', function() { orientation: 2, }); - console.table(city.dbpf.props); - await city.save({ file: out }); }); From 1ffad65565c710e54f86280053f20435cab892d4 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Mon, 16 Mar 2020 18:48:18 +0100 Subject: [PATCH 31/39] Create lot index --- lib/lot-index.js | 234 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 lib/lot-index.js diff --git a/lib/lot-index.js b/lib/lot-index.js new file mode 100644 index 0000000..f104f96 --- /dev/null +++ b/lib/lot-index.js @@ -0,0 +1,234 @@ +// # lot-index.js +"use strict"; +const bsearch = require('binary-search-bounds'); +const { FileType } = require('./enums.js'); +const { hex } = require('./util.js'); + +// Exemplar properties we'll be using. +const LotConfigPropertySize = 0x88edc790; +const OccupantSize = 0x27812810; +const OccupantGroups = 0xaa1dd396; + +// # LotIndex +// A helper class that we use to index lots by a few important properties. +// They're sorted by height and such they will also remain so. This means that +// when filtering, you can rest assured that they remain sorted by height as +// well! +class LotIndex { + + // ## constructor(index) + // Creates the lot index from the given file index. + constructor(index) { + + // Store the file index, we'll still need it. + this.fileIndex = index; + this.lots = []; + + // Loop every exemplar. If it's a lot configurations exemplar, then + // read it so that we can find the building that appears on the lot. + for (let entry of index.exemplars) { + let file = entry.read(); + if (this.getPropertyValue(file, 0x10) !== 0x10) { + continue; + } + + // Cool, add the lot. + this.add(entry); + + } + + // Now it's time to set up all our indices. For now we'll only index + // by height though. + this.height = IndexedArray.create({ + compare(a, b) { + return a.height - b.height; + }, + entries: this.lots, + }); + + } + + // ## add(entry) + // Adds the given lot exemplar to the index. + add(entry) { + + // Find the building on the lot. + let file = entry.read(); + let { lotObjects } = file; + let rep = lotObjects.find(({ type }) => type === 0x00); + let IID = rep.IID; + let building = this.getBuilding(IID); + + // Read in the lot size. + let size = this.getPropertyValue(file, LotConfigPropertySize); + + // Read in the building size as that is our main indexing property. + let [width, height, depth] = this.getPropertyValue( + building, OccupantSize, + ); + + // At last, create our entry. + this.lots.push({ + lot: entry, + height, + size, + }); + + } + + // ## getBuilding(IID) + getBuilding(IID) { + let buildings = this.fileIndex + .findAllTI(FileType.Exemplar, IID) + .filter(entry => { + let file = entry.read(); + let type = this.getPropertyValue(file, 0x10); + return type === 0x02; + }); + if (buildings.length) { + return buildings[0].read(); + } + + // No building found? Don't worry, check the families. + let family = this.fileIndex.family(IID); + if (!family) { + throw new Error([ + `No building found with IID ${ hex(IID) }!` + ]); + } + return family[0].read(); + + } + + // ## getPropertyValue(file, prop) + // Helper function for quickly reading property values. + getPropertyValue(file, prop) { + return this.fileIndex.getPropertyValue(file, prop); + } + +} +module.exports = LotIndex; + +// ## IndexedArray +// An extension of an array that takes into account that the array is sorted +// using a certain comparator - which is to be specified by *extending* the +// IndexedArray. Very useful to perform efficient range queries. +class IndexedArray extends Array { + + // ## static extend(compare) + static extend(compare) { + const Child = class IndexedArray extends this {}; + Child.prototype.compare = compare; + return Child; + } + + // ## static create(opts) + static create(opts) { + let { compare, entries = [] } = opts; + const Child = this.extend(compare); + let index = new Child(...entries); + index.sort(); + return index; + } + + // ## getRangeIndices(min, max) + getRangeIndices(min, max) { + const { compare } = this; + let first = bsearch.le(this, min, compare)+1; + let last = bsearch.ge(this, max, compare); + return [first, last]; + } + + // ## range(min, max) + // Filters down the subselection to only include the given height range. + // Note: perhaps that we should find a way to change the index criterion + // easily, that's for later on though. + range(min, max) { + let [first, last] = this.getRangeIndices(min, max); + return this.slice(first, last); + } + + // ## *it(min, max) + // Helper function which allows a range to be used as an iterator. + *it(min, max) { + let [first, last] = this.getRangeIndices(min, max); + for (let i = first; i < last; i++) { + yield this[i]; + } + } + + // ## query(query) + // Helper function for carrying out a query using the normal array filter + // method. Only exact queries are possible for the moment, no range + // queries though that should be possible as well - see MongoDB for + // example. + query(query) { + + // First of all we'll build the query. Building the query means that + // we're creating an array of functions which *all* need to pass in + // order to evaluate to true. This means an "and" condition. + let filters = []; + for (let key in query) { + let def = query[key]; + + // If the definition is an array, we'll check whether the value is + // within the array. + if (Array.isArray(def)) { + filters.push($oneOf(key, def)); + } else { + filters.push($equals(key, def)); + } + + } + + const keys = Object.keys(query); + return this.filter(function(entry) { + for (let fn of filters) { + if (!fn(entry)) { + return false; + } + } + return true; + }); + } + + // ## sort() + // Sorts the indexed array. Normally this shouldn't be necessary to call + // manually by the way, but we need call this once upon creation! + sort() { + return super.sort(this.compare); + } + +} + +// ## compare(a, b) +// The function we use to sort all lots by height. +function compare(a, b) { + return a.height - b.height; +} + +// ## contains(arr, it) +// Helper function for checking if the given array includes *one* of the +// elements in the given iterator. +function contains(arr, it) { + for (let el of it) { + if (arr.includes(el)) { + return true; + } + } + return false; +} + +// ## $equals(key, value) +function $equals(key, value) { + return function(entry) { + return entry[key] === value; + } +} + +// ## $oneOf(key, arr) +function $oneOf(key, arr) { + return function(entry) { + return arr.includes(entry[key]); + } +} From 5b65979a3c05ba16c0b10def40de729e8235b2e2 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Mon, 16 Mar 2020 21:51:47 +0100 Subject: [PATCH 32/39] Use lot index to create skyline --- lib/lot-index.js | 28 ++++++- lib/skyline.js | 204 ++++++++++++++++------------------------------- 2 files changed, 93 insertions(+), 139 deletions(-) diff --git a/lib/lot-index.js b/lib/lot-index.js index f104f96..1b2431a 100644 --- a/lib/lot-index.js +++ b/lib/lot-index.js @@ -67,11 +67,20 @@ class LotIndex { building, OccupantSize, ); + // Get the occupant groups as well. + let groups = this.getPropertyValue(building, OccupantGroups) || []; + + // Get the zone types. + let zoneTypes = this.getPropertyValue(file, 0x88edc793); + // At last, create our entry. this.lots.push({ lot: entry, height, + building: [width, height, depth], size, + occupantGroups: groups, + zoneTypes, }); } @@ -222,13 +231,28 @@ function contains(arr, it) { // ## $equals(key, value) function $equals(key, value) { return function(entry) { - return entry[key] === value; + let x = entry[key]; + if (Array.isArray(x)) { + return x.includes(value); + } else { + return x === value; + } } } // ## $oneOf(key, arr) function $oneOf(key, arr) { return function(entry) { - return arr.includes(entry[key]); + let x = entry[key]; + if (Array.isArray(x)) { + for (let el of x) { + if (arr.includes(el)) { + return true; + } + } + return false; + } else { + return arr.includes(entry[key]); + } } } diff --git a/lib/skyline.js b/lib/skyline.js index 98c7fce..22dac12 100644 --- a/lib/skyline.js +++ b/lib/skyline.js @@ -1,6 +1,7 @@ // # skyline.js "use strict"; const bsearch = require('binary-search-bounds'); +const LotIndex = require('./lot-index.js'); const { FileType } = require('./enums.js'); const { hex } = require('./util.js'); const LotConfigPropertySize = 0x88edc790; @@ -16,15 +17,68 @@ function skyline(opts) { radius } = opts; - // Make sure to index all lots by height. - let lots = indexLots(city); + // Create the master index for all lots order by height and then filter to + // only include RC buildings. + let lots = new LotIndex(city.index).height + .query({ + occupantGroups: [ + 0x11010, + 0x11020, + 0x11030, + 0x13110, + 0x13120, + 0x13130, + 0x13320, + 0x13330, + ], + }) + .filter(entry => { + + // Filter out buildings with an insufficient filling degree. + let [x, z] = entry.size; + if (Math.max(x, z) > 5) { + return false; + } + + // Calculate the filling degree. + if (x*z > 2) { + let [width, height, depth] = entry.building + let fill = (width*depth) / (x*z*16*16); + if (fill < 0.5 + 0.5*height/400) { + return false; + } + } + + return true; + + }); + + // Filter out the residential lots for the suburbs. + let suburbs = lots + .filter(entry => entry.occupantGroups.includes(0x11020)) + .filter(entry => entry.zoneTypes.length === 3); // Create our skyline function that will determine the maximum height // based on the distance from the center. - let fn = makeSkylineFunction({ + let cluster = makeSkylineFunction({ center, radius, }); + let fn = (x, z) => Math.max(10, cluster(x, z)); + // let f = 2; + // let a = makeSkylineFunction({ + // center: [64-f*10, 64-f*5], + // radius: 30, + // max: 400, + // }); + // let b = makeSkylineFunction({ + // center: [64+f*10, 64+f*5], + // radius: 45, + // max: 100, + // }); + // const fn = function(x, z) { + // return Math.max(10, a(x, z) + b(x, z)); + // }; // Now loop all city tiles & plop away. let zones = city.dbpf.zones; @@ -39,27 +93,25 @@ function skyline(opts) { continue; } - // Random voids. - // if (Math.random() < 0.1) { - // continue; - // } - // Calculate the maximum height for this tile & select the // appropriate lot range. let max = fn(x, z); - let last = bsearch.ge(lots, { height: max }, compare); - let first = bsearch.le(lots, { height: 0.25*max }, compare)+1; - let diff = last - first; + let db; + if (max < 11) { + db = suburbs; + } else { + db = lots.range({ height: 0.25*max }, { height: max }); + } // No suitable lots found to plop? Pity, go on. - if (diff === 0) { + if (db.length === 0) { continue; } // Now pick a random lot from the suitable lots. We will favor // higher lots more to create a nicer effect. - let index = Math.floor(diff * Math.random()**0.75); - let lot = lots[first + index].entry; + let index = Math.floor(db.length * Math.random()**0.75); + let { lot } = db[index]; // Cool, an appropriate lot was found, but we're not done yet. // It's possible if there's no space to plop the lot, we're not @@ -116,133 +168,11 @@ function makeSkylineFunction(opts) { cx = 32, cz = 32, ], - min = 15, max = 400, radius = 32, } = opts; - let diff = (max-min); return function(x, z) { let t = Math.sqrt((cx-x)**2 + (cz-z)**2) / radius; - return min+diff*Math.exp(-((2*t)**2)); + return max*Math.exp(-((2*t)**2)); }; } - -// ## indexLots(city) -// Creates an index of all lots by height that are available to the city. -function indexLots(city) { - let { index } = city; - let lots = []; - - // Loop every exemplar that we have indexed. If its a lot configurations - // exemplar, then read it so that we can find the building that appears on - // the lot. - for (let entry of index.exemplars) { - let file = entry.read(); - - // Not a Lot Configurations exemplar? Don't bother. - if (index.getPropertyValue(file, 0x10) !== 0x10) { - continue; - } - - // Check the lot size. We're not going to include lots that are too - // big. - let [xSize, zSize] = index.getPropertyValue(file, LotConfigPropertySize); - if (Math.max(xSize, zSize) > 5) { - continue; - } - - // Find the building on the lot. - let lotObjects = file.lotObjects; - let rep = lotObjects.find(({ type }) => type === 0x00); - let IID = rep.IID; - - // Check if the building belongs to a family. - let family = index.family(IID); - let building; - if (family) { - let pivot = family[0].read(); - if (!pivot) { - console.warn('Could not find the size for a lot!'); - continue; - } - building = pivot; - } else { - - // Cool, the building on the lot does not belong to a family. In - // that case just find it. - let buildings = index.findAllTI(FileType.Exemplar, IID) - .filter(entry => { - let file = entry.read(); - let type = index.getPropertyValue(file, 0x10); - return type === 0x02; - }); - if (buildings.length === 0) { - console.warn([ - `No building found with IID ${ hex(IID) }, but referenced on lot ${ hex(entry.instance) }!` - ].join(' ')); - continue; - } - building = buildings[0].read(); - - } - - // Now check the occupant groups of the building. We'll only want - // residential and commerical for now. - let groups = index.getPropertyValue(building, OccupantGroups); - if (!groups) { - continue; - } - let required = [ - // 0x00011010, - 0x00011020, - 0x00011030, - // 0x00013110, - // 0x00013120, - 0x00013130, - // 0x00013320, - 0x00013330, - ]; - let filter = groups.filter(group => { - return required.includes(group); - }); - - // Filter out the New York style. - let filter2 = groups.filter(group => { - return true; - // return group === 0x2000 || group === 0x2001; - // return group === 0x2001 || group === 0x2002; - }); - - if (filter.length === 0 || filter2.length === 0) { - continue; - } - - let prop = index.getProperty(building, OccupantSize); - let [width, height, depth] = prop.value; - - // Check the filling degree of the building. If that's not - // sufficiently high, don't include the building. - if (xSize * zSize > 2) { - let fill = (width*depth) / (xSize*zSize*16*16); - if (fill < 0.5 + 0.5*height/400) { - continue; - } - } - - lots.push({ - entry, - height, - }); - - } - - lots.sort(compare); - return lots; - -} - -// ## compare(a, b) -// The function we use to sort all lots. -function compare(a, b) { - return a.height - b.height; -} From 8a54dacfb539a2ae73c200c28ec8d2d9dcc2f8ff Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Mon, 16 Mar 2020 23:34:48 +0100 Subject: [PATCH 33/39] Hilbert curve --- package.json | 1 + test/plop-test.js | 95 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 3c0acfb..f007ff0 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "chai": "^4.2.0", "chai-spies": "^1.0.0", "electron": "^5.0.6", + "hilbert-curve": "^2.0.5", "mocha": "^6.2.2", "node-gyp": "^4.0.0", "prebuildify": "^3.0.4", diff --git a/test/plop-test.js b/test/plop-test.js index 14dcf10..b746091 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -295,7 +295,7 @@ describe('A city manager', function() { }); - it.only('builds a skyline', async function() { + it.skip('builds a skyline', async function() { this.timeout(0); @@ -522,7 +522,7 @@ describe('A city manager', function() { }); - it.only('plops a lot with an ATC prop', async function() { + it.skip('plops a lot with an ATC prop', async function() { this.timeout(); let dir = path.join(__dirname, 'files'); @@ -553,4 +553,95 @@ describe('A city manager', function() { }); + it.only('plops a Hilbert curve', async function() { + + const curve = require('hilbert-curve'); + + this.timeout(); + let dir = path.join(__dirname, 'files'); + let source = path.join(dir, 'City - Large Tile.sc4'); + let out = path.join(REGION, 'City - Large Tile.sc4'); + + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + ], + dirs: [ + path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), + ], + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(source); + let zones = city.dbpf.zones; + + const order = 6; + const n = 2**order; + const nn = n**2; + let data = []; + const length = 4; + for (let i = 0; i < nn; i++) { + let point = curve.indexToPoint(i, order); + data.push({ + x: (length-1)*point.x, + z: (length-1)*point.y, + }); + } + + let matrix = Array(length*n).fill().map(() => { + return Array(length*n).fill(); + }); + + for (let i = 1; i < data.length; i++) { + let P = data[i-1]; + let Q = data[i]; + let d = { + x: (Q.x - P.x)/(length-1), + z: (Q.z - P.z)/(length-1), + }; + for (let j = 0; j < length; j++) { + let xx = P.x + d.x*j + 2; + let zz = P.z + d.z*j + 2; + matrix[xx][zz] = true; + } + } + + // Loop the entire curve and grow a lot to the left or to the right. + for (let i = 1; i < data.length; i++) { + let P = data[i-1]; + let Q = data[i]; + let d = { + x: (Q.x - P.x)/(length-1), + z: (Q.z - P.z)/(length-1), + }; + let n = { + x: d.z, + z: -d.x, + }; + for (let j = 0; j < length; j++) { + let xx = P.x + d.x*j + 2; + let zz = P.z + d.z*j + 2; + for (let s of [-1, 1]) { + let x = xx + n.x*s; + let z = zz + n.z*s; + if (zones.cells[x][z]) continue; + if (x < 0 || z < 0) continue; + if (matrix[x][z]) continue; + + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x, + z, + }); + + } + } + } + + // Save baby. + await city.dbpf.save({ file: out }); + + }); + }); From 74e7b6d266fa3b7d59301fec476fe23b7e4bc6bf Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Tue, 17 Mar 2020 18:00:25 +0100 Subject: [PATCH 34/39] Fix lot IID --- lib/city-manager.js | 24 ++++--- test/plop-test.js | 154 +++++++++++++++----------------------------- 2 files changed, 64 insertions(+), 114 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 1287667..a0ee8fe 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -238,18 +238,12 @@ class CityManager { orientation = 0, } = opts; - // It's important to be able to know building properties to pass to - // the lot, such as the wealth etc. Therefore, read in the building. - let file = building.read(); - let zoneWealth = this.getPropertyValue(file, Wealth) || 0x00; - let lot = this.createLot({ exemplar: lotExemplar, - building: building.instance, + building, x: opts.x, z: opts.z, orientation, - zoneWealth, }); // Loop all objects on the lot such and insert them. @@ -312,13 +306,17 @@ class CityManager { LotConfigPropertyZoneTypes, ); + // Determine the zoneWealth as well. Note that this is to be taken + // **from the building**. + let buildingFile = building.read(); + let zoneWealth = this.getPropertyValue(buildingFile, Wealth); + // Cool, we can now create a new lot entry. Note that we will need to // take into account the let lot = new Lot({ mem: this.mem(), - IID: building, - buildingIID: building, - zoneType: 0x0f, + IID: exemplar.instance, + buildingIID: building.instance, // For now, just put at y = 270. In the future we'll need to read // in the terrain here. @@ -332,7 +330,7 @@ class CityManager { width, depth, orientation, - zoneWealth: opts.zoneWealth || 0x00, + zoneWealth: zoneWealth || 0x00, zoneType: zoneTypes[0] || 0x0f, // Apparently jobCapacities is also required, otherwise CTD! The @@ -497,8 +495,8 @@ class CityManager { // TODO: This is only for flat cities, should use terrain queries // later on! - minY: lot.minY, - maxY: lot.maxY + INSET, + minY: lot.yPos, + maxY: lot.yPos + INSET, }); setTract(texture); diff --git a/test/plop-test.js b/test/plop-test.js index b746091..9c0d188 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -359,6 +359,57 @@ describe('A city manager', function() { }); + it.only('sets the ZoneType', async function() { + + let dir = path.join(__dirname, 'files'); + let out = path.join(REGION, 'City - Empty City.sc4'); + let source = path.join(dir, 'City - Empty City.sc4'); + // let source = out; + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + path.join(c, 'SimCity_2.dat'), + path.join(c, 'SimCity_3.dat'), + path.join(c, 'SimCity_4.dat'), + path.join(c, 'SimCity_5.dat'), + ], + dirs: [ + path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), + ], + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(source); + + for (let i = 0; i < 10; i++) { + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x: i, + z: 0, + orientation: 2, + }); + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x: i, + z: 2, + orientation: 0, + }); + } + for (let lot of city.dbpf.lots) { + lot.jobsCapacities = [{ demandSourceIndex: 4112, capacity: 5 }]; + lot.jobsTotalCapacities = [{ demandSourceIndex: 4112, capacity: 5 }]; + } + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0x60000b70], + x: 11, + z: 2, + }); + + await city.save({ file: out }); + + }); + it.skip('includes the textures when plopping', async function() { this.timeout(0); @@ -454,106 +505,7 @@ describe('A city manager', function() { }); - it.skip('positions a 1x2 lot', async function() { - - this.timeout(); - let dir = path.join(__dirname, 'files'); - let out = path.join(REGION, 'City - Base Textures.sc4'); - let source = path.join(dir, 'City - Base Textures.sc4'); - - let index = new FileIndex({ - files: [ - path.join(c, 'SimCity_1.dat'), - ], - }); - await index.build(); - - let city = new CityManager({ index }); - city.load(source); - - city.grow({ - tgi: [0x6534284a,0xa8fbd372,0x600045d0], - x: 2, - z: 2, - orientation: 0, - }); - - await city.save({ file: out }); - - }); - - it.skip('fixes the brown boxes', async function() { - - this.timeout(); - - let dir = path.join(__dirname, 'files'); - let out = path.join(REGION, 'City - Base Textures.sc4'); - let source = path.join(dir, 'City - Base Textures.sc4'); - - let index = new FileIndex({ - files: [ - path.join(c, 'SimCity_1.dat'), - ], - }); - await index.build(); - - let entry = index.find([0x29A5D1EC,0x2A2458F9,0x213B0000]); - console.log(entry.dbpf.file); - - return; - - let city = new CityManager({ index }); - city.load(source); - - for (let i = 2; i < 64; i++) { - for (let j = 2; j < 32; j++) { - city.grow({ - tgi: [0x6534284a,0xa8fbd372,0x600029b0], - x: i, - z: 2*j, - orientation: 0, - }); - break; - } - break; - } - - await city.save({ file: out }); - - }); - - it.skip('plops a lot with an ATC prop', async function() { - - this.timeout(); - let dir = path.join(__dirname, 'files'); - let source = path.join(dir, 'City - ATC.sc4'); - let out = path.join(REGION, 'City - ATC.sc4'); - - let index = new FileIndex({ - files: [ - path.join(c, 'SimCity_1.dat'), - ], - dirs: [ - path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), - ], - }); - await index.build(); - - let city = new CityManager({ index }); - city.load(source); - - city.grow({ - tgi: [0x6534284a,0xa8fbd372,0xa706ed25], - x: 2, - z: 1, - orientation: 2, - }); - - await city.save({ file: out }); - - }); - - it.only('plops a Hilbert curve', async function() { + it.skip('plops a Hilbert curve', async function() { const curve = require('hilbert-curve'); @@ -580,7 +532,7 @@ describe('A city manager', function() { const n = 2**order; const nn = n**2; let data = []; - const length = 4; + const length = 5; for (let i = 0; i < nn; i++) { let point = curve.indexToPoint(i, order); data.push({ From 239c467cce7c8a55ed89e75453410caa2a8b3b7f Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Tue, 17 Mar 2020 23:09:09 +0100 Subject: [PATCH 35/39] Create zones --- lib/city-manager.js | 58 +++++++++++++++++++++++++++++++++++++++++ lib/lot-base-texture.js | 24 ++++++++--------- test/plop-test.js | 55 +++++++++----------------------------- 3 files changed, 83 insertions(+), 54 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index a0ee8fe..649c5b4 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -286,6 +286,64 @@ class CityManager { } + // ## zone(opts) + // The function responsible for creating RCI zones. Note that we **don't** + // use the createLot function underneath as that + zone(opts) { + let { + x = 0, + z = 0, + width = 1, + depth = 1, + orientation = 0, + zoneType = 0x01, + } = opts; + let { dbpf } = this; + let { lots, zones } = dbpf; + + // Create the lot with the zone. + let lot = new Lot({ + mem: this.mem(), + flag1: 0x10, + yPos: 270, + minX: x, + maxX: x+width-1, + minZ: z, + maxZ: z+depth-1, + commuteX: x, + commuteZ: z, + width, + depth, + orientation, + zoneType, + + jobCapacities: [{ + demandSourceIndex: 0x00003320, + capacity: 0, + }], + + }); + lots.push(lot); + + // Put in the zone developer file. + for (let x = lot.minX; x <= lot.maxX; x++) { + for (let z = lot.minZ; z <= lot.maxZ; z++) { + zones.cells[x][z] = { + mem: lot.mem, + type: FileType.LotFile, + }; + } + } + + // At last update the com serializer. + let com = dbpf.COMSerializerFile; + com.set(FileType.LotFile, lots.length); + + // Return the created zone. + return lot; + + } + // ## createLot(opts) // Creates a new lot object from the given options when plopping a lot. createLot(opts) { diff --git a/lib/lot-base-texture.js b/lib/lot-base-texture.js index 4370fc7..a81d3fe 100644 --- a/lib/lot-base-texture.js +++ b/lib/lot-base-texture.js @@ -190,10 +190,10 @@ class Texture { this.z = this.x = 0; this.orientation = 0; this.priority = 0x00; - this.u2 = 0xff; - this.u3 = 0xff; - this.u4 = 0xff; - this.u5 = 0xff; + this.r = 0xff; + this.g = 0xff; + this.b = 0xff; + this.alpha = 0xff; this.u6 = 0xff; this.u7 = 0x00; Object.assign(this, opts); @@ -206,10 +206,10 @@ class Texture { this.z = rs.byte(); this.orientation = rs.byte(); this.priority = rs.byte(); - this.u2 = rs.byte(); - this.u3 = rs.byte(); - this.u4 = rs.byte(); - this.u5 = rs.byte(); + this.r = rs.byte(); + this.g = rs.byte(); + this.b = rs.byte(); + this.alpha = rs.byte(); this.u6 = rs.byte(); this.u7 = rs.byte(); return this; @@ -229,10 +229,10 @@ class Texture { ws.byte(this.z); ws.byte(this.orientation); ws.byte(this.priority); - ws.byte(this.u2); - ws.byte(this.u3); - ws.byte(this.u4); - ws.byte(this.u5); + ws.byte(this.r); + ws.byte(this.g); + ws.byte(this.b); + ws.byte(this.alpha); ws.byte(this.u6); ws.byte(this.u7); yield buff; diff --git a/test/plop-test.js b/test/plop-test.js index 9c0d188..0e58825 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -20,6 +20,7 @@ const HOME = process.env.HOMEPATH; const PLUGINS = path.resolve(HOME, 'documents/SimCity 4/plugins'); const REGION = path.resolve(HOME, 'documents/SimCity 4/regions/experiments'); const c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; +const dir = path.join(__dirname, 'files'); describe('A city manager', function() { @@ -295,7 +296,7 @@ describe('A city manager', function() { }); - it.skip('builds a skyline', async function() { + it.only('builds a skyline', async function() { this.timeout(0); @@ -359,53 +360,23 @@ describe('A city manager', function() { }); - it.only('sets the ZoneType', async function() { + it.skip('creates RCI zones', async function() { - let dir = path.join(__dirname, 'files'); - let out = path.join(REGION, 'City - Empty City.sc4'); - let source = path.join(dir, 'City - Empty City.sc4'); - // let source = out; - let index = new FileIndex({ - files: [ - path.join(c, 'SimCity_1.dat'), - path.join(c, 'SimCity_2.dat'), - path.join(c, 'SimCity_3.dat'), - path.join(c, 'SimCity_4.dat'), - path.join(c, 'SimCity_5.dat'), - ], - dirs: [ - path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), - ], - }); - await index.build(); + this.timeout(0); - let city = new CityManager({ index }); + let source = path.join(dir, 'City - Zone me.sc4'); + let out = path.join(REGION, 'City - Zone me.sc4'); + let city = new CityManager({}); city.load(source); - for (let i = 0; i < 10; i++) { - city.grow({ - tgi: [0x6534284a,0xa8fbd372,0xa706ed25], - x: i, - z: 0, - orientation: 2, - }); - city.grow({ - tgi: [0x6534284a,0xa8fbd372,0xa706ed25], - x: i, - z: 2, - orientation: 0, - }); - } - for (let lot of city.dbpf.lots) { - lot.jobsCapacities = [{ demandSourceIndex: 4112, capacity: 5 }]; - lot.jobsTotalCapacities = [{ demandSourceIndex: 4112, capacity: 5 }]; - } - city.grow({ - tgi: [0x6534284a,0xa8fbd372,0x60000b70], - x: 11, - z: 2, + // Create a new zone. + city.zone({ + x: 1, + z: 0, + orientation: 2, }); + // console.log(city.dbpf.textures); await city.save({ file: out }); }); From 1f6283c0a21bd9c93a5baadd8aa0f44caad4771d Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Wed, 18 Mar 2020 13:46:21 +0100 Subject: [PATCH 36/39] Create appropriate zone types in SimGrids --- lib/city-manager.js | 10 ++++++++-- lib/savegame.js | 7 +++++++ lib/sim-grid-file.js | 19 ++++++++++++++----- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 649c5b4..49fb93f 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -20,6 +20,7 @@ const LotResourceKey = 0xea260589; const LotConfigPropertySize = 0x88edc790; const LotConfigPropertyZoneTypes = 0x88edc793; const Wealth = 0x27812832; +const ZoneData = 0x41800000; const INSET = 0.1; // # CityManager @@ -325,9 +326,11 @@ class CityManager { }); lots.push(lot); - // Put in the zone developer file. + // Put in the zone developer file & update the Zone View Sim Grid. + let grid = dbpf.getSimGrid(FileType.SimGridSint8, ZoneData); for (let x = lot.minX; x <= lot.maxX; x++) { for (let z = lot.minZ; z <= lot.maxZ; z++) { + grid.set(x, z, zoneType); zones.cells[x][z] = { mem: lot.mem, type: FileType.LotFile, @@ -363,6 +366,7 @@ class CityManager { file, LotConfigPropertyZoneTypes, ); + let zoneType = zoneTypes[0] || 0x0f; // Determine the zoneWealth as well. Note that this is to be taken // **from the building**. @@ -389,7 +393,7 @@ class CityManager { depth, orientation, zoneWealth: zoneWealth || 0x00, - zoneType: zoneTypes[0] || 0x0f, + zoneType, // Apparently jobCapacities is also required, otherwise CTD! The // capacity is stored I guess in the LotConfig exemplar, or @@ -407,12 +411,14 @@ class CityManager { // Now put the lot in the zone developer file as well. TODO: We should // actually check first and ensure that no building exists yet here! let zones = dbpf.zoneDeveloperFile; + let grid = dbpf.getSimGrid(FileType.SimGridSint8, ZoneData); for (let x = lot.minX; x <= lot.maxX; x++) { for (let z = lot.minZ; z <= lot.maxZ; z++) { zones.cells[x][z] = { mem: lot.mem, type: FileType.LotFile, }; + grid.set(x, z, zoneType); } } diff --git a/lib/savegame.js b/lib/savegame.js index 9d1387e..f4e8ffd 100644 --- a/lib/savegame.js +++ b/lib/savegame.js @@ -77,6 +77,13 @@ class Savegame extends DBPF { return this.readByType(FileType.COMSerializerFile); } + // ## getSimGrid(type, dataId) + // Finds & return a SimGrid based on type and data id. + getSimGrid(type, dataId) { + let { grids } = this.readByType(type); + return grids.find(grid => grid.dataId === dataId); + } + // # getByType(type) // This method returns an entry in the savegame by type. If it doesn't // exist yet, it is created. diff --git a/lib/sim-grid-file.js b/lib/sim-grid-file.js index c6cab98..bcf035e 100644 --- a/lib/sim-grid-file.js +++ b/lib/sim-grid-file.js @@ -129,11 +129,6 @@ const SimGrid = SimGridFile.SimGrid = class SimGrid { this.u8 = rs.dword(); this.u9 = rs.dword(); - // let format = '4 4 4 2 1 4 4 4 4 4 4 4 4 4'.split(' ').map(x => 2*x); - // let header = buff.slice(0, 55).toString('hex'); - // console.log(chunk(format, header)); - // console.log('rest', Math.sqrt(buff.byteLength-55)); - // Don't know if multiple values are possible here, the SInt8 does // some pretty weird stuff... Anyway, for now we'll just read in the // rest into the appropriate underlying array type. @@ -203,6 +198,20 @@ const SimGrid = SimGridFile.SimGrid = class SimGrid { } + // ## get(x, z) + // Returns the value stored in cell (x, z) + get(x, z) { + let { zSize } = this; + return this.data[ x*zSize + z ]; + } + + // ## set(x, z) + // Sets the value stored in cell (x, z) + set(x, z, value) { + this.data[ x*this.zSize+z ] = value; + return this; + } + // ## createProxy() // Creates a data proxy so that we can access the data in an array-like // way. From 1b4202768d9a359fd4d6210f9c596513a3a28cdb Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Wed, 18 Mar 2020 16:50:11 +0100 Subject: [PATCH 37/39] Parse zone manager --- lib/dbpf.js | 1 + lib/file-types.js | 1 + lib/pointer.js | 20 ++++++ lib/savegame.js | 9 ++- lib/stream.js | 9 +++ lib/type.js | 9 +++ lib/write-stream.js | 8 +++ lib/zone-manager.js | 128 ++++++++++++++++++++++++++++++++++++++ test/zone-manager-test.js | 96 ++++++++++++++++++++++++++++ 9 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 lib/pointer.js create mode 100644 lib/zone-manager.js create mode 100644 test/zone-manager-test.js diff --git a/lib/dbpf.js b/lib/dbpf.js index f92e0ac..9aaf525 100644 --- a/lib/dbpf.js +++ b/lib/dbpf.js @@ -542,6 +542,7 @@ DBPF.register([ require('./zone-developer-file'), require('./lot-developer-file'), require('./com-serializer-file'), + require('./zone-manager.js'), ]); DBPF.register(FileType.Cohort, Exemplar); diff --git a/lib/file-types.js b/lib/file-types.js index e356e1a..34d78df 100644 --- a/lib/file-types.js +++ b/lib/file-types.js @@ -48,6 +48,7 @@ const TYPES = module.exports = { "PropDeveloperFile": 0x89c48f47, "COMSerializerFile": 0x499b23fe, "NetworkFile": 0xc9c05c6e, + "ZoneManager": 0x298f9b2d, // Sim grids "SimGridFloat32": 0x49b9e60a, diff --git a/lib/pointer.js b/lib/pointer.js new file mode 100644 index 0000000..e700210 --- /dev/null +++ b/lib/pointer.js @@ -0,0 +1,20 @@ +// # pointer.js +// Small helper class that represents a pointer to a certain record in the +// subfile. +const { hex } = require('./util.js'); +class Pointer { + + // ## constructor(type, address) + constructor(type, address = 0x00000000) { + this.type = type; + this.address = address; + } + + // ## get [Symbol.toPrimitive](hint) + // Allows you to get the numerical value of the pointer by using +pointer. + [Symbol.toPrimitive](hint) { + return hint === 'number' ? this.address : hex(this.address); + } + +} +module.exports = Pointer; diff --git a/lib/savegame.js b/lib/savegame.js index f4e8ffd..a25a73e 100644 --- a/lib/savegame.js +++ b/lib/savegame.js @@ -72,6 +72,11 @@ class Savegame extends DBPF { return this.readByType(FileType.FloraFile); } + // ## get zoneManager() + get zoneManager() { + return this.readByType(FileType.ZoneManager); + } + // ## get COMSerializerFile() get COMSerializerFile() { return this.readByType(FileType.COMSerializerFile); @@ -80,8 +85,8 @@ class Savegame extends DBPF { // ## getSimGrid(type, dataId) // Finds & return a SimGrid based on type and data id. getSimGrid(type, dataId) { - let { grids } = this.readByType(type); - return grids.find(grid => grid.dataId === dataId); + let grids = this.readByType(type); + return grids.get(dataId); } // # getByType(type) diff --git a/lib/stream.js b/lib/stream.js index 469e5a1..b8d288f 100644 --- a/lib/stream.js +++ b/lib/stream.js @@ -1,5 +1,6 @@ // # stream.js "use strict"; +const Pointer = require('./pointer.js'); // Monkey-patch big ints. if (!Buffer.prototype.readBigInt64LE) { @@ -98,6 +99,14 @@ class Stream { dword() { return this.uint32(); } bool() { return Boolean(this.uint8()); } + // Helper function for reading a pointer. Those are given as [pointer, + // Type ID]. + pointer() { + let address = this.dword(); + let type = this.dword(); + return new Pointer(type, address); + } + } module.exports = Stream; \ No newline at end of file diff --git a/lib/type.js b/lib/type.js index 0917e3b..e4951e3 100644 --- a/lib/type.js +++ b/lib/type.js @@ -41,6 +41,15 @@ class Type { return this.constructor.toString(); } + // ## [Symbol.toPrimitive](hint) + [Symbol.toPrimitive](hint) { + if (hint === 'number') { + return this.mem; + } else { + return hex(this.mem); + } + } + // ## toBuffer(opts) // Most types share the same method of creating a buffer, being by using a // binary generator under the hood. diff --git a/lib/write-stream.js b/lib/write-stream.js index ba8bb3b..188b051 100644 --- a/lib/write-stream.js +++ b/lib/write-stream.js @@ -77,5 +77,13 @@ class WriteStream { return this; } + // ## pointer(ptr) + // Writes a pointer to the buffer. + pointer(ptr) { + this.dword(ptr.address); + this.dword(ptr.type); + return this; + } + } module.exports = WriteStream; \ No newline at end of file diff --git a/lib/zone-manager.js b/lib/zone-manager.js new file mode 100644 index 0000000..779c92c --- /dev/null +++ b/lib/zone-manager.js @@ -0,0 +1,128 @@ +// # zone-manager.js +const Stream = require('./stream.js'); +const WriteStream = require('./write-stream.js'); +const crc32 = require('./crc.js'); +const Type = require('./type.js'); +const Pointer = require('./pointer.js'); +const { FileType } = require('./enums.js'); + +// Some type ids. We should put them within the FileType's though! +const cSC4SimGridSint8 = 0x49b9e603; +const cSC4City = 0x8990c372; +const cSC4OccupantManager = 0x098f964d; +const cSTETerrain = 0xe98f9525; +const cSC4PollutionSimulator = 0x8990c065; +const cSC4BudgetSimulator = 0xe990be01; + +// # ZoneManager +// Not sure what it does yet, still decoding. +class ZoneManager extends Type(FileType.ZoneManager) { + + // ## constructor(opts) + constructor(opts) { + super(); + this.crc = 0x00000000; + this.mem = 0x00000000; + this.major = 0x0001; + + // Pointer to the ZoneView grid. + this.grid = new Pointer(cSC4SimGridSint8); + this.u1 = Array(16).fill(0x00000000); + + // Note even though the size is repeated multiple times when parsing, + // we'll only assign it once. + this.size = 0x00000000; + + // From now on follows a part that always seems to be fixed. + this.u2 = Buffer.from(fixed, 'hex'); + + // Pointers to other subfiles. + this.city = new Pointer(cSC4City); + this.occupantManager = new Pointer(cSC4OccupantManager); + this.terrain = new Pointer(cSTETerrain); + this.pollutionSimulator = new Pointer(cSC4PollutionSimulator); + this.budgetSimulator = new Pointer(cSC4BudgetSimulator); + Object.assign(this, opts); + } + + // ## parse(buff) + parse(buff) { + let rs = new Stream(buff); + let size = rs.dword(); + this.crc = rs.dword(); + this.mem = rs.dword(); + this.major = rs.word(); + this.grid = rs.pointer(); + + // Read in the unknowns. + let arr = this.u1 = []; + for (let i = 0; i < 16; i++) { + arr.push(rs.dword()); + } + + // Read in the size, but skip the fact that it's repeated. + this.size = rs.dword(); + rs.skip(32*4); + + // Read in the next 533 bytes. Seems like they're always fixed. + this.u2 = rs.read(533); + + // More pointers follow now. + this.city = rs.pointer(); + this.occupantManager = rs.pointer(); + this.terrain = rs.pointer(); + this.pollutionSimulator = rs.pointer(); + this.budgetSimulator = rs.pointer(); + + } + + *bgen() { + let buff = Buffer.allocUnsafe(791); + let ws = new WriteStream(buff); + ws.dword(buff.byteLength); + ws.jump(8); + ws.dword(this.mem); + ws.word(this.major); + ws.pointer(this.grid); + for (let dword of this.u1) { + ws.dword(dword); + } + for (let i = 0; i < 33; i++) { + ws.dword(this.size); + } + ws.write(this.u2); + ws.pointer(this.city); + ws.pointer(this.occupantManager); + ws.pointer(this.terrain); + ws.pointer(this.pollutionSimulator); + ws.pointer(this.budgetSimulator); + + // Write crc & we're done. + buff.writeUInt32LE(this.crc = crc32(buff, 8), 4); + yield buff; + + } + +} +module.exports = ZoneManager; + +const fixed = ` +00000000 00000000 00000000 00000000 01000000 +01000000 01000000 01000000 01000000 01000000 04000000 01000000 01000000 +01000000 01000000 01000000 01000000 02000000 01000000 00800000 20000000 +20000000 20000000 20000000 20000000 20000000 30000000 20000000 20000000 +20000000 20000000 20000000 20000000 20000000 20000000 00000000 00000000 +0a000000 00000000 14000000 00000000 32000000 00000000 0a000000 00000000 +14000000 00000000 32000000 00000000 0a000000 00000000 14000000 00000000 +32000000 00000000 01000000 00000000 01000000 00000000 01000000 00000000 +01000000 00000000 32000000 00000000 01000000 00000000 00000000 00000000 +01000000 00000000 01000000 00000000 01000000 00000000 01000000 00000000 +01000000 00000000 01000000 00000000 01000000 00000000 01000000 00000000 +01000000 00000000 01000000 00000000 01000000 00000000 01000000 00000000 +01000000 00000000 01000000 00000000 01000000 00000000 19100000 15100000 +16100000 14100000 03100000 04100000 02100000 10100000 11100000 0f100000 +12100000 00100000 18100000 1b100000 05100000 1a100000 ff0000ff ff00c400 +ff009a00 ff007200 ffff774f ffff5320 ffe22014 ff32dbff ff33b2ff ff1f90ce +00000000 00000000 00000000 00000000 00000000 00000000 aa000000 00000000 +00 +`.replace(/( |\n)/g, ''); diff --git a/test/zone-manager-test.js b/test/zone-manager-test.js new file mode 100644 index 0000000..08a0abb --- /dev/null +++ b/test/zone-manager-test.js @@ -0,0 +1,96 @@ +// # zone-manager-test.js +"use strict"; +const { expect } = require('chai'); +const fs = require('fs'); +const path = require('path'); +const Stream = require('../lib/stream.js'); +const Savegame = require('../lib/savegame.js'); +const { hex, chunk } = require('../lib/util'); +const { FileType, cClass } = require('../lib/enums.js'); +const REGION = path.resolve(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments'); +const dir = path.resolve(__dirname, 'files'); + +describe('The zone manager file', function() { + + it('is parsed & serialized correctly', function() { + this.timeout(0); + let file = path.join(dir, 'city.sc4'); + let buff = fs.readFileSync(file); + let dbpf = new Savegame(buff); + let entry = dbpf.getByType(FileType.ZoneManager); + let zm = entry.read(); + + // Check that the input buffer matches the out buffer exactly. + let { crc } = zm; + let out = zm.toBuffer(); + expect(out).to.eql(entry.decompress()); + + }); + + it('is decoded', async function() { + + // let file = path.join(REGION, 'City - Growth.sc4'); + // let dbpf = new Savegame(file); + let one = new Savegame(path.join(REGION, 'City - Growth.sc4')); + // let one = new Savegame(path.resolve(REGION, '../New Delphina/City - Strateigia.sc4')); + // let two = new Savegame(path.join(REGION, 'City - Growth.sc4')); + + const crc32 = require('../lib/crc.js'); + let entry = one.entries.find(entry => entry.type === 0x298f9b2d); + // for (let entry of one) { + let buff = entry.decompress(); + let size = buff.readUInt32LE(); + let slice = buff.slice(0, size); + let crc = crc32(slice, 8); + if (crc !== buff.readUInt32LE(4)) { + console.log(types[ entry.type ]); + } + // } + + let rs = new Stream(buff); + + // let header = buff.slice(0, 14); + let header = rs.read(14); + console.log(chunk([8, 8, 8, 4], header.toString('hex'))); + + let n = 23; + for (let i = 0; i < n; i++) { + // let slice = body.slice(32*i, 32*i+32); + let slice = rs.read(32); + let format = Array(8).fill(8); + console.log(chunk(format, slice.toString('hex'))); + } + console.log(rs.read(1).toString('hex')); + for (let i = 0; i < 5; i++) { + let slice = rs.read(8); + console.log(chunk([8, 8], slice.toString('hex'))); + } + + // console.log(buff.toString('hex')); + // console.log(slice.toString('hex').length); + + return; + + for (let type of [FileType.SimGridSint8]) { + + let { grids } = one.readByType(type); + console.log('LENGTH', grids.length); + console.log(grids[0].dataId.toString(16)); + + function fill(grid, x=1) { + let { dataId } = grid; + // console.log('DATA ID', hex(dataId)); + grid.data.fill(x); + } + fill(grids[0], 1); + // for (let i = 0; i < grids.length; i++) { + // fill(grids[i], 1); + // } + + } + + await one.save({ file: path.join(REGION, 'City - Growth.sc4') }); + + }); + +}); \ No newline at end of file From 96392b249473ede0e8705c4703d2cf0fcf427233 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Thu, 19 Mar 2020 12:09:19 +0100 Subject: [PATCH 38/39] Fix growable zones --- lib/city-manager.js | 8 +++ test/zone-manager-test.js | 100 +++++++++++++++----------------------- 2 files changed, 46 insertions(+), 62 deletions(-) diff --git a/lib/city-manager.js b/lib/city-manager.js index 49fb93f..0231382 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -318,6 +318,14 @@ class CityManager { orientation, zoneType, + // An empty growable lot has this flag set to 0, but to 1 when + // it's powered. In theory we need to check hence if the lot is + // reachable by power, but apparently the game does this by + // itself! The only thing we need to make sure is that the second + // bit is **never** set to 1! Otherwise the lot is considered as + // being built! + flag2: 0b00000001, + jobCapacities: [{ demandSourceIndex: 0x00003320, capacity: 0, diff --git a/test/zone-manager-test.js b/test/zone-manager-test.js index 08a0abb..f537f93 100644 --- a/test/zone-manager-test.js +++ b/test/zone-manager-test.js @@ -7,7 +7,10 @@ const Stream = require('../lib/stream.js'); const Savegame = require('../lib/savegame.js'); const { hex, chunk } = require('../lib/util'); const { FileType, cClass } = require('../lib/enums.js'); -const REGION = path.resolve(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments'); +const CityManager = require('../lib/city-manager.js'); +const HOME = process.env.HOMEPATH; +const PLUGINS = path.resolve(HOME, 'documents/SimCity 4/plugins'); +const REGION = path.resolve(HOME, 'documents/SimCity 4/regions/experiments'); const dir = path.resolve(__dirname, 'files'); describe('The zone manager file', function() { @@ -27,69 +30,42 @@ describe('The zone manager file', function() { }); - it('is decoded', async function() { - - // let file = path.join(REGION, 'City - Growth.sc4'); - // let dbpf = new Savegame(file); - let one = new Savegame(path.join(REGION, 'City - Growth.sc4')); - // let one = new Savegame(path.resolve(REGION, '../New Delphina/City - Strateigia.sc4')); - // let two = new Savegame(path.join(REGION, 'City - Growth.sc4')); - - const crc32 = require('../lib/crc.js'); - let entry = one.entries.find(entry => entry.type === 0x298f9b2d); - // for (let entry of one) { - let buff = entry.decompress(); - let size = buff.readUInt32LE(); - let slice = buff.slice(0, size); - let crc = crc32(slice, 8); - if (crc !== buff.readUInt32LE(4)) { - console.log(types[ entry.type ]); - } - // } - - let rs = new Stream(buff); - - // let header = buff.slice(0, 14); - let header = rs.read(14); - console.log(chunk([8, 8, 8, 4], header.toString('hex'))); - - let n = 23; - for (let i = 0; i < n; i++) { - // let slice = body.slice(32*i, 32*i+32); - let slice = rs.read(32); - let format = Array(8).fill(8); - console.log(chunk(format, slice.toString('hex'))); - } - console.log(rs.read(1).toString('hex')); - for (let i = 0; i < 5; i++) { - let slice = rs.read(8); - console.log(chunk([8, 8], slice.toString('hex'))); - } - - // console.log(buff.toString('hex')); - // console.log(slice.toString('hex').length); - - return; - - for (let type of [FileType.SimGridSint8]) { - - let { grids } = one.readByType(type); - console.log('LENGTH', grids.length); - console.log(grids[0].dataId.toString(16)); - - function fill(grid, x=1) { - let { dataId } = grid; - // console.log('DATA ID', hex(dataId)); - grid.data.fill(x); - } - fill(grids[0], 1); - // for (let i = 0; i < grids.length; i++) { - // fill(grids[i], 1); - // } - + it.only('is decoded', async function() { + + const FileIndex = require('../lib/file-index.js'); + + let out = path.join(REGION, 'City - Growth.sc4'); + let one = path.join(dir, 'City - Growth - 1.sc4'); + let two = new Savegame(path.join(dir, 'City - Growth - 2.sc4')); + + let c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; + // let index = new FileIndex(nybt); + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + path.join(c, 'SimCity_2.dat'), + path.join(c, 'SimCity_3.dat'), + path.join(c, 'SimCity_4.dat'), + path.join(c, 'SimCity_5.dat'), + ], + dirs: [ + PLUGINS, + ], + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(one); + for (let i = 0; i < 10; i++) { + let lot = city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x: 3+i, + z: 1, + orientation: 2, + }); } - await one.save({ file: path.join(REGION, 'City - Growth.sc4') }); + await city.save({ file: out }); }); From 7130274d437cb0014997d73f13b017dc63b48f4e Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Mon, 13 Jul 2020 15:44:12 +0200 Subject: [PATCH 39/39] Allow growified residentials to be redeveloped --- lib/api.js | 22 +++++++++++++++++++--- test/api-test.js | 32 ++++++++++++++++---------------- test/victoria-test.js | 17 +++++++++++++++++ test/zone-manager-test.js | 2 +- 4 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 test/victoria-test.js diff --git a/lib/api.js b/lib/api.js index 2cb2563..49c99c8 100644 --- a/lib/api.js +++ b/lib/api.js @@ -64,16 +64,32 @@ exports.growify = async function(opts) { opts = Object.assign(Object.create(baseOptions), opts); let dbpf = open(opts.dbpf); + // Get the SimGrid with the ZoneData information because that needs to be + // updated as well. + const ZoneData = 0x41800000; + let grid = dbpf.getSimGrid(FileType.SimGridSint8, ZoneData); + + // Helper function that will update the zoneType in the SimGrid as well + // when growifying. + function setType(lot, zoneType) { + lot.zoneType = zoneType; + for (let x = lot.minX; x <= lot.maxX; x++) { + for (let z = lot.minZ; z <= lot.maxZ; z++) { + grid.set(x, z, zoneType); + } + } + } + let rCount = 0, iCount = 0, aCount = 0; for (let lot of dbpf.lotFile) { if (opts.residential && lot.isPloppedResidential) { - lot.zoneType = opts.residential; + setType(lot, opts.residential); rCount++; } else if (opts.industrial && lot.isPloppedIndustrial) { - lot.zoneType = opts.industrial; + setType(lot, opts.industrial); iCount++; } else if (opts.agricultural && lot.isPloppedAgricultural) { - lot.zoneType = opts.agricultural; + setType(lot, opts.agricultural); aCount++; } } diff --git a/test/api-test.js b/test/api-test.js index 224d206..adeb9f6 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -16,8 +16,8 @@ describe('#historical()', function() { it('should make all buildings in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "all": true + dbpf: path.join(files, 'city.sc4'), + all: true }); // Check the dbpf file now. Everything should be historical. @@ -29,8 +29,8 @@ describe('#historical()', function() { it('should make all residentials in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "residential": true + dbpf: path.join(files, 'city.sc4'), + residential: true }); for (let lot of dbpf.lotFile) { @@ -41,8 +41,8 @@ describe('#historical()', function() { it('should make all commercials in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "commercial": true + dbpf: path.join(files, 'city.sc4'), + commercial: true }); for (let lot of dbpf.lotFile) { @@ -53,8 +53,8 @@ describe('#historical()', function() { it('should make all industrials in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "industrial": true + dbpf: path.join(files, 'city.sc4'), + industrial: true }); for (let lot of dbpf.lotFile) { @@ -65,8 +65,8 @@ describe('#historical()', function() { it('should make all agriculturals in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "agricultural": true + dbpf: path.join(files, 'city.sc4'), + agricultural: true }); for (let lot of dbpf.lotFile) { @@ -90,8 +90,8 @@ describe('#growify', function() { expect(plopped.size).to.be.above(0); await growify({ - "dbpf": dbpf, - "residential": ZoneType.RMedium + dbpf, + residential: ZoneType.RMedium, }); for (let lot of dbpf.lotFile) { @@ -114,8 +114,8 @@ describe('#growify', function() { expect(plopped.size).to.be.above(0); await growify({ - "dbpf": dbpf, - "industrial": ZoneType.IHigh + dbpf, + industrial: ZoneType.IHigh }); for (let lot of dbpf.lotFile) { @@ -138,8 +138,8 @@ describe('#growify', function() { expect(plopped.size).to.be.above(0); await growify({ - "dbpf": dbpf, - "agricultural": ZoneType.ILow + dbpf, + agricultural: ZoneType.ILow, }); for (let lot of dbpf.lotFile) { diff --git a/test/victoria-test.js b/test/victoria-test.js new file mode 100644 index 0000000..48b21f7 --- /dev/null +++ b/test/victoria-test.js @@ -0,0 +1,17 @@ +// # victoria-test.js +const path = require('path'); +const Savegame = require('../lib/savegame.js'); + +let city = new Savegame(path.resolve(__dirname, 'files/City - Victoria.sc4')); +let { lots } = city; +for (let lot of lots) { + if (lot.minX === 2752/16 && lot.minZ === 3120/16) { + lot.zoneWealth = 0x03; + // console.log(lot); + } + // if (lot.zoneWealth === 0) { + // console.log(lot); + // } +} + +city.save(path.resolve(__dirname, 'files/City - Victoria restored.sc4')); diff --git a/test/zone-manager-test.js b/test/zone-manager-test.js index f537f93..377fa79 100644 --- a/test/zone-manager-test.js +++ b/test/zone-manager-test.js @@ -30,7 +30,7 @@ describe('The zone manager file', function() { }); - it.only('is decoded', async function() { + it('is decoded', async function() { const FileIndex = require('../lib/file-index.js');