diff --git a/lib/api.js b/lib/api.js index 2cb2563..49c99c8 100644 --- a/lib/api.js +++ b/lib/api.js @@ -64,16 +64,32 @@ exports.growify = async function(opts) { opts = Object.assign(Object.create(baseOptions), opts); let dbpf = open(opts.dbpf); + // Get the SimGrid with the ZoneData information because that needs to be + // updated as well. + const ZoneData = 0x41800000; + let grid = dbpf.getSimGrid(FileType.SimGridSint8, ZoneData); + + // Helper function that will update the zoneType in the SimGrid as well + // when growifying. + function setType(lot, zoneType) { + lot.zoneType = zoneType; + for (let x = lot.minX; x <= lot.maxX; x++) { + for (let z = lot.minZ; z <= lot.maxZ; z++) { + grid.set(x, z, zoneType); + } + } + } + let rCount = 0, iCount = 0, aCount = 0; for (let lot of dbpf.lotFile) { if (opts.residential && lot.isPloppedResidential) { - lot.zoneType = opts.residential; + setType(lot, opts.residential); rCount++; } else if (opts.industrial && lot.isPloppedIndustrial) { - lot.zoneType = opts.industrial; + setType(lot, opts.industrial); iCount++; } else if (opts.agricultural && lot.isPloppedAgricultural) { - lot.zoneType = opts.agricultural; + setType(lot, opts.agricultural); aCount++; } } diff --git a/lib/building.js b/lib/building.js index 6d3fe63..f3edc58 100644 --- a/lib/building.js +++ b/lib/building.js @@ -12,8 +12,8 @@ const Type = require('./type'); // Represents a single building from the building file. const Building = class Building extends Type(FileType.BuildingFile) { - // ## constructor() - constructor() { + // ## constructor(opts) + constructor(opts) { super(); this.crc = 0x00000000; this.mem = 0x00000000; @@ -21,7 +21,7 @@ const Building = class Building extends Type(FileType.BuildingFile) { this.minor = 0x0004; this.zotWord = 0x0; this.unknown1 = 0x00; - this.appearance = 0x04; + this.appearance = 0b00000101; this.xMinTract = 0x00; this.zMinTract = 0x00; this.xMaxTract = 0x00; @@ -29,12 +29,13 @@ const Building = class Building extends Type(FileType.BuildingFile) { this.xTractSize = 0x0002; this.zTractSize = 0x0002; this.sgprops = []; - this.unknown2 = 0x01; + this.unknown2 = 0x00; this.IID1 = this.IID = this.TID = this.GID = 0x00000000; this.minZ = this.minY = this.minX = 0; this.maxZ = this.maxY = this.maxX = 0; this.orientation = 0x00; - this.scaffold = 0; + this.scaffold = 0x01; + Object.assign(this, opts); } // ## move(dx, dy, dz) @@ -100,11 +101,13 @@ const Building = class Building extends Type(FileType.BuildingFile) { prop.parse(rs); } - this.unknown2 = rs.byte(); + // There seems to be an error in the Wiki. The unknown byte should + // come **after** the IID1, otherwise they're incorrect. this.GID = rs.dword(); this.TID = rs.dword(); this.IID = rs.dword(); this.IID1 = rs.dword(); + this.unknown2 = rs.byte(); this.minX = rs.float(); this.minY = rs.float(); this.minZ = rs.float(); @@ -158,11 +161,11 @@ const Building = class Building extends Type(FileType.BuildingFile) { // Serialize the fixed remainder. let two = Buffer.allocUnsafe(46); ws = new WriteStream(two); - ws.byte(this.unknown2); ws.dword(this.GID); ws.dword(this.TID); ws.dword(this.IID); ws.dword(this.IID1); + ws.byte(this.unknown2); ws.float(this.minX); ws.float(this.minY); ws.float(this.minZ); diff --git a/lib/city-manager.js b/lib/city-manager.js index cab7312..0231382 100644 --- a/lib/city-manager.js +++ b/lib/city-manager.js @@ -2,59 +2,731 @@ "use strict"; const path = require('path'); const fs = require('fs'); -const Savegame = require('./savegame'); -const Index = require('./index'); +const Savegame = require('./savegame.js'); +const Lot = require('./lot.js'); +const Building = require('./building.js'); +const Prop = require('./prop.js'); +const BaseTexture = require('./lot-base-texture.js'); +const { FileType } = require('./enums.js'); const SC4 = path.resolve(process.env.HOMEPATH, 'documents/SimCity 4'); const regions = path.join(SC4, 'regions'); -const plugins = path.join(SC4, 'plugins'); + +// Hex constants to make the code more readable. +const ExemplarType = 0x10; +const Buildings = 0x02; +const OccupantSize = 0x27812810; +const LotConfigurations = 0x10; +const LotResourceKey = 0xea260589; +const LotConfigPropertySize = 0x88edc790; +const LotConfigPropertyZoneTypes = 0x88edc793; +const Wealth = 0x27812832; +const ZoneData = 0x41800000; +const INSET = 0.1; // # CityManager +// A class for performing operations on a certain city, such as plopping +// arbitrary lots etc. Have a look at https://sc4devotion.com/forums/ +// index.php?topic=5656.0, contains a lot of relevant info. class CityManager { - // ## constructor(dbpf) + // ## constructor(opts) // Sets up the city manager. - constructor(dbpf) { + constructor(opts = {}) { - // If we've received a string, treat it as a path. - if (typeof dbpf === 'string') { - let file = path.resolve(regions, dbpf); - - // No extension given? Add .sc4 - let ext = path.extname(file); - if (ext !== '.sc4') file += '.sc4'; - - // Check if the file exists. If it doesn't exist, then try again - // with "City - " in front. - if (!fs.existsSync(file)) { - let name = path.basename(file); - let dir = path.dirname(file); - file = path.join(dir, 'City - '+name); - if (!fs.existsSync(file)) { - throw new Error(`City "${dbpf}" could not be found!`); - } + // Pre-initialize the "private" fields that cannot be modified by the + // options. + this.memRefs = null; + this.$mem = 1; + + // Setup the "public" fields. + this.dbpf = opts.dbpf || null; + this.index = opts.index || null; + + } + + // ## setFileIndex(index) + // Stores the file index to be used for looking up TGI's etc. That's + // required if you want to plop lot's etc. because in that case we need to + // know where to look for the resources! + setFileIndex(index) { + this.index = index; + } + + // ## load(file) + // Loads the given savegame into the city manager. + load(file) { + + let full = path.resolve(regions, file); + + // No extension given? Add .sc4 + let ext = path.extname(full); + if (ext !== '.sc4') { + full += '.sc4'; + } + + // Check if the file exists. If it doesn't exist, then try again + // with "City - " in front. + if (!fs.existsSync(full)) { + let name = path.basename(full); + let dir = path.dirname(full); + full = path.join(dir, 'City - '+name); + if (!fs.existsSync(full)) { + throw new Error(`City "${file}" could not be found!`); + } + } + + // Create the city. + this.dbpf = new Savegame(fs.readFileSync(full)); + + // Index the *initial* mem refs as well. We need to make sure we do + // this before creating any new files in the dbpf, otherwise we're + // indexing mem refs we might have created ourselves. + let set = this.memRefs = new Set(); + for (let { mem } of this.dbpf.memRefs()) { + set.add(mem); + } + + } + + // ## save(opts) + // Saves the city to the given file. + save(opts) { + return this.dbpf.save(opts); + } + + // ## mem() + // Returns an unused memory address. This is useful if we add new stuff to + // a city - such as buildings etc. - because we need to make sure that the + // memory addresses for every record are unique. + mem() { + + // Create a new memory reference, but make sure it doesn't exist yet. + let ref = this.$mem++; + while (this.memRefs.has(ref)) { + ref = this.$mem++; + } + this.memRefs.add(ref); + return ref; + + } + + // ## getProperty(file, key) + // Helper function for getting a property from an exemplar, taking into + // account the inheritance chain. It's the index that is actually + // responsible for this though. + getProperty(file, key) { + return this.index.getProperty(file, key); + } + + // ## getPropertyValue(file, prop) + // Returns the direct value for the given property. + getPropertyValue(file, key) { + return this.index.getPropertyValue(file, key); + } + + // ## plop(opts) + // Behold, the mother of all functions. This function allows to plop any + // lot anywhere in the city. Note that this function expects a *building* + // exemplar, which means it only works for *ploppable* buildings. For + // growable buildings the process is different, in that case you have to + // use the "grow" method. + plop(opts = {}) { + + // (1) First of all we need to find the T10 exemplar file with the + // information to plop the lot. Most of the time this resides in an + // .sc4lot file, but it doesn't have to. + let { tgi, building } = opts; + if (!building && tgi) { + building = this.index.find(tgi); + if (!building) { + throw new Error( + `Exemplar ${ JSON.stringify(tgi) } not found!`, + ); } + } + + // Check what type of exemplar we're dealing with. As explained by + // RippleJet, there's a fundamental difference between ploppable and + // growable buildings. Apparently ploppable buildings start from a + // building exemplar and then we can look up according + // LotConfiguration exemplar. + let file = building.read(); + if (this.getPropertyValue(file, ExemplarType) !== Buildings) { + throw new Error([ + 'The exemplar is not a building exemplar!', + 'The `.plop()` function expects a ploppable building exemplar!' + ].join(' ')); + } + + // Find the lot resource key, which is the IID where we can find the + // LotResourceKey & then based on that find the appropriate Building + // exemplar. Note that we currently have no other choice than finding + // everything with the same instance ID... + let IID = this.getPropertyValue(file, LotResourceKey); + let lotExemplar = this.findExemplarOfType(IID, LotConfigurations); - // Create the city. - dbpf = new Savegame(fs.readFileSync(file)); + // Cool, we have both the building & the lot exemplar. Create the lot. + let lot = this.build({ + lot: lotExemplar, + building, + x: opts.x, + z: opts.z, + orientation: opts.orientation, + }); + } + + // ## grow(opts) + // This method is similar to the `plop()` method, but this time it starts + // from a *Lot Configurations* exemplar, not a ploppable building exemplar + // - which is how the game does it. From then on the logic is pretty much + // the same. + grow(opts) { + + let { tgi, exemplar } = opts; + let record = exemplar ? exemplar : this.index.find(tgi); + if (!record) { + throw new Error( + `Exemplar ${ JSON.stringify(tgi) } not found!`, + ); + } + + // Ensure that the exemplar that was specified. + let props = record.read(); + if (+props.value(ExemplarType) !== LotConfigurations) { + throw new Error([ + 'The exemplar is not a lot configurations exemplar!', + 'The `.grow()` function expects a lot exemplar!' + ].join(' ')); + } + + // Find the appropriate building exemplar. Note that it's possible + // that the building belongs to a family. In that case we'll pick a + // random building from the family. + let IIDs = props.lotObjects.find(({ type }) => type === 0x00).IIDs; + let IID = rand(IIDs); + let buildingExemplar = this.findExemplarOfType(IID, 0x02); + + // Now that we have both the building exemplar and as well as the lot + // exemplar we can create the lot and insert everything on it into the + // city. + let { x, z, orientation } = opts; + let lot = this.build({ + building: buildingExemplar, + lot: record, + x, + z, + orientation, + }); + + } + + // ## build(opts) + // This method is responsible for inserting all *physical* entities into + // the city such as a lot, a building, the props on the lot, the textures + // etc. It's not really meant for public use, you should use the `.plop()` + // or `.grow()` methods instead. It requires a lot exemplar and a building + // exemplar to be specified. It's the `.plop()` and `.grow()` methods that + // are responsible for deciding what building will be inserted. + build(opts) { + + // First of all create the lot record & insert it into the city. + let { + lot: lotExemplar, + building, + x = 0, + z = 0, + orientation = 0, + } = opts; + + let lot = this.createLot({ + exemplar: lotExemplar, + building, + x: opts.x, + z: opts.z, + orientation, + }); + + // Loop all objects on the lot such and insert them. + let { lotObjects } = lotExemplar.read(); + let textures = []; + for (let lotObject of lotObjects) { + switch (lotObject.type) { + case 0x00: + this.createBuilding({ + lot, + lotObject, + exemplar: building, + }); + break; + case 0x01: + this.createProp({ + lot, + lotObject, + }); + break; + case 0x02: + + // Note: We can't handle textures right away because they + // need to be put in a *single* BaseTexture entry. As such + // we'll simply collect them for now. + textures.push(lotObject); + break; + } } + // Create the textures. + this.createTexture({ + lot, + textures, + }); + + // At last return the created lot so that the calling function can + // modify the properties such as capcity, zoneWealth, zoneDensity etc. + return lot; + } - // ## loadPlugins(opts) - async loadPlugins(opts) { - if (!opts) { - opts = { - "dirs": [plugins] + // ## zone(opts) + // The function responsible for creating RCI zones. Note that we **don't** + // use the createLot function underneath as that + zone(opts) { + let { + x = 0, + z = 0, + width = 1, + depth = 1, + orientation = 0, + zoneType = 0x01, + } = opts; + let { dbpf } = this; + let { lots, zones } = dbpf; + + // Create the lot with the zone. + let lot = new Lot({ + mem: this.mem(), + flag1: 0x10, + yPos: 270, + minX: x, + maxX: x+width-1, + minZ: z, + maxZ: z+depth-1, + commuteX: x, + commuteZ: z, + width, + depth, + orientation, + zoneType, + + // An empty growable lot has this flag set to 0, but to 1 when + // it's powered. In theory we need to check hence if the lot is + // reachable by power, but apparently the game does this by + // itself! The only thing we need to make sure is that the second + // bit is **never** set to 1! Otherwise the lot is considered as + // being built! + flag2: 0b00000001, + + jobCapacities: [{ + demandSourceIndex: 0x00003320, + capacity: 0, + }], + + }); + lots.push(lot); + + // Put in the zone developer file & update the Zone View Sim Grid. + let grid = dbpf.getSimGrid(FileType.SimGridSint8, ZoneData); + for (let x = lot.minX; x <= lot.maxX; x++) { + for (let z = lot.minZ; z <= lot.maxZ; z++) { + grid.set(x, z, zoneType); + zones.cells[x][z] = { + mem: lot.mem, + type: FileType.LotFile, + }; } } - // Build the index. - let index = this.plugins = new Index(opts); - await index.build(); + // At last update the com serializer. + let com = dbpf.COMSerializerFile; + com.set(FileType.LotFile, lots.length); + + // Return the created zone. + return lot; } + // ## createLot(opts) + // Creates a new lot object from the given options when plopping a lot. + createLot(opts) { + + // Read in the size of the lot because we'll still need it. + let { dbpf } = this; + let lots = dbpf.lotFile; + let { exemplar, x, z, building, orientation = 0 } = opts; + let file = exemplar.read(); + let [width, depth] = this.getPropertyValue( + file, + LotConfigPropertySize + ); + + // Determine the zone type. + let zoneTypes = this.getPropertyValue( + file, + LotConfigPropertyZoneTypes, + ); + let zoneType = zoneTypes[0] || 0x0f; + + // Determine the zoneWealth as well. Note that this is to be taken + // **from the building**. + let buildingFile = building.read(); + let zoneWealth = this.getPropertyValue(buildingFile, Wealth); + + // Cool, we can now create a new lot entry. Note that we will need to + // take into account the + let lot = new Lot({ + mem: this.mem(), + IID: exemplar.instance, + buildingIID: building.instance, + + // For now, just put at y = 270. In the future we'll need to read + // in the terrain here. + yPos: 270, + minX: x, + maxX: x+(orientation % 2 === 1 ? depth : width)-1, + minZ: z, + maxZ: z+(orientation % 2 === 1 ? width : depth)-1, + commuteX: x, + commuteZ: z, + width, + depth, + orientation, + zoneWealth: zoneWealth || 0x00, + zoneType, + + // Apparently jobCapacities is also required, otherwise CTD! The + // capacity is stored I guess in the LotConfig exemplar, or + // perhaps in the building exemplar. + jobCapacities: [{ + demandSourceIndex: 0x00003320, + capacity: 0, + }], + + }); + + // Push the lot in the lotFile. + lots.push(lot); + + // Now put the lot in the zone developer file as well. TODO: We should + // actually check first and ensure that no building exists yet here! + let zones = dbpf.zoneDeveloperFile; + let grid = dbpf.getSimGrid(FileType.SimGridSint8, ZoneData); + for (let x = lot.minX; x <= lot.maxX; x++) { + for (let z = lot.minZ; z <= lot.maxZ; z++) { + zones.cells[x][z] = { + mem: lot.mem, + type: FileType.LotFile, + }; + grid.set(x, z, zoneType); + } + } + + // Don't forget to update the COMSerializer to include the updated + // length! Otherwise the lot won't show up! + let com = dbpf.COMSerializerFile; + com.set(FileType.LotFile, lots.length); + + // Return the lot that we've just created. + return lot; + + } + + // ## createBuilding(opts) + // Creates a new building record and inserts it into the savegame. + createBuilding(opts) { + let { lot, lotObject, exemplar } = opts; + let file = exemplar.read(); + let [width, height, depth] = this.getPropertyValue(file, OccupantSize); + let { orientation, y } = lotObject; + + // Create the building. + let building = new Building({ + mem: this.mem(), + + // Now use the **rotated** building rectangle and use it to + // position the building appropriately. + ...position(lotObject, lot), + minY: lot.yPos + y, + maxY: lot.yPos + y + height, + orientation: (orientation + lot.orientation) % 4, + + // Store the TGI of the building exemplar. + TID: exemplar.type, + GID: exemplar.group, + IID: exemplar.instance, + IID1: exemplar.instance, + + }); + setTract(building); + + // Put the building in the index at the correct spot. + let { dbpf } = this; + this.addToItemIndex(building, FileType.BuildingFile); + + // Push in the file with all buildings. + let buildings = dbpf.buildingFile; + buildings.push(building); + + // Add to the lot developer file as well. + let dev = dbpf.lotDeveloperFile; + dev.buildings.push({ + mem: building.mem, + type: FileType.BuildingFile, + }); + + // At last update the COMSerializer file. + let com = dbpf.COMSerializerFile; + com.set(FileType.BuildingFile, buildings.length); + return building; + + } + + // ## createProp(opts) + // Creates a new prop record in and inserts it into the save game. Takes + // into account the position it should take up in a lot. + createProp({ lot, lotObject }) { + + // Note: in contrast to the building, we don't know yet what prop + // we're going to insert if we're dealing with a prop family. As such + // we'll check the families first. + let { OID, IIDs, orientation, y } = lotObject; + let IID = rand(IIDs); + let exemplar = this.findExemplarOfType(IID, 0x1e); + + // Missing props? Just ignore them. + if (!exemplar) { + return; + } + + // Get the dimensions of the prop bounding box. + let file = exemplar.read(); + let [width, height, depth] = this.getPropertyValue(file, OccupantSize); + + // Create the prop & position correctly. + let prop = new Prop({ + mem: this.mem(), + + ...position(lotObject, lot), + minY: lot.yPos + y, + maxY: lot.yPos + y + height, + orientation: (orientation + lot.orientation) % 4, + + // Store the TGI of the prop. + TID: exemplar.type, + GID: exemplar.group, + IID: exemplar.instance, + IID1: exemplar.instance, + OID, + + appearance: 5, + state: 0, + + }); + setTract(prop); + + // Push in the file with all props. + let { dbpf } = this; + let { props } = dbpf; + props.push(prop); + + // Put the prop in the index. + this.addToItemIndex(prop, FileType.PropFile, lotObject); + + // Update the COM serializer and we're done. + let com = dbpf.COMSerializerFile; + com.set(FileType.PropFile, props.length); + return props; + + } + + // ## createTexture(opts) + // Creates a texture entry in the BaseTexture file of the city for the + // given lot. + createTexture(opts) { + + // Create a new texture instance and copy some lot properties in it. + let { lot, textures } = opts; + let texture = new BaseTexture({ + mem: this.mem(), + + // Apparently the game requires "insets" on the texture - which it + // sets to 0.1, which get rounded to Float32's by the way. + minX: 16*lot.minX + INSET, + maxX: 16*(lot.maxX+1) - INSET, + minZ: 16*lot.minZ + INSET, + maxZ: 16*(lot.maxZ+1) - INSET, + + // TODO: This is only for flat cities, should use terrain queries + // later on! + minY: lot.yPos, + maxY: lot.yPos + INSET, + + }); + setTract(texture); + + // Add all required textures. + for (let def of textures) { + let { orientation, x, z, IID } = def; + let [xx, zz] = orient([x, z], lot, { bare: true }); + + // Note: the orientation is given in **lot** coordinates, + // but orientation 0 in the city is 2 in the lot, so add 2 to it. + // Additionally we'll also need to handle mirroring. + let mirrored = orientation >= 0x80000000; + orientation %= 0x80000000; + orientation = (lot.orientation + orientation) % 4; + if (mirrored) { + orientation += 4; + } + + // Create the texture at last. + texture.add({ + IID, + x: lot.minX + Math.floor(xx), + z: lot.minZ + Math.floor(zz), + orientation, + }); + + } + + // Cool, now push the base texture in the city & update the + // COMSerializer as well. + let { dbpf } = this; + dbpf.textures.push(texture); + let com = dbpf.COMSerializerFile; + com.set(FileType.BaseTextureFile, dbpf.textures.length); + + // Update the item index as well. + this.addToItemIndex(texture, FileType.BaseTextureFile); + + // Return the base texture that we've created. + return texture; + + } + + // ## addToItemIndex(obj, type) + // Helper function for adding the given object - that exposes the tract + // coordinates - to the item index. + addToItemIndex(obj, type, euh) { + let { dbpf } = this; + let index = dbpf.itemIndexFile; + for (let x = obj.xMinTract; x <= obj.xMaxTract; x++) { + for (let z = obj.zMinTract; z <= obj.zMaxTract; z++) { + if (!index[x][z]) { + console.log(obj, euh.minX, euh.maxX, euh.minZ, euh.maxZ); + } + index[x][z].push({ + mem: obj.mem, + type: type, + }); + } + } + } + + // ## findExemplarOfType(IID, type) + // Helper function that can find an exemplar with the given instance of + // the given type. It will make use of the families we have as well. + findExemplarOfType(IID, type) { + let { index } = this; + let family = index.family(IID); + const filter = entry => { + let file = entry.read(); + return index.getPropertyValue(file, 0x10) === type; + }; + if (family) { + let exemplars = family.filter(filter); + return exemplars[ Math.random()*exemplars.length | 0 ]; + } else { + let exemplars = index + .findAllTI(FileType.Exemplar, IID) + .filter(filter); + return exemplars[ exemplars.length-1 ]; + } + } + } -module.exports = CityManager; \ No newline at end of file +module.exports = CityManager; + +// ## setTract(obj) +// Helper function for setting the correct "Tract" values in the given object +// based on its bounding box. +function setTract(obj) { + const xSize = 16 * 2**obj.xTractSize; + const zSize = 16 * 2**obj.zTractSize; + obj.xMinTract = Math.max(64, 64 + Math.floor(obj.minX / xSize)); + obj.xMaxTract = 64 + Math.floor(obj.maxX / xSize); + obj.zMinTract = Math.max(64, 64 + Math.floor(obj.minZ / zSize)); + obj.zMaxTract = 64 + Math.floor(obj.maxZ / zSize); +} + +// ## orient([x, y], lot, opts) +// Helper function for transforming the point [x, y] that is given in +// **local** lot coordinates into global **city** coordinates. Note that local +// lot coordinates use an origin in the bottom-left corner of the lot with an +// y axis that is going up. This means that we'll need to invert properly! +function orient([x, y], lot, opts = {}) { + let { width, depth } = lot; + + // First of all we need to swap because orientation 0 in the city is "up", + // while orientation 0 in the is "down", and that's also how the + // coordinates are expressed! + [x, y] = [width-x, depth-y]; + + // Based on the lot orientation, position correctly. + switch (lot.orientation) { + case 0x01: + [x, y] = [depth-y, x]; + break; + case 0x02: + [x, y] = [width-x, depth-y]; + break; + case 0x03: + [x, y] = [y, width-x]; + break; + + } + + // If we didn't request bare coordinates explicitly, transform to city + // coordinates. + if (opts.bare) { + return [x, y]; + } else { + return [ + 16*(lot.minX + x), + 16*(lot.minZ + y), + ]; + } + +} + +// ## position(lotObject, lot) +// Returns the rectangle we need to position the given lotObject on, taken +// into account it's positioned on the given lot. +function position(lotObject, lot) { + let { minX, maxX, minZ, maxZ } = lotObject; + [minX, minZ] = orient([minX, minZ], lot); + [maxX, maxZ] = orient([maxX, maxZ], lot); + if (minX > maxX) { + [minX, maxX] = [maxX, minX]; + } + if (minZ > maxZ) { + [minZ, maxZ] = [maxZ, minZ]; + } + return { minX, maxX, minZ, maxZ }; +} + +// ## rand(arr) +// Helper function that randomly selects a value from a given array. +function rand(arr) { + return arr[Math.random()*arr.length | 0]; +} diff --git a/lib/com-serializer-file.js b/lib/com-serializer-file.js index 4d91777..521d957 100644 --- a/lib/com-serializer-file.js +++ b/lib/com-serializer-file.js @@ -1,6 +1,5 @@ // # com-serializer-file.js "use strict"; -const bsearch = require('binary-search'); const Stream = require('./stream'); const WriteStream = require('./write-stream'); const { FileType } = require('./enums'); diff --git a/lib/dbpf.js b/lib/dbpf.js index 527cd15..69ca517 100644 --- a/lib/dbpf.js +++ b/lib/dbpf.js @@ -1,6 +1,8 @@ // # dbpf.js "use strict"; +const util = require('util'); const fs = require('fs'); +const { EventEmitter } = require('events'); const TYPES = require('./file-types'); const Stream = require('./stream'); const crc32 = require('./crc'); @@ -12,15 +14,35 @@ const { hex, tgi } = require('./util'); // A class that represents a DBPF file. A DBPF file is basically just a custom // file archive format, a bit like .zip etc. as it contains other files that // might be compressed etc. -const DBPF = module.exports = class DBPF { - - // ## constructor(buff) - constructor(buff) { +const DBPF = module.exports = class DBPF extends EventEmitter { + + // ## constructor(file) + // Constructs the dbpf for the given file. For backwards compatibility and + // ease of use, a DBPF is synchronous by default, but we should support + // async parsing as well: that's important if we're reading in an entire + // plugins directory for example. Note that DBPF's are always constructed + // from files, **not** from buffers. As such we don't have to keep the + // entire buffer in memory and we can read the required parts of the file + // "on the fly". That's what the DBPF format was designed for! + constructor(file = null, opts = {}) { // DBPF Magic number. By default this is DBPF, but we'll allow the // user to change this so that a dbpf can be de-activated. + super(); this.id = 'DBPF'; + // If the file specified is actually a buffer, store that we don't + // have a file. Note that this is not recommended: we need to be able + // to load and unload DBPFs on the fly because we simply cannot load + // them all into memory! + if (Buffer.isBuffer(file)) { + this.file = null; + this.buffer = file; + } else { + this.file = file; + this.buffer = null; + } + // Versioning of DBPF. In SC4 we always use DBPF 1.0. this.major = 1; this.minor = 0; @@ -34,19 +56,26 @@ const DBPF = module.exports = class DBPF { // TGI - is found where. this.entries = []; this.index = new Index(this); + this.indexCount = 0; + this.indexOffset = 0; + this.indexSize = 0; - // If a buffer was specified, parse the dbpf from it. - if (buff) { - if (typeof buff === 'string') { - buff = fs.readFileSync(buff); - } - this.parse(buff); + // If the user specified a file, parse the DBPF right away. + if (file) { + this.parse(); } + } - // ## add(tfi, file) + // ## find(...args) + // Proxies to entries.find() + find(...args) { + return this.entries.find(...args); + } + + // ## add(tgi, file) // Adds a new entry to the dbpf file. - add(tfi, file) { + add(tgi, file) { let entry = new Entry(); entry.tgi = tgi; this.entries.push(entry); @@ -58,12 +87,83 @@ const DBPF = module.exports = class DBPF { return entry; } - // ## parse(buff) - // Decodes the DBPF file from the given buffer. - parse(buff) { + // ## load() + // If the buffer is not loaded yet, load it. + load() { + if (!this.buffer) { + this.buffer = fs.readFileSync(this.file); + } + return this.buffer; + } + + // ## free() + // This method unloads the underlying buffer of the dbpf so that it can be + // garbage collected to free up some memory. This is useful if we just + // needed the DBPF for indexing but are not planning to use it soon. In + // that case the cache can decide to free up the dbpf and only read it in + // again upon the next read. + free() { + if (!this.file) { + console.warn([ + 'No file is set.', + 'This means you will no longer be able to use this DBPF!' + ].join(' ')); + } + + // Delete the buffer & loop all our entries so that we unload those as + // well. + this.buffer = null; + for (let entry of this) { + entry.free(); + } + return this; + + } + + // ## readBytes(offset, length) + // Returns a buffer contain the bytes starting at offset and with the + // given length. We use this method so that we can use a buffer or a file + // as underlying source interchangeably. + readBytes(offset, length) { + + // If we don't have a buffer, but we do have a file - which can happen + // if the DBPF was "unloaded" to free up memory - we'll have to load + // it in memory again. Note that it's your responsibility to free up + // memory the DBPF is taking up. You can use the `.free()` method for + // that. + let buffer = this.load(); + return Buffer.from(buffer.buffer, buffer.offset+offset, length); + + } + + // ## parse() + // Reads in the DBPF in a *synchronous* way. That's useful if you're + // testing stuff out, but for bulk reading you should use the async + // reading. + parse() { + + // First of all we need to read the header, and only the header. From + // this we can derive where to find the index so that we can parse the + // entries from it. + this.parseHeader(this.readBytes(0, 96)); + + // Header is parsed which means we now know the offset of the index. + // Let's parse the index as well then. + let buffer = this.readBytes(this.indexOffset, this.indexSize); + let rs = new Stream(buffer); + this.index.parse(rs); + + } + + // ## parseHeader(buff) + // Parses the header of the DBPF file from the given buffer. + parseHeader(buff) { let rs = new Stream(buff); this.id = rs.string(4); + if (this.id !== 'DBPF') { + throw new Error(`${ this.file } is not a valid DBPF file!`); + } this.major = rs.uint32(); this.minor = rs.uint32(); @@ -81,9 +181,9 @@ const DBPF = module.exports = class DBPF { // Read in where we can find the file index and the holes. Note that // this is specific to the serialization of a dbpf, we only need this // when parsing so we won't store these values on the dbpf itself. - const indexCount = rs.uint32(); - const indexOffset = rs.uint32(); - const indexSize = rs.uint32(); + const indexCount = this.indexCount = rs.uint32(); + const indexOffset = this.indexOffset = rs.uint32(); + const indexSize = this.indexSize = rs.uint32(); const holesCount = rs.uint32(); const holesOffset = rs.uint32(); const holesSize = rs.uint32(); @@ -91,18 +191,12 @@ const DBPF = module.exports = class DBPF { // Read in the minor version of the index, for some weird reason this // comes here in the header. index.minor = rs.uint32(); - rs.skip(4); // Read in all entries from the file index. It's very important that // we update the length of all entries first so that the index knows // how much entries it should parse! - const entries = this.entries; - entries.length = 0; - entries.length = indexCount; - rs.jump(indexOffset); - index.parse(rs); - - return this; + const entries = this.entries = []; + entries.length = this.indexCount; } @@ -111,7 +205,7 @@ const DBPF = module.exports = class DBPF { // it's just easier. save(opts) { if (typeof opts === 'string') { - opts = {"file": opts}; + opts = { file: opts }; } this.modified = new Date(); let buff = this.toBuffer(opts); @@ -171,31 +265,37 @@ const DBPF = module.exports = class DBPF { dir.push(list.length); } list.push({ - "type": entry.type, - "group": entry.group, - "instance": entry.instance, - "buffer": buffer, - "compressed": entry.compressed, - "fileSize": fileSize, - "compressedSize": buffer.byteLength + type: entry.type, + group: entry.group, + instance: entry.instance, + buffer: buffer, + compressed: entry.compressed, + fileSize: fileSize, + compressedSize: buffer.byteLength }); - } else if (entry.raw) { + } else { + + // Allright, the entry has not been decoded into a file yet. + // Check if has even been read. If not the case we will need + // to do this first. + if (!entry.raw) { + entry.readRaw(); + } + if (entry.compressed) { dir.push(list.length); } list.push({ - "type": entry.type, - "group": entry.group, - "instance": entry.instance, - "buffer": entry.raw, - "compressed": entry.compressed, - "fileSize": entry.fileSize, - "compressedSize": entry.compressedSize + type: entry.type, + group: entry.group, + instance: entry.instance, + buffer: entry.raw, + compressed: entry.compressed, + fileSize: entry.fileSize, + compressedSize: entry.compressedSize }); - } else { - throw new Error('Entry has no buffer and no file! Can\'t serialize it!'); } } @@ -215,13 +315,13 @@ const DBPF = module.exports = class DBPF { } list.push({ - "type": 0xE86B1EEF, - "group": 0xE86B1EEF, - "instance": 0x286B1F03, - "buffer": buff, - "compressed": false, - "fileSize": buff.byteLength, - "compressedSize": buff.byteLength + type: 0xE86B1EEF, + group: 0xE86B1EEF, + instance: 0x286B1F03, + buffer: buff, + compressed: false, + fileSize: buff.byteLength, + compressedSize: buff.byteLength }); } @@ -327,10 +427,10 @@ const DBPF = module.exports = class DBPF { // Allright, first entry is of type "SIZE MEM CRC", we assume that // all following entries are as well. all.push({ - "mem": slice.readUInt32LE(8), - "type": entry.type, - "entry": entry, - "index": 0 + mem: slice.readUInt32LE(8), + type: entry.type, + entry: entry, + index: 0 }); let index = size; buff = buff.slice(size); @@ -339,10 +439,10 @@ const DBPF = module.exports = class DBPF { let slice = buff.slice(0, size); let mem = slice.readUInt32LE(8); all.push({ - "mem": slice.readUInt32LE(8), - "type": entry.type, - "entry": entry, - "index": index + mem: slice.readUInt32LE(8), + type: entry.type, + entry, + index, }); index += size; buff = buff.slice(size); @@ -380,9 +480,9 @@ const DBPF = module.exports = class DBPF { let index = raw.indexOf(hex); if (index > -1) { out[i].push({ - "class": cClass[entry.type], - "entry": entry, - "index": index + class: cClass[entry.type], + entry, + index, }); } } @@ -428,8 +528,9 @@ DBPF.FileTypes = Object.create(null); // Register our known filetypes. For now only Exemplars and LotFiles. More // might be added. +const Exemplar = require('./exemplar.js'); DBPF.register([ - require('./exemplar'), + Exemplar, require('./lot').Array, require('./building').Array, require('./prop').Array, @@ -441,8 +542,10 @@ DBPF.register([ require('./zone-developer-file'), require('./lot-developer-file'), require('./com-serializer-file'), + require('./zone-manager.js'), require('./tract-developer.js'), ]); +DBPF.register(FileType.Cohort, Exemplar); // Register the different sim grids. We use the same class for multiple type // ids, so we need to register under id manually. @@ -464,11 +567,14 @@ class Index { // An index is always tied to a DBPF file, so we'll have mutual // references. - Object.defineProperty(this, 'dbpf', {"value": dbpf}); + Object.defineProperty(this, 'dbpf', { + value: dbpf, + enumerable: false, + writable: false, + }); // Store the raw array of all entries. This array should always be // treated by reference because it is shared with the DBPF class. - this.entries = dbpf.entries; this.entriesById = Object.create(null); // Versioning of the index. By default we use 7.0 in SC4. @@ -477,6 +583,11 @@ class Index { } + // ## get entries() + get entries() { + return this.dbpf.entries; + } + // ## get(tgi) // Returns an entry by tgi get(tgi) { @@ -489,16 +600,15 @@ class Index { // buffer! This is because an index is inherently intertwined with a DBPF. parse(rs, opts = {}) { - let entries = this.entries; + let { dbpf, entries } = this; let byId = this.entriesById; let length = entries.length; let dir = null; for (let i = 0; i < length; i++) { - let entry = entries[i] = new Entry(); + let entry = entries[i] = new Entry(dbpf); entry.parse(rs, { - "major": this.major, - "minor": this.minor, - "buffer": rs.buffer + major: this.major, + minor: this.minor, }); byId[ entry.id ] = entry; @@ -514,14 +624,14 @@ class Index { // accordingly. if (dir) { let parsed = parseDir(dir, { - "major": this.major, - "minor": this.minor + major: this.major, + minor: this.minor }); // Now find all compressed entries and update their compressed // sizes so that we can store the raw buffers correctly afterwards. for (let id in parsed) { - let {size} = parsed[id]; + let { size } = parsed[id]; let entry = this.get(id); entry.compressed = true; entry.fileSize = size; @@ -542,8 +652,13 @@ class Index { // it will be parsed appropriately. class Entry { - // ## constructor() - constructor() { + // ## constructor(dbpf) + constructor(dbpf) { + Object.defineProperty(this, 'dbpf',{ + value: dbpf, + enumerable: false, + witable: false, + }); this.type = 0; this.group = 0; this.instance = 0; @@ -562,6 +677,15 @@ class Entry { } + // ## free() + // Frees up the memory the entry is taking up. Useful when DBPFs get + // unloaded to not take up too much memory. + free() { + this.raw = null; + this.file = null; + return this; + } + // ## get tgi() get tgi() { return [ this.type, this.group, this.instance ]; } set tgi(tgi) { @@ -618,6 +742,9 @@ class Entry { // Returns the decompressed raw entry buffer. If the entry is not // compressed, then the buffer is returned as is. decompress() { + if (!this.raw) { + this.readRaw(); + } let buff = this.raw; if (this.compressed) { buff = decompress(buff); @@ -625,25 +752,36 @@ class Entry { return buff; } + // ## readRaw() + // **Synchronously** reads the entry's raw buffer and stores it in the + // "raw" property. + readRaw() { + + // Find the raw entry buffer inside the DBPF file. We'll do this in a + // sync way, but we should support doing this asynchronously as well. + this.raw = this.dbpf.readBytes(this.offset, this.compressedSize); + + } + // ## read() // Tries to convert the raw buffer of the entry into a known file type. If // this fails, we'll simply return the raw buffer, but decompressed if the // entry was compressed. read() { - // If the entry was already read, don't read it again. + // If the entry was already read, don't read it again. Note that it's + // possible to dispose the entry to free up some memory if required. if (this.file) { return this.file; } - // No raw buffer stored? Unable to read the entry then. - if (!this.raw) { - throw new Error( - 'No raw buffer set for the entry! Hence cannot read it!' - ); - } + // Before reading the entry, we'll emit an event on the owning DBPF. + // As such a cache can keep track of which DBPF files are read from + // most often. + this.dbpf.emit('read'); // If the entry is compressed, decompress it. + this.readRaw(); let buff = this.decompress(); // Now check for known file types. @@ -654,7 +792,7 @@ class Entry { return buff; } else { let file = this.file = new Klass(); - file.parse(buff, {"entry": this}); + file.parse(buff, { entry: this }); return file; } @@ -665,6 +803,23 @@ class Entry { return tgi(this.type, this.group, this.instance); } + // ## [util.inspect.custom]() + // If we console.log an entry in node we want to convert the TGI to their + // hex equivalents so that it's easier to debug. + [util.inspect.custom]() { + return { + type: hex(this.type), + group: hex(this.group), + instance: hex(this.instance), + fileSize: this.fileSize, + compressedSize: this.compressedSize, + offset: this.offset, + compressed: this.compressed, + file: String(this.file), + raw: this.raw ? '[Object Buffer]' : null, + }; + } + } // Export on the DBPF class. @@ -673,25 +828,39 @@ DBPF.Entry = Entry; // # parseDir(dir, opts) // Helper function for parsing the "dir" entry. Returns a json object. function parseDir(dir, opts = {}) { - const { major = 7, minor = 0} = opts; - const rs = new Stream(dir.raw); - const byteLength = major === 7 && minor === 1 ? 20 : 16; - const n = dir.fileSize / byteLength; + const { + major = 7, + minor = 0, + } = opts; + let { dbpf } = dir; + + // Read in the bytes for the dir. + let size = dir.fileSize; + let offset = dir.offset; + let byteLength = major === 7 && minor === 1 ? 20 : 16; + let n = size / byteLength; + let buffer = dbpf.readBytes(offset, size); + let rs = new Stream(buffer); + + // Now create the dir entry. let out = Object.create(null); for (let i = 0; i < n; i++) { let type = rs.uint32(); let group = rs.uint32(); let instance = rs.uint32(); let size = rs.uint32(); - if (byteLength > 16) this.skip(4); + if (byteLength > 16) { + rs.skip(4); + } let id = tgi(type, group, instance); out[id] = { - "type": type, - "group": group, - "instance": instance, - "id": id, - "size": size + type, + group, + instance, + id, + size, }; } return out; -} \ No newline at end of file + +} diff --git a/lib/exemplar.js b/lib/exemplar.js index 5950848..fc3b1ac 100644 --- a/lib/exemplar.js +++ b/lib/exemplar.js @@ -68,6 +68,14 @@ class Exemplar { return FileType.Exemplar; } + // ## constructor() + constructor() { + this.id = 'EQZB1###'; + this.parent = [0,0,0]; + this.props = []; + this.table = null; + } + // ## get fileType() get fileType() { return FileType.Exemplar; @@ -88,12 +96,17 @@ class Exemplar { return out; } - // ## constructor() - constructor() { - this.id = 'EQZB1###'; - this.parent = [0,0,0]; - this.props = []; - this.table = null; + // ## prop(key) + // Helper function for accessing a property. + prop(key) { + return this.table[ key ]; + } + + // ## value(key) + // Helper function for directly accessing the value of a property. + value(key) { + let prop = this.prop(key); + return prop ? prop.value : undefined; } // ## parse(buff) @@ -224,6 +237,12 @@ class Property { this.value = value; } + // ## [Symbol.toPrimitive]() + // Casting the prop to a number will return the numeric value. + [Symbol.toPrimitive](hint) { + return hint === 'number' ? this.value : this.hex; + } + // ## get hex() // Computed property that shows the hex value of the property name. Useful // when comparing this with Reader, because Reader shows everything in hex diff --git a/lib/file-index.js b/lib/file-index.js new file mode 100644 index 0000000..15db5b5 --- /dev/null +++ b/lib/file-index.js @@ -0,0 +1,296 @@ +// # file-index.js +"use strict"; +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const bsearch = require('binary-search-bounds'); +const LRUCache = require('lru-cache'); +const {default:PQueue} = require('p-queue'); +const DBPF = require('./dbpf.js'); +const { FileType } = require('./enums.js'); + +const Family = 0x27812870; + +// Patch fs promises. +if (!fs.promises) { + const util = require('util'); + fs.promises = { readFile: util.promisify(fs.readFile) }; +} + +// # FileIndex +// The file index is a data structure that scans a list of dbpf files and +// builds up an index of all files in it by their TGI's. This should mimmick +// how the game scans the plugins folder as well. We obivously cannot keep +// everything in memory so we'll keep pointers to where we can find each file +// **on the disk**. Note: we should make use of node's async nature here so +// that we can read in as much files as possible in parallel! +class FileIndex { + + // ## constructor(opts) + constructor(opts = {}) { + + // Set up the cache that we'll use to free up memory of DBPF files + // that are not read often. + let { mem = Number(os.totalmem) } = opts; + this.cache = new LRUCache({ + max: 0.5*mem, + length(n, dbpf) { + return dbpf.buffer.byteLength; + }, + dispose(dbpf, n) { + dbpf.free(); + }, + }); + + // Our array containing all our records. This array will be sorted by + // tgi. + this.records = null; + + // We'll also keep track of all exemplar records because we'll often + // need them and it can be cumbersome to loop just about everything. + this.exemplars = []; + + // A map that contains references to all building & prop families. + // Note that we cannot use a separate data structure for this because + // the order in which files get loaded is important here! + this.families = new Map(); + + // If the options are simply given as a string, consider it to be a + // directory. + if (typeof opts === 'string') { + opts = { dirs: [opts] }; + } + + // No directory or files specified? Use the default plugins directory. + // Note that we should look for the SimCity_1.dat core files as well + // of course. + if (!opts.files && !opts.dirs) { + let plugins = path.join( + process.env.HOMEPATH, + 'Documents/SimCity 4/Plugins', + ); + opts = Object.assign({ + dirs: [plugins], + }, opts); + } + + let files = this.files = []; + if (opts.files) { + files.push(...opts.files); + } + + // Scan directories as well. + if (opts.dirs) { + for (let dir of opts.dirs) { + collect(dir, files); + } + } + + } + + // ## get length() + get length() { + return this.records.length; + } + + // ## async build(opts) + // Builds up the index. + async build(opts = {}) { + + // Initialize our records array. + this.records = []; + + // Limit the amount of reads that we carry out. + let cpus = os.cpus(); + let max = Math.max(2, cpus.length, opts.concurrency || 0); + let Q = new PQueue({ concurrency: max }); + let all = []; + for (let file of this.files) { + + // Note: SC4 doesn't work this way, but we are going to ignore any + // extensions other than .dat, sc4desc, sc4model & sc4lot for now. + let ext = path.extname(file).toLowerCase().slice(1); + if (!extRegex.test(ext)) continue; + + // Add to the index. + let task = Q.add(() => this.addToIndex(file)); + all.push(task); + + } + await Promise.all(all); + + // Allright we now have all records. Time to sort them in their + // respective arrays. + this.records.sort(compare); + + // Loop all exemplars so that we can find the building and prop + // families. + for (let entry of this.exemplars) { + let file = entry.read(); + let family = this.getPropertyValue(file, Family); + if (family) { + let [IID] = family; + let arr = this.families.get(IID); + if (!arr) { + this.families.set(IID, arr = []); + } + arr.push(entry); + } + } + + } + + // ## async addToIndex(file) + // Asynchronously adds the given file to the index. Note that this is not + // meant for external use as this doesn't sort the records! + async addToIndex(file) { + + // Asynchronously load read in the file to add. As such we can make + // use of the OS's multithreading for reading files, even though JS is + // single threaded. + let buff = await fs.promises.readFile(file); + + // Ensure that the file is a dbpf file, ignore otherwise. + if (buff.toString('utf8', 0, 4) !== 'DBPF') { + return; + } + + // Parse the DBPF. + let dbpf = new DBPF(buff); + dbpf.file = file; + for (let entry of dbpf.entries) { + this.records.push(entry); + + // If the entry is an exemplar, push it in all our exemplars. Note + // that we don't read it yet and check for a family: we need to + // have access to **all** exemplars first! + if (entry.type === FileType.Exemplar) { + this.exemplars.push(entry); + } + + } + + // Important! Make sure to free the buffer, otherwise the `read` + // handler still has access to it and won't ever be cleared from + // memory! + buff = null; + + // Note done yet. Listen to how many times the entries of the DBPF are + // read so that we can update the cache with it. + this.cache.set(dbpf); + dbpf.on('read', () => { + if (!this.cache.has(dbpf)) { + if (!dbpf.buffer) { + dbpf.load(); + } + this.cache.set(dbpf); + } + }); + + } + + // ## find(type, group, instance) + // Finds the record identified by the given tgi + find(type, group, instance) { + if (Array.isArray(type)) { + [type, group, instance] = type; + } else if (typeof type === 'object') { + ({type, group, instance} = type); + } + let query = { + type, + group, + instance, + }; + + let index = bsearch.eq(this.records, query, compare); + return index > -1 ? this.records[index] : null; + + } + + // ## findAllTI(type, instance) + // Finds all entries with the given Type and Instance ID. + findAllTI(type, instance) { + let query = { type, instance, group: 0 }; + let index = bsearch.lt(this.records, query, compare); + let out = []; + let record; + while ( + (record = this.records[++index]) && + record.type === type && + record.instance === instance + ) { + out.push(record); + } + return out; + } + + // ## family(id) + // Checks if the a prop or building family exists with the given IID and + // if so returns the family array. + family(id) { + let arr = this.families.get(id); + return arr || null; + } + + // ## getProperty(exemplar, key) + // This function accepts a parsed exemplar file and looks up the property + // with the given key. If the property doesn't exist, then tries to look + // it up in the parent cohort and so on all the way up. + getProperty(exemplar, key) { + let original = exemplar; + let prop = exemplar.prop(key); + while (!prop && exemplar.parent[0]) { + let { parent } = exemplar; + let entry = this.find(parent); + if (!entry) { + break; + } + exemplar = entry.read(); + prop = exemplar.prop(key); + } + return prop; + } + + // ## getPropertyValue(exemplar, key) + // Directly returns the value for the given property in the exemplar. If + // it doesn't exist, looks it up in the parent cohort. + getPropertyValue(exemplar, key) { + let prop = this.getProperty(exemplar, key); + return prop ? prop.value : undefined; + } + +} +module.exports = FileIndex; + +// # compare(a, b) +// The function that we use for sorting all files in our index. This +// effectively creates an *index* on all the records so that we can use a +// binary search algorithm for finding them. Given that the instance id (IID) +// is what we'll use the most, this will be the main sorting key, followed by +// type id (TID) and then by group id (GID). +function compare(a, b) { + return a.instance - b.instance || a.type - b.type || a.group - b.group; +} + +// # extRegex +const extRegex = /^(sc4desc)|(sc4lot)|(sc4model)|(dat)$/; + +// # collect(dir, all) +// Recursively crawls the given directory and collects all files within it. +// Note that we use the **sync** version of readdir here because the operation +// is relatively inexpensive and we have to do it anyway. +function collect(dir, all) { + all = all || []; + let list = fs.readdirSync(dir); + for (let file of list) { + file = path.join(dir, file); + let stat = fs.statSync(file); + if (stat.isDirectory()) { + collect(file, all); + } else { + all.push(file); + } + } + return all; +} diff --git a/lib/file-types.js b/lib/file-types.js index e2f630f..a87c8c8 100644 --- a/lib/file-types.js +++ b/lib/file-types.js @@ -29,7 +29,9 @@ // RUL TypeID=0A5BCF4B // Cohort TypeID=05342861 const TYPES = module.exports = { + Exemplar: 0x6534284A, + Cohort: 0x05342861, DIR: 0xE86B1EEF, PNG: 0x856DDBAC, diff --git a/lib/gui/gui.js b/lib/gui/gui.js index b7f01d6..e9a8dc3 100644 --- a/lib/gui/gui.js +++ b/lib/gui/gui.js @@ -7,8 +7,8 @@ const Savegame = require('../savegame'); // Do some stuff. let renderer = window.r = new Renderer({ - "width": 1000, - "height": 750, + "width": window.innerWidth-2, + "height": window.innerHeight-4, "position": [16*64, 1000, 16*64], "target": [16*64,0,16*64], "perspective": true, @@ -19,7 +19,9 @@ document.body.append(renderer.el); // Append a box. -let file = path.resolve(__dirname, '../../test/files/city.sc4'); +// let file = path.resolve(__dirname, '../../test/files/city.sc4'); +// let file = path.resolve(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments/City - Skyline.sc4'); +let file = path.resolve(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments/City - Plopsaland.sc4'); let dbpf = new Savegame(fs.readFileSync(file)); // Read all buildings. @@ -30,6 +32,7 @@ let material = new three.MeshStandardMaterial({ }); let buildings = dbpf.buildingFile; +let box = new three.BoxBufferGeometry(1, 1, 1); for (let building of buildings) { let width = building.maxX - building.minX; @@ -42,8 +45,8 @@ for (let building of buildings) { let z = building.minZ + depth/2; // Create a box for it. - let box = new three.BoxBufferGeometry(width, height, depth); let mesh = new three.Mesh(box, material); + mesh.scale.set(width, height, depth); renderer.add(mesh); mesh.position.set(x, y, z); } diff --git a/lib/gui/index.html b/lib/gui/index.html index 682b6b6..9c3c544 100644 --- a/lib/gui/index.html +++ b/lib/gui/index.html @@ -3,6 +3,12 @@ + diff --git a/lib/index.js b/lib/index.js deleted file mode 100644 index 40b16c5..0000000 --- a/lib/index.js +++ /dev/null @@ -1,240 +0,0 @@ -// # index.js -"use strict"; -const fs = require('fs'); -const path = require('path'); -const os = require('os'); -const bsearch = require('binary-search'); -const DBPF = require('./dbpf'); -const { Entry } = DBPF; -const {default:PQueue} = require('p-queue'); - -// Patch fs promises. -if (!fs.promises) { - const util = require('util'); - fs.promises = { - "readFile": util.promisify(fs.readFile) - }; -} - -// # Index -// The index is a data structure that scans a list of dbpf files and builds up -// an index of all files in it by their TGI's. This should mimmick how the -// game scans the plugins folder as well. We obivously cannot keep everything -// in memory so we'll keep pointers to where we can find each file **on the -// disk**. Note: we should make use of node's async nature here so that we can -// read in as much files as possible in parallel! -class Index { - - // ## constructor(opts) - constructor(opts) { - - // Our array containing all our records. This array will be sorted by - // tgi. - this.records = null; - this.git = null; - this.igt = null; - - let files = this.files = []; - if (opts.files) { - files.push(...opts.files); - } - - // Scan directories as well. - if (opts.dirs) { - for (let dir of opts.dirs) { - collect(dir, files); - } - } - - } - - // ## get length() - get length() { - return this.records.length; - } - - // ## async build(opts) - // Builds up the index. - async build(opts = {}) { - - // Initialize our records array. - this.records = []; - - // Limit the amount of reads that we carry out. - let cpus = os.cpus(); - let max = Math.max(2, cpus.length, opts.concurrency || 0); - let Q = new PQueue({"concurrency": max}); - let all = []; - for (let file of this.files) { - - // Note: SC4 doesn't work this way, but we are going to ignore any - // extensions other than .dat, sc4desc, sc4model & sc4lot for now. - let ext = path.extname(file).toLowerCase().slice(1); - if (!extRegex.test(ext)) continue; - - // Add to the index. - let task = Q.add(() => this.addToIndex(file)); - all.push(task); - - } - await Promise.all(all); - - // Allright we now have all records. Time to sort them in their - // respective arrays. - this.git = this.records.map(x => x).sort(git); - this.igt = this.records.map(x => x).sort(igt); - this.records.sort(tgi); - - } - - // ## async addToIndex(file) - // Asynchronously adds the given file to the index. - async addToIndex(file) { - - // Read in the file. - let buff = await fs.promises.readFile(file); - const source = new Source(file); - - // Ensure that the file is a dbpf file, ignore otherwise. - if (buff.toString('utf8', 0, 4) !== 'DBPF') return; - - // Parse. - let dbpf = new DBPF(buff); - for (let entry of dbpf.entries) { - let record = new Record(entry); - record.source = source; - this.records.push(record); - } - - } - - // ## find(type, group, instance) - // Finds the record identified by the given tgi - find(type, group, instance) { - if (Array.isArray(type)) { - [type, group, instance] = type; - } else if (typeof type === 'object') { - ({type, group, instance} = type); - } - query.type = type; - query.group = group; - query.instance = instance; - - let index = bsearch(this.records, query, tgi); - if (index < 0) return null; - return this.records[index]; - - } - -} -module.exports = Index; - -// Object that we'll re-use to query so that we don't have to recreate it all -// the time. -const query = { - "type": 0, - "group": 0, - "instance": 0 -}; - -// # Record -// Represents an record in the index. -class Record extends Entry { - - // ## constructor(entry) - // Creates the record from the underlying dbpf entry. The difference here - // is that we no longer store the raw buffer because we don't want to keep - // everything in memory. We'll provide functionality to read the file by - // ourselves. - constructor(entry) { - super(); - this.type = entry.type; - this.group = entry.group; - this.instance = entry.instance; - this.fileSize = entry.fileSize; - this.compressedSize = entry.compressedSize; - this.offset = entry.offset; - this.compressed = entry.compressed; - - // Additionally store a reference to the source file where we can find - // the entry as well. - this.source = null; - - } - - // ## read() - // Overrides the DBPF's read method because we no longer have a raw buffer - // set. We need to read that one in first. - read() { - - // Entry already read? Don't read it again. - if (this.file) return this.file; - if (this.raw) return this.raw; - - // Read from the file, but at the correct offset (and in a syncronous way). - let file = String(this.source); - let fd = fs.openSync(file, 'r'); - let buff = Buffer.allocUnsafe(this.compressedSize); - fs.readSync(fd, buff, 0, buff.byteLength, this.offset); - fs.closeSync(fd); - this.raw = buff; - - // Call the super method. - return super.read(); - - } - -} - -// # Source -// Represents a source file. We use this so that we don't have to include the -// long filename everywhere. Saves on memory because we can work with -// pointers, yeah well references. -class Source { - constructor(file) { - this.file = file; - } - toString() { - return this.file; - } -} - -// # extRegex -const extRegex = /^(sc4desc)|(sc4lot)|(sc4model)|(dat)$/; - -// # tgi(a, b) -// TGI comparator that will sort by type, then group and then instance. -function tgi(a, b) { - return a.type - b.type || a.group - b.group || a.instance - b.instance; -} - -// # git(a, b) -// TGI comparator that will sort by group, then instance, then type. -function git(a, b) { - return a.group - b.group || a.instance - b.instance || a.type - b.type; -} - -// # igt(a, b) -// TGI comperator that will sort by instance, then group, then type. -function igt(a, b) { - return a.instance - b.instance || a.group - b.group || a.type - b.type; -} - -// # collect(dir, all) -// Recursively crawls the given directory and collects all files within it. -// Note that we use the **sync** version of readdir here because the operation -// is relatively inexpensive and we have to do it anyway. -function collect(dir, all) { - all = all || []; - let list = fs.readdirSync(dir); - for (let file of list) { - file = path.join(dir, file); - let stat = fs.statSync(file); - if (stat.isDirectory()) { - collect(file, all); - } else { - all.push(file); - } - } - return all; -} \ No newline at end of file diff --git a/lib/lot-base-texture.js b/lib/lot-base-texture.js index b141cad..a81d3fe 100644 --- a/lib/lot-base-texture.js +++ b/lib/lot-base-texture.js @@ -10,8 +10,8 @@ const Type = require('./type'); // # LotBaseTexture class LotBaseTexture extends Type(FileType.BaseTextureFile) { - // ## constructor() - constructor() { + // ## constructor(opts) + constructor(opts) { super(); this.crc = 0x00000000; this.mem = 0x00000000; @@ -20,8 +20,8 @@ class LotBaseTexture extends Type(FileType.BaseTextureFile) { this.u1 = 0x00; this.u2 = 0x00; this.u3 = 0x00; - this.u4 = 0x00; - this.u5 = 0x00000000; + this.u4 = 0x05; + this.u5 = 0x497f6d9d; this.xMinTract = 0x40; this.zMinTract = 0x40; this.xMaxTract = 0x40; @@ -36,6 +36,7 @@ class LotBaseTexture extends Type(FileType.BaseTextureFile) { this.maxZ = this.maxY = this.maxX = 0; this.u10 = 0x02; this.textures = []; + Object.assign(this, opts); } // ## move(dx, dz) @@ -162,6 +163,14 @@ class LotBaseTexture extends Type(FileType.BaseTextureFile) { } + // ## add(opts) + // Adds a single texture into the array of all textures. + add(opts) { + let texture = new Texture(opts); + this.textures.push(texture); + return texture; + } + } module.exports = LotBaseTexture; @@ -175,18 +184,19 @@ Object.defineProperty(LotBaseTexture.Array.prototype, 'textures', { // # Texture class Texture { - // ## constructor() - constructor() { + // ## constructor(opts) + constructor(opts) { this.IID = 0x00000000; this.z = this.x = 0; this.orientation = 0; this.priority = 0x00; - this.u2 = 0x00; - this.u3 = 0x00; - this.u4 = 0x00; - this.u5 = 0x00; - this.u6 = 0x00; + this.r = 0xff; + this.g = 0xff; + this.b = 0xff; + this.alpha = 0xff; + this.u6 = 0xff; this.u7 = 0x00; + Object.assign(this, opts); } // ## parse(rs) @@ -196,10 +206,10 @@ class Texture { this.z = rs.byte(); this.orientation = rs.byte(); this.priority = rs.byte(); - this.u2 = rs.byte(); - this.u3 = rs.byte(); - this.u4 = rs.byte(); - this.u5 = rs.byte(); + this.r = rs.byte(); + this.g = rs.byte(); + this.b = rs.byte(); + this.alpha = rs.byte(); this.u6 = rs.byte(); this.u7 = rs.byte(); return this; @@ -219,10 +229,10 @@ class Texture { ws.byte(this.z); ws.byte(this.orientation); ws.byte(this.priority); - ws.byte(this.u2); - ws.byte(this.u3); - ws.byte(this.u4); - ws.byte(this.u5); + ws.byte(this.r); + ws.byte(this.g); + ws.byte(this.b); + ws.byte(this.alpha); ws.byte(this.u6); ws.byte(this.u7); yield buff; diff --git a/lib/lot-index.js b/lib/lot-index.js new file mode 100644 index 0000000..1b2431a --- /dev/null +++ b/lib/lot-index.js @@ -0,0 +1,258 @@ +// # lot-index.js +"use strict"; +const bsearch = require('binary-search-bounds'); +const { FileType } = require('./enums.js'); +const { hex } = require('./util.js'); + +// Exemplar properties we'll be using. +const LotConfigPropertySize = 0x88edc790; +const OccupantSize = 0x27812810; +const OccupantGroups = 0xaa1dd396; + +// # LotIndex +// A helper class that we use to index lots by a few important properties. +// They're sorted by height and such they will also remain so. This means that +// when filtering, you can rest assured that they remain sorted by height as +// well! +class LotIndex { + + // ## constructor(index) + // Creates the lot index from the given file index. + constructor(index) { + + // Store the file index, we'll still need it. + this.fileIndex = index; + this.lots = []; + + // Loop every exemplar. If it's a lot configurations exemplar, then + // read it so that we can find the building that appears on the lot. + for (let entry of index.exemplars) { + let file = entry.read(); + if (this.getPropertyValue(file, 0x10) !== 0x10) { + continue; + } + + // Cool, add the lot. + this.add(entry); + + } + + // Now it's time to set up all our indices. For now we'll only index + // by height though. + this.height = IndexedArray.create({ + compare(a, b) { + return a.height - b.height; + }, + entries: this.lots, + }); + + } + + // ## add(entry) + // Adds the given lot exemplar to the index. + add(entry) { + + // Find the building on the lot. + let file = entry.read(); + let { lotObjects } = file; + let rep = lotObjects.find(({ type }) => type === 0x00); + let IID = rep.IID; + let building = this.getBuilding(IID); + + // Read in the lot size. + let size = this.getPropertyValue(file, LotConfigPropertySize); + + // Read in the building size as that is our main indexing property. + let [width, height, depth] = this.getPropertyValue( + building, OccupantSize, + ); + + // Get the occupant groups as well. + let groups = this.getPropertyValue(building, OccupantGroups) || []; + + // Get the zone types. + let zoneTypes = this.getPropertyValue(file, 0x88edc793); + + // At last, create our entry. + this.lots.push({ + lot: entry, + height, + building: [width, height, depth], + size, + occupantGroups: groups, + zoneTypes, + }); + + } + + // ## getBuilding(IID) + getBuilding(IID) { + let buildings = this.fileIndex + .findAllTI(FileType.Exemplar, IID) + .filter(entry => { + let file = entry.read(); + let type = this.getPropertyValue(file, 0x10); + return type === 0x02; + }); + if (buildings.length) { + return buildings[0].read(); + } + + // No building found? Don't worry, check the families. + let family = this.fileIndex.family(IID); + if (!family) { + throw new Error([ + `No building found with IID ${ hex(IID) }!` + ]); + } + return family[0].read(); + + } + + // ## getPropertyValue(file, prop) + // Helper function for quickly reading property values. + getPropertyValue(file, prop) { + return this.fileIndex.getPropertyValue(file, prop); + } + +} +module.exports = LotIndex; + +// ## IndexedArray +// An extension of an array that takes into account that the array is sorted +// using a certain comparator - which is to be specified by *extending* the +// IndexedArray. Very useful to perform efficient range queries. +class IndexedArray extends Array { + + // ## static extend(compare) + static extend(compare) { + const Child = class IndexedArray extends this {}; + Child.prototype.compare = compare; + return Child; + } + + // ## static create(opts) + static create(opts) { + let { compare, entries = [] } = opts; + const Child = this.extend(compare); + let index = new Child(...entries); + index.sort(); + return index; + } + + // ## getRangeIndices(min, max) + getRangeIndices(min, max) { + const { compare } = this; + let first = bsearch.le(this, min, compare)+1; + let last = bsearch.ge(this, max, compare); + return [first, last]; + } + + // ## range(min, max) + // Filters down the subselection to only include the given height range. + // Note: perhaps that we should find a way to change the index criterion + // easily, that's for later on though. + range(min, max) { + let [first, last] = this.getRangeIndices(min, max); + return this.slice(first, last); + } + + // ## *it(min, max) + // Helper function which allows a range to be used as an iterator. + *it(min, max) { + let [first, last] = this.getRangeIndices(min, max); + for (let i = first; i < last; i++) { + yield this[i]; + } + } + + // ## query(query) + // Helper function for carrying out a query using the normal array filter + // method. Only exact queries are possible for the moment, no range + // queries though that should be possible as well - see MongoDB for + // example. + query(query) { + + // First of all we'll build the query. Building the query means that + // we're creating an array of functions which *all* need to pass in + // order to evaluate to true. This means an "and" condition. + let filters = []; + for (let key in query) { + let def = query[key]; + + // If the definition is an array, we'll check whether the value is + // within the array. + if (Array.isArray(def)) { + filters.push($oneOf(key, def)); + } else { + filters.push($equals(key, def)); + } + + } + + const keys = Object.keys(query); + return this.filter(function(entry) { + for (let fn of filters) { + if (!fn(entry)) { + return false; + } + } + return true; + }); + } + + // ## sort() + // Sorts the indexed array. Normally this shouldn't be necessary to call + // manually by the way, but we need call this once upon creation! + sort() { + return super.sort(this.compare); + } + +} + +// ## compare(a, b) +// The function we use to sort all lots by height. +function compare(a, b) { + return a.height - b.height; +} + +// ## contains(arr, it) +// Helper function for checking if the given array includes *one* of the +// elements in the given iterator. +function contains(arr, it) { + for (let el of it) { + if (arr.includes(el)) { + return true; + } + } + return false; +} + +// ## $equals(key, value) +function $equals(key, value) { + return function(entry) { + let x = entry[key]; + if (Array.isArray(x)) { + return x.includes(value); + } else { + return x === value; + } + } +} + +// ## $oneOf(key, arr) +function $oneOf(key, arr) { + return function(entry) { + let x = entry[key]; + if (Array.isArray(x)) { + for (let el of x) { + if (arr.includes(el)) { + return true; + } + } + return false; + } else { + return arr.includes(entry[key]); + } + } +} diff --git a/lib/lot-object.js b/lib/lot-object.js index 022243d..1032ae0 100644 --- a/lib/lot-object.js +++ b/lib/lot-object.js @@ -33,6 +33,44 @@ class LotObject { get z() { return this.values[5]/scale; } set z(value) { this.values[5] = Math.round(scale*value); } + get minX() { return signed(this.values[6])/scale; } + get minZ() { return signed(this.values[7])/scale; } + get maxX() { return signed(this.values[8])/scale; } + get maxZ() { return signed(this.values[9])/scale; } + get usage() { return this.values[10]; } + + // ## get OID() + // The Object id is rep 12. The wiki says about this: + // 0xA0000000 = 0,2,4,6,8,a,c,e - Random one of these characters in case + // the other ID's are the same. + // 0x0BBBB000 = Object Family + // 0x00000CCC = Unique Object ID for this family. Incremental for similar + // objects. + get OID() { + return this.values[11]; + } + + // ## get IID() + get IID() { + return this.values[12]; + } + + // ## get IIDs() + // Note: every rep starting from 13 can be an IID, that's another way to + // create families. We should take this into account as well hence. + get IIDs() { + return this.values.slice(12); + } + } -module.exports = LotObject; \ No newline at end of file +module.exports = LotObject; + +// ## signed(x) +// Helper function that ensures certain 32 bit integers are considered as +// signed. +let arr = new Int32Array([0]); +function signed(x) { + arr[0] = x; + return arr[0]; +} diff --git a/lib/lot.js b/lib/lot.js index fe87be3..62617a4 100644 --- a/lib/lot.js +++ b/lib/lot.js @@ -22,7 +22,7 @@ const Lot = class Lot extends Type(FileType.LotFile) { // Pre-initialize the lot properties with correct types to produce better // optimized vm code. This is inherent to how V8 optimizes Object // properties. - constructor() { + constructor(opts) { super(); this.crc = 0x00000000; this.mem = 0x00000000; @@ -53,6 +53,7 @@ const Lot = class Lot extends Type(FileType.LotFile) { this.commutes = []; this.commuteBuffer = null; this.debug = 0x00; + Object.assign(this, opts); } // ## get/set historical() diff --git a/lib/pointer.js b/lib/pointer.js new file mode 100644 index 0000000..e700210 --- /dev/null +++ b/lib/pointer.js @@ -0,0 +1,20 @@ +// # pointer.js +// Small helper class that represents a pointer to a certain record in the +// subfile. +const { hex } = require('./util.js'); +class Pointer { + + // ## constructor(type, address) + constructor(type, address = 0x00000000) { + this.type = type; + this.address = address; + } + + // ## get [Symbol.toPrimitive](hint) + // Allows you to get the numerical value of the pointer by using +pointer. + [Symbol.toPrimitive](hint) { + return hint === 'number' ? this.address : hex(this.address); + } + +} +module.exports = Pointer; diff --git a/lib/prop.js b/lib/prop.js index 9fa4e73..93ec8c7 100644 --- a/lib/prop.js +++ b/lib/prop.js @@ -11,8 +11,8 @@ const Type = require('./type'); // Represents a single prop from the prop file. class Prop extends Type(FileType.PropFile) { - // ## constructor() - constructor() { + // ## constructor(opts) + constructor(opts) { super(); this.crc = 0x00000000; this.mem = 0x00000000; @@ -20,7 +20,7 @@ class Prop extends Type(FileType.PropFile) { this.minor = 0x0004; this.zot = 0x0000; this.unknown1 = 0x00; - this.appearance = 0x00; + this.appearance = 0b00000101; this.unknown2 = 0xA823821E; this.zMinTract = this.xMinTract = 0x00; this.zMaxTract = this.xMaxTract = 0x00; @@ -39,6 +39,7 @@ class Prop extends Type(FileType.PropFile) { this.lotType = 0x02; this.OID = 0x00000000; this.condition = 0x00; + Object.assign(this, opts); } // ## parse(rs) diff --git a/lib/savegame.js b/lib/savegame.js index e013120..a25a73e 100644 --- a/lib/savegame.js +++ b/lib/savegame.js @@ -6,65 +6,113 @@ const FileType = require('./file-types'); // # Savegame() // A class specifically designed for some Savegame functionality. Obviously // extends the DBPF class because savegames are dbpf files. -module.exports = class Savegame extends DBPF { +class Savegame extends DBPF { + + // ## get GID() + get GID() { + return this.entries[0].group; + } // ## get lotFile() get lotFile() { - let entry = this.entries.find(x => x.type === FileType.LotFile); - return entry ? entry.read() : null; + return this.readByType(FileType.LotFile); + } + get lots() { + return this.lotFile; } // ## get buildingFile() get buildingFile() { - let entry = this.entries.find(x => x.type === FileType.BuildingFile); - return entry ? entry.read() : null; + return this.readByType(FileType.BuildingFile); + } + get buildings() { + return this.buildingFile; } // ## get propFile() + // Getter for the propfile. If it doesn't exist yet, we'll create it. get propFile() { - let entry = this.entries.find(x => x.type === FileType.PropFile); - return entry ? entry.read() : null; + return this.readByType(FileType.PropFile); + } + get props() { + return this.propFile; } // ## get baseTextureFile() get baseTextureFile() { - let entry = this.getByType(FileType.BaseTextureFile); - return entry ? entry.read() : null; + return this.readByType(FileType.BaseTextureFile); + } + get textures() { + return this.baseTextureFile; } // ## get itemIndexFile() get itemIndexFile() { - let entry = this.getByType(FileType.ItemIndexFile); - return entry ? entry.read() : null; + return this.readByType(FileType.ItemIndexFile); + } + get itemIndex() { + return this.itemIndexFile; } // ## get zoneDeveloperFile() get zoneDeveloperFile() { - let entry = this.getByType(FileType.ZoneDeveloperFile); - return entry ? entry.read() : null; + return this.readByType(FileType.ZoneDeveloperFile); + } + get zones() { + return this.zoneDeveloperFile; } // ## get lotDeveloperFile() get lotDeveloperFile() { - let entry = this.getByType(FileType.LotDeveloperFile); - return entry ? entry.read() : null; + return this.readByType(FileType.LotDeveloperFile); } // ## get floraFile() get floraFile() { - let entry = this.getByType(FileType.FloraFile); - return entry ? entry.read() : null; + return this.readByType(FileType.FloraFile); + } + + // ## get zoneManager() + get zoneManager() { + return this.readByType(FileType.ZoneManager); } // ## get COMSerializerFile() get COMSerializerFile() { - let entry = this.getByType(FileType.COMSerializerFile); - return entry ? entry.read() : null; + return this.readByType(FileType.COMSerializerFile); + } + + // ## getSimGrid(type, dataId) + // Finds & return a SimGrid based on type and data id. + getSimGrid(type, dataId) { + let grids = this.readByType(type); + return grids.get(dataId); } // # getByType(type) + // This method returns an entry in the savegame by type. If it doesn't + // exist yet, it is created. getByType(type) { - return this.entries.find(x => x.type === type); + let entry = this.entries.find(x => x.type === type); + if (!entry) { + const Klass = DBPF.FileTypes[type]; + if (Klass) { + if (Klass.Array) { + Klass = Klass.Array; + } + let tgi = [type, this.GID, 0]; + entry = this.add(tgi, new Klass()); + } + } + return entry || null; + } + + // ## readByType(type) + // Helper function that reads an entry when it can be returned. + readByType(type) { + let entry = this.getByType(type); + return entry ? entry.read() : null; } -}; \ No newline at end of file +}; +module.exports = Savegame; diff --git a/lib/sim-grid-file.js b/lib/sim-grid-file.js index 571254b..bcf035e 100644 --- a/lib/sim-grid-file.js +++ b/lib/sim-grid-file.js @@ -1,6 +1,6 @@ // # sim-grid.js "use strict"; -const bsearch = require('binary-search'); +const bsearch = require('binary-search-bounds'); const Stream = require('./stream'); const WriteStream = require('./write-stream'); const crc32 = require('./crc'); @@ -71,7 +71,7 @@ class SimGridFile { // ## get(dataId) // Returns the grid for the given data Id. get(dataId) { - let index = bsearch(this.sorted, {"dataId": dataId}, compare); + let index = bsearch.eq(this.sorted, { dataId }, compare); return index > -1 ? this.sorted[index] : null; } @@ -129,11 +129,6 @@ const SimGrid = SimGridFile.SimGrid = class SimGrid { this.u8 = rs.dword(); this.u9 = rs.dword(); - // let format = '4 4 4 2 1 4 4 4 4 4 4 4 4 4'.split(' ').map(x => 2*x); - // let header = buff.slice(0, 55).toString('hex'); - // console.log(chunk(format, header)); - // console.log('rest', Math.sqrt(buff.byteLength-55)); - // Don't know if multiple values are possible here, the SInt8 does // some pretty weird stuff... Anyway, for now we'll just read in the // rest into the appropriate underlying array type. @@ -203,6 +198,20 @@ const SimGrid = SimGridFile.SimGrid = class SimGrid { } + // ## get(x, z) + // Returns the value stored in cell (x, z) + get(x, z) { + let { zSize } = this; + return this.data[ x*zSize + z ]; + } + + // ## set(x, z) + // Sets the value stored in cell (x, z) + set(x, z, value) { + this.data[ x*this.zSize+z ] = value; + return this; + } + // ## createProxy() // Creates a data proxy so that we can access the data in an array-like // way. diff --git a/lib/skyline.js b/lib/skyline.js new file mode 100644 index 0000000..22dac12 --- /dev/null +++ b/lib/skyline.js @@ -0,0 +1,178 @@ +// # skyline.js +"use strict"; +const bsearch = require('binary-search-bounds'); +const LotIndex = require('./lot-index.js'); +const { FileType } = require('./enums.js'); +const { hex } = require('./util.js'); +const LotConfigPropertySize = 0x88edc790; +const OccupantSize = 0x27812810; +const OccupantGroups = 0xaa1dd396; + +// # skyline(opts) +// Just for fun: plop a random skyline. +function skyline(opts) { + let { + city, + center = [], + radius + } = opts; + + // Create the master index for all lots order by height and then filter to + // only include RC buildings. + let lots = new LotIndex(city.index).height + .query({ + occupantGroups: [ + 0x11010, + 0x11020, + 0x11030, + 0x13110, + 0x13120, + 0x13130, + 0x13320, + 0x13330, + ], + }) + .filter(entry => { + + // Filter out buildings with an insufficient filling degree. + let [x, z] = entry.size; + if (Math.max(x, z) > 5) { + return false; + } + + // Calculate the filling degree. + if (x*z > 2) { + let [width, height, depth] = entry.building + let fill = (width*depth) / (x*z*16*16); + if (fill < 0.5 + 0.5*height/400) { + return false; + } + } + + return true; + + }); + + // Filter out the residential lots for the suburbs. + let suburbs = lots + .filter(entry => entry.occupantGroups.includes(0x11020)) + .filter(entry => entry.zoneTypes.length === 3); + + // Create our skyline function that will determine the maximum height + // based on the distance from the center. + let cluster = makeSkylineFunction({ + center, + radius, + }); + let fn = (x, z) => Math.max(10, cluster(x, z)); + // let f = 2; + // let a = makeSkylineFunction({ + // center: [64-f*10, 64-f*5], + // radius: 30, + // max: 400, + // }); + // let b = makeSkylineFunction({ + // center: [64+f*10, 64+f*5], + // radius: 45, + // max: 100, + // }); + // const fn = function(x, z) { + // return Math.max(10, a(x, z) + b(x, z)); + // }; + + // Now loop all city tiles & plop away. + let zones = city.dbpf.zones; + const { xSize, zSize } = zones; + for (let x = 0; x < xSize; x++) { + outer: + for (let z = 0; z < zSize; z++) { + + // First of all make sure there's no lot yet on this tile. Not + // going to overplop lots. + if (zones.cells[x][z]) { + continue; + } + + // Calculate the maximum height for this tile & select the + // appropriate lot range. + let max = fn(x, z); + let db; + if (max < 11) { + db = suburbs; + } else { + db = lots.range({ height: 0.25*max }, { height: max }); + } + + // No suitable lots found to plop? Pity, go on. + if (db.length === 0) { + continue; + } + + // Now pick a random lot from the suitable lots. We will favor + // higher lots more to create a nicer effect. + let index = Math.floor(db.length * Math.random()**0.75); + let { lot } = db[index]; + + // Cool, an appropriate lot was found, but we're not done yet. + // It's possible if there's no space to plop the lot, we're not + // going to bother. Perhaps that we can retry later, but ok. + // Note: we'll still have to take into account the rotation as + // well here! + let orientation = Math.random()*4 | 0; + let [width, depth] = lot.read().value(LotConfigPropertySize); + if (orientation % 2 === 1) { + [width, depth] = [depth, width]; + } + for (let i = 0; i < width; i++) { + for (let j = 0; j < depth; j++) { + let xx = x + i; + let zz = z + j; + if (xx >= xSize || zz >= zSize) { + continue outer; + } + let row = zones.cells[xx][zz]; + if (row) { + continue outer; + } + + // Mock a 8x8 city grid. + let grid = 16; + let half = Math.floor(grid/2); + if (xx % grid === half || zz % grid === half) { + continue outer; + } + + } + } + + // Cool, we got space left to plop the lot. Just do it baby. + city.grow({ + exemplar: lot, + x, + z, + orientation, + }); + + } + } + +} +module.exports = skyline; + +// ## makeSkylineFunction(opts) +// Factory for the skyline function that returns an appropriate height for the +// given (x, z) tile. +function makeSkylineFunction(opts) { + let { + center: [ + cx = 32, + cz = 32, + ], + max = 400, + radius = 32, + } = opts; + return function(x, z) { + let t = Math.sqrt((cx-x)**2 + (cz-z)**2) / radius; + return max*Math.exp(-((2*t)**2)); + }; +} diff --git a/lib/stream.js b/lib/stream.js index 469e5a1..b8d288f 100644 --- a/lib/stream.js +++ b/lib/stream.js @@ -1,5 +1,6 @@ // # stream.js "use strict"; +const Pointer = require('./pointer.js'); // Monkey-patch big ints. if (!Buffer.prototype.readBigInt64LE) { @@ -98,6 +99,14 @@ class Stream { dword() { return this.uint32(); } bool() { return Boolean(this.uint8()); } + // Helper function for reading a pointer. Those are given as [pointer, + // Type ID]. + pointer() { + let address = this.dword(); + let type = this.dword(); + return new Pointer(type, address); + } + } module.exports = Stream; \ No newline at end of file diff --git a/lib/type.js b/lib/type.js index 0917e3b..e4951e3 100644 --- a/lib/type.js +++ b/lib/type.js @@ -41,6 +41,15 @@ class Type { return this.constructor.toString(); } + // ## [Symbol.toPrimitive](hint) + [Symbol.toPrimitive](hint) { + if (hint === 'number') { + return this.mem; + } else { + return hex(this.mem); + } + } + // ## toBuffer(opts) // Most types share the same method of creating a buffer, being by using a // binary generator under the hood. diff --git a/lib/write-stream.js b/lib/write-stream.js index ba8bb3b..188b051 100644 --- a/lib/write-stream.js +++ b/lib/write-stream.js @@ -77,5 +77,13 @@ class WriteStream { return this; } + // ## pointer(ptr) + // Writes a pointer to the buffer. + pointer(ptr) { + this.dword(ptr.address); + this.dword(ptr.type); + return this; + } + } module.exports = WriteStream; \ No newline at end of file diff --git a/lib/zone-manager.js b/lib/zone-manager.js new file mode 100644 index 0000000..779c92c --- /dev/null +++ b/lib/zone-manager.js @@ -0,0 +1,128 @@ +// # zone-manager.js +const Stream = require('./stream.js'); +const WriteStream = require('./write-stream.js'); +const crc32 = require('./crc.js'); +const Type = require('./type.js'); +const Pointer = require('./pointer.js'); +const { FileType } = require('./enums.js'); + +// Some type ids. We should put them within the FileType's though! +const cSC4SimGridSint8 = 0x49b9e603; +const cSC4City = 0x8990c372; +const cSC4OccupantManager = 0x098f964d; +const cSTETerrain = 0xe98f9525; +const cSC4PollutionSimulator = 0x8990c065; +const cSC4BudgetSimulator = 0xe990be01; + +// # ZoneManager +// Not sure what it does yet, still decoding. +class ZoneManager extends Type(FileType.ZoneManager) { + + // ## constructor(opts) + constructor(opts) { + super(); + this.crc = 0x00000000; + this.mem = 0x00000000; + this.major = 0x0001; + + // Pointer to the ZoneView grid. + this.grid = new Pointer(cSC4SimGridSint8); + this.u1 = Array(16).fill(0x00000000); + + // Note even though the size is repeated multiple times when parsing, + // we'll only assign it once. + this.size = 0x00000000; + + // From now on follows a part that always seems to be fixed. + this.u2 = Buffer.from(fixed, 'hex'); + + // Pointers to other subfiles. + this.city = new Pointer(cSC4City); + this.occupantManager = new Pointer(cSC4OccupantManager); + this.terrain = new Pointer(cSTETerrain); + this.pollutionSimulator = new Pointer(cSC4PollutionSimulator); + this.budgetSimulator = new Pointer(cSC4BudgetSimulator); + Object.assign(this, opts); + } + + // ## parse(buff) + parse(buff) { + let rs = new Stream(buff); + let size = rs.dword(); + this.crc = rs.dword(); + this.mem = rs.dword(); + this.major = rs.word(); + this.grid = rs.pointer(); + + // Read in the unknowns. + let arr = this.u1 = []; + for (let i = 0; i < 16; i++) { + arr.push(rs.dword()); + } + + // Read in the size, but skip the fact that it's repeated. + this.size = rs.dword(); + rs.skip(32*4); + + // Read in the next 533 bytes. Seems like they're always fixed. + this.u2 = rs.read(533); + + // More pointers follow now. + this.city = rs.pointer(); + this.occupantManager = rs.pointer(); + this.terrain = rs.pointer(); + this.pollutionSimulator = rs.pointer(); + this.budgetSimulator = rs.pointer(); + + } + + *bgen() { + let buff = Buffer.allocUnsafe(791); + let ws = new WriteStream(buff); + ws.dword(buff.byteLength); + ws.jump(8); + ws.dword(this.mem); + ws.word(this.major); + ws.pointer(this.grid); + for (let dword of this.u1) { + ws.dword(dword); + } + for (let i = 0; i < 33; i++) { + ws.dword(this.size); + } + ws.write(this.u2); + ws.pointer(this.city); + ws.pointer(this.occupantManager); + ws.pointer(this.terrain); + ws.pointer(this.pollutionSimulator); + ws.pointer(this.budgetSimulator); + + // Write crc & we're done. + buff.writeUInt32LE(this.crc = crc32(buff, 8), 4); + yield buff; + + } + +} +module.exports = ZoneManager; + +const fixed = ` +00000000 00000000 00000000 00000000 01000000 +01000000 01000000 01000000 01000000 01000000 04000000 01000000 01000000 +01000000 01000000 01000000 01000000 02000000 01000000 00800000 20000000 +20000000 20000000 20000000 20000000 20000000 30000000 20000000 20000000 +20000000 20000000 20000000 20000000 20000000 20000000 00000000 00000000 +0a000000 00000000 14000000 00000000 32000000 00000000 0a000000 00000000 +14000000 00000000 32000000 00000000 0a000000 00000000 14000000 00000000 +32000000 00000000 01000000 00000000 01000000 00000000 01000000 00000000 +01000000 00000000 32000000 00000000 01000000 00000000 00000000 00000000 +01000000 00000000 01000000 00000000 01000000 00000000 01000000 00000000 +01000000 00000000 01000000 00000000 01000000 00000000 01000000 00000000 +01000000 00000000 01000000 00000000 01000000 00000000 01000000 00000000 +01000000 00000000 01000000 00000000 01000000 00000000 19100000 15100000 +16100000 14100000 03100000 04100000 02100000 10100000 11100000 0f100000 +12100000 00100000 18100000 1b100000 05100000 1a100000 ff0000ff ff00c400 +ff009a00 ff007200 ffff774f ffff5320 ffe22014 ff32dbff ff33b2ff ff1f90ce +00000000 00000000 00000000 00000000 00000000 00000000 aa000000 00000000 +00 +`.replace(/( |\n)/g, ''); diff --git a/package.json b/package.json index 10d4991..0683487 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "chai": "^4.2.0", "chai-spies": "^1.0.0", "electron": "^5.0.6", + "hilbert-curve": "^2.0.5", "mocha": "^6.2.2", "node-gyp": "^4.0.0", "prebuildify": "^3.0.4", @@ -34,11 +35,12 @@ "three": "^0.106.2" }, "dependencies": { - "binary-search": "^1.3.6", + "binary-search-bounds": "^2.0.4", "chalk": "^2.4.2", "commander": "^2.20.3", "ini": "^1.3.5", "inquirer": "^6.5.2", + "lru-cache": "^5.1.1", "node-addon-api": "^1.7.1", "node-gyp-build": "^4.2.1", "ora": "^3.4.0", diff --git a/test/api-test.js b/test/api-test.js index 224d206..adeb9f6 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -16,8 +16,8 @@ describe('#historical()', function() { it('should make all buildings in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "all": true + dbpf: path.join(files, 'city.sc4'), + all: true }); // Check the dbpf file now. Everything should be historical. @@ -29,8 +29,8 @@ describe('#historical()', function() { it('should make all residentials in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "residential": true + dbpf: path.join(files, 'city.sc4'), + residential: true }); for (let lot of dbpf.lotFile) { @@ -41,8 +41,8 @@ describe('#historical()', function() { it('should make all commercials in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "commercial": true + dbpf: path.join(files, 'city.sc4'), + commercial: true }); for (let lot of dbpf.lotFile) { @@ -53,8 +53,8 @@ describe('#historical()', function() { it('should make all industrials in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "industrial": true + dbpf: path.join(files, 'city.sc4'), + industrial: true }); for (let lot of dbpf.lotFile) { @@ -65,8 +65,8 @@ describe('#historical()', function() { it('should make all agriculturals in a city historical', async function() { let dbpf = await historical({ - "dbpf": path.join(files, 'city.sc4'), - "agricultural": true + dbpf: path.join(files, 'city.sc4'), + agricultural: true }); for (let lot of dbpf.lotFile) { @@ -90,8 +90,8 @@ describe('#growify', function() { expect(plopped.size).to.be.above(0); await growify({ - "dbpf": dbpf, - "residential": ZoneType.RMedium + dbpf, + residential: ZoneType.RMedium, }); for (let lot of dbpf.lotFile) { @@ -114,8 +114,8 @@ describe('#growify', function() { expect(plopped.size).to.be.above(0); await growify({ - "dbpf": dbpf, - "industrial": ZoneType.IHigh + dbpf, + industrial: ZoneType.IHigh }); for (let lot of dbpf.lotFile) { @@ -138,8 +138,8 @@ describe('#growify', function() { expect(plopped.size).to.be.above(0); await growify({ - "dbpf": dbpf, - "agricultural": ZoneType.ILow + dbpf, + agricultural: ZoneType.ILow, }); for (let lot of dbpf.lotFile) { diff --git a/test/city-manager-test.js b/test/city-manager-test.js index 4bba2fd..0c02294 100644 --- a/test/city-manager-test.js +++ b/test/city-manager-test.js @@ -1,9 +1,9 @@ // # city-manager-test.js "use strict"; -const chai = require('chai'); -const expect = chai.expect; -const CityManager = require('../lib/city-manager'); +const { expect } = require('chai'); const path = require('path'); +const FileIndex = require('../lib/file-index.js'); +const CityManager = require('../lib/city-manager.js'); describe('A city manager', function() { @@ -24,4 +24,126 @@ describe('A city manager', function() { }); -}); \ No newline at end of file + context('#mem()', function() { + + it('returns an unused memory address', async function() { + + let file = path.resolve(__dirname, 'files/City - RCI.sc4'); + let city = new CityManager(); + city.load(file); + + expect(city.mem()).to.equal(1); + city.memRefs.add(2); + expect(city.mem()).to.equal(3); + expect(city.mem()).to.equal(4); + + }); + + }); + + context('#grow()', function() { + + const regions = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments'); + + it('grows a lot', async function() { + + this.slow(1000); + + let dir = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Plugins'); + let index = new FileIndex(dir); + await index.build(); + + // Create the city manager. + let game = path.join(__dirname, 'files/City - 432.sc4'); + let city = new CityManager({ index }); + city.load(game); + + // console.log(city.dbpf.textures[0]); + + // Grow a lot. + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0x8fcc0f62], + x: 10, + z: 10, + orientation: 0, + }); + + let file = path.join(regions, 'City - Plopsaland.sc4'); + await city.save({ file }); + + }); + + it.only('grows a lot with props', async function() { + + this.slow(1000); + + const dir = 'c:/GOG Games/Simcity 4 Deluxe Edition'; + let index = new FileIndex({ + files: [ + path.join(dir, 'Simcity_1.dat'), + path.join(dir, 'Simcity_2.dat'), + path.join(dir, 'Simcity_3.dat'), + path.join(dir, 'Simcity_4.dat'), + path.join(dir, 'Simcity_5.dat'), + ], + }); + await index.build(); + + // Create the city manager. + let game = path.join(__dirname, 'files/City - 432.sc4'); + let city = new CityManager({ index }); + city.load(game); + + // Grow a lot. + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0x600040d0], + x: 10, + z: 10, + orientation: 0, + }); + + let file = path.join(regions, 'City - Plopsaland.sc4'); + await city.save({ file }); + + }); + + }); + + context('#plop()', function() { + + it('plops a ploppable lot', async function() { + + this.slow(1000); + + // First of all we need to build up a file index that the city + // manager can use. + // let dir = path.join(__dirname, 'files/DarkNight_11KingStreetWest'); + // let dir = path.join(__dirname,'files/DiegoDL-432ParkAvenue-LM-DN'); + let dir = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Plugins'); + let index = new FileIndex(dir); + await index.build(); + + // Create the city manager. + let game = path.join(__dirname, 'files/City - 432.sc4'); + let city = new CityManager({ index }); + city.load(game); + + // Plop it baby. + for (let i = 0; i < 1; i++) { + city.plop({ + tgi: [0x6534284a, 0xd60100c4, 0x483248bb], + // tgi: [0x6534284a,0x76fbb03a,0x290dc058], + x: (1+i)*8, + z: 8, + orientation: i % 4, + }); + } + let regions = path.join(process.env.HOMEPATH, 'Documents/SimCity 4/Regions/Experiments'); + let file = path.join(regions, 'City - Plopsaland.sc4'); + await city.save({ file }); + + }); + + }); + +}); diff --git a/test/dbpf-test.js b/test/dbpf-test.js index aea8f41..cf7a88c 100644 --- a/test/dbpf-test.js +++ b/test/dbpf-test.js @@ -15,19 +15,81 @@ const { ZoneType, cClass } = require('../lib/enums'); describe('A DBPF file', function() { - it('should be parsed', function() { + it('parses from a file', function() { let file = path.resolve(__dirname, 'files/cement.sc4lot'); - let buff = fs.readFileSync(file); // Parse the dbpf. + let dbpf = new DBPF(file); + + // Find an entry and verify that it gets read correctly. + let entry = dbpf.find(entry => { + return ( + entry.type === 0x6534284a && + entry.group === 0xa8fbd372 && + entry.instance === 0x8a73e853 + ); + }); + let exemplar = entry.read(); + expect(+exemplar.prop(0x10)).to.equal(0x10); + + }); + + it('parses from a buffer', function() { + + let file = path.resolve(__dirname, 'files/cement.sc4lot'); + let buff = fs.readFileSync(file); let dbpf = new DBPF(buff); + // Find an entry and verify that it gets read correctly. + let entry = dbpf.find(entry => { + return ( + entry.type === 0x6534284a && + entry.group === 0xa8fbd372 && + entry.instance === 0x8a73e853 + ); + }); + let exemplar = entry.read(); + expect(+exemplar.prop(0x10)).to.equal(0x10); + + }); + + it('frees memory the DBPF is taking up', function() { + + // Read in the DBPF and make sure all entries are properly read. + let file = path.resolve(__dirname, 'files/cement.sc4lot'); + let dbpf = new DBPF(file); + expect(dbpf.buffer).to.be.ok; + for (let entry of dbpf) { + entry.read(); + expect(entry.raw).to.be.ok; + } + let entry = dbpf.find(entry => { + return ( + entry.type === 0x6534284a && + entry.group === 0xa8fbd372 && + entry.instance === 0x8a73e853 + ); + }); + + // Free up the DBPF memory. + dbpf.free(); + expect(dbpf.buffer).to.be.null; + for (let entry of dbpf) { + expect(entry.raw).to.be.null; + expect(entry.file).to.be.null; + } + + // Check that the DBPF gets automatically reloaded if we request to + // read an entry. + let exemplar = entry.read(); + expect(+exemplar.prop(0x10)).to.equal(0x10); + }); it('should be serialized to a buffer', function() { let file = path.resolve(__dirname, 'files/cement.sc4lot'); - let dbpf = new DBPF(fs.readFileSync(file)); + let dbpf = new DBPF(file); // Serialize the DBPF into a buffer and immediately parse again so // that we can compare. diff --git a/test/file-index-test.js b/test/file-index-test.js new file mode 100644 index 0000000..6ef95b2 --- /dev/null +++ b/test/file-index-test.js @@ -0,0 +1,66 @@ +// # file-index-test.js +"use strict"; +const path = require('path'); +const { expect } = require('chai'); +const Index = require('../lib/file-index.js'); +const FileType = require('../lib/file-types.js'); +const dir = path.join(__dirname, 'files'); + +describe('The file index', function() { + + it('should index all files in a directory', async function() { + + let index = new Index( + path.resolve(__dirname, 'files/DarkNight_11KingStreetWest') + ); + + // Build up the index. This is done asynchronously so that files can + // be read in parallel while parsing. + await index.build(); + + let record = index.find(0x6534284a, 0xa8fbd372, 0xe001a291); + expect(record).to.be.ok; + expect(record.fileSize).to.equal(2378); + expect(record.compressedSize).to.equal(2378); + expect(record.compressed).to.be.false; + + // Read the file. Should be an exemplar. + let file = record.read(); + expect(file.fileType).to.equal(FileType.Exemplar); + expect(file.table).to.have.property('LotConfigPropertyLotObject'); + expect(file.table).to.have.property(0x88EDC900); + + let building = file.lotObjects.find(x => x.type === 0x00); + expect(building.x).to.equal(1); + expect(building.y).to.equal(0); + expect(building.z).to.equal(1.5); + + }); + + it('uses a memory limit for the cache', async function() { + + let nybt = path.join(dir, 'NYBT/Aaron Graham/NYBT Gracie Manor'); + let index = new Index({ + dirs: [nybt], + mem: 1500000, + }); + await index.build(); + for (let entry of index.records) { + entry.read(); + } + + }); + + it('indexes all building and prop families', async function() { + + let nybt = path.join(dir, 'NYBT/Aaron Graham/NYBT Gracie Manor'); + let index = new Index(nybt); + await index.build(); + let { families } = index; + expect(families).to.have.length(2); + expect(index.family(0x5484CA20)).to.have.length(4); + expect(index.family(0x5484CA1F)).to.have.length(4); + + }); + +}); diff --git a/test/index-test.js b/test/index-test.js deleted file mode 100644 index 9ada9bb..0000000 --- a/test/index-test.js +++ /dev/null @@ -1,41 +0,0 @@ -// # index-test.js -"use strict"; -const chai = require('chai'); -const expect = chai.expect; -const path = require('path'); - -const Index = require('../lib/index'); -const FileType = require('../lib/file-types'); - -describe('The file index', function() { - - it.skip('should index all files in a directory', async function() { - - let index = new Index({ - "dirs": [ - path.resolve(__dirname, 'files/DarkNight_11KingStreetWest') - ] - }); - - // Build up the index. This is done asynchronously so that files can - // be read in parallel while parsing. - await index.build(); - - let record = index.find(0x6534284a, 0xa8fbd372, 0xe001a291); - expect(record).to.be.ok; - expect(record.fileSize).to.equal(2378); - expect(record.compressedSize).to.equal(2378); - expect(record.compressed).to.be.false; - - // Read the file. Should be an exemplar. - let file = record.read(); - expect(file.fileType).to.equal(FileType.Exemplar); - expect(file.table).to.have.property('LotConfigPropertyLotObject'); - expect(file.table).to.have.property(0x88EDC900); - - let building = file.lotObjects.find(x => x.type === 0x00); - console.log(building.x, building.y, building.z); - - }); - -}); \ No newline at end of file diff --git a/test/plop-test.js b/test/plop-test.js index 6b4c7dc..0e58825 100644 --- a/test/plop-test.js +++ b/test/plop-test.js @@ -10,14 +10,19 @@ const Stream = require('../lib/stream'); const crc32 = require('../lib/crc'); const { hex, chunk, split } = require('../lib/util'); const { ZoneType, FileType, cClass, SimGrid } = require('../lib/enums'); -const Index = require('../lib/index'); +const CityManager = require('../lib/city-manager.js'); +const FileIndex = require('../lib/file-index.js'); const Savegame = require('../lib/savegame'); const Lot = require('../lib/lot'); const Building = require('../lib/building'); +const skyline = require('../lib/skyline.js'); const HOME = process.env.HOMEPATH; +const PLUGINS = path.resolve(HOME, 'documents/SimCity 4/plugins'); const REGION = path.resolve(HOME, 'documents/SimCity 4/regions/experiments'); +const c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; +const dir = path.join(__dirname, 'files'); -describe.skip('A city manager', function() { +describe('A city manager', function() { it.skip('should decode the cSC4Occupant class', function() { @@ -145,7 +150,7 @@ describe.skip('A city manager', function() { }); - it('should play with the grids in an established city', async function() { + it.skip('should play with the grids in an established city', async function() { let buff = fs.readFileSync(path.resolve(__dirname, 'files/City - Established.sc4')); let dbpf = new Savegame(buff); @@ -216,7 +221,7 @@ describe.skip('A city manager', function() { // Beware!! If the tracts are not set correctly we've created immortal // flora. Probably when deleting within a tract the game only looks for // stuff in that tract. That's quite logical actually. - it('should create forested streets', async function() { + it.skip('should create forested streets', async function() { let buff = fs.readFileSync(path.resolve(__dirname, 'files/City - Million Trees.sc4')); let dbpf = new Savegame(buff); @@ -257,7 +262,7 @@ describe.skip('A city manager', function() { }); - it('should move a building', async function() { + it.skip('should move a building', async function() { let buff = fs.readFileSync(path.resolve(__dirname, 'files/City - Move bitch.sc4')); let dbpf = new Savegame(buff); @@ -291,4 +296,275 @@ describe.skip('A city manager', function() { }); -}); \ No newline at end of file + it.only('builds a skyline', async function() { + + this.timeout(0); + + let dir = path.join(__dirname, 'files'); + let file = path.join(dir, 'City - Plopsaland.sc4'); + let nybt = path.join(PLUGINS, 'NYBT'); + + let c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; + // let index = new FileIndex(nybt); + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + path.join(c, 'SimCity_2.dat'), + path.join(c, 'SimCity_3.dat'), + path.join(c, 'SimCity_4.dat'), + path.join(c, 'SimCity_5.dat'), + ] + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(file); + + // Create the skyline in the city. + skyline({ city }); + + // Save the city. + let out = path.join(REGION, 'City - Plopsaland.sc4'); + await city.save({ file: out }); + + }); + + it.skip('builds a skyline on a medium tile', async function() { + + this.timeout(0); + + let dir = path.join(__dirname, 'files'); + let file = path.join(dir, 'City - Medium tile.sc4'); + + let c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + path.join(c, 'SimCity_2.dat'), + path.join(c, 'SimCity_3.dat'), + path.join(c, 'SimCity_4.dat'), + path.join(c, 'SimCity_5.dat'), + ] + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(file); + + // Create the skyline in the city. + skyline({ city }); + + // Save the city. + let out = path.join(REGION, 'City - Medium tile.sc4'); + await city.save({ file: out }); + + }); + + it.skip('creates RCI zones', async function() { + + this.timeout(0); + + let source = path.join(dir, 'City - Zone me.sc4'); + let out = path.join(REGION, 'City - Zone me.sc4'); + let city = new CityManager({}); + city.load(source); + + // Create a new zone. + city.zone({ + x: 1, + z: 0, + orientation: 2, + }); + + // console.log(city.dbpf.textures); + await city.save({ file: out }); + + }); + + it.skip('includes the textures when plopping', async function() { + + this.timeout(0); + + let dir = path.join(__dirname, 'files'); + let file = path.join(dir, 'City - Textures.sc4'); + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + ], + dirs: [ + path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), + ], + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(file); + // console.log(city.dbpf.textures[0]); + + for (let i = 0; i < 9; i++) { + for (let j = 0; j < 9; j++) { + if (i === 1 && j === 1) { + continue; + } + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x: i, + z: j, + orientation: (i+j) % 4, + }); + } + } + + // console.table(city.dbpf.textures, [ + // 'u1', 'u2', 'u3', 'u4', 'u5', 'u6', 'u7', 'u8', 'u9', + // ]); + + // console.table([ + // city.dbpf.textures[0].textures[0], + // city.dbpf.textures[1].textures[0], + // ]); + + // console.table([ + // city.dbpf.textures[0].textures[1], + // city.dbpf.textures[1].textures[1], + // ]); + + // Save + let out = path.join(REGION, 'City - Textures.sc4'); + await city.save({ file: out }); + + }); + + it.skip('includes the base texture when plopping', async function() { + + this.timeout(0); + let dir = path.join(__dirname, 'files'); + let out = path.join(REGION, 'City - Base Textures.sc4'); + let source = path.join(dir, 'City - Base Textures.sc4'); + // let source = out; + + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + ], + dirs: [ + path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), + ], + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(source); + + for (let i = 0; i < 5; i++) { + for (let j = 0; j < 5; j++) { + if (i === 1 && j === 1) { + continue; + } + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x: i, + z: j, + orientation: (i+j) % 4, + }); + // break; + } + // break; + } + + await city.save({ file: out }); + + }); + + it.skip('plops a Hilbert curve', async function() { + + const curve = require('hilbert-curve'); + + this.timeout(); + let dir = path.join(__dirname, 'files'); + let source = path.join(dir, 'City - Large Tile.sc4'); + let out = path.join(REGION, 'City - Large Tile.sc4'); + + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + ], + dirs: [ + path.join(PLUGINS, 'Two Simple 1 x 1 Residential Lots v2'), + ], + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(source); + let zones = city.dbpf.zones; + + const order = 6; + const n = 2**order; + const nn = n**2; + let data = []; + const length = 5; + for (let i = 0; i < nn; i++) { + let point = curve.indexToPoint(i, order); + data.push({ + x: (length-1)*point.x, + z: (length-1)*point.y, + }); + } + + let matrix = Array(length*n).fill().map(() => { + return Array(length*n).fill(); + }); + + for (let i = 1; i < data.length; i++) { + let P = data[i-1]; + let Q = data[i]; + let d = { + x: (Q.x - P.x)/(length-1), + z: (Q.z - P.z)/(length-1), + }; + for (let j = 0; j < length; j++) { + let xx = P.x + d.x*j + 2; + let zz = P.z + d.z*j + 2; + matrix[xx][zz] = true; + } + } + + // Loop the entire curve and grow a lot to the left or to the right. + for (let i = 1; i < data.length; i++) { + let P = data[i-1]; + let Q = data[i]; + let d = { + x: (Q.x - P.x)/(length-1), + z: (Q.z - P.z)/(length-1), + }; + let n = { + x: d.z, + z: -d.x, + }; + for (let j = 0; j < length; j++) { + let xx = P.x + d.x*j + 2; + let zz = P.z + d.z*j + 2; + for (let s of [-1, 1]) { + let x = xx + n.x*s; + let z = zz + n.z*s; + if (zones.cells[x][z]) continue; + if (x < 0 || z < 0) continue; + if (matrix[x][z]) continue; + + city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x, + z, + }); + + } + } + } + + // Save baby. + await city.dbpf.save({ file: out }); + + }); + +}); diff --git a/test/victoria-test.js b/test/victoria-test.js new file mode 100644 index 0000000..48b21f7 --- /dev/null +++ b/test/victoria-test.js @@ -0,0 +1,17 @@ +// # victoria-test.js +const path = require('path'); +const Savegame = require('../lib/savegame.js'); + +let city = new Savegame(path.resolve(__dirname, 'files/City - Victoria.sc4')); +let { lots } = city; +for (let lot of lots) { + if (lot.minX === 2752/16 && lot.minZ === 3120/16) { + lot.zoneWealth = 0x03; + // console.log(lot); + } + // if (lot.zoneWealth === 0) { + // console.log(lot); + // } +} + +city.save(path.resolve(__dirname, 'files/City - Victoria restored.sc4')); diff --git a/test/zone-manager-test.js b/test/zone-manager-test.js new file mode 100644 index 0000000..377fa79 --- /dev/null +++ b/test/zone-manager-test.js @@ -0,0 +1,72 @@ +// # zone-manager-test.js +"use strict"; +const { expect } = require('chai'); +const fs = require('fs'); +const path = require('path'); +const Stream = require('../lib/stream.js'); +const Savegame = require('../lib/savegame.js'); +const { hex, chunk } = require('../lib/util'); +const { FileType, cClass } = require('../lib/enums.js'); +const CityManager = require('../lib/city-manager.js'); +const HOME = process.env.HOMEPATH; +const PLUGINS = path.resolve(HOME, 'documents/SimCity 4/plugins'); +const REGION = path.resolve(HOME, 'documents/SimCity 4/regions/experiments'); +const dir = path.resolve(__dirname, 'files'); + +describe('The zone manager file', function() { + + it('is parsed & serialized correctly', function() { + this.timeout(0); + let file = path.join(dir, 'city.sc4'); + let buff = fs.readFileSync(file); + let dbpf = new Savegame(buff); + let entry = dbpf.getByType(FileType.ZoneManager); + let zm = entry.read(); + + // Check that the input buffer matches the out buffer exactly. + let { crc } = zm; + let out = zm.toBuffer(); + expect(out).to.eql(entry.decompress()); + + }); + + it('is decoded', async function() { + + const FileIndex = require('../lib/file-index.js'); + + let out = path.join(REGION, 'City - Growth.sc4'); + let one = path.join(dir, 'City - Growth - 1.sc4'); + let two = new Savegame(path.join(dir, 'City - Growth - 2.sc4')); + + let c = 'c:/GOG Games/SimCity 4 Deluxe Edition'; + // let index = new FileIndex(nybt); + let index = new FileIndex({ + files: [ + path.join(c, 'SimCity_1.dat'), + path.join(c, 'SimCity_2.dat'), + path.join(c, 'SimCity_3.dat'), + path.join(c, 'SimCity_4.dat'), + path.join(c, 'SimCity_5.dat'), + ], + dirs: [ + PLUGINS, + ], + }); + await index.build(); + + let city = new CityManager({ index }); + city.load(one); + for (let i = 0; i < 10; i++) { + let lot = city.grow({ + tgi: [0x6534284a,0xa8fbd372,0xa706ed25], + x: 3+i, + z: 1, + orientation: 2, + }); + } + + await city.save({ file: out }); + + }); + +}); \ No newline at end of file