diff --git a/src/tools/Extractor_projects/map-extractor/System.cpp b/src/tools/Extractor_projects/map-extractor/System.cpp index 96bb28292..36f940a47 100644 --- a/src/tools/Extractor_projects/map-extractor/System.cpp +++ b/src/tools/Extractor_projects/map-extractor/System.cpp @@ -28,6 +28,11 @@ #include #include #include +#include +#include +#include +#include +#include #include "dbcfile.h" // The following is a temp fix until the extractor is merged with the unified extractor @@ -89,6 +94,13 @@ char output_path[128] = "."; /**< TODO */ char input_path[128] = "."; /**< TODO */ uint32 maxAreaId = 0; /**< TODO */ uint32 CONF_max_build = 0; +uint32 CONF_threads = 0; ///< Worker threads for tile conversion; 0 = auto-detect cores, 1 = serial. + +/// Serializes MPQ archive reads. StormLib mutates a shared per-archive file +/// position with no internal locking, so concurrent reads from one handle +/// race (unsafe on Linux). Held only around the archive read in ConvertADT; +/// the parse/pack/write that follows runs fully parallel. +std::mutex g_mpqReadMutex; /** * @brief Data types which can be extracted * @@ -142,6 +154,8 @@ void Usage(char* prg) printf(" size, but also accuracy\n"); printf(" -e, --extract # extract specified client data. 1 = maps, 2 = DBCs,\n"); printf(" 3 = both. Defaults to extracting both.\n"); + printf(" -t, --threads # worker threads for map conversion. 0 = auto-detect\n"); + printf(" cores (default), 1 = serial.\n"); printf("\n"); printf(" Example:\n"); printf(" - use input path and do not flatten maps:\n"); @@ -231,6 +245,16 @@ void HandleArgs(int argc, char* arg[]) Usage(arg[0]); } break; + case 't': + if (c + 1 < argc) // all ok + { + CONF_threads = atoi(arg[(c++) + 1]); + } + else + { + Usage(arg[0]); + } + break; default: Usage(arg[0]); break; @@ -583,19 +607,23 @@ float selectUInt16StepStore(float maxDiff) return 65535 / maxDiff; } -uint16 area_flags[ADT_CELLS_PER_GRID][ADT_CELLS_PER_GRID]; /**< Temporary grid data store */ +// Per-tile working buffers. thread_local so each worker thread owns a private +// copy: with one thread this is identical to the former plain globals, and it +// lets ConvertADT run on several tiles concurrently without trampling shared +// scratch. ConvertADT fully (re)writes/resets these per tile before reading. +thread_local uint16 area_flags[ADT_CELLS_PER_GRID][ADT_CELLS_PER_GRID]; /**< Temporary grid data store */ -float V8[ADT_GRID_SIZE][ADT_GRID_SIZE]; /**< TODO */ -float V9[ADT_GRID_SIZE + 1][ADT_GRID_SIZE + 1]; /**< TODO */ -uint16 uint16_V8[ADT_GRID_SIZE][ADT_GRID_SIZE]; /**< TODO */ -uint16 uint16_V9[ADT_GRID_SIZE + 1][ADT_GRID_SIZE + 1]; /**< TODO */ -uint8 uint8_V8[ADT_GRID_SIZE][ADT_GRID_SIZE]; /**< TODO */ -uint8 uint8_V9[ADT_GRID_SIZE + 1][ADT_GRID_SIZE + 1]; /**< TODO */ +thread_local float V8[ADT_GRID_SIZE][ADT_GRID_SIZE]; /**< TODO */ +thread_local float V9[ADT_GRID_SIZE + 1][ADT_GRID_SIZE + 1]; /**< TODO */ +thread_local uint16 uint16_V8[ADT_GRID_SIZE][ADT_GRID_SIZE]; /**< TODO */ +thread_local uint16 uint16_V9[ADT_GRID_SIZE + 1][ADT_GRID_SIZE + 1]; /**< TODO */ +thread_local uint8 uint8_V8[ADT_GRID_SIZE][ADT_GRID_SIZE]; /**< TODO */ +thread_local uint8 uint8_V9[ADT_GRID_SIZE + 1][ADT_GRID_SIZE + 1]; /**< TODO */ -uint16 liquid_entry[ADT_CELLS_PER_GRID][ADT_CELLS_PER_GRID]; /**< TODO */ -uint8 liquid_flags[ADT_CELLS_PER_GRID][ADT_CELLS_PER_GRID]; /**< TODO */ -bool liquid_show[ADT_GRID_SIZE][ADT_GRID_SIZE]; /**< TODO */ -float liquid_height[ADT_GRID_SIZE + 1][ADT_GRID_SIZE + 1]; /**< TODO */ +thread_local uint16 liquid_entry[ADT_CELLS_PER_GRID][ADT_CELLS_PER_GRID]; /**< TODO */ +thread_local uint8 liquid_flags[ADT_CELLS_PER_GRID][ADT_CELLS_PER_GRID]; /**< TODO */ +thread_local bool liquid_show[ADT_GRID_SIZE][ADT_GRID_SIZE]; /**< TODO */ +thread_local float liquid_height[ADT_GRID_SIZE + 1][ADT_GRID_SIZE + 1]; /**< TODO */ /** * @brief @@ -610,15 +638,34 @@ bool ConvertADT(char* filename, char* filename2, uint32 build) { ADT_file adt; - if (!adt.loadFile(filename, true)) { - printf("Error: Failed to load ADT file: %s\n", filename); - return false; + // loadFile reads the whole ADT into memory; everything after this is + // in-memory work, so the MPQ lock is held only for the read itself. + std::lock_guard mpqLock(g_mpqReadMutex); + if (!adt.loadFile(filename, true)) + { + printf("Error: Failed to load ADT file: %s\n", filename); + return false; + } } + // Zero every per-tile scratch buffer up front so a tile's output can never + // inherit residual data from a previously processed tile. Cells a tile does + // not fully overwrite (e.g. liquid height outside the liquid bounds) are + // dead to the server (gated by the liquid flags), but they still land in the + // .map file -- leaving them stale made the bytes depend on tile order, which + // is non-deterministic once tiles are processed across threads. + memset(area_flags, 0, sizeof(area_flags)); + memset(V8, 0, sizeof(V8)); + memset(V9, 0, sizeof(V9)); + memset(uint16_V8, 0, sizeof(uint16_V8)); + memset(uint16_V9, 0, sizeof(uint16_V9)); + memset(uint8_V8, 0, sizeof(uint8_V8)); + memset(uint8_V9, 0, sizeof(uint8_V9)); memset(liquid_show, 0, sizeof(liquid_show)); memset(liquid_flags, 0, sizeof(liquid_flags)); memset(liquid_entry, 0, sizeof(liquid_entry)); + memset(liquid_height, 0, sizeof(liquid_height)); // Prepare map header map_fileheader map; @@ -1273,8 +1320,6 @@ bool ConvertADT(char* filename, char* filename2, uint32 build) */ void ExtractMapsFromMpq(uint32 build, const int locale) { - char mpq_filename[1024]; - char output_filename[1024]; char mpq_map_name[1024]; printf("\nExtracting maps...\n"); @@ -1290,17 +1335,17 @@ void ExtractMapsFromMpq(uint32 build, const int locale) std::string path = output_path; path += "/maps/"; - if (!CreateDir(path)) - { - printf("Warning: Output directory '%s' already exists or could not be created\n", path.c_str()); - } - else - { - printf("Created output directory: %s\n", path.c_str()); - } + if (!CreateDir(path)) + { + printf("Warning: Output directory '%s' already exists or could not be created\n", path.c_str()); + } + else + { + printf("Created output directory: %s\n", path.c_str()); + } - uint32 success_count = 0; - uint32 failed_count = 0; + std::atomic success_count{0}; + std::atomic failed_count{0}; printf("\n Converting map files\n"); for (uint32 z = 0; z < map_count; ++z) @@ -1324,57 +1369,108 @@ void ExtractMapsFromMpq(uint32 build, const int locale) continue; } - uint32 adt_count = 0; - uint32 adt_exist_in_wdt = 0; + uint32 adt_count = 0; + uint32 adt_exist_in_wdt = 0; for (uint32 y = 0; y < WDT_MAP_SIZE; ++y) { for (uint32 x = 0; x < WDT_MAP_SIZE; ++x) { - // Check bit 0 only (some WDT versions use the full uint32, check only the lower bit) - if (wdt.main->adt_list[y][x].exist & 0x1) + // Check bit 0 only: some WDT versions use the full uint32 + if (wdt.main->adt_list[y][x].exist & 0x1) { - ++adt_exist_in_wdt; - } - } - } - printf(" WDT indicates %u ADT files exist for this map\n", adt_exist_in_wdt); + ++adt_exist_in_wdt; + } + } + } + printf(" WDT indicates %u ADT files exist for this map\n", adt_exist_in_wdt); - for (uint32 y = 0; y < WDT_MAP_SIZE; ++y) - { - for (uint32 x = 0; x < WDT_MAP_SIZE; ++x) - { - // Check bit 0 only - this is the ADT existence flag - if (!(wdt.main->adt_list[y][x].exist & 0x1)) - { - continue; + // Collect the existing tiles, then convert them across CONF_threads + // workers (0 = auto-detect cores). Each tile writes its own independent + // .map file, so the output is identical regardless of thread count or + // completion order. + std::queue > tileQueue; // (tileX, tileY) + for (uint32 y = 0; y < WDT_MAP_SIZE; ++y) + { + for (uint32 x = 0; x < WDT_MAP_SIZE; ++x) + { + // Check bit 0 only: this is the ADT existence flag + if (wdt.main->adt_list[y][x].exist & 0x1) + { + tileQueue.push(std::make_pair(x, y)); } - ++adt_count; - sprintf(mpq_filename, "World\\Maps\\%s\\%s_%u_%u.adt", map_ids[z].name, map_ids[z].name, x, y); - sprintf(output_filename, "%s/maps/%04u%02u%02u.map", output_path, map_ids[z].id, y, x); - if (ConvertADT(mpq_filename, output_filename, build)) - { - ++success_count; - } - else - { - ++failed_count; - } - } - // draw progress bar - printf(" Processing........................%d%%\r", (100 * (y + 1)) / WDT_MAP_SIZE); + } } - if (adt_count > 0) - { - printf("\n Found %u ADT files for map %s\n", adt_count, map_ids[z].name); - } - else - { - printf("\n WARNING: No ADT files found for map %s (map ID: %u)\n", map_ids[z].name, map_ids[z].id); - } - } - printf("\n\nMap extraction complete!\n"); - printf("Successfully converted: %u tiles\n", success_count); - printf("Failed to convert: %u tiles\n", failed_count); + adt_count = static_cast(tileQueue.size()); + + uint32 nThreads = CONF_threads ? CONF_threads : std::thread::hardware_concurrency(); + if (nThreads < 1) + { + nThreads = 1; + } + + std::mutex queueMutex; + auto worker = [&]() + { + char tile_mpq[1024]; + char tile_out[1024]; + while (true) + { + uint32 tileX, tileY; + { + std::lock_guard lock(queueMutex); + if (tileQueue.empty()) + { + break; + } + tileX = tileQueue.front().first; + tileY = tileQueue.front().second; + tileQueue.pop(); + } + sprintf(tile_mpq, "World\\Maps\\%s\\%s_%u_%u.adt", map_ids[z].name, map_ids[z].name, tileX, tileY); + sprintf(tile_out, "%s/maps/%04u%02u%02u.map", output_path, map_ids[z].id, tileY, tileX); + if (ConvertADT(tile_mpq, tile_out, build)) + { + ++success_count; + } + else + { + ++failed_count; + } + } + }; + + if (nThreads <= 1) + { + // Serial path: one worker on this thread, identical to the + // pre-threading behaviour. + worker(); + } + else + { + std::vector workers; + workers.reserve(nThreads); + for (uint32 t = 0; t < nThreads; ++t) + { + workers.emplace_back(worker); + } + for (std::thread& w : workers) + { + w.join(); + } + } + + if (adt_count > 0) + { + printf("\n Found %u ADT files for map %s\n", adt_count, map_ids[z].name); + } + else + { + printf("\n WARNING: No ADT files found for map %s (map ID: %u)\n", map_ids[z].name, map_ids[z].id); + } + } + printf("\n\nMap extraction complete!\n"); + printf("Successfully converted: %u tiles\n", success_count.load()); + printf("Failed to convert: %u tiles\n", failed_count.load()); delete [] areas; delete [] map_ids; } diff --git a/src/tools/Extractor_projects/vmap-extractor/adtfile.cpp b/src/tools/Extractor_projects/vmap-extractor/adtfile.cpp index 213abbb40..fd8e5239e 100644 --- a/src/tools/Extractor_projects/vmap-extractor/adtfile.cpp +++ b/src/tools/Extractor_projects/vmap-extractor/adtfile.cpp @@ -123,7 +123,13 @@ bool ADTFile::init(uint32 map_num, uint32 tileX, uint32 tileY, StringSet& failed string AdtMapNumber = xMap + ' ' + yMap + ' ' + GetPlainName((char*)AdtFilename.c_str()); - std::string dirname = std::string(szWorkDirWmo) + "/dir_bin"; + // Write this tile's placement records to a per-tile temp file rather than + // the shared dir_bin. A placement is several fwrites, so concurrent tiles + // sharing one handle would interleave (corrupt) records. ParsMapFiles + // concatenates these temps into dir_bin in tile order afterwards. + char tileSuffix[32]; + sprintf(tileSuffix, "/dir_bin.%u_%u", tileX, tileY); + std::string dirname = std::string(szWorkDirWmo) + tileSuffix; FILE* dirfile; dirfile = fopen(dirname.c_str(), "ab"); if (!dirfile) diff --git a/src/tools/Extractor_projects/vmap-extractor/gameobject_extract.cpp b/src/tools/Extractor_projects/vmap-extractor/gameobject_extract.cpp index a938f7e33..7ad2607db 100644 --- a/src/tools/Extractor_projects/vmap-extractor/gameobject_extract.cpp +++ b/src/tools/Extractor_projects/vmap-extractor/gameobject_extract.cpp @@ -5,6 +5,20 @@ #include #include +#include +#include +#include + +// Dedup model extraction across parallel tile workers. The old +// FileExists()-then-write check is a TOCTOU race once tiles run concurrently. +// One worker extracts a given model; the rest must WAIT until its file is on +// disk, because the placement written straight after (ModelInstance) opens that +// file and silently drops the spawn if it is missing. A plain "claimed" flag is +// not enough -- the file has to exist before any referencing worker proceeds. +static std::mutex s_modelExtractMutex; +static std::condition_variable s_modelExtractCv; +static std::set s_modelsInProgress; +static std::set s_modelsDone; bool ExtractSingleModel(std::string& origPath, std::string& fixedName, StringSet& failedPaths) { @@ -26,18 +40,39 @@ bool ExtractSingleModel(std::string& origPath, std::string& fixedName, StringSet output += "/"; output += fixedName; - if (FileExists(output.c_str())) { - return true; + std::unique_lock lock(s_modelExtractMutex); + if (s_modelsDone.count(fixedName)) + { + return true; + } + if (s_modelsInProgress.count(fixedName)) + { + // Another worker is extracting this model; block until its file is + // written so the placement that follows can read it. + s_modelExtractCv.wait(lock, [&] { return s_modelsDone.count(fixedName) != 0; }); + return true; + } + if (FileExists(output.c_str())) + { + s_modelsDone.insert(fixedName); + return true; + } + // Claim it, then extract outside the lock so distinct models convert in + // parallel. + s_modelsInProgress.insert(fixedName); } Model mdl(origPath); // Possible changed fname - if (!mdl.open(failedPaths)) + bool ok = mdl.open(failedPaths) && mdl.ConvertToVMAPModel(output.c_str()); + { - return false; + std::lock_guard lock(s_modelExtractMutex); + s_modelsInProgress.erase(fixedName); + s_modelsDone.insert(fixedName); } - - return mdl.ConvertToVMAPModel(output.c_str()); + s_modelExtractCv.notify_all(); + return ok; } extern HANDLE LocaleMpq; diff --git a/src/tools/Extractor_projects/vmap-extractor/mpqfile.cpp b/src/tools/Extractor_projects/vmap-extractor/mpqfile.cpp index dae0318e3..833ebd9f5 100644 --- a/src/tools/Extractor_projects/vmap-extractor/mpqfile.cpp +++ b/src/tools/Extractor_projects/vmap-extractor/mpqfile.cpp @@ -1,14 +1,23 @@ #include "mpqfile.h" #include #include +#include #include "StormLib.h" +// StormLib mutates a shared per-archive file position on every read with no +// internal locking, so concurrent reads from one archive handle race (and +// return wrong data on Linux). Every MPQ read in this tool flows through this +// constructor, so one mutex here serialises all archive I/O; the parsing and +// geometry conversion that follow run fully in parallel. +std::mutex g_mpqReadMutex; + MPQFile::MPQFile(HANDLE mpq, const char* filename): eof(false), buffer(0), pointer(0), size(0) { + std::lock_guard mpqLock(g_mpqReadMutex); HANDLE file; if (!SFileOpenFileEx(mpq, filename, SFILE_OPEN_FROM_MPQ, &file)) { diff --git a/src/tools/Extractor_projects/vmap-extractor/vmapexport.cpp b/src/tools/Extractor_projects/vmap-extractor/vmapexport.cpp index 37dc39db7..3462b656e 100644 --- a/src/tools/Extractor_projects/vmap-extractor/vmapexport.cpp +++ b/src/tools/Extractor_projects/vmap-extractor/vmapexport.cpp @@ -27,6 +27,11 @@ #include #include #include +#include +#include +#include +#include +#include #if defined WIN32 #include @@ -106,6 +111,7 @@ char output_path[128] = "."; char input_path[1024] = "."; bool preciseVectorData = true; uint32 CONF_max_build = 0; +uint32 CONF_threads = 0; ///< Worker threads for tile extraction; 0 = auto-detect cores, 1 = serial. // Constants @@ -417,6 +423,7 @@ void Usage(char* prg) printf(" -l : large size, ~500MB more vmap data. (might contain more details)\n"); printf(" -d : Path to the vector data source folder.\n"); printf(" -b : target build (default %u)", CONF_TargetBuild); + printf(" -t <#>: worker threads for tile extraction. 0 = auto-detect cores (default), 1 = serial.\n"); printf(" -? : This message.\n"); } @@ -438,22 +445,102 @@ void ParsMapFiles() WDTFile WDT(fn, map_ids[i].name); if (WDT.init(id, map_ids[i].id)) { - printf(" Processing Map %u (%s)\n[", map_ids[i].id, map_ids[i].name); + printf(" Processing Map %u (%s)\n", map_ids[i].id, map_ids[i].name); + + // Queue every tile slot; GetMap()/init() fast-fail the absent ones. + std::queue > tileQueue; for (int x = 0; x < 64; ++x) { for (int y = 0; y < 64; ++y) { + tileQueue.push(std::make_pair(x, y)); + } + } + + uint32 nThreads = CONF_threads ? CONF_threads : std::thread::hardware_concurrency(); + if (nThreads < 1) + { + nThreads = 1; + } + + uint32 mapId = map_ids[i].id; + std::mutex queueMutex; + std::mutex failedMutex; + auto worker = [&]() + { + StringSet localFailed; + while (true) + { + int x, y; + { + std::lock_guard lock(queueMutex); + if (tileQueue.empty()) + { + break; + } + x = tileQueue.front().first; + y = tileQueue.front().second; + tileQueue.pop(); + } if (ADTFile* ADT = WDT.GetMap(x, y)) { - //sprintf(id_filename,"%02u %02u %03u",x,y,map_ids[i].id);//!!!!!!!!! - ADT->init(map_ids[i].id, x, y, failedPaths); + ADT->init(mapId, x, y, localFailed); delete ADT; } } - printf("#"); - fflush(stdout); + std::lock_guard lock(failedMutex); + failedPaths.insert(localFailed.begin(), localFailed.end()); + }; + + if (nThreads <= 1) + { + // Serial path: one worker on this thread. + worker(); + } + else + { + std::vector workers; + workers.reserve(nThreads); + for (uint32 t = 0; t < nThreads; ++t) + { + workers.emplace_back(worker); + } + for (std::thread& w : workers) + { + w.join(); + } + } + + // Concatenate the per-tile temp files into dir_bin in tile order, + // so dir_bin is assembled in the same order no matter which worker + // wrote which tile, then remove the temps. + std::string dirBin = std::string(szWorkDirWmo) + "/dir_bin"; + if (FILE* out = fopen(dirBin.c_str(), "ab")) + { + char copyBuf[8192]; + for (int x = 0; x < 64; ++x) + { + for (int y = 0; y < 64; ++y) + { + char tileSuffix[32]; + sprintf(tileSuffix, "/dir_bin.%u_%u", x, y); + std::string tilePath = std::string(szWorkDirWmo) + tileSuffix; + FILE* in = fopen(tilePath.c_str(), "rb"); + if (!in) + { + continue; + } + size_t n; + while ((n = fread(copyBuf, 1, sizeof(copyBuf), in)) > 0) + { + fwrite(copyBuf, 1, n, out); + } + fclose(in); + remove(tilePath.c_str()); + } + } + fclose(out); } - printf("]\n"); } } @@ -687,6 +774,18 @@ bool processArgv(int argc, char** argv) CONF_TargetBuild = atoi(argv[i++ + 1]); } } + else if (strcmp("-t", argv[i]) == 0) + { + if (i + 1 < argc) // all ok + { + CONF_threads = atoi(argv[i + 1]); + ++i; + } + else + { + result = false; + } + } else { result = false; diff --git a/src/tools/Extractor_projects/vmap-extractor/wmo.cpp b/src/tools/Extractor_projects/vmap-extractor/wmo.cpp index e19cc29df..9dc739981 100644 --- a/src/tools/Extractor_projects/vmap-extractor/wmo.cpp +++ b/src/tools/Extractor_projects/vmap-extractor/wmo.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #undef min #undef max #include "mpqfile.h" @@ -660,9 +661,15 @@ WMOInstance::WMOInstance(MPQFile& f, const char* WmoInstName, uint32 mapID, uint /// uint32 range, so they cannot collide with client MODF/MDDF unique ids. static uint32 GenerateDoodadUniqueId(uint32 wmoUniqueId, uint32 doodadIndex) { + static std::mutex s_doodadIdMutex; static std::map, uint32> doodadIdMap; static uint32 nextId = 0x80000000; + // Guards the shared map + counter against concurrent tile workers. The id a + // given (wmo, doodad) gets still depends on first-seen order, so threaded + // runs assign different (but equally valid, unique) ids than serial runs -- + // the values are opaque spawn tags, so output stays functionally correct. + std::lock_guard lock(s_doodadIdMutex); uint32& uid = doodadIdMap[std::make_pair(wmoUniqueId, doodadIndex)]; if (uid == 0) {