Skip to content

Commit

Permalink
Uoconvert improvements 3 (#561)
Browse files Browse the repository at this point in the history

---------

Co-authored-by: Kevin Eady <8634912+KevinEady@users.noreply.github.com>
  • Loading branch information
frozenblit and KevinEady committed Nov 23, 2023
1 parent 5e5f405 commit 4ac5eac
Show file tree
Hide file tree
Showing 23 changed files with 608 additions and 167 deletions.
1 change: 1 addition & 0 deletions cmake/Boost.cmake
Expand Up @@ -65,6 +65,7 @@ if(NOT EXISTS ${BOOST_REGEX_LIB} OR NOT EXISTS ${BOOST_SYSTEM_LIB} OR NOT EXISTS
LOG_CONFIGURE 1
BUILD_IN_SOURCE 1
LOG_OUTPUT_ON_FAILURE 1
DOWNLOAD_EXTRACT_TIMESTAMP 1
)
set_target_properties (boost_extract PROPERTIES FOLDER 3rdParty)

Expand Down
1 change: 1 addition & 0 deletions cmake/Curl.cmake
Expand Up @@ -26,6 +26,7 @@ if(NOT EXISTS "${CURL_LIB}")
LOG_BUILD 1
LOG_INSTALL 1
LOG_OUTPUT_ON_FAILURE 1
DOWNLOAD_EXTRACT_TIMESTAMP 1
)
set_target_properties (libcurl PROPERTIES FOLDER 3rdParty)
else()
Expand Down
55 changes: 45 additions & 10 deletions cmake/core_tests.cmake
Expand Up @@ -12,7 +12,51 @@ else()
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
endif()
set_tests_properties(cleantestdir PROPERTIES FIXTURES_SETUP client)
set_tests_properties(cleantestdir PROPERTIES FIXTURES_SETUP "client;initial_cleanup")

# uoconvert tests

add_test(NAME uoconvert_test_map0_hsa
COMMAND poltool testfiles
outdir=coretest/convert_test/client
width=768
height=4096
mapid=0
hsa=1
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
set_tests_properties(uoconvert_test_map0_hsa PROPERTIES FIXTURES_REQUIRED initial_cleanup)
set_tests_properties(uoconvert_test_map0_hsa PROPERTIES FIXTURES_SETUP uoconvert_tests_map0)

add_test(NAME uoconvert_test_setup_map1_no_hsa
COMMAND poltool testfiles
outdir=coretest/convert_test/client
width=640
height=4096
mapid=1
hsa=0
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
set_tests_properties(uoconvert_test_setup_map1_no_hsa PROPERTIES FIXTURES_REQUIRED initial_cleanup)
set_tests_properties(uoconvert_test_setup_map1_no_hsa PROPERTIES FIXTURES_SETUP uoconvert_tests_map1)

add_test(NAME uoconvert_test_convert_map0
COMMAND uoconvert map realm=britannia
mapid=0
uodata=client
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest/convert_test
)
set_tests_properties( uoconvert_test_convert_map0 PROPERTIES FIXTURES_REQUIRED uoconvert_tests_map0)

add_test(NAME uoconvert_test_convert_map1
COMMAND uoconvert map realm=britannia2
mapid=1
uodata=client
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest/convert_test
)
set_tests_properties( uoconvert_test_convert_map1 PROPERTIES FIXTURES_REQUIRED uoconvert_tests_map1)



# generate client files, minimal distro and needed core cfgs
add_test(NAME testenv_map1
Expand Down Expand Up @@ -84,7 +128,6 @@ add_test(NAME uoconvert_map
COMMAND uoconvert map realm=britannia
width=192 height=192
uodata=client
maxtileid=0x3fff
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
)
set_tests_properties( uoconvert_map PROPERTIES FIXTURES_REQUIRED client)
Expand All @@ -93,7 +136,6 @@ set_tests_properties( uoconvert_map PROPERTIES FIXTURES_SETUP uoconvert)
add_test(NAME uoconvert_statics
COMMAND uoconvert statics realm=britannia
uodata=client
maxtileid=0x3fff
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
)
set_tests_properties( uoconvert_statics PROPERTIES DEPENDS uoconvert_map)
Expand All @@ -103,7 +145,6 @@ set_tests_properties( uoconvert_statics PROPERTIES FIXTURES_SETUP uoconvert)
add_test(NAME uoconvert_maptile
COMMAND uoconvert maptile realm=britannia
uodata=client
maxtileid=0x3fff
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
)
set_tests_properties( uoconvert_maptile PROPERTIES FIXTURES_REQUIRED client)
Expand All @@ -116,7 +157,6 @@ add_test(NAME uoconvert_map2
height=1600
mapid=1
uodata=client
maxtileid=0x3fff
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
)
set_tests_properties( uoconvert_map2 PROPERTIES DEPENDS uoconvert_maptile)
Expand All @@ -127,7 +167,6 @@ add_test(NAME uoconvert_statics2
COMMAND uoconvert statics realm=britannia2
uodata=client
mapid=1
maxtileid=0x3fff
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
)
set_tests_properties( uoconvert_statics2 PROPERTIES DEPENDS uoconvert_map2)
Expand All @@ -138,7 +177,6 @@ add_test(NAME uoconvert_maptile2
COMMAND uoconvert maptile realm=britannia2
uodata=client
mapid=1
maxtileid=0x3fff
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
)
set_tests_properties( uoconvert_maptile2 PROPERTIES FIXTURES_REQUIRED client)
Expand All @@ -148,7 +186,6 @@ set_tests_properties( uoconvert_maptile2 PROPERTIES FIXTURES_SETUP uoconvert)
add_test(NAME uoconvert_tiles
COMMAND uoconvert tiles
uodata=client
maxtileid=0x3fff
outdir=config
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
)
Expand All @@ -158,7 +195,6 @@ set_tests_properties( uoconvert_tiles PROPERTIES FIXTURES_SETUP uoconvert)
add_test(NAME uoconvert_landtiles
COMMAND uoconvert landtiles
uodata=client
maxtileid=0x3fff
outdir=config
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
)
Expand All @@ -168,7 +204,6 @@ set_tests_properties( uoconvert_landtiles PROPERTIES FIXTURES_SETUP uoconvert)
add_test(NAME uoconvert_multis
COMMAND uoconvert multis
uodata=client
maxtileid=0x3fff
outdir=config
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coretest
)
Expand Down
8 changes: 2 additions & 6 deletions docs/docs.polserver.com/pol100/configfiles.xml
Expand Up @@ -197,11 +197,6 @@ Mounts
{
Tiles (int ItemID) [(int ItemID) ...]
}

ClientOptions
{
UseNewHSAFormat (integer 0/1 {default 0})
}
</structure>
<explain>MultiIDs are defined in multi.mul/idx. A multi can either be a "house", "boat" or "stairs" (decorations such as stairs, curtains, carpets, etc).</explain>
<explain>ItemIDs are defined in anim.mul and tiledata.mul.</explain>
Expand Down Expand Up @@ -966,7 +961,7 @@ PidFilePath=(where POL will write its .pid file {default ./})
[AssertionFailureAction=(abort/continue/shutdown/shutdown-nosave/shutdown-save-full/shutdown-save-incremental {default abort})]
[ShutdownSaveType=(full/incremental {default full})]
[TimestampEveryLine=(1/0 {default 0})]
[MaxTileID=(0x3FFF/0x7FFF {default 0x3FFF})]
[MaxTileID=(0/0x3FFF/0x7FFF/0xFFFF {default is 0})]
[MaxObjtype=(0x20000/0xFFFFFFFF {default 0x20000})]
[DiscardOldEvents=(1/0 {default 0})]
[UseSingleThreadLogin=(1/0 {default 0})]
Expand Down Expand Up @@ -1002,6 +997,7 @@ PidFilePath=(where POL will write its .pid file {default ./})
<explain>ShowWarningGump: will show unexpected gump responses and B1 packet overflow messages on the console.</explain>
<explain>ShowWarningItem: will show equip item and drop item warning messages on the console.</explain>
<explain>ShowWarningCursorSequence: will show a warning when a player sends click packets out of sequence, this is usually due to the player running some sort of macro or client injection program.</explain>
<explain>MaxTileID: maximum tile id. If 0, it will be chosen according to the graphics in tiles.cfg.</explain>
</cfgfile>


Expand Down
15 changes: 13 additions & 2 deletions docs/docs.polserver.com/pol100/corechanges.xml
Expand Up @@ -5,11 +5,22 @@
<datemodified>11-19-2023</datemodified>
</header>
<version name="POL100.1.0">
<entry>
<entry>
<date>11-19-2023</date>
<author>Turley:</author>
<change type="Fixed">possible deadlock in packethooks</change>
</entry>
</entry>
<entry>
<date>11-16-2023</date>
<author>Nando:</author>
<change type="Added">UoConvert smartness:<br/>
- MaxTileID is now detected from tiledata.mul (uoconvert) and tiles.cfg (POL).<br/>
- Default width/height for each map id<br/>
- map0 and map1 width can be estimated automatically based on a height of 4096<br/>
- better error handling</change>
<change type="Changed">usedif=1 is now the default in uoconvert</change>
<change type="Removed">UseNewHSAFormat in uoconvert.cfg. This will be detected automatically from<br/>
tiledata.mul size.</change>
<entry>
<date>11-12-2023</date>
<author>Kevin:</author>
Expand Down
9 changes: 9 additions & 0 deletions pol-core/doc/core-changes.txt
@@ -1,6 +1,15 @@
-- POL100.1.0 --
11-19-2023 Turley:
Fixed: possible deadlock in packethooks
11-16-2023 Nando:
Added: UoConvert smartness:
- MaxTileID is now detected from tiledata.mul (uoconvert) and tiles.cfg (POL).
- Default width/height for each map id
- map0 and map1 width can be estimated automatically based on a height of 4096
- better error handling
Changed: usedif=1 is now the default in uoconvert
Removed: UseNewHSAFormat in uoconvert.cfg. This will be detected automatically from
tiledata.mul size.
11-12-2023 Kevin:
Fixed: crash when control scripts end for a destroyed item
11-08-2023 Turley:
Expand Down
2 changes: 2 additions & 0 deletions pol-core/plib/CMakeSources.cmake
Expand Up @@ -21,6 +21,8 @@ set (plib_sources # sorted !
maptileserver.h
mapwriter.cpp
mapwriter.h
mul/map.h
mul/tiledata.h
pkg.cpp
pkg.h
polfile.h
Expand Down
32 changes: 15 additions & 17 deletions pol-core/plib/RawMap.cpp
Expand Up @@ -125,27 +125,25 @@ unsigned int RawMap::load_full_map( int uo_mapid, std::istream& ifs )
std::map<uint64_t, uop_t::file_t*> filemap;

// First pass over the data to build up filehash -> file mapping and calculate total map size.
// NOTE: This only reads the first block of the UOP file, which is all that is used for the
// current version (~2018). If that ever changes, you need to access the next blocks in
// sequence:
// currentblock = currentblock->block_body()->next_addr()
// until next_addr() == nullptr.
//
uop_t::block_addr_t* currentblock = uopfile.header()->firstblock();

for ( auto file : *currentblock->block_body()->files() )
do
{
if ( file == nullptr )
continue;
if ( file->decompressed_size() == 0 )
continue;
for ( auto file : *currentblock->block_body()->files() )
{
if ( file == nullptr )
continue;
if ( file->decompressed_size() == 0 )
continue;

passert_r( file->compression_type() == uop_t::COMPRESSION_TYPE_NO_COMPRESSION,
"This map is zlib compressed and we can't handle that yet." );
passert_r( file->compression_type() == uop_t::COMPRESSION_TYPE_NO_COMPRESSION,
"This map is zlib compressed and we can't handle that yet." );

filemap[file->filehash()] = file;
totalSize += file->decompressed_size();
}
currentblock = currentblock->block_body()->next_addr();
} while ( currentblock != nullptr && filemap.size() < uopfile.header()->nfiles() );

filemap[file->filehash()] = file;
totalSize += file->decompressed_size();
}
if ( uopfile.header()->nfiles() != filemap.size() )
INFO_PRINT << "Warning: not all chunks read (" << filemap.size() << "/"
<< uopfile.header()->nfiles() << ")\n";
Expand Down
124 changes: 124 additions & 0 deletions pol-core/plib/mul/map.h
@@ -0,0 +1,124 @@
#pragma once

#include <stdexcept>

namespace Pol::Plib::MUL
{
struct Map
{
static constexpr size_t blockWidth = 8;
static constexpr size_t blockHeight = 8;
static constexpr size_t cellSize = 3;

// DWORD + 64 cells (3 bytes) per block
static constexpr size_t blockSize = 4 + cellSize * blockWidth * blockHeight;

static constexpr size_t expected_blocks( int width, int height )
{
return ( width / blockWidth ) * ( height / blockHeight );
}

static constexpr bool valid_size( size_t filesize, int width, int height )
{
if ( width % blockWidth != 0 )
return false;

if ( height % blockHeight != 0 )
return false;

size_t nblocks = filesize / Map::blockSize;
return expected_blocks( width, height ) == nblocks;
}
};
static_assert( Map::blockSize == 196, "Size mismatch" );
static_assert( Map::valid_size( 77070336, 6144, 4096 ) ); // Legacy map
static_assert( Map::valid_size( 89915392, 7168, 4096 ) ); // New maps
static_assert( Map::valid_size( 77070336 - 196, 6144, 4096 ) == false ); // Wrong map 1
static_assert( Map::valid_size( 89915392 - 196, 7168, 4096 ) == false ); // Wrong map 2

class MapInfo
{
int _width = 0;
int _height = 0;

bool _guessed_correctly = false;

void set_default_size();
bool guess_size( size_t filesize );

public:
int width() { return _width; }
int height() { return _height; }

bool guessed() { return _guessed_correctly; }
bool matches_filesize() { return Map::valid_size( filesize, _width, _height ); }

const int mapid = -1;
const size_t filesize = 0;

MapInfo( int mapid ) : mapid( mapid ) { set_default_size(); }

MapInfo( int mapid, size_t filesize ) : mapid( mapid ), filesize( filesize )
{
set_default_size();
if ( _height == 0 && _width == 0 )
{
_guessed_correctly = guess_size( filesize );
}
}
};


inline bool MapInfo::guess_size( size_t map_size )
{
// Guessing is only implemented for britannia now
if ( mapid != 0 && mapid != 1 )
return false;

constexpr int map0_height = 4096;

size_t mapblocks = map_size / Map::blockSize;
constexpr int map0_blocks = map0_height / Map::blockHeight;

// Number of blocks has to be divisible by either dimension
if ( ( mapblocks % map0_blocks ) != 0 )
return false;

_height = map0_height;
_width = static_cast<int>( Map::blockWidth * ( mapblocks / map0_blocks ) );

return true;
}


inline void MapInfo::set_default_size()
{
_width = 0;
_height = 0;

switch ( mapid )
{
case 0: // britannia / britannia-alt
case 1:
// depends on the filesize
break;
case 2: // ilshenar:
_width = 2304;
_height = 1600;
break;
case 3: // malas
_width = 2560;
_height = 2048;
break;
case 4: // tokuno
_width = 1448;
_height = 1448;
break;
case 5: // termur
_width = 1280;
_height = 4096;
break;
}
}

}; // namespace Pol::Plib::MUL

0 comments on commit 4ac5eac

Please sign in to comment.