Merge #3201

3201: Map optimizer r=def- a=Jupeyy

Wrote a map optimizer(very aggressive, explain later why).
It reduces map sizes by ~40% (optimized the whole ddnet-maps repo and it was around 40% smaller afterwards)

Test and idea by mind, credits to him

What it does, and why I called it aggressive:

- It removes all unused tiles from embedded images
- It clears transparent pixels and dilates them as 2D array textures, instead of simple 2D textures.
- It uses zlibs best compression

So yeah, it changes the output map(keeps the source file working, output is in "out" dir), its not the same as before, it's a nice tool for optimizing size with a lot of embedded images as a final release, or when using higher resolution textures.

Co-authored-by: jupeyy <jupeyy@am-6036ddb5ef41.intern>

CMakeLists.txt

@@ -2066,6 +2066,7 @@ set_src(TOOLS GLOB src/tools
   map_convert_07.cpp
   map_diff.cpp
   map_extract.cpp
+  map_optimize.cpp
   map_replace_image.cpp
   map_resave.cpp
   packetgen.cpp
@@ -2078,7 +2079,7 @@ foreach(ABS_T ${TOOLS})
     string(REGEX REPLACE "\\.cpp$" "" TOOL "${T}")
     set(TOOL_DEPS ${DEPS})
     set(TOOL_LIBS ${LIBS})
-    if(TOOL MATCHES "^(tileset_.*|dilate|map_convert_07|map_extract|map_replace_image)$")
+    if(TOOL MATCHES "^(tileset_.*|dilate|map_convert_07|map_optimize|map_extract|map_replace_image)$")
       list(APPEND TOOL_DEPS ${PNGLITE_DEP})
       list(APPEND TOOL_LIBS ${PNGLITE_LIBRARIES})
       list(APPEND TOOL_INCLUDE_DIRS ${PNGLITE_INCLUDE_DIRS})

src/engine/shared/datafile.cpp

@@ -10,8 +10,6 @@
 
 #include "uuid_manager.h"
 
-#include <zlib.h>
-
 static const int DEBUG = 0;
 
 enum
@@ -692,15 +690,15 @@ int CDataFileWriter::AddItem(int Type, int ID, int Size, void *pData)
 	return m_NumItems - 1;
 }
 
-int CDataFileWriter::AddData(int Size, void *pData)
+int CDataFileWriter::AddData(int Size, void *pData, int CompressionLevel)
 {
 	dbg_assert(m_NumDatas < 1024, "too much data");
 
 	CDataInfo *pInfo = &m_pDatas[m_NumDatas];
 	unsigned long s = compressBound(Size);
 	void *pCompData = malloc(s); // temporary buffer that we use during compression
 
-	int Result = compress((Bytef *)pCompData, &s, (Bytef *)pData, Size); // ignore_convention
+	int Result = compress2((Bytef *)pCompData, &s, (Bytef *)pData, Size, CompressionLevel); // ignore_convention
 	if(Result != Z_OK)
 	{
 		dbg_msg("datafile", "compression error %d", Result);

src/engine/shared/datafile.h

@@ -8,6 +8,8 @@
 #include <base/hash.h>
 #include <base/system.h>
 
+#include <zlib.h>
+
 // raw datafile access
 class CDataFileReader
 {
@@ -100,7 +102,7 @@ class CDataFileWriter
 	void Init();
 	bool OpenFile(class IStorage *pStorage, const char *pFilename, int StorageType = IStorage::TYPE_SAVE);
 	bool Open(class IStorage *pStorage, const char *pFilename, int StorageType = IStorage::TYPE_SAVE);
-	int AddData(int Size, void *pData);
+	int AddData(int Size, void *pData, int CompressionLevel = Z_DEFAULT_COMPRESSION);
 	int AddDataSwapped(int Size, void *pData);
 	int AddItem(int Type, int ID, int Size, void *pData);
 	int Finish();

src/engine/shared/image_manipulation.cpp

@@ -2,7 +2,7 @@
 #include <base/math.h>
 #include <base/system.h>
 
-static void Dilate(int w, int h, int BPP, unsigned char *pSrc, unsigned char *pDest, unsigned char AlphaThreshold = 30)
+static void Dilate(int w, int h, int BPP, unsigned char *pSrc, unsigned char *pDest, unsigned char AlphaThreshold = TW_DILATE_ALPHA_THRESHOLD)
 {
 	int ix, iy;
 	const int aDirX[] = {0, -1, 1, 0};
@@ -67,26 +67,50 @@ static void CopyColorValues(int w, int h, int BPP, unsigned char *pSrc, unsigned
 }
 
 void DilateImage(unsigned char *pImageBuff, int w, int h, int BPP)
+{
+	DilateImageSub(pImageBuff, w, h, BPP, 0, 0, w, h);
+}
+
+void DilateImageSub(unsigned char *pImageBuff, int w, int h, int BPP, int x, int y, int sw, int sh)
 {
 	unsigned char *apBuffer[2] = {NULL, NULL};
 
-	apBuffer[0] = (unsigned char *)malloc((size_t)w * h * sizeof(unsigned char) * BPP);
-	apBuffer[1] = (unsigned char *)malloc((size_t)w * h * sizeof(unsigned char) * BPP);
+	apBuffer[0] = (unsigned char *)malloc((size_t)sw * sh * sizeof(unsigned char) * BPP);
+	apBuffer[1] = (unsigned char *)malloc((size_t)sw * sh * sizeof(unsigned char) * BPP);
+	unsigned char *pBufferOriginal = (unsigned char *)malloc((size_t)sw * sh * sizeof(unsigned char) * BPP);
 
 	unsigned char *pPixelBuff = (unsigned char *)pImageBuff;
 
-	Dilate(w, h, BPP, pPixelBuff, apBuffer[0]);
+	for(int Y = 0; Y < sh; ++Y)
+	{
+		int SrcImgOffset = ((y + Y) * w * BPP) + (x * BPP);
+		int DstImgOffset = (Y * sw * BPP);
+		int CopySize = sw * BPP;
+		mem_copy(&pBufferOriginal[DstImgOffset], &pPixelBuff[SrcImgOffset], CopySize);
+	}
+
+	Dilate(sw, sh, BPP, pBufferOriginal, apBuffer[0]);
 
 	for(int i = 0; i < 5; i++)
 	{
-		Dilate(w, h, BPP, apBuffer[0], apBuffer[1]);
-		Dilate(w, h, BPP, apBuffer[1], apBuffer[0]);
+		Dilate(sw, sh, BPP, apBuffer[0], apBuffer[1]);
+		Dilate(sw, sh, BPP, apBuffer[1], apBuffer[0]);
 	}
 
-	CopyColorValues(w, h, BPP, apBuffer[0], pPixelBuff);
+	CopyColorValues(sw, sh, BPP, apBuffer[0], pBufferOriginal);
 
 	free(apBuffer[0]);
 	free(apBuffer[1]);
+
+	for(int Y = 0; Y < sh; ++Y)
+	{
+		int SrcImgOffset = ((y + Y) * w * BPP) + (x * BPP);
+		int DstImgOffset = (Y * sw * BPP);
+		int CopySize = sw * BPP;
+		mem_copy(&pPixelBuff[SrcImgOffset], &pBufferOriginal[DstImgOffset], CopySize);
+	}
+
+	free(pBufferOriginal);
 }
 
 static float CubicHermite(float A, float B, float C, float D, float t)

src/engine/shared/image_manipulation.h

@@ -4,7 +4,10 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#define TW_DILATE_ALPHA_THRESHOLD 30
+
 void DilateImage(unsigned char *pImageBuff, int w, int h, int BPP);
+void DilateImageSub(unsigned char *pImageBuff, int w, int h, int BPP, int x, int y, int sw, int sh);
 
 // returned pointer is allocated with malloc
 uint8_t *ResizeImage(const uint8_t *pImgData, int Width, int Height, int NewWidth, int NewHeight, int BPP);