Skip to content

Commit

Permalink
Merge branch 'master' into feature/libraryOfProcessedFiles
Browse files Browse the repository at this point in the history
  • Loading branch information
Alex A. Yermoshenko committed Oct 13, 2022
2 parents 17e0f33 + 66d25e4 commit 224b0ee
Show file tree
Hide file tree
Showing 28 changed files with 11,922 additions and 7,983 deletions.
6 changes: 0 additions & 6 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,13 @@ jobs:
strategy:
matrix:
os:
- ubuntu-16.04
- ubuntu-18.04
- ubuntu-20.04
- macos-10.15
compiler:
- { cc: gcc, cxx: g++ }
- { cc: clang, cxx: clang++ }

exclude:
# gcc 5 doesn't support C++17
- os: ubuntu-16.04
compiler: { cc: gcc, cxx: g++ }

fail-fast: false

env:
Expand Down
24 changes: 13 additions & 11 deletions Leanify.vcxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,11 +321,21 @@
<ClCompile Include="formats\jpeg.cpp" />
<ClCompile Include="formats\lua.cpp" />
<ClCompile Include="formats\mime.cpp" />
<ClCompile Include="formats\pe.cpp" />
<ClCompile Include="formats\png.cpp" />
<ClCompile Include="formats\rdb.cpp" />
<ClCompile Include="formats\swf.cpp" />
<ClCompile Include="formats\tar.cpp" />
<ClCompile Include="formats\vcf.cpp" />
<ClCompile Include="formats\xml.cpp" />
<ClCompile Include="formats\zip.cpp" />
<ClCompile Include="leanify.cpp" />
<ClCompile Include="library.cpp" />
<ClCompile Include="lib\LZMA\Alloc.c" />
<ClCompile Include="lib\LZMA\CpuArch.c" />
<ClCompile Include="lib\LZMA\LzFind.c" />
<ClCompile Include="lib\LZMA\LzFindMt.c" />
<ClCompile Include="lib\LZMA\LzFindOpt.c" />
<ClCompile Include="lib\LZMA\LzmaDec.c" />
<ClCompile Include="lib\LZMA\LzmaEnc.c" />
<ClCompile Include="lib\LZMA\Threads.c" />
Expand Down Expand Up @@ -355,17 +365,7 @@
<ClCompile Include="lib\mozjpeg\jmemnobs.c" />
<ClCompile Include="lib\mozjpeg\jsimd_none.c" />
<ClCompile Include="lib\mozjpeg\jutils.c" />
<ClCompile Include="formats\pe.cpp" />
<ClCompile Include="formats\png.cpp" />
<ClCompile Include="formats\rdb.cpp" />
<ClCompile Include="formats\swf.cpp" />
<ClCompile Include="formats\tar.cpp" />
<ClCompile Include="lib\pugixml\pugixml.cpp" />
<ClCompile Include="formats\xml.cpp" />
<ClCompile Include="formats\zip.cpp" />
<ClCompile Include="lib\zopflipng\lodepng\lodepng.cpp" />
<ClCompile Include="lib\zopflipng\lodepng\lodepng_util.cpp" />
<ClCompile Include="lib\zopflipng\zopflipng_lib.cc" />
<ClCompile Include="lib\zopfli\blocksplitter.c" />
<ClCompile Include="lib\zopfli\cache.c" />
<ClCompile Include="lib\zopfli\deflate.c" />
Expand All @@ -378,7 +378,9 @@
<ClCompile Include="lib\zopfli\util.c" />
<ClCompile Include="lib\zopfli\zlib_container.c" />
<ClCompile Include="lib\zopfli\zopfli_lib.c" />
<ClCompile Include="leanify.cpp" />
<ClCompile Include="lib\zopflipng\lodepng\lodepng_util.cpp" />
<ClCompile Include="lib\zopflipng\lodepng\lodepng.cpp" />
<ClCompile Include="lib\zopflipng\zopflipng_lib.cc" />
<ClCompile Include="main.cpp" />
<ClCompile Include="utils.cpp" />
</ItemGroup>
Expand Down
41 changes: 16 additions & 25 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
LEANIFY_SRC := leanify.cpp main.cpp utils.cpp library.cpp $(wildcard formats/*.cpp)
LZMA_OBJ := lib/LZMA/Alloc.o lib/LZMA/LzFind.o lib/LZMA/LzmaDec.o lib/LZMA/LzmaEnc.o
LEANIFY_OBJ := leanify.o main.o utils.o library.o $(patsubst %.cpp,%.o,$(wildcard formats/*.cpp))
LZMA_OBJ := lib/LZMA/Alloc.o lib/LZMA/LzFind.o lib/LZMA/LzFindMt.o lib/LZMA/LzFindOpt.o lib/LZMA/LzmaDec.o lib/LZMA/LzmaEnc.o lib/LZMA/Threads.o
MOZJPEG_OBJ := lib/mozjpeg/jaricom.o lib/mozjpeg/jcapimin.o lib/mozjpeg/jcarith.o lib/mozjpeg/jcext.o lib/mozjpeg/jchuff.o lib/mozjpeg/jcmarker.o lib/mozjpeg/jcmaster.o lib/mozjpeg/jcomapi.o lib/mozjpeg/jcparam.o lib/mozjpeg/jcphuff.o lib/mozjpeg/jctrans.o lib/mozjpeg/jdapimin.o lib/mozjpeg/jdarith.o lib/mozjpeg/jdatadst.o lib/mozjpeg/jdatasrc.o lib/mozjpeg/jdcoefct.o lib/mozjpeg/jdhuff.o lib/mozjpeg/jdinput.o lib/mozjpeg/jdmarker.o lib/mozjpeg/jdphuff.o lib/mozjpeg/jdtrans.o lib/mozjpeg/jerror.o lib/mozjpeg/jmemmgr.o lib/mozjpeg/jmemnobs.o lib/mozjpeg/jsimd_none.o lib/mozjpeg/jutils.o
PUGIXML_OBJ := lib/pugixml/pugixml.o
ZOPFLI_OBJ := lib/zopfli/hash.o lib/zopfli/squeeze.o lib/zopfli/gzip_container.o lib/zopfli/katajainen.o lib/zopfli/zopfli_lib.o lib/zopfli/cache.o lib/zopfli/zlib_container.o lib/zopfli/util.o lib/zopfli/tree.o lib/zopfli/deflate.o lib/zopfli/blocksplitter.o lib/zopfli/lz77.o
ZOPFLIPNG_OBJ := lib/zopflipng/lodepng/lodepng.o lib/zopflipng/lodepng/lodepng_util.o lib/zopflipng/zopflipng_lib.o

CFLAGS += -Wall -Wextra -Wno-unused-parameter -Werror -O3 -msse2 -mfpmath=sse -flto
CFLAGS += -Wall -Werror -O3 -msse2 -mfpmath=sse -flto
CPPFLAGS += -I./lib
CXXFLAGS += $(CFLAGS) -std=c++17 -fno-rtti
LDFLAGS += -flto -lpthread
LDFLAGS += -flto -lpthread -lstdc++fs

ifeq ($(OS), Windows_NT)
SYSTEM := Windows
Expand All @@ -17,42 +17,33 @@ else
SYSTEM := $(shell uname -s)
endif

# Gold linker only supports Linux
ifeq ($(SYSTEM), Linux)
LDFLAGS += -fuse-ld=gold
endif

# -lstdc++fs supported only on Linux
ifeq ($(SYSTEM), Linux)
LDFLAGS += -lstdc++fs
endif

ifeq ($(SYSTEM), Darwin)
LDLIBS += -liconv
else
# -s is "obsolete" on mac
LDFLAGS += -s
endif

# Multithread in LZMA SDK is only supported on Windows
ifeq ($(SYSTEM), Windows)
LEANIFY_SRC += fileio_win.cpp
LZMA_OBJ += lib/LZMA/LzFindMt.o lib/LZMA/Threads.o
LEANIFY_OBJ += fileio_win.o
else
LEANIFY_SRC += fileio_linux.cpp
LZMA_CFLAGS := -D _7ZIP_ST
LEANIFY_OBJ += fileio_linux.o
endif

.PHONY: leanify clean
.PHONY: leanify asan clean

leanify: $(LEANIFY_SRC) $(LZMA_OBJ) $(MOZJPEG_OBJ) $(PUGIXML_OBJ) $(ZOPFLI_OBJ) $(ZOPFLIPNG_OBJ)
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $^ $(LDFLAGS) $(LDLIBS) -o $@
leanify: $(LEANIFY_OBJ) $(LZMA_OBJ) $(MOZJPEG_OBJ) $(PUGIXML_OBJ) $(ZOPFLI_OBJ) $(ZOPFLIPNG_OBJ)
$(CXX) $^ $(LDFLAGS) $(LDLIBS) -o $@

$(LZMA_OBJ): CFLAGS := $(filter-out -Wextra,$(CFLAGS)) $(LZMA_CFLAGS) -Wno-parentheses
$(LEANIFY_OBJ): CFLAGS += -Wextra -Wno-unused-parameter

$(MOZJPEG_OBJ): CFLAGS := $(filter-out -Wextra,$(CFLAGS))
$(LZMA_OBJ): CFLAGS += -Wno-unknown-warning-option -Wno-dangling-pointer

$(ZOPFLI_OBJ): CFLAGS += -Wno-unused-function

asan: CFLAGS += -g -fsanitize=address -fno-omit-frame-pointer
asan: LDFLAGS := -fsanitize=address $(filter-out -s,$(LDFLAGS))
asan: leanify

clean:
rm -f $(LZMA_OBJ) $(MOZJPEG_OBJ) $(PUGIXML_OBJ) $(ZOPFLI_OBJ) $(ZOPFLIPNG_OBJ) leanify
rm -f $(LEANIFY_OBJ) $(LZMA_OBJ) $(MOZJPEG_OBJ) $(PUGIXML_OBJ) $(ZOPFLI_OBJ) $(ZOPFLIPNG_OBJ) leanify
5 changes: 5 additions & 0 deletions SECURITY.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Security Policy

## Reporting a Vulnerability

Please report security vulnerabilities to jayxon@gmail.com
33 changes: 27 additions & 6 deletions formats/rdb.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,28 @@ size_t Rdb::Leanify(size_t size_leanified /*= 0*/) {
}

depth++;
uint8_t* p_read;
uint8_t* p_read = fp_;
size_t rdb_size_leanified = 0;

// header
p_read = fp_;
fp_ -= size_leanified;

// total number of files including directory
uint32_t file_num = *(uint32_t*)(p_read + 0x10);

uint64_t index_offset = *(uint64_t*)(p_read + 0x14);

if (index_offset > size_) {
cerr << "index offset out of range: " << index_offset << endl;
return Format::Leanify(size_leanified);
}

uint64_t content_offset = index_offset + *(uint64_t*)(p_read + 0x1C);

if (content_offset < index_offset || content_offset > size_) {
cerr << "content offset out of range: " << content_offset << endl;
return Format::Leanify(size_leanified);
}

// move header and indexes
fp_ -= size_leanified;
memmove(fp_, p_read, (size_t)content_offset);

uint8_t* p_index = fp_ + index_offset;
Expand All @@ -50,12 +57,26 @@ size_t Rdb::Leanify(size_t size_leanified /*= 0*/) {
// note that on Linux wchar_t is 4 bytes instead of 2
// so I can't use wcslen
// p_index += (wcslen(file_name) + 1) * 2;
while (*(uint16_t*)p_index) {
while (p_index + 2 < fp_ + content_offset && *(uint16_t*)p_index) {
p_index += 2;
}
p_index += 2;

size_t remaining_size = fp_ + size_leanified + size_ - p_read;
if (p_index + 8 > fp_ + content_offset) {
cerr << "index is overlapping with content" << endl;
memmove(p_read - rdb_size_leanified - size_leanified, p_read, remaining_size);
p_read += remaining_size;
break;
}

uint64_t file_size = *(uint64_t*)(p_index + 8);
if (file_size > static_cast<uint64_t>(remaining_size)) {
cerr << "file size out of range: " << file_size << endl;
memmove(p_read - rdb_size_leanified - size_leanified, p_read, remaining_size);
p_read += remaining_size;
break;
}

*(uint64_t*)p_index -= rdb_size_leanified;

Expand Down
59 changes: 52 additions & 7 deletions formats/swf.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ size_t GetRECTSize(uint8_t* rect) {

bool LZMACompress(const uint8_t* src, size_t src_len, vector<uint8_t>* out) {
// Reserve enough space.
out->resize(src_len + src_len / 8);
out->resize(src_len + src_len / 8 + LZMA_PROPS_SIZE);

CLzmaEncProps props;
LzmaEncProps_Init(&props);
Expand Down Expand Up @@ -81,8 +81,18 @@ size_t Swf::Leanify(size_t size_leanified /*= 0*/) {
if (is_fast && compression != 'F')
return Format::Leanify(size_leanified);

if (size_ < 8) {
cerr << "SWF file too small!" << endl;
return Format::Leanify(size_leanified);
}

uint8_t* in_buffer = fp_ + 8;
uint32_t in_len = *(uint32_t*)(fp_ + 4) - 8;
uint32_t in_len = *(uint32_t*)(fp_ + 4);
if (in_len < 9) {
cerr << "invalid file size in header: " << in_len << endl;
return Format::Leanify(size_leanified);
}
in_len -= 8;

// if SWF is compressed, decompress it first
if (compression == 'C') {
Expand All @@ -101,6 +111,10 @@ size_t Swf::Leanify(size_t size_leanified /*= 0*/) {
} else if (compression == 'Z') {
// LZMA
VerbosePrint("SWF is compressed with LZMA.");
if (size_ < 4 + 12 + LZMA_PROPS_SIZE) {
cerr << "SWF file too small!" << endl;
return Format::Leanify(size_leanified);
}
// | 4 bytes | 4 bytes | 4 bytes | 5 bytes | n bytes | 6 bytes |
// | 'ZWS' + version | scriptLen | compressedLen | LZMA props | LZMA data | LZMA end marker |
uint8_t* dst_buffer = new uint8_t[in_len];
Expand All @@ -114,31 +128,50 @@ size_t Swf::Leanify(size_t size_leanified /*= 0*/) {
in_buffer = dst_buffer;
} else {
VerbosePrint("SWF is not compressed.");
if (in_len + 8 > size_) {
cerr << "SWF file too small!" << endl;
return Format::Leanify(size_leanified);
}
}

// parsing SWF tags
uint8_t* p = in_buffer + GetRECTSize(in_buffer); // skip FrameSize which is a RECT
p += 4; // skip FrameRate(2 Byte) + FrameCount(2 Byte) = 4 Byte
size_t tag_size_leanified = 0;
do {
while (p + 2 <= in_buffer + in_len) {
uint16_t tag_type = *(uint16_t*)p >> 6;
uint32_t tag_length = *p & 0x3F;
size_t tag_header_length = 2;
if (tag_length == 0x3F) {
if (p + 6 > in_buffer + in_len)
break;
tag_length = *(uint32_t*)(p + 2);
tag_header_length += 4;
}

memmove(p - tag_size_leanified, p, tag_header_length);
p += tag_header_length;

if (tag_length > in_len || p - in_buffer + tag_length > in_len) {
VerbosePrint("SWF tag too long: ", tag_length);
break;
}

switch (tag_type) {
// DefineBitsLossless
case 20:
// DefineBitsLossless2
case 36: {
size_t header_size = 7 + (p[3] == 3);
VerbosePrint("DefineBitsLossless tag found.");
if (3 > tag_length) {
VerbosePrint("SWF tag too short: ", tag_length);
break;
}
size_t header_size = 7 + (p[3] == 3);
if (header_size > tag_length) {
VerbosePrint("SWF tag too short: ", tag_length);
break;
}
memmove(p - tag_size_leanified, p, header_size);

// recompress Zlib bitmap data
Expand All @@ -151,6 +184,10 @@ size_t Swf::Leanify(size_t size_leanified /*= 0*/) {
// DefineBitsJPEG2
case 21: {
VerbosePrint("DefineBitsJPEG2 tag found.");
if (2 > tag_length) {
VerbosePrint("SWF tag too short: ", tag_length);
break;
}
// copy id
*(uint16_t*)(p - tag_size_leanified) = *(uint16_t*)p;

Expand All @@ -165,13 +202,21 @@ size_t Swf::Leanify(size_t size_leanified /*= 0*/) {
case 35:
// DefineBitsJPEG4
case 90: {
size_t header_size = tag_type == 90 ? 8 : 6;
VerbosePrint("DefineBitsJPEG", header_size / 2, " tag found.");
if (header_size > tag_length) {
VerbosePrint("SWF tag too short: ", tag_length);
break;
}
// copy id
*(uint16_t*)(p - tag_size_leanified) = *(uint16_t*)p;

uint32_t img_size = *(uint32_t*)(p + 2);
size_t header_size = tag_type == 90 ? 8 : 6;
if (img_size > tag_length || header_size + img_size > tag_length) {
VerbosePrint("SWF tag too short: ", tag_length);
break;
}

VerbosePrint("DefineBitsJPEG", header_size / 2, " tag found.");
// Leanify embedded image
size_t new_img_size = LeanifyFile(p + header_size, img_size, tag_size_leanified);
*(uint32_t*)(p + 2 - tag_size_leanified) = new_img_size;
Expand All @@ -198,7 +243,7 @@ size_t Swf::Leanify(size_t size_leanified /*= 0*/) {
memmove(p - tag_size_leanified, p, tag_length);
}
p += tag_length;
} while (p < in_buffer + in_len);
}

VerbosePrint("Uncompressed SWF tags leanified: ", tag_size_leanified);
in_len -= tag_size_leanified;
Expand Down

0 comments on commit 224b0ee

Please sign in to comment.