Add support for Android saves

Android save support now working.

Android saves use version 0x13 for the save file (vs 0x10 for the PC
version of EW). The Android saves differ from PC in several key ways:

- Android saves have no header checksum, just the compressed data
checksum. It occurs in the same place as for PC, and is the same CRC32B
algorithm.

- The compressed data section of Android saves are compressed using zlib
instead of lzo.

- There is also a small bit of data after the checksum in the header but
before the compressed data. This may be Android profile related?

.gitignore

@@ -1 +1,3 @@
 build/
+test/
+.vs/

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "json11"]
 	path = json11
 	url = https://github.com/dropbox/json11
+[submodule "zlib"]
+	path = zlib
+	url = https://github.com/madler/zlib

CMakeLists.txt

@@ -11,13 +11,14 @@ endif (UNIX)
 
 if (WIN32)
 add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE)
-set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -MTd -EHsc")
-set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -MT -EHsc")
+set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -EHsc")
+set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -EHsc")
 endif (WIN32)
 
-include_directories("minilzo-2.09" "json11")
+include_directories("minilzo-2.09" "json11" "zlib" "build/zlib")
 add_library (xcomsave ${xcomsave_sources} ${xcomsave_headers})
-set_target_properties (xcomsave PROPERTIES LINKER_LANGUAGE CXX)
+target_link_libraries(xcomsave zlibstatic)
+set_target_properties(xcomsave PROPERTIES LINKER_LANGUAGE CXX)
 
 set (xcom2json_sources xcom2json.cpp)
 set (xcom2json_headers)
@@ -31,3 +32,4 @@ add_executable (json2xcom ${json2xcom_sources} ${json2xcom_headers})
 set_target_properties (json2xcom PROPERTIES LINKER_LANGUAGE CXX)
 target_link_libraries(json2xcom xcomsave)
 
+add_subdirectory(zlib)

json2xcom.cpp

@@ -88,7 +88,11 @@ header build_header(const Json& json)
                 "Error reading json file: header format does not match\n");
     }
 
-    hdr.version = json["version"].int_value();
+    hdr.version = static_cast<xcom_version>(json["version"].int_value());
+    if (!supported_version(hdr.version)) {
+        throw std::runtime_error("Unsupported version number in json file");
+    }
+
     hdr.uncompressed_size = json["uncompressed_size"].int_value();
     hdr.game_number = json["game_number"].int_value();
     hdr.save_number = json["save_number"].int_value();

util.h

@@ -34,7 +34,9 @@ namespace xcom
 
     namespace util
     {
+        unsigned int crc32(const unsigned char *message, long len);
         unsigned int crc32b(const unsigned char *message, long len);
+        unsigned int adler32(const unsigned char *message, long len);
 
         std::string iso8859_1_to_utf8(const std::string& in);
         std::string utf8_to_iso8859_1(const std::string& in);

xcom.h

@@ -30,8 +30,25 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 namespace xcom
 {
-    // The supported saved game version
-    static const int save_version = 0x10;
+    // The supported saved game version(s)
+    enum class xcom_version : uint32_t {
+        invalid = 0,
+        enemy_within = 0x10,
+        enemy_within_android = 0x13,
+        // xcom2 = 0x14 (not supported)
+    };
+
+    static bool supported_version(xcom_version ver)
+    {
+        switch (ver)
+        {
+        case xcom_version::enemy_within:
+        case xcom_version::enemy_within_android:
+            return true;
+        default:
+            return false;
+        }
+    }
 
     // A buffer object for performing file IO. An xcom save may be read from
     // or written to a buffer instance or a file.
@@ -85,8 +102,8 @@ namespace xcom
     // 1024-end : Compressed data
     struct header
     {
-        // XCom save version (always 0x10)
-        int32_t version;
+        // XCom save version
+        xcom_version version;
 
         // From the upk packages this looks like it should be the total
         // uncompressed size of the save data. In practice It's always all zeros.

xcom2json.cpp

@@ -528,7 +528,7 @@ void buildJson(const saved_game& save, json_writer& w)
 
     w.write_key("header");
     w.begin_object();
-    w.write_int("version", hdr.version);
+    w.write_int("version", static_cast<uint32_t>(hdr.version));
     w.write_int("uncompressed_size", hdr.uncompressed_size);
     w.write_int("game_number", hdr.game_number);
     w.write_int("save_number", hdr.save_number);

xcomreader.cpp

@@ -18,6 +18,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 */
 
 #include "minilzo.h"
+#include "zlib.h"
 #include "xcomio.h"
 #include "util.h"
 
@@ -36,13 +37,21 @@ namespace xcom
     header read_header(xcom_io &r)
     {
         header hdr;
-        hdr.version = r.read_int();
-        if (hdr.version != save_version) {
+        hdr.version = static_cast<xcom_version>(r.read_int());
+        switch (hdr.version)
+        {
+        case xcom_version::enemy_within:
+        case xcom_version::enemy_within_android:
+            // Yay! A supported save!
+            break;
+        default:
+            // Not supported.
             fprintf(stderr, 
-                "Error: Data does not appear to be an xcom save: expected file version %d but got %d\n", 
-                save_version, hdr.version);
-            return{ 0 };
+                "Error: File does not appear to be a supported xcom save: got version %d\n",
+                hdr.version);
+            return{ xcom_version::invalid };
         }
+
         hdr.uncompressed_size = r.read_int();
         hdr.game_number = r.read_int();
         hdr.save_number = r.read_int();
@@ -57,16 +66,20 @@ namespace xcom
         uint32_t compressed_crc = (uint32_t)r.read_int();
 
         // Compute the CRC of the header
-        r.seek(xcom_io::seek_kind::start, 1016);
-        int32_t hdr_size = r.read_int();
-        uint32_t hdr_crc = (uint32_t)r.read_int();
-
-        // CRC the first hdr_size bytes of the buffer
-        r.seek(xcom_io::seek_kind::start, 0);
-        uint32_t computed_hdr_crc = r.crc(hdr_size);
-        if (hdr_crc != computed_hdr_crc)
-        {
-            throw std::runtime_error("CRC mismatch in header. Bad save?\n");
+        // Note the android version includes only a single checksum over the compressed data,
+        // the header is not checked.
+        if (hdr.version != xcom_version::enemy_within_android) {
+            r.seek(xcom_io::seek_kind::start, 1016);
+            int32_t hdr_size = r.read_int();
+            uint32_t hdr_crc = (uint32_t)r.read_int();
+
+            // CRC the first hdr_size bytes of the buffer
+            r.seek(xcom_io::seek_kind::start, 0);
+            uint32_t computed_hdr_crc = r.crc(hdr_size);
+            if (hdr_crc != computed_hdr_crc)
+            {
+                throw std::runtime_error("CRC mismatch in header. Bad save?\n");
+            }
         }
 
         // CRC the compressed data
@@ -607,23 +620,62 @@ namespace xcom
         return uncompressed_size;
     }
 
-    buffer<unsigned char> decompress(xcom_io &r)
+    unsigned long decompress_one_chunk(xcom_version version, const unsigned char *compressed_start, unsigned long compressed_size, unsigned char *decompressed_start, unsigned long decompressed_size)
+    {
+        switch (version)
+        {
+            case xcom_version::enemy_within:
+            {
+                lzo_init();
+                if (lzo1x_decompress_safe(compressed_start, compressed_size, decompressed_start,
+                    &decompressed_size, nullptr) != LZO_E_OK) {
+                    throw std::runtime_error("LZO decompress of save data failed\n");
+                }
+
+                return decompressed_size;
+            }
+
+            case xcom_version::enemy_within_android:
+            {
+                z_stream stream;
+                stream.zalloc = Z_NULL;
+                stream.zfree = Z_NULL;
+                stream.opaque = Z_NULL;
+                stream.avail_in = compressed_size;
+                stream.next_in = (Bytef*)compressed_start;
+                stream.avail_out = decompressed_size;
+                stream.next_out = (Bytef*)decompressed_start;
+                inflateInit(&stream);
+                inflate(&stream, Z_NO_FLUSH);
+                inflateEnd(&stream);
+                return stream.total_out;
+            }
+            break;
+
+            default:
+                throw std::runtime_error("Unexpected version\n");
+        }
+    }
+
+    buffer<unsigned char> decompress(xcom_io &r, xcom_version version)
     {
-        size_t uncompressed_size = calculate_uncompressed_size(r);
-        if (uncompressed_size < 0) {
-            throw format_exception(r.offset(), "Found no uncompressed data in save.\n");
+        size_t total_uncompressed_size = calculate_uncompressed_size(r);
+        if (total_uncompressed_size < 0) {
+            throw format_exception(r.offset(), "Found no uncompressed data in save\n");
         }
 
-        std::unique_ptr<unsigned char[]> buf = std::make_unique<unsigned char[]>(uncompressed_size);
+        std::unique_ptr<unsigned char[]> buf = std::make_unique<unsigned char[]>(total_uncompressed_size);
         // Start back at the beginning of the compressed data.
         r.seek(xcom_io::seek_kind::start, compressed_data_start);
 
         unsigned char *outp = buf.get();
+        unsigned long bytes_remaining = total_uncompressed_size;
+
         do
         {
             // Expect the magic header value 0x9e2a83c1 at the start of each chunk
             if (r.read_int() != UPK_Magic) {
-                throw format_exception(r.offset(), 
+                throw format_exception(r.offset(),
                         "Failed to find compressed chunk header\n");
             }
 
@@ -635,15 +687,7 @@ namespace xcom
 
             // Uncompressed size is at p+12
             int32_t uncompressed_size = r.read_int();
-
-            unsigned long decomp_size = uncompressed_size;
-
-            if (lzo1x_decompress_safe(r.pointer() + 8, compressed_size, outp, 
-                                      &decomp_size, nullptr) != LZO_E_OK) {
-                throw format_exception(r.offset(), 
-                        "LZO decompress of save data failed\n");
-            }
-
+            unsigned long decomp_size = decompress_one_chunk(version, r.pointer() + 8, compressed_size, outp, bytes_remaining);
             if (decomp_size != uncompressed_size)
             {
                 throw format_exception(r.offset(), "Failed to decompress chunk\n");
@@ -653,9 +697,10 @@ namespace xcom
             // compressedSize bytes later.
             r.seek(xcom_io::seek_kind::current, compressed_size + 8);
             outp += uncompressed_size;
+            bytes_remaining -= uncompressed_size;
         } while (!r.eof());
 
-        return{ std::move(buf), uncompressed_size };
+        return{ std::move(buf), total_uncompressed_size };
     }
 
     buffer<unsigned char> read_file(const std::string& filename)
@@ -691,7 +736,7 @@ namespace xcom
 
         xcom_io rdr{ std::move(b) };
         save.hdr = read_header(rdr);
-        buffer<unsigned char> uncompressed_buf = decompress(rdr);
+        buffer<unsigned char> uncompressed_buf = decompress(rdr, static_cast<xcom_version>(save.hdr.version));
 #ifdef _DEBUG
         FILE *fp = fopen("output.dat", "wb");
         fwrite(uncompressed_buf.buf.get(), 1, uncompressed_buf.length, fp);