ImHex pattern scripts

[sehnryr]

Attempts to extract images from the cache files were made a long time ago on XeNTaX, but their understanding of the header was somewhat lacking. To do so effectively, one must have a basic understanding of each file header.

I was initially overwhelmed when looking at the bytes of the cache header but, thanks to @Puxtril and his work, I was able to recognize patterns and make the statement that the entire image information must be contained within it. This enabled me to make an attempt at understanding the header and extracting the source images from the cache.

Here is my pattern script for the ImHex hex editor to highlight bytes in meaningful groups:

#include <std/io.pat>
#include <std/math.pat>
#include <std/mem.pat>

struct Path {
    u32 len;
    u8 name[len];
};

// Random unique hash of the file
u128 HASH @ $;

// This file can be the merge of multiple other files
// on different channels
u32 merged_file_count @ $;
// File paths that compose the current file
Path file_path[merged_file_count] @ $;

// Arguments that specifies how the file should be understand
// They might specify the channels of the merged files if they exist
u32 arguments_length @ $;
u8 arguments[arguments_length] @ $;

// Arguments must end by 0x00
if (arguments_length != 0) {
    $ += 1;
}

// File type code which is not an hex signature
/*
0xA3: Other ? Emission map || Tint mask || PNG image
0xB8: Normal map
0xBC: PBR map
*/
u32 file_type @ $;

// Unknown byte
$ += 1;

// Number of images stored in the F.cache
u8 image_count @ $;

// Unknown byte
$ += 1;

// DDS compression format
/*
0x00: Default
0x01: DXT1
0x02: DXT3
0x03: DXT5
0x06: ATI1
0x07: ATI2
*/
u8 DDS_FORMAT @ $;

// Number of mip-maps stored in the F.cache
u32 mip_map_count @ $;

// Images offset in the uncompressed F.cache
u32 image_offset[mip_map_count] @ $;

// Width and height ratios the are not the sizes of the image
u16 width_ratio @ $;
u16 height_ratio @ $;

// Width and height of the largest image stored in the B.cache
u16 B_max_width @ $;
u16 B_max_height @ $;

// Largest side size of the main image
u32 largest_side_size @ $;

// Real image sizes
u32 image_width;
u32 image_height;

if (width_ratio > height_ratio) {
    image_width = largest_side_size;
    image_height = largest_side_size * height_ratio / width_ratio;
} else {
    image_width = largest_side_size * width_ratio / height_ratio;
    image_height = largest_side_size;
}

// unknown data
$ += 39;

// Weird repetition of the merged file paths
u32 len @ $;
u8 data[len] @ $;

// Calculate the DDS image size
fn dds_size(u32 width, u32 height, u8 dds_format) {
    u32 _width = (width + 3) >> 2;
    u32 _height = (height + 3) >> 2;
    u64 size = std::math::max(1, _width) * std::math::max(1, _height);

    u8 factor;
    if (dds_format == 0x00 || 
        dds_format == 0x01 ||
        dds_format == 0x06) {
        factor = 8;
    } else if (
        dds_format == 0x02 ||
        dds_format == 0x03 ||
        dds_format == 0x07) {
        factor = 16;
    }

    return size * factor;
};

std::print(
    "Width: {}, Height: {}",
    image_width,
    image_height
);

std::print(
    "DDS image size: {}",
    dds_size(image_width, image_height, DDS_FORMAT)
);

if (image_count == 0) {
    std::print("Image is contained in the B.cache");

    // Here, inverse offset means that the image has a real offset of
    //   decomp_len - inverse_offset
    u32 inverse_offset = dds_size(B_max_width, B_max_height, DDS_FORMAT);
    std::print("Decompressed image inverse offset: {}", inverse_offset);
} else {
    std::print("Image is contained in the F.cache");

    // We need the image compressed length.
    // We can compute this length using:
    //   comp_len - offset
    u32 offset = 0;
    if (mip_map_count > 0) {
        offset = image_offset[mip_map_count - 1];
    }
    std::print("Compressed image offset: {}", offset);
}

I will periodically update this script when I find new information or potential improvements.

[sehnryr]

Here's a pattern script for highlighting the different blocks in the F and B cache:

#include <std/io.pat>
#include <std/mem.pat>

struct Block {
    u64 offset = $;
    u8 buffer[8] @ $;
    $ += 8;

    u32 num1 = buffer[0] << 24 | buffer[1] << 16 | buffer[2] << 8 | buffer[3];
    u32 num2 = buffer[4] << 24 | buffer[5] << 16 | buffer[6] << 8 | buffer[7];
    
    u32 block_comp_len = num1 >> 2 & 0xFFFFFF;
    u32 block_decomp_len = num2 >> 5 & 0xFFFFFF;

    u8 data[block_comp_len] @ $;
    $ += block_comp_len;
    
    bool is_oodle = data[0] == 0x8C;

    std::print(
        "0x{:X} -> 0x{:X} (is oodle: {}, offset: 0x{:X})",
        block_comp_len,
        block_decomp_len,
        is_oodle,
        offset
    );
};

Block block[while(std::mem::read_unsigned($, 1) == 0x80)] @ $;

[sehnryr]

Here's a pattern for highlighting the TOC file:

#pragma pattern_limit 1000000
#include <std/mem.pat>

// Ignore first 8 bytes
$ += 8;

struct RawTocEntry {
    s64 cache_offset @ $;
    s64 timestamp @ $ + 8;
    s32 comp_len @ $ + 16 ;
    s32 len @ $ + 20 ;
    s32 reserved @ $ + 24 ;
    s32 parent_dir_index @ $ + 28 ;
    u8 name[64] @ $ + 32 ;
    $ += 32 + 64;
};

RawTocEntry entry[while(std::mem::read_unsigned($, 16) != 0)] @ $;

The pattern limit needs to be upped as the file is quite big. Also, the pattern runs very slowly (~2s on my setup)

[sehnryr]

Script for highlighting DDS headers.

using DWORD = be u32;
using UINT = be u32;

struct DDS_PIXELFORMAT {
  DWORD dwSize;
  DWORD dwFlags;
  DWORD dwFourCC;
  DWORD dwRGBBitCount;
  DWORD dwRBitMask;
  DWORD dwGBitMask;
  DWORD dwBBitMask;
  DWORD dwABitMask;
};

struct DDS_HEADER {
  DWORD           dwSize;
  DWORD           dwFlags;
  DWORD           dwHeight;
  DWORD           dwWidth;
  DWORD           dwPitchOrLinearSize;
  DWORD           dwDepth;
  DWORD           dwMipMapCount;
  DWORD           dwReserved1[11];
  DDS_PIXELFORMAT ddspf;
  DWORD           dwCaps;
  DWORD           dwCaps2;
  DWORD           dwCaps3;
  DWORD           dwCaps4;
  DWORD           dwReserved2;
};

enum DXGI_FORMAT : u32 {
  DXGI_FORMAT_UNKNOWN = 0,
  DXGI_FORMAT_R32G32B32A32_TYPELESS = 1,
  DXGI_FORMAT_R32G32B32A32_FLOAT = 2,
  DXGI_FORMAT_R32G32B32A32_UINT = 3,
  DXGI_FORMAT_R32G32B32A32_SINT = 4,
  DXGI_FORMAT_R32G32B32_TYPELESS = 5,
  DXGI_FORMAT_R32G32B32_FLOAT = 6,
  DXGI_FORMAT_R32G32B32_UINT = 7,
  DXGI_FORMAT_R32G32B32_SINT = 8,
  DXGI_FORMAT_R16G16B16A16_TYPELESS = 9,
  DXGI_FORMAT_R16G16B16A16_FLOAT = 10,
  DXGI_FORMAT_R16G16B16A16_UNORM = 11,
  DXGI_FORMAT_R16G16B16A16_UINT = 12,
  DXGI_FORMAT_R16G16B16A16_SNORM = 13,
  DXGI_FORMAT_R16G16B16A16_SINT = 14,
  DXGI_FORMAT_R32G32_TYPELESS = 15,
  DXGI_FORMAT_R32G32_FLOAT = 16,
  DXGI_FORMAT_R32G32_UINT = 17,
  DXGI_FORMAT_R32G32_SINT = 18,
  DXGI_FORMAT_R32G8X24_TYPELESS = 19,
  DXGI_FORMAT_D32_FLOAT_S8X24_UINT = 20,
  DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS = 21,
  DXGI_FORMAT_X32_TYPELESS_G8X24_UINT = 22,
  DXGI_FORMAT_R10G10B10A2_TYPELESS = 23,
  DXGI_FORMAT_R10G10B10A2_UNORM = 24,
  DXGI_FORMAT_R10G10B10A2_UINT = 25,
  DXGI_FORMAT_R11G11B10_FLOAT = 26,
  DXGI_FORMAT_R8G8B8A8_TYPELESS = 27,
  DXGI_FORMAT_R8G8B8A8_UNORM = 28,
  DXGI_FORMAT_R8G8B8A8_UNORM_SRGB = 29,
  DXGI_FORMAT_R8G8B8A8_UINT = 30,
  DXGI_FORMAT_R8G8B8A8_SNORM = 31,
  DXGI_FORMAT_R8G8B8A8_SINT = 32,
  DXGI_FORMAT_R16G16_TYPELESS = 33,
  DXGI_FORMAT_R16G16_FLOAT = 34,
  DXGI_FORMAT_R16G16_UNORM = 35,
  DXGI_FORMAT_R16G16_UINT = 36,
  DXGI_FORMAT_R16G16_SNORM = 37,
  DXGI_FORMAT_R16G16_SINT = 38,
  DXGI_FORMAT_R32_TYPELESS = 39,
  DXGI_FORMAT_D32_FLOAT = 40,
  DXGI_FORMAT_R32_FLOAT = 41,
  DXGI_FORMAT_R32_UINT = 42,
  DXGI_FORMAT_R32_SINT = 43,
  DXGI_FORMAT_R24G8_TYPELESS = 44,
  DXGI_FORMAT_D24_UNORM_S8_UINT = 45,
  DXGI_FORMAT_R24_UNORM_X8_TYPELESS = 46,
  DXGI_FORMAT_X24_TYPELESS_G8_UINT = 47,
  DXGI_FORMAT_R8G8_TYPELESS = 48,
  DXGI_FORMAT_R8G8_UNORM = 49,
  DXGI_FORMAT_R8G8_UINT = 50,
  DXGI_FORMAT_R8G8_SNORM = 51,
  DXGI_FORMAT_R8G8_SINT = 52,
  DXGI_FORMAT_R16_TYPELESS = 53,
  DXGI_FORMAT_R16_FLOAT = 54,
  DXGI_FORMAT_D16_UNORM = 55,
  DXGI_FORMAT_R16_UNORM = 56,
  DXGI_FORMAT_R16_UINT = 57,
  DXGI_FORMAT_R16_SNORM = 58,
  DXGI_FORMAT_R16_SINT = 59,
  DXGI_FORMAT_R8_TYPELESS = 60,
  DXGI_FORMAT_R8_UNORM = 61,
  DXGI_FORMAT_R8_UINT = 62,
  DXGI_FORMAT_R8_SNORM = 63,
  DXGI_FORMAT_R8_SINT = 64,
  DXGI_FORMAT_A8_UNORM = 65,
  DXGI_FORMAT_R1_UNORM = 66,
  DXGI_FORMAT_R9G9B9E5_SHAREDEXP = 67,
  DXGI_FORMAT_R8G8_B8G8_UNORM = 68,
  DXGI_FORMAT_G8R8_G8B8_UNORM = 69,
  DXGI_FORMAT_BC1_TYPELESS = 70,
  DXGI_FORMAT_BC1_UNORM = 71,
  DXGI_FORMAT_BC1_UNORM_SRGB = 72,
  DXGI_FORMAT_BC2_TYPELESS = 73,
  DXGI_FORMAT_BC2_UNORM = 74,
  DXGI_FORMAT_BC2_UNORM_SRGB = 75,
  DXGI_FORMAT_BC3_TYPELESS = 76,
  DXGI_FORMAT_BC3_UNORM = 77,
  DXGI_FORMAT_BC3_UNORM_SRGB = 78,
  DXGI_FORMAT_BC4_TYPELESS = 79,
  DXGI_FORMAT_BC4_UNORM = 80,
  DXGI_FORMAT_BC4_SNORM = 81,
  DXGI_FORMAT_BC5_TYPELESS = 82,
  DXGI_FORMAT_BC5_UNORM = 83,
  DXGI_FORMAT_BC5_SNORM = 84,
  DXGI_FORMAT_B5G6R5_UNORM = 85,
  DXGI_FORMAT_B5G5R5A1_UNORM = 86,
  DXGI_FORMAT_B8G8R8A8_UNORM = 87,
  DXGI_FORMAT_B8G8R8X8_UNORM = 88,
  DXGI_FORMAT_R10G10B10_XR_BIAS_A2_UNORM = 89,
  DXGI_FORMAT_B8G8R8A8_TYPELESS = 90,
  DXGI_FORMAT_B8G8R8A8_UNORM_SRGB = 91,
  DXGI_FORMAT_B8G8R8X8_TYPELESS = 92,
  DXGI_FORMAT_B8G8R8X8_UNORM_SRGB = 93,
  DXGI_FORMAT_BC6H_TYPELESS = 94,
  DXGI_FORMAT_BC6H_UF16 = 95,
  DXGI_FORMAT_BC6H_SF16 = 96,
  DXGI_FORMAT_BC7_TYPELESS = 97,
  DXGI_FORMAT_BC7_UNORM = 98,
  DXGI_FORMAT_BC7_UNORM_SRGB = 99,
  DXGI_FORMAT_AYUV = 100,
  DXGI_FORMAT_Y410 = 101,
  DXGI_FORMAT_Y416 = 102,
  DXGI_FORMAT_NV12 = 103,
  DXGI_FORMAT_P010 = 104,
  DXGI_FORMAT_P016 = 105,
  DXGI_FORMAT_420_OPAQUE = 106,
  DXGI_FORMAT_YUY2 = 107,
  DXGI_FORMAT_Y210 = 108,
  DXGI_FORMAT_Y216 = 109,
  DXGI_FORMAT_NV11 = 110,
  DXGI_FORMAT_AI44 = 111,
  DXGI_FORMAT_IA44 = 112,
  DXGI_FORMAT_P8 = 113,
  DXGI_FORMAT_A8P8 = 114,
  DXGI_FORMAT_B4G4R4A4_UNORM = 115,
  DXGI_FORMAT_P208 = 130,
  DXGI_FORMAT_V208 = 131,
  DXGI_FORMAT_V408 = 132,
  DXGI_FORMAT_SAMPLER_FEEDBACK_MIN_MIP_OPAQUE,
  DXGI_FORMAT_SAMPLER_FEEDBACK_MIP_REGION_USED_OPAQUE,
  DXGI_FORMAT_FORCE_UINT = 0xffffffff
};

enum D3D10_RESOURCE_DIMENSION : u32 {
  D3D10_RESOURCE_DIMENSION_UNKNOWN = 0,
  D3D10_RESOURCE_DIMENSION_BUFFER = 1,
  D3D10_RESOURCE_DIMENSION_TEXTURE1D = 2,
  D3D10_RESOURCE_DIMENSION_TEXTURE2D = 3,
  D3D10_RESOURCE_DIMENSION_TEXTURE3D = 4
};

struct DDS_HEADER_DXT10 {
  DXGI_FORMAT              dxgiFormat;
  D3D10_RESOURCE_DIMENSION resourceDimension;
  UINT                     miscFlag;
  UINT                     arraySize;
  UINT                     miscFlags2;
};

$ += 4;
DDS_HEADER header @ $;
// DDS_HEADER_DXT10 header10 @ $;

[sehnryr]

Audio ADPCM

struct Path {
    u32 len;
    u8 name[len];
};

// Random unique hash of the file
u128 HASH @ $;

// This file can be the merge of multiple other files
// on different channels
u32 merged_file_count @ $;
// File paths that compose the current file
Path file_path[merged_file_count] @ $;

// Arguments that specifies how the file should be understand
// They might specify the channels of the merged files if they exist
u32 arguments_length @ $;
u8 arguments[arguments_length] @ $;

// Arguments must end by 0x00
if (arguments_length != 0) {
    $ += 1;
}

// File type code which is not an hex signature
u32 file_type @ $;

$ += 4;

u32 formatTag @ $; // 0x01: PCM, 0x02: ADPCM
$ += 24;

u32 samplesPerSec @ $;
u8 bitsPerSample @ $;
u8 channels @ $;

$ += 4;

u32 avgBytesPerSec @ $;
u16 blockAlign @ $;
u16 samplesPerBlock @ $;

$ += 12;

u32 dwChunkSize @ $;