Skip to content

Commit

Permalink
Prepare Vdex processing backend for Android-P
Browse files Browse the repository at this point in the history 
Vdex version shipped with Android P preview images appears to fully break
backwards compatibility. Therefore, the common Vdex structures from Oreo
cannot be used anymore. After this commit, each supported Vdex version is
required to define a different frontend (Vdex header/structs), backend (processor)
and decompiler.

The functions required to be exported to the main tool, will be initialised as an
env struct with function pointers according to the found Vdex version.

Signed-off-by: Anestis Bechtsoudis <anestis@census-labs.com>
  • Loading branch information
@anestisb
anestisb committed Jun 10, 2018
1 parent 02a2204 commit fe078c9
Show file tree
Hide file tree
Showing 23 changed files with 960 additions and 571 deletions.
4 changes: 2 additions & 2 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
src/*.o
src/*.dSYM
*.o
*.dSYM
src/vdexExtractor
obj
libs
2 changes: 1 addition & 1 deletion jni/Android.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := vdexExtractor
SRC := ../src
SRC_FILE_LIST := $(wildcard $(LOCAL_PATH)/$(SRC)/*.c)
SRC_FILE_LIST := $(wildcard $(LOCAL_PATH)/$(SRC)/*.c) $(wildcard $(LOCAL_PATH)/$(SRC)/*/*.c)
LOCAL_SRC_FILES := $(SRC_FILE_LIST:$(LOCAL_PATH)/%=%)
LOCAL_CFLAGS += -c -std=c11 -D_GNU_SOURCE \
-Wall -Wextra -Werror
Expand Down
3 changes: 2 additions & 1 deletion src/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ endif
default: $(TARGET)
all: default

OBJECTS = $(patsubst %.c, %.o, $(wildcard *.c))
OBJECTS = $(patsubst %.c, %.o, $(wildcard *.c) $(wildcard */*.c))
HEADERS = $(wildcard *.h)

%.o: %.c $(HEADERS)
Expand All @@ -49,6 +49,7 @@ $(TARGET): $(OBJECTS)

clean:
-rm -f *.o
-rm -f */*.o
-rm -f $(TARGET)

format:
Expand Down
8 changes: 4 additions & 4 deletions src/dex_instruction.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -677,16 +677,16 @@ void dexInstr_getVarArgs(u2 *code_ptr, u4 arg[kMaxVarArgRegs]) {
switch (count) {
case 5:
arg[4] = InstA(code_ptr);
/* fall through */
/* fall through */
case 4:
arg[3] = (regList >> 12) & 0x0f;
/* fall through */
/* fall through */
case 3:
arg[2] = (regList >> 8) & 0x0f;
/* fall through */
/* fall through */
case 2:
arg[1] = (regList >> 4) & 0x0f;
/* fall through */
/* fall through */
case 1:
arg[0] = regList & 0x0f;
break;
Expand Down
2 changes: 2 additions & 0 deletions src/log.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
#ifndef _LOG_H_
#define _LOG_H_

#include "common.h"

typedef enum { l_FATAL = 0, l_ERROR, l_WARN, l_INFO, l_DEBUG, l_MAX_LEVEL } log_level_t;

void log_setMinLevel(log_level_t);
Expand Down
6 changes: 6 additions & 0 deletions src/utils.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -381,3 +381,9 @@ char *utils_fileBasename(char const *path) {
return strdup(s + 1);
}
}

bool utils_isDir(const char *path) {
struct stat buf;
stat(path, &buf);
return S_ISDIR(buf.st_mode);
}
1 change: 1 addition & 0 deletions src/utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,5 +45,6 @@ long utils_endTimer(struct timespec *);
u4 *utils_processFileWithCsums(const char *, int *);

char *utils_fileBasename(char const *);
bool utils_isDir(const char *);

#endif
251 changes: 42 additions & 209 deletions src/vdex.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,209 +22,31 @@

#include <sys/mman.h>

#include "log.h"
#include "out_writer.h"
#include "utils.h"
#include "vdex.h"
#include "vdex_backend_v10.h"
#include "vdex_backend_v6.h"

static void *(*initDepsInfoPtr)(const u1 *);
static void (*destroyDepsInfoPtr)(const void *);
static void (*dumpDepsInfoPtr)(const u1 *, const void *);
static int (*processPtr)(const char *, const u1 *, const runArgs_t *);

void vdex_backendInit(VdexBackend ver) {
switch (ver) {
case kBackendV6:
initDepsInfoPtr = &vdex_initDepsInfo_v6;
destroyDepsInfoPtr = &vdex_destroyDepsInfo_v6;
dumpDepsInfoPtr = &vdex_dumpDepsInfo_v6;
processPtr = &vdex_process_v6;
break;
case kBackendV10:
initDepsInfoPtr = &vdex_initDepsInfo_v10;
destroyDepsInfoPtr = &vdex_destroyDepsInfo_v10;
dumpDepsInfoPtr = &vdex_dumpDepsInfo_v10;
processPtr = &vdex_process_v10;
break;
default:
LOGMSG(l_FATAL, "Invalid Vdex backend version");
}
}

bool vdex_isMagicValid(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return (memcmp(pVdexHeader->magic, kVdexMagic, sizeof(kVdexMagic)) == 0);
}

bool vdex_isVersionValid(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
for (u4 i = 0; i < kNumVdexVersions; i++) {
if (memcmp(pVdexHeader->version, kVdexMagicVersions[i], kVdexVersionLen) == 0) {
LOGMSG(l_DEBUG, "Vdex version '%s' detected", pVdexHeader->version);
return true;
}
}
return false;
}

bool vdex_isValidVdex(const u1 *cursor) {
return vdex_isMagicValid(cursor) && vdex_isVersionValid(cursor);
}

bool vdex_hasDexSection(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return pVdexHeader->dexSize != 0;
}

u4 vdex_GetSizeOfChecksumsSection(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return sizeof(VdexChecksum) * pVdexHeader->numberOfDexFiles;
}

const u1 *vdex_DexBegin(const u1 *cursor) {
return cursor + sizeof(vdexHeader) + vdex_GetSizeOfChecksumsSection(cursor);
}

u4 vdex_DexBeginOffset(const u1 *cursor) {
return sizeof(vdexHeader) + vdex_GetSizeOfChecksumsSection(cursor);
}

const u1 *vdex_DexEnd(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return vdex_DexBegin(cursor) + pVdexHeader->dexSize;
}

u4 vdex_DexEndOffset(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return vdex_DexBeginOffset(cursor) + pVdexHeader->dexSize;
}

// TODO: Cache embedded Dex file offsets so that we don't have to parse from scratch when we
// want to iterate over all files.
const u1 *vdex_GetNextDexFileData(const u1 *cursor, u4 *offset) {
if (*offset == 0) {
if (vdex_hasDexSection(cursor)) {
const u1 *dexBuf = vdex_DexBegin(cursor);
*offset = sizeof(vdexHeader) + vdex_GetSizeOfChecksumsSection(cursor);
LOGMSG(l_DEBUG, "Processing first Dex file at offset:0x%x", *offset);

// Adjust offset to point at the end of current Dex file
dexHeader *pDexHeader = (dexHeader *)(dexBuf);
*offset += pDexHeader->fileSize;
return dexBuf;
} else {
return NULL;
}
#include "vdex/vdex_006.h"
#include "vdex/vdex_010.h"

bool vdex_initEnv(const u1 *cursor, vdex_env_t *env) {
// Check if a supported Vdex version is found
if (vdex_006_isValidVdex(cursor)) {
LOGMSG(l_DEBUG, "Initializing environment for Vdex version '006'");
env->dumpHeaderInfo = vdex_006_dumpHeaderInfo;
env->dumpDepsInfo = vdex_006_dumpDepsInfo;
env->process = vdex_006_process;
} else if (vdex_010_isValidVdex(cursor)) {
LOGMSG(l_DEBUG, "Initializing environment for Vdex version '010'");
env->dumpHeaderInfo = vdex_010_dumpHeaderInfo;
env->dumpDepsInfo = vdex_010_dumpDepsInfo;
env->process = vdex_010_process;
} else {
dexHeader *pDexHeader = (dexHeader *)(cursor + *offset);

// Check boundaries
const u1 *dexBuf = cursor + *offset;
const u1 *dexBufMax = dexBuf + pDexHeader->fileSize;
if (dexBufMax == vdex_DexEnd(cursor)) {
LOGMSG(l_DEBUG, "Processing last Dex file at offset:0x%x", *offset);
} else if (dexBufMax <= vdex_DexEnd(cursor)) {
LOGMSG(l_DEBUG, "Processing Dex file at offset:0x%x", *offset);
} else {
LOGMSG(l_ERROR, "Invalid cursor offset '0x%x'", *offset);
return NULL;
}

// Adjust offset to point at the end of current Dex file
*offset += pDexHeader->fileSize;
return dexBuf;
}
}

u4 vdex_GetLocationChecksum(const u1 *cursor, u4 fileIdx) {
u4 *checksums = (u4 *)(cursor + sizeof(vdexHeader));
return checksums[fileIdx];
}

void vdex_SetLocationChecksum(const u1 *cursor, u4 fileIdx, u4 value) {
u4 *checksums = (u4 *)(cursor + sizeof(vdexHeader));
checksums[fileIdx] = value;
}

const u1 *vdex_GetVerifierDepsData(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return vdex_DexBegin(cursor) + pVdexHeader->dexSize;
}

u4 vdex_GetVerifierDepsDataOffset(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return vdex_DexBeginOffset(cursor) + pVdexHeader->dexSize;
}

u4 vdex_GetVerifierDepsDataSize(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return pVdexHeader->verifierDepsSize;
}

const u1 *vdex_GetQuickeningInfo(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return vdex_GetVerifierDepsData(cursor) + pVdexHeader->verifierDepsSize;
}

u4 vdex_GetQuickeningInfoOffset(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return vdex_GetVerifierDepsDataOffset(cursor) + pVdexHeader->verifierDepsSize;
}

u4 vdex_GetQuickeningInfoSize(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;
return pVdexHeader->quickeningInfoSize;
}

void vdex_dumpHeaderInfo(const u1 *cursor) {
const vdexHeader *pVdexHeader = (const vdexHeader *)cursor;

LOGMSG_RAW(l_DEBUG, "------ Vdex Header Info ------\n");
LOGMSG_RAW(l_DEBUG, "magic header & version : %.4s-%.4s\n", pVdexHeader->magic,
pVdexHeader->version);
LOGMSG_RAW(l_DEBUG, "number of dex files : %" PRIx32 " (%" PRIu32 ")\n",
pVdexHeader->numberOfDexFiles, pVdexHeader->numberOfDexFiles);
LOGMSG_RAW(l_DEBUG, "dex size (overall) : %" PRIx32 " (%" PRIu32 ")\n",
pVdexHeader->dexSize, pVdexHeader->dexSize);
LOGMSG_RAW(l_DEBUG, "verifier dependencies size : %" PRIx32 " (%" PRIu32 ")\n",
vdex_GetVerifierDepsDataSize(cursor), vdex_GetVerifierDepsDataSize(cursor));
LOGMSG_RAW(l_DEBUG, "verifier dependencies offset: %" PRIx32 " (%" PRIu32 ")\n",
vdex_GetVerifierDepsDataOffset(cursor), vdex_GetVerifierDepsDataOffset(cursor));
LOGMSG_RAW(l_DEBUG, "quickening info size : %" PRIx32 " (%" PRIu32 ")\n",
vdex_GetQuickeningInfoSize(cursor), vdex_GetQuickeningInfoSize(cursor));
LOGMSG_RAW(l_DEBUG, "quickening info offset : %" PRIx32 " (%" PRIu32 ")\n",
vdex_GetQuickeningInfoOffset(cursor), vdex_GetQuickeningInfoOffset(cursor));
LOGMSG_RAW(l_DEBUG, "dex files info :\n");

for (u4 i = 0; i < pVdexHeader->numberOfDexFiles; ++i) {
LOGMSG_RAW(l_DEBUG, " [%" PRIu32 "] location checksum : %" PRIx32 " (%" PRIu32 ")\n", i,
vdex_GetLocationChecksum(cursor, i), vdex_GetLocationChecksum(cursor, i));
LOGMSG(l_ERROR, "Unsupported Vdex version");
return false;
}
LOGMSG_RAW(l_DEBUG, "---- EOF Vdex Header Info ----\n");
}

int vdex_process(const char *VdexFileName, const u1 *cursor, const runArgs_t *pRunArgs) {
// Measure time spend to process all Dex files of a Vdex file
struct timespec timer;
utils_startTimer(&timer);

// Process Vdex file
int ret = (*processPtr)(VdexFileName, cursor, pRunArgs);

// Get elapsed time in ns
long timeSpend = utils_endTimer(&timer);
LOGMSG(l_DEBUG, "Took %ld ms to process Vdex file", timeSpend / 1000000);

return ret;
}

void *vdex_initDepsInfo(const u1 *vdexFileBuf) { return (*initDepsInfoPtr)(vdexFileBuf); }

void vdex_destroyDepsInfo(const void *dataPtr) { (*destroyDepsInfoPtr)(dataPtr); }

void vdex_dumpDepsInfo(const u1 *vdexFileBuf, const void *dataPtr) {
(*dumpDepsInfoPtr)(vdexFileBuf, dataPtr);
return true;
}

bool vdex_updateChecksums(const char *inVdexFileName,
Expand All @@ -242,20 +64,31 @@ bool vdex_updateChecksums(const char *inVdexFileName,
return ret;
}

if (!vdex_isValidVdex(buf)) {
LOGMSG(l_WARN, "'%s' is an invalid Vdex file", inVdexFileName);
goto fini;
}
if (vdex_006_isValidVdex(buf)) {
const vdexHeader_006 *pVdexHeader = (const vdexHeader_006 *)buf;
if ((u4)nCsums != pVdexHeader->numberOfDexFiles) {
LOGMSG(l_ERROR, "%d checksums loaded from file, although Vdex has %" PRIu32 " Dex entries",
nCsums, pVdexHeader->numberOfDexFiles)
goto fini;
}

const vdexHeader *pVdexHeader = (const vdexHeader *)buf;
if ((u4)nCsums != pVdexHeader->numberOfDexFiles) {
LOGMSG(l_ERROR, "%d checksums loaded from file, although Vdex has %" PRIu32 " Dex entries",
nCsums, pVdexHeader->numberOfDexFiles)
goto fini;
}
for (u4 i = 0; i < pVdexHeader->numberOfDexFiles; ++i) {
vdex_006_SetLocationChecksum(buf, i, checksums[i]);
}
} else if (vdex_010_isValidVdex(buf)) {
const vdexHeader_010 *pVdexHeader = (const vdexHeader_010 *)buf;
if ((u4)nCsums != pVdexHeader->numberOfDexFiles) {
LOGMSG(l_ERROR, "%d checksums loaded from file, although Vdex has %" PRIu32 " Dex entries",
nCsums, pVdexHeader->numberOfDexFiles)
goto fini;
}

for (u4 i = 0; i < pVdexHeader->numberOfDexFiles; ++i) {
vdex_SetLocationChecksum(buf, i, checksums[i]);
for (u4 i = 0; i < pVdexHeader->numberOfDexFiles; ++i) {
vdex_010_SetLocationChecksum(buf, i, checksums[i]);
}
} else {
LOGMSG(l_ERROR, "Unsupported Vdex version - updateChecksums failed");
goto fini;
}

if (!outWriter_VdexFile(pRunArgs, inVdexFileName, buf, fileSz)) {
Expand Down
Loading

4 comments on commit fe078c9

@frap129
Copy link

@frap129 frap129 commented on fe078c9 Aug 7, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The native oatdump implimentation released in P can handle this also, yes?

@anestisb
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@frap129 yes (https://android.googlesource.com/platform/art/+/android-9.0.0_r3/oatdump/oatdump.cc#611)

@frap129
Copy link

@frap129 frap129 commented on fe078c9 Aug 8, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like google isn't too set on maintaining that, as it doesnt work.
oatdump E 08-08 17:12:36 49256 49256 oatdump.cc:3106] Failed to open oat file from '/tmp/tmp.FtKDBekPj3/com.google.android.camera.experimental2017.vdex': Failed to find expected EI_CLASS value 1 or 2 in /tmp/tmp.FtKDBekPj3/com.google.android.camera.experimental2017.vdex, found 48
Looks like the header format got changed but no one informed oatdump. Not asking for support or anything since I can just use odex instead. just thought you might be interested to know.

@anestisb
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Damn it. The whole reason I bothered to upstream the oatdump patches is that they are maintained while ART evolves. Thanks for letting me know (saved some time from expected broken tools).

Please sign in to comment.