Skip to content
Permalink
Browse files

Fixed many bugs in the c++ implementation. (#45)

Bugs were introduced with the recent standards patch.

* The standard renamed the old AFF4_IMAGE_TYPE to AFF4_IMAGESTREAM_TYPE so the code registered the wrong implementation.
* Bevy indexes were not written in the standard format (offset, length).

We still need to implement the new AFF4_IMAGE_TYPE as a proxy to another stream via the dataStream attribute.
  • Loading branch information...
scudette committed Dec 25, 2017
1 parent 2cbfe9d commit b88446e99b4e19219519bedb4f5bb66562f60aac
Showing with 970 additions and 907 deletions.
  1. +2 −1 pyaff4/pyaff4/zip.py
  2. +74 −57 src/aff4_image.cc
  3. +1 −0 src/aff4_imager_utils.cc
  4. +16 −27 src/aff4_map.cc
  5. +0 −5 src/aff4_registry.h
  6. +686 −649 src/data_store.cc
  7. +47 −45 src/data_store.h
  8. +9 −2 src/lexicon.h
  9. +70 −70 src/libaff4-c.cc
  10. +65 −51 src/zip.cc
@@ -156,7 +156,8 @@ def empty(self):
return [] == [d for _, _, d in self.fields[2:] if d is not None]

def Pack(self):
self.data_size = self.sizeof()
# Size of extra less the header.
self.Set("data_size", self.sizeof() - 4)
return struct.pack(self.format_string(),
*[v for t, _, v in self.fields if v is not None])

@@ -31,7 +31,7 @@ AFF4ScopedPtr<AFF4Image> AFF4Image::NewAFF4Image(
// Inform the volume that we have a new image stream contained within it.
volume->children.insert(image_urn.SerializeToString());

resolver->Set(image_urn, AFF4_TYPE, new URN(AFF4_IMAGE_TYPE));
resolver->Set(image_urn, AFF4_TYPE, new URN(AFF4_IMAGESTREAM_TYPE));
resolver->Set(image_urn, AFF4_STORED, new URN(volume_urn));

// We need to use the resolver here instead of just making a new object, in
@@ -50,46 +50,45 @@ AFF4ScopedPtr<AFF4Image> AFF4Image::NewAFF4Image(
*/
AFF4Status AFF4Image::LoadFromURN() {
if (resolver->Get(urn, AFF4_STORED, volume_urn) != STATUS_OK) {
if (resolver->Get(urn, AFF4_LEGACY_STORED, volume_urn) != STATUS_OK) {
return NOT_FOUND;
}
if (resolver->Get(urn, AFF4_LEGACY_STORED, volume_urn) != STATUS_OK) {
return NOT_FOUND;
}
}

// Determine if this is an AFF4:ImageStream (AFF4 Standard) or
// a aff4:stream (AFF4 Legacy)
URN rdfType (AFF4_LEGACY_IMAGESTREAM_TYPE);
isAFF4Legacy = (resolver->Has(urn, AFF4_TYPE, rdfType) == STATUS_OK);


// Configure the stream parameters.
XSDInteger value;

if(!isAFF4Legacy){
// AFF4 Standard
if (resolver->Get(urn, AFF4_STREAM_CHUNK_SIZE, value) == STATUS_OK) {
chunk_size = value.value;
}

if (resolver->Get(urn, AFF4_STREAM_CHUNKS_PER_SEGMENT, value) == STATUS_OK) {
chunks_per_segment = value.value;
}

if (resolver->Get(urn, AFF4_STREAM_SIZE, value) == STATUS_OK) {
size = value.value;
}
// AFF4 Standard
if (resolver->Get(urn, AFF4_STREAM_CHUNK_SIZE, value) == STATUS_OK) {
chunk_size = value.value;
}

if (resolver->Get(urn, AFF4_STREAM_CHUNKS_PER_SEGMENT, value) == STATUS_OK) {
chunks_per_segment = value.value;
}

if (resolver->Get(urn, AFF4_STREAM_SIZE, value) == STATUS_OK) {
size = value.value;
}
} else {
// AFF4 Legacy
if (resolver->Get(urn, AFF4_LEGACY_STREAM_CHUNK_SIZE, value) == STATUS_OK) {
chunk_size = value.value;
}
// AFF4 Legacy
if (resolver->Get(urn, AFF4_LEGACY_STREAM_CHUNK_SIZE, value) == STATUS_OK) {
chunk_size = value.value;
}

if (resolver->Get(urn, AFF4_LEGACY_STREAM_CHUNKS_PER_SEGMENT, value) == STATUS_OK) {
chunks_per_segment = value.value;
}
if (resolver->Get(urn, AFF4_LEGACY_STREAM_CHUNKS_PER_SEGMENT, value) == STATUS_OK) {
chunks_per_segment = value.value;
}

if (resolver->Get(urn, AFF4_LEGACY_STREAM_SIZE, value) == STATUS_OK) {
size = value.value;
}
if (resolver->Get(urn, AFF4_LEGACY_STREAM_SIZE, value) == STATUS_OK) {
size = value.value;
}
}

// Load the compression scheme. If it is not set we just default to ZLIB.
@@ -185,7 +184,9 @@ AFF4Status AFF4Image::_FlushBevy() {
* @return Status.
*/
AFF4Status AFF4Image::FlushChunk(const char* data, size_t length) {
uint32_t bevy_offset = bevy.Tell();
uint64_t bevy_offset = bevy.Tell();
uint32_t bevy_chunk_length = 0;

std::string output;

AFF4Status result;
@@ -211,11 +212,18 @@ AFF4Status AFF4Image::FlushChunk(const char* data, size_t length) {
return IO_ERROR;
}

// The index contains the offset and the length of each chunk.
if (bevy_index.Write(
reinterpret_cast<char*>(&bevy_offset), sizeof(bevy_offset)) < 0) {
return IO_ERROR;
}

bevy_chunk_length = output.size();
if (bevy_index.Write(
reinterpret_cast<char*>(&bevy_chunk_length), sizeof(bevy_chunk_length)) < 0) {
return IO_ERROR;
}

if (bevy.Write(output) < 0) {
return IO_ERROR;
}
@@ -313,7 +321,8 @@ class _CompressorStream: public AFF4Stream {
public:
StringIO bevy_index;
uint32_t chunk_count_in_bevy = 0;
uint32_t bevy_length = 0;
uint64_t bevy_length = 0;
uint32_t chunk_length = 0;

_CompressorStream(DataStore* resolver, AFF4Image* owner, AFF4Stream* stream):
AFF4Stream(resolver), stream(stream), owner(owner) {}
@@ -366,7 +375,13 @@ class _CompressorStream: public AFF4Stream {

if (result == STATUS_OK) {
if (bevy_index.Write(
reinterpret_cast<char*>(&bevy_length), sizeof(bevy_length)) < 0) {
reinterpret_cast<char*>(&bevy_length), sizeof(bevy_length)) < 0) {
return "";
}

chunk_length = output.size();
if (bevy_index.Write(
reinterpret_cast<char*>(&chunk_length), sizeof(chunk_length)) < 0) {
return "";
}

@@ -553,33 +568,33 @@ int AFF4Image::_ReadPartial(unsigned int chunk_id, int chunks_to_read,
URN bevy_index_urn = (!isAFF4Legacy) ? bevy_urn.value + (".index") : bevy_urn.value + ("/index");

AFF4ScopedPtr<AFF4Stream> bevy_index = resolver->AFF4FactoryOpen
<AFF4Stream>(bevy_index_urn);
<AFF4Stream>(bevy_index_urn);

AFF4ScopedPtr<AFF4Stream> bevy = resolver->AFF4FactoryOpen<AFF4Stream>(
bevy_urn);
bevy_urn);

if (!bevy_index || !bevy) {
LOG(ERROR) << "Unable to open bevy " <<
bevy_urn.SerializeToString();
bevy_urn.SerializeToString();
return -1;
}

uint32_t index_size = bevy_index->Size() / sizeof(BevvyIndex);
std::string bevy_index_data = bevy_index->Read(bevy_index->Size());

if (isAFF4Legacy) {
// Massage the bevvy data format from the old into the new.
bevy_index_data = _FixupBevvyData(&bevy_index_data);
index_size = bevy_index->Size() / sizeof(uint32_t);
}
if (isAFF4Legacy) {
// Massage the bevvy data format from the old into the new.
bevy_index_data = _FixupBevvyData(&bevy_index_data);
index_size = bevy_index->Size() / sizeof(uint32_t);
}

BevvyIndex* bevy_index_array = reinterpret_cast<BevvyIndex*>(
const_cast<char*>(bevy_index_data.data()));
const_cast<char*>(bevy_index_data.data()));

while (chunks_to_read > 0) {
// Read a full chunk from the bevy.
AFF4Status res = _ReadChunkFromBevy(
result, chunk_id, bevy, bevy_index_array, index_size);
result, chunk_id, bevy, bevy_index_array, index_size);

if (res != STATUS_OK) {
return res;
@@ -686,21 +701,23 @@ static AFF4Registrar<AFF4Image> r2(AFF4_LEGACY_IMAGESTREAM_TYPE);
void aff4_image_init() {}

std::string AFF4Image::_FixupBevvyData(std::string* data){
uint32_t index_size = data->length() / sizeof(uint32_t);
BevvyIndex* bevy_index_array = new BevvyIndex[index_size];
uint32_t* bevy_index_data = reinterpret_cast<uint32_t*>(const_cast<char*>(data->data()));

uint64_t cOffset = 0;
uint32_t cLength = 0;

for(off_t offset = 0; offset < index_size; offset++){
cLength = bevy_index_data[offset];
bevy_index_array[offset].offset = cOffset;
bevy_index_array[offset].length = cLength - cOffset;
cOffset += bevy_index_array[offset].length;
}

std::string result = std::string(reinterpret_cast<char*>(bevy_index_array), index_size * sizeof(BevvyIndex));
delete[] bevy_index_array;
return result;
uint32_t index_size = data->length() / sizeof(uint32_t);
BevvyIndex* bevy_index_array = new BevvyIndex[index_size];
uint32_t* bevy_index_data = reinterpret_cast<uint32_t*>(const_cast<char*>(data->data()));

uint64_t cOffset = 0;
uint32_t cLength = 0;

for(off_t offset = 0; offset < index_size; offset++){
cLength = bevy_index_data[offset];
bevy_index_array[offset].offset = cOffset;
bevy_index_array[offset].length = cLength - cOffset;
cOffset += bevy_index_array[offset].length;
}

std::string result = std::string(
reinterpret_cast<char*>(bevy_index_array),
index_size * sizeof(BevvyIndex));
delete[] bevy_index_array;
return result;
}
@@ -9,6 +9,7 @@

//using std::cout;


// This will be flipped by the signal handler.
AFF4Status ImageStream(DataStore& resolver, std::vector<URN>& input_urns,
URN output_urn,
@@ -47,29 +47,11 @@ AFF4ScopedPtr<AFF4Map> AFF4Map::NewAFF4Map(


AFF4Status AFF4Map::LoadFromURN() {
/*
* We could be type aff4:Image and/or aff4:Map. If we don't have aff4:Map as a type, then
* we are an aff4:Image and need to locate the corresponding aff4:Map via aff4:dataStream.
*/
URN rdfType (AFF4_MAP_TYPE);
URN map_urn = urn;
if (resolver->Has(urn, AFF4_TYPE, rdfType) != STATUS_OK) {
rdfType = URN(AFF4_LEGACY_MAP_TYPE);
if (resolver->Has(urn, AFF4_TYPE, rdfType) != STATUS_OK) {
URN dataStream;
if (resolver->Get(urn, AFF4_DATASTREAM, dataStream) != STATUS_OK ){
LOG(ERROR) << "Image URN doesn't contain reference to dataStream ";
return NOT_FOUND;
}
map_urn = dataStream;
}
}

URN map_stream_urn = map_urn.Append("map");
URN map_idx_urn = map_urn.Append("idx");
URN map_stream_urn = urn.Append("map");
URN map_idx_urn = urn.Append("idx");

AFF4ScopedPtr<AFF4Stream> map_idx = resolver->AFF4FactoryOpen<AFF4Stream>(
map_idx_urn);
map_idx_urn);

// Parse the map out of the map stream. If the stream does not exist yet we
// just start with an empty map.
@@ -78,10 +60,10 @@ AFF4Status AFF4Map::LoadFromURN() {
std::istringstream f(map_idx->Read(map_idx->Size()));
std::string line;
while (std::getline(f, line)) {
// deal with CRLF EOL. (hack).
if(*line.rbegin() == '\r'){
line.erase(line.length()-1, 1);
}
// deal with CRLF EOL. (hack).
if(*line.rbegin() == '\r'){
line.erase(line.length()-1, 1);
}
target_idx_map[line] = targets.size();
targets.push_back(line);
}
@@ -93,7 +75,7 @@ AFF4Status AFF4Map::LoadFromURN() {
// Clear the map so we start with a fresh map.
Clear();
} else {
// Note: The legacy AFF4 version, and AFF4 Standard use the same map format.
// Note: The legacy AFF4 version, and AFF4 Standard use the same map format.
BinaryRange binary_range;
while (map_stream->ReadIntoBuffer(
&binary_range, sizeof(binary_range)) == sizeof(binary_range)) {
@@ -697,7 +679,14 @@ AFF4Status AFF4Map::WriteStream(AFF4Map* source, ProgressContext* progress) {
static AFF4Registrar<AFF4Map> map1(AFF4_MAP_TYPE);
static AFF4Registrar<AFF4Map> map2(AFF4_LEGACY_MAP_TYPE);
// AFF4 Standard
static AFF4Registrar<AFF4Map> image1(AFF4_IMAGE_TYPE);

// In the AFF4 Standard, AFF4_IMAGE_TYPE is an abstract concept which
// delegates the data stream implementation to some other AFF4 stream
// (image or map) via the DataStore attribute.

// TODO: Implement this kind of object for reading.

//static AFF4Registrar<AFF4Map> image1(AFF4_IMAGE_TYPE);
static AFF4Registrar<AFF4Map> image2(AFF4_DISK_IMAGE_TYPE);
static AFF4Registrar<AFF4Map> image3(AFF4_VOLUME_IMAGE_TYPE);
static AFF4Registrar<AFF4Map> image4(AFF4_MEMORY_IMAGE_TYPE);
@@ -14,11 +14,6 @@
#include <iostream>
#include <functional>

//using std::string;
//using std::unique_ptr;
//using std::unordered_map;
//using std::function;

class DataStore;
class URN;

0 comments on commit b88446e

Please sign in to comment.
You can’t perform that action at this time.