Snappy module error when decompressing memory dump with avml-convert

[ekt0-syn]

AVML version: 0.17.0

When I try to decompress a memory dump with avml-convert, the following error is produced:

Error: error: unable to create snapshot
caused by:
    0: io error: unable to copy compressed data
    1: snappy: corrupt input (expected stream header but got unexpected chunk type byte 65)

The output dump is around 1GB or 2GB depending on serveral tests I performed. Steps to reproduce:

$ sudo ./avml --compress test.avml
$ sudo ./avml-convert test.avml test.lime
Error: error: unable to create snapshot
caused by:
    0: io error: unable to copy compressed data
    1: snappy: corrupt input (expected stream header but got unexpected chunk type byte 65)
$ ls -l test*
-rw------- 1 root root 17106358360 15 mars  12:53 test.avml
-rw------- 1 root root  1057613855 15 mars  12:53 test.lime

Tested both AVML commands on Debian Trixie and Ubuntu 24.04.

[demoray]

I can replicate this locally. Investigating.

[demoray]

I've committed a fix for generating new snapshots that does not have this issue. We'll be publishing a new release with this fix included soon.

In the meantime, the following should enable you to fix images that were created with this issue.

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#![deny(clippy::unwrap_used)]
#![deny(clippy::expect_used)]
#![deny(clippy::panic)]
#![deny(clippy::manual_assert)]
#![deny(clippy::indexing_slicing)]

use avml::image::{self, Header, Image, MAX_BLOCK_SIZE};
use clap::Parser;
use core::{
    error::Error as StdError,
    fmt::{Debug as FmtDebug, Formatter, Result as FmtResult},
};
use snap::read::FrameDecoder;
use std::{
    fs::{File, metadata},
    io::{Cursor, Read, Seek, SeekFrom, Write, copy},
    path::{Path, PathBuf},
};

const HEADER_LEN: usize = 32;
const COMPRESSED_TRAILER_LEN: i64 = 8;

#[derive(Parser)]
/// Convert malformed AVML/LiME images with broken null blocks to a valid LiME image
#[command(author, version, about, long_about = None)]
struct Config {
    /// Source AVML/LiME image with broken null blocks
    src: PathBuf,

    /// Fixed LiME output file
    dst: PathBuf,
}

#[derive(thiserror::Error)]
enum Error {
    #[error(transparent)]
    Image(#[from] image::Error),

    #[error("io error: {1}")]
    Io(#[source] std::io::Error, &'static str),

    #[error("unsupported block version: {0}")]
    UnsupportedBlockVersion(u32),
}

impl FmtDebug for Error {
    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
        write!(f, "error: {self}")?;

        let mut source = self.source();
        if self.source().is_some() {
            writeln!(f, "\ncaused by:")?;
            let mut i = 0usize;
            while let Some(inner) = source {
                writeln!(f, "{i: >5}: {inner}")?;
                source = inner.source();
                i = i.saturating_add(1);
            }
        }

        Ok(())
    }
}

type Result<T> = core::result::Result<T, Error>;

fn convert(src: &Path, dst: &Path) -> Result<()> {
    let src_len = metadata(src)
        .map_err(|e| Error::Io(e, "unable to read source size"))?
        .len();
    let mut image = Image::<File, File>::new(1, src, dst)?;
    convert_image_with_broken_null_blocks_to_lime(&mut image, src_len)
}

fn range_len(range: &core::ops::Range<u64>) -> u64 {
    range.end.saturating_sub(range.start)
}

fn looks_like_broken_null_block<R: Read + Seek>(src: &mut R, header: &Header) -> Result<bool> {
    if range_len(&header.range) > MAX_BLOCK_SIZE {
        return Ok(false);
    }

    let payload_offset = src
        .stream_position()
        .map_err(|e| Error::Io(e, "unable to get payload offset"))?;
    let end = src
        .seek(SeekFrom::End(0))
        .map_err(|e| Error::Io(e, "unable to get source size"))?;
    src.seek(SeekFrom::Start(payload_offset))
        .map_err(|e| Error::Io(e, "unable to restore payload offset"))?;

    if payload_offset == end {
        return Ok(true);
    }
    if end.saturating_sub(payload_offset) < HEADER_LEN as u64 {
        return Ok(false);
    }

    let mut bytes = [0; HEADER_LEN];
    src.read_exact(&mut bytes)
        .map_err(|e| Error::Io(e, "unable to inspect following block"))?;
    src.seek(SeekFrom::Start(payload_offset))
        .map_err(|e| Error::Io(e, "unable to restore payload offset"))?;

    Ok(match Header::read(Cursor::new(bytes)) {
        Ok(next) => next.version == header.version && next.range.start >= header.range.end,
        Err(_) => false,
    })
}

fn copy_compressed_block<R: Read + Seek, W: Write>(
    src: &mut R,
    dst: &mut W,
    len: u64,
) -> Result<()> {
    let mut decoder = FrameDecoder::new(&mut *src).take(len);
    copy(&mut decoder, dst).map_err(|e| Error::Io(e, "unable to decompress snappy block"))?;
    src.seek(SeekFrom::Current(COMPRESSED_TRAILER_LEN))
        .map_err(|e| Error::Io(e, "unable to skip compressed length trailer"))?;
    Ok(())
}

fn write_lime_header<W: Write>(dst: &mut W, range: core::ops::Range<u64>) -> Result<()> {
    Header { range, version: 1 }.write(dst)?;
    Ok(())
}

fn convert_image_with_broken_null_blocks_to_lime<R, W>(
    image: &mut image::Image<R, W>,
    src_len: u64,
) -> Result<()>
where
    R: Read + Seek,
    W: Write,
{
    image.version = 1;

    loop {
        let current = image
            .src
            .stream_position()
            .map_err(|e| Error::Io(e, "unable to get current input offset"))?;
        if current >= src_len {
            break;
        }

        let header = image.read_header()?;

        // Malformed images with broken null blocks wrote a block header before
        // deciding an all-zero range could be skipped, so the next bytes are
        // another header or EOF.
        if looks_like_broken_null_block(&mut image.src, &header)? {
            continue;
        }

        match header.version {
            1 => image.copy_block(header.range)?,
            2 => {
                write_lime_header(&mut image.dst, header.range.clone())?;
                copy_compressed_block(&mut image.src, &mut image.dst, range_len(&header.range))?;
            }
            other => return Err(Error::UnsupportedBlockVersion(other)),
        }
    }

    Ok(())
}

fn main() -> Result<()> {
    let config = Config::parse();
    convert(&config.src, &config.dst)
}

#[cfg(test)]
mod tests {
    use crate::{Error, Result, convert_image_with_broken_null_blocks_to_lime};
    use avml::{
        image::{Header, Image},
        io::snappy::SnapCountWriter,
    };
    use std::{
        io::{Cursor, Write},
        ops::Range,
    };

    fn append_header(bytes: &mut Vec<u8>, range: Range<u64>, version: u32) -> Result<()> {
        Header { range, version }.write(&mut *bytes)?;
        Ok(())
    }

    fn append_raw_block(bytes: &mut Vec<u8>, range: Range<u64>, data: &[u8]) -> Result<()> {
        append_header(bytes, range, 1)?;
        bytes
            .write_all(data)
            .map_err(|e| Error::Io(e, "unable to write test raw block"))?;
        Ok(())
    }

    fn append_compressed_block(bytes: &mut Vec<u8>, range: Range<u64>, data: &[u8]) -> Result<()> {
        append_header(bytes, range, 2)?;
        let mut writer = SnapCountWriter::new(&mut *bytes);
        writer
            .write_all(data)
            .map_err(|e| Error::Io(e, "unable to write test compressed block"))?;
        writer
            .finalize()
            .map_err(|e| Error::Io(e, "unable to finalize test compressed block"))?;
        Ok(())
    }

    #[test]
    fn converts_broken_null_compressed_blocks_to_sparse_lime_output() -> Result<()> {
        let block = vec![0x5a; 0x1000];
        let mut input = Vec::new();
        append_header(&mut input, 0..0x1000, 2)?;
        append_compressed_block(&mut input, 0x1000..0x2000, &block)?;
        append_header(&mut input, 0x2000..0x3000, 2)?;

        let src_len = input.len() as u64;
        let mut image = Image {
            version: 1,
            align_src: false,
            src: Cursor::new(input),
            dst: Cursor::new(Vec::new()),
        };

        convert_image_with_broken_null_blocks_to_lime(&mut image, src_len)?;

        let output = image.dst.into_inner();
        let mut expected = Vec::new();
        append_raw_block(&mut expected, 0x1000..0x2000, &block)?;

        assert_eq!(output, expected);
        Ok(())
    }

    #[test]
    fn converts_broken_null_raw_blocks_to_sparse_lime_output() -> Result<()> {
        let block = vec![0xa5; 0x800];
        let mut input = Vec::new();
        append_header(&mut input, 0..0x1000, 1)?;
        append_raw_block(&mut input, 0x1000..0x1800, &block)?;
        append_header(&mut input, 0x1800..0x2000, 1)?;

        let src_len = input.len() as u64;
        let mut image = Image {
            version: 1,
            align_src: false,
            src: Cursor::new(input),
            dst: Cursor::new(Vec::new()),
        };

        convert_image_with_broken_null_blocks_to_lime(&mut image, src_len)?;

        let output = image.dst.into_inner();
        let mut expected = Vec::new();
        append_raw_block(&mut expected, 0x1000..0x1800, &block)?;

        assert_eq!(output, expected);
        Ok(())
    }
}