diff --git a/src/config/config.c b/src/config/config.c index 2ca05dff7d..443e338904 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -182,6 +182,9 @@ REQUIRE_OBJECT ( efi_image ); #ifdef IMAGE_SDI REQUIRE_OBJECT ( sdi ); #endif +#ifdef IMAGE_GZIP +REQUIRE_OBJECT ( gzip ); +#endif /* * Drag in all requested commands diff --git a/src/config/general.h b/src/config/general.h index 3c14a2cd06..18035f0363 100644 --- a/src/config/general.h +++ b/src/config/general.h @@ -116,6 +116,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #define IMAGE_PNG /* PNG image support */ #define IMAGE_DER /* DER image support */ #define IMAGE_PEM /* PEM image support */ +#define IMAGE_GZIP /* GZIP image support */ /* * Command-line commands to include diff --git a/src/crypto/deflate.c b/src/crypto/deflate.c index e1c87d5fe1..1ff42c4bbd 100644 --- a/src/crypto/deflate.c +++ b/src/crypto/deflate.c @@ -30,6 +30,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include #include #include +#include /** @file * @@ -385,6 +386,40 @@ static int deflate_extract ( struct deflate *deflate, struct deflate_chunk *in, return data; } +/** + * Attempt to get and extract a fixed number of bytes from input stream + * + * @v deflate Decompressor + * @v in Compressed input data + * @v len Number of bytes to extract + * @ret data Pointer to extracted data (or NULL if not available) + * + * No accumulated bits are allowed + */ +static void * deflate_extract_buffer ( struct deflate *deflate, struct deflate_chunk *in, + unsigned int len ) { + size_t offset, remaining; + + /* Sanity check */ + assert ( deflate->bits == 0 ); + + /* Return immediately if we are attempting to extract zero bytes */ + if ( len == 0 ) + return NULL; + + /* Attempt to get len bytes */ + offset = in->offset; + remaining = ( in->len - offset ); + if ( len > remaining ) + return NULL; + + in->offset += len; + + DBGCP ( deflate, "DEFLATE %p extracted %d bytes\n", deflate, len ); + + return user_to_virt ( in->data, offset ); +} + /** * Attempt to decode a Huffman-coded symbol from input stream * @@ -453,9 +488,9 @@ static void deflate_discard_to_byte ( struct deflate *deflate ) { * @v offset Starting offset within source data * @v len Length to copy */ -static void deflate_copy ( struct deflate_chunk *out, +static void deflate_copy ( struct deflate *deflate, struct deflate_chunk *out, userptr_t start, size_t offset, size_t len ) { - size_t out_offset = out->offset; + size_t in_offset = offset, out_offset = out->offset; size_t copy_len; /* Copy data one byte at a time, to allow for overlap */ @@ -465,10 +500,33 @@ static void deflate_copy ( struct deflate_chunk *out, copy_len = len; while ( copy_len-- ) { memcpy_user ( out->data, out_offset++, - start, offset++, 1 ); + start, in_offset++, 1 ); + } + } + + /* Copy data within gzip window */ + if ( deflate->window ) { + copy_len = len; + in_offset = offset; + out_offset = out->offset; + if ( out->data != start ){ + while ( copy_len --> 0 ) { + memcpy_user ( deflate->window, ( out_offset++ ) % GZIP_WSIZE, + start, in_offset++, 1 ); + } + deflate->checksum = crc32_le( deflate->checksum, + user_to_virt ( start, offset ), len ); + } else { + while ( copy_len --> 0 ) { + deflate->checksum = crc32_le( deflate->checksum, + user_to_virt ( deflate->window, in_offset % GZIP_WSIZE ), 1 ); + memcpy_user ( deflate->window, ( out_offset++ ) % GZIP_WSIZE, + deflate->window, ( in_offset++ ) % GZIP_WSIZE, 1 ); + } } } out->offset += len; + deflate->total_length += len; } /** @@ -501,6 +559,13 @@ int deflate_inflate ( struct deflate *deflate, } else switch ( deflate->format ) { case DEFLATE_RAW: goto block_header; case DEFLATE_ZLIB: goto zlib_header; + case DEFLATE_GZIP: + if ( deflate->window == UNULL ) { + DBGC ( deflate, "DEFLATE %p window buffer not available", deflate ); + return -EINVAL; + } + deflate->checksum = 0xffffffff; + goto gzip_header; default: assert ( 0 ); } @@ -532,6 +597,123 @@ int deflate_inflate ( struct deflate *deflate, goto block_header; } + gzip_header: { + uint8_t * header; + + /* Extract header */ + header = deflate_extract_buffer( deflate, in, GZIP_HEADER_BYTES ); + if ( header == NULL ) { + deflate->resume = &&gzip_header; + return 0; + } + + if ( header [0] != 0x1f || header [1] != 0x8b ) { + DBGC ( deflate, "DEFLATE %p invalid GZIP format\n", deflate ); + return -EINVAL; + } + + if ( header [2] != GZIP_HEADER_CM_DEFLATE ) { + DBGC ( deflate, "DEFLATE %p unsupported GZIP " + "compression method %d\n", deflate, header [2] ); + return -ENOTSUP; + } + + /* Save flags */ + deflate->header = header [3]; + + /* Process GZIP members */ + goto gzip_fextra_xlen; + } + + gzip_fextra_xlen: { + if ( deflate->header & GZIP_HEADER_FLG_FEXTRA ) { + uint8_t * xlen; + + /* Extract XLEN field */ + xlen = deflate_extract_buffer( deflate, in, GZIP_HEADER_XLEN_BYTES ); + if ( xlen == NULL ) { + deflate->resume = &&gzip_fextra_xlen; + return 0; + } + + deflate->remaining = xlen [0] | ( xlen [1] << 8 ); + } else { + /* Process FNAME */ + goto gzip_fname; + } + } + + gzip_fextra_data: { + size_t in_remaining; + size_t len; + + /* Calculate available amount of FEXTRA data */ + in_remaining = ( in->len - in->offset ); + len = deflate->remaining; + if ( len > in_remaining ) + len = in_remaining; + + /* Discard data from input buffer */ + in->offset += len; + deflate->remaining -= len; + + /* Finish processing if we are blocked */ + if ( deflate->remaining ) { + deflate->resume = &&gzip_fextra_data; + return 0; + } + + /* Otherwise, finish FEXTRA member */ + } + + + gzip_fname: { + if ( deflate->header & GZIP_HEADER_FLG_FNAME ) { + char * name; + + /* Extract FNAME member */ + do { + /* Extract one char of FNAME */ + name = deflate_extract_buffer( deflate, in, 1 ); + if ( name == NULL ) { + deflate->resume = &&gzip_fname; + return 0; + } + } while ( * name != '\0' ); + } + } + + gzip_fcomment: { + if ( deflate->header & GZIP_HEADER_FLG_FCOMMENT ) { + char * comment; + + /* Extract FCOMMENT member */ + do { + /* Extract char of FNAME */ + comment = deflate_extract_buffer( deflate, in, 1 ); + if ( comment == NULL ) { + deflate->resume = &&gzip_fcomment; + return 0; + } + } while ( * comment != '\0' ); + } + } + + gzip_fhcrc: { + if ( deflate->header & GZIP_HEADER_FLG_FHCRC ) { + uint8_t * fhcrc; + + /* Extract FHCRC member */ + fhcrc = deflate_extract_buffer( deflate, in, GZIP_HEADER_FHCRC_BYTES ); + if ( fhcrc == NULL ) { + deflate->resume = &&gzip_fhcrc; + return 0; + } + } + + /* Process first block header */ + } + block_header: { int header; int bfinal; @@ -617,7 +799,7 @@ int deflate_inflate ( struct deflate *deflate, len = in_remaining; /* Copy data to output buffer */ - deflate_copy ( out, in->data, in->offset, len ); + deflate_copy ( deflate, out, in->data, in->offset, len ); /* Consume data from input buffer */ in->offset += len; @@ -844,7 +1026,7 @@ int deflate_inflate ( struct deflate *deflate, DBGCP ( deflate, "DEFLATE %p literal %#02x " "('%c')\n", deflate, byte, ( isprint ( byte ) ? byte : '.' ) ); - deflate_copy ( out, virt_to_user ( &byte ), 0, + deflate_copy ( deflate, out, virt_to_user ( &byte ), 0, sizeof ( byte ) ); } else if ( code == DEFLATE_LITLEN_END ) { @@ -934,8 +1116,8 @@ int deflate_inflate ( struct deflate *deflate, } /* Copy data, allowing for overlap */ - deflate_copy ( out, out->data, ( out->offset - dup_distance ), - dup_len ); + deflate_copy ( deflate, out, out->data, + ( out->offset - dup_distance ), dup_len ); /* Process next literal/length symbol */ goto lzhuf_litlen; @@ -953,6 +1135,7 @@ int deflate_inflate ( struct deflate *deflate, switch ( deflate->format ) { case DEFLATE_RAW: goto finished; case DEFLATE_ZLIB: goto zlib_footer; + case DEFLATE_GZIP: goto gzip_footer; default: assert ( 0 ); } } @@ -982,6 +1165,48 @@ int deflate_inflate ( struct deflate *deflate, goto finished; } + gzip_footer: { + + /* Discard any bits up to the next byte boundary */ + deflate_discard_to_byte ( deflate ); + + /* Return any remaining bytes to the input */ + in->offset -= deflate->bits / 8; + + deflate->bits = 0; + deflate->checksum ^= 0xffffffff; + } + + gzip_crc32_isize: { + uint8_t * footer; + uint32_t crc32, isize; + + /* Extract footer */ + footer = deflate_extract_buffer( deflate, in, + GZIP_FOOTER_CRC32_BYTES + GZIP_FOOTER_ISIZE_BYTES ); + if ( footer == NULL ) { + deflate->resume = &&gzip_crc32_isize; + return 0; + } + + crc32 = footer [0] | ( footer [1] << 8 ) | ( footer [2] << 16 ) | ( footer [3] << 24 ); + if ( deflate->checksum != crc32 ) { + DBGCP ( deflate, "DEFLATE %p invalid GZIP CRC 0x%08x/0x%08x\n", + deflate, deflate->checksum, crc32 ); + return -EINVAL; + } + + isize = footer [4] | ( footer [5] << 8 ) | ( footer [6] << 16 ) | ( footer [7] << 24 ); + if ( deflate->total_length != isize ) { + DBGCP ( deflate, "DEFLATE %p invalid GZIP ISIZE 0x%08x/0x%08x\n", + deflate, deflate->checksum, crc32 ); + return -EINVAL; + } + + /* Finish processing */ + goto finished; + } + finished: { /* Mark as finished and terminate */ DBGCP ( deflate, "DEFLATE %p finished\n", deflate ); diff --git a/src/image/gzip.c b/src/image/gzip.c new file mode 100644 index 0000000000..55f82333bc --- /dev/null +++ b/src/image/gzip.c @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2020 Miao Wang . + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** + * @file + * + * gzip files + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * Unpack gzip image and execute it + * + * @v image Gzip image + * @ret rc Return status code + */ +static int gzip_exec ( struct image *image ) { + struct image *unzipped; + struct deflate_chunk in, out; + struct deflate deflate; + int rc = 0; + userptr_t gzip_window; + + /* Inflate into an empty chunk to calc the size after inflation */ + deflate_init ( &deflate, DEFLATE_GZIP ); + deflate.window = gzip_window = umalloc ( GZIP_WSIZE ); + if ( ! deflate.window ) { + DBGC ( image, "GZIP %s could not allocate the buffer for inflation\n", + image->name ); + rc = -ENOMEM; + goto err_wndalloc; + } + deflate_chunk_init ( &in, image->data, 0, image->len ); + deflate_chunk_init ( &out, UNULL, 0, 0 ); + printf ( "GZIP: trying to decompress: %s ", image->name ); + if ( ( rc = deflate_inflate ( &deflate, &in, &out ) ) != 0 ) { + printf ( "[failed]\n" ); + printf ( "GZIP: %s could not decompress: %s\n", + image->name, strerror ( rc ) ); + goto err_deflate; + } + if ( ! deflate_finished ( &deflate ) ) { + printf ( "[failed]\n" ); + printf ( "GZIP: %s unexpected EOF\n", image->name ); + rc = -EINVAL; + goto err_deflate; + } + printf ( "%zd bytes\n", out.offset ); + + /* Allocate the new image */ + unzipped = alloc_image( image->uri ); + if ( ! unzipped ){ + DBGC ( image, "GZIP %s could not allocate the new image\n", + image->name ); + rc = -ENOMEM; + goto err_alloc_img; + } + + /* Construct the new image */ + if ( ( rc = image_set_name ( unzipped, image->name ) ) != 0 ) { + DBGC ( image, "GZIP %s could not name the new image\n", + image->name ); + goto err_copy_prop; + }; + if ( ( rc = image_set_cmdline ( unzipped, image->cmdline ) ) != 0 ) { + DBGC ( image, "GZIP %s could not set cmdline for the new image\n", + image->name ); + goto err_copy_prop; + }; + /* Allocate buffer */ + unzipped->data = umalloc ( out.offset ); + if ( ! unzipped->data ) { + DBGC ( image, "GZIP %s could not allocate data buffer\n", + image->name ); + rc = -ENOMEM; + goto err_alloc_buf; + } + unzipped->len = out.offset; + + /* Decompress */ + printf ( "GZIP: decompressing: %s ", image->name ); + deflate_init ( &deflate, DEFLATE_GZIP ); + deflate.window = gzip_window; + deflate_chunk_init ( &in, image->data, 0, image->len); + deflate_chunk_init ( &out, unzipped->data, 0, unzipped->len); + /* Since we've already inflated, it cannot fail */ + rc = deflate_inflate ( &deflate, &in, &out ); + assert ( rc == 0 ); + assert ( deflate_finished ( &deflate ) ); + assert ( out.offset == unzipped->len ); + printf ( "[ok]\n" ); + + if ( ( rc = register_image ( unzipped ) ) != 0 ) { + DBGC ( image, "GZIP %s could not register the new image\n", + image->name ); + goto err_reg; + } + + unregister_image ( image ); + + if ( ( rc = image_replace ( unzipped ) ) != 0 ) { + DBGC ( image, "GZIP %s could not replace with the new image\n", + image->name ); + goto err_replace; + } + + /* Success */ + rc = 0; + + err_replace: + err_reg: + err_alloc_buf: + err_copy_prop: + image_put ( unzipped ); + err_alloc_img: + err_deflate: + ufree ( deflate.window ); + err_wndalloc: + return rc; +} + +/** + * Probe gzip image + * + * @v image Gzip image + * @ret rc Return status code + */ +static int gzip_probe ( struct image *image ) { + uint8_t gzip_header[GZIP_HEADER_BYTES]; + + /* Sanity check */ + if ( image->len < sizeof ( gzip_header ) ) { + DBGC ( image, "GZIP %s is too short\n", image->name ); + return -ENOEXEC; + } + + /* Extract header */ + copy_from_user ( gzip_header, image->data, 0, sizeof ( gzip_header ) ); + + if ( gzip_header [0] != 0x1f || gzip_header [1] != 0x8b ) { + DBGC ( image, "GZIP %s invalid GZIP format\n", image->name ); + return -ENOEXEC; + } + if ( gzip_header [2] != GZIP_HEADER_CM_DEFLATE ) { + DBGC ( image, "GZIP %s unsupported GZIP " + "compression method %d\n", image->name, gzip_header [2] ); + return -ENOTSUP; + } + + return 0; +} + +/** Gzip image type */ +struct image_type gzip_image_type __image_type ( PROBE_NORMAL ) = { + .name = "gzip", + .probe = gzip_probe, + .exec = gzip_exec, +}; + diff --git a/src/include/ipxe/deflate.h b/src/include/ipxe/deflate.h index b751aa9a37..1550be583a 100644 --- a/src/include/ipxe/deflate.h +++ b/src/include/ipxe/deflate.h @@ -19,6 +19,8 @@ enum deflate_format { DEFLATE_RAW, /** ZLIB header and footer */ DEFLATE_ZLIB, + /** GZIP header and footer */ + DEFLATE_GZIP, }; /** Block header length (in bits) */ @@ -111,6 +113,34 @@ enum deflate_format { /** ZLIB ADLER32 length (in bits) */ #define ZLIB_ADLER32_BITS 32 +/** GZIP header length (in bytes) */ +#define GZIP_HEADER_BYTES 10 + +/** GZIP header compression method: DEFLATE */ +#define GZIP_HEADER_CM_DEFLATE 8 + +/** GZIP header flags */ +#define GZIP_HEADER_FLG_FTEXT 0x01 +#define GZIP_HEADER_FLG_FHCRC 0x02 +#define GZIP_HEADER_FLG_FEXTRA 0x04 +#define GZIP_HEADER_FLG_FNAME 0x08 +#define GZIP_HEADER_FLG_FCOMMENT 0x10 + +/** GZIP header XLEN bytes */ +#define GZIP_HEADER_XLEN_BYTES 2 + +/** GZIP header FHCRC bytes */ +#define GZIP_HEADER_FHCRC_BYTES 2 + +/** GZIP footer CRC32 bytes */ +#define GZIP_FOOTER_CRC32_BYTES 4 + +/** GZIP footer ISIZE bytes */ +#define GZIP_FOOTER_ISIZE_BYTES 4 + +/** GZIP Window size--must be a power of two, and at least 32K */ +#define GZIP_WSIZE 0x8000 + /** A Huffman-coded set of symbols of a given length */ struct deflate_huf_symbols { /** Length of Huffman-coded symbols */ @@ -235,6 +265,19 @@ struct deflate { uint8_t lengths[ ( ( DEFLATE_LITLEN_MAX_CODE + 1 ) + ( DEFLATE_DISTANCE_MAX_CODE + 1 ) + 1 /* round up */ ) / 2 ]; + + /** ZLIB/GZIP checksum */ + uint32_t checksum; + + /** Total inflated length */ + unsigned int total_length; + + /** Window buffer needed for gzip + * This should be allocated by the caller in the length of + * GZIP_WSIZE before calling deflate_inflate () when + * decompressing GZIP streams. + */ + userptr_t window; }; /** A chunk of data */ diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h index 242f91f820..42334f0178 100644 --- a/src/include/ipxe/errfile.h +++ b/src/include/ipxe/errfile.h @@ -295,6 +295,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #define ERRFILE_png ( ERRFILE_IMAGE | 0x00070000 ) #define ERRFILE_der ( ERRFILE_IMAGE | 0x00080000 ) #define ERRFILE_pem ( ERRFILE_IMAGE | 0x00090000 ) +#define ERRFILE_gzip ( ERRFILE_IMAGE | 0x000a0000 ) #define ERRFILE_asn1 ( ERRFILE_OTHER | 0x00000000 ) #define ERRFILE_chap ( ERRFILE_OTHER | 0x00010000 ) diff --git a/src/tests/deflate_test.c b/src/tests/deflate_test.c index 20ff5b9a2b..c62c8d225e 100644 --- a/src/tests/deflate_test.c +++ b/src/tests/deflate_test.c @@ -133,6 +133,21 @@ DEFLATE ( zlib, DEFLATE_ZLIB, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e ) ); +/* "GZIP file format specification version 4.3" */ +DEFLATE ( gzip, DEFLATE_GZIP, + DATA ( 0x1f, 0x8b, 0x08, 0x08, 0x72, 0x4b, 0x54, 0x5c, 0x02, 0x0b, + 0x67, 0x7a, 0x69, 0x70, 0x2d, 0x74, 0x65, 0x73, 0x74, 0x00, + 0x73, 0x8f, 0xf2, 0x0c, 0x50, 0x48, 0xcb, 0xcc, 0x49, 0x55, + 0x48, 0xcb, 0x2f, 0xca, 0x4d, 0x2c, 0x51, 0x28, 0x2e, 0x48, + 0x4d, 0xce, 0x4c, 0xcb, 0x4c, 0x4e, 0x2c, 0xc9, 0xcc, 0xcf, + 0x53, 0x28, 0x4b, 0x2d, 0x2a, 0x06, 0xd1, 0x26, 0x7a, 0xc6, + 0x00, 0xde, 0x2b, 0xcf, 0xca, 0x2a, 0x00, 0x00, 0x00 ), + DATA ( 0x47, 0x5a, 0x49, 0x50, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x20, + 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x73, 0x70, 0x65, + 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x34, + 0x2e, 0x33 ) ); + /* "ZLIB Compressed Data Format Specification" fragment list */ static struct deflate_test_fragments zlib_fragments[] = { { { -1UL, } }, @@ -163,9 +178,11 @@ static void deflate_okx ( struct deflate *deflate, size_t offset = 0; size_t remaining = test->compressed_len; unsigned int i; + uint8_t gzip_window[ GZIP_WSIZE ]; /* Initialise decompressor */ deflate_init ( deflate, test->format ); + deflate->window = virt_to_user ( gzip_window ); /* Initialise output chunk */ deflate_chunk_init ( &out, virt_to_user ( data ), 0, sizeof ( data ) ); @@ -231,6 +248,7 @@ static void deflate_test_exec ( void ) { deflate_ok ( deflate, &hello_hello_world, NULL ); deflate_ok ( deflate, &rfc_sentence, NULL ); deflate_ok ( deflate, &zlib, NULL ); + deflate_ok ( deflate, &gzip, NULL ); /* Test fragmentation */ for ( i = 0 ; i < ( sizeof ( zlib_fragments ) /