Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

archive-zip: streaming for deflated files

After an entry has been streamed out, its CRC and sizes are written as
part of a data descriptor.

For simplicity, we make the buffer for the compressed chunks twice as
big as for the uncompressed ones, to be sure the result fit in even
if deflate makes them bigger.

t5000 verifies output. t1050 makes sure the command always respects
core.bigfilethreshold

Signed-off-by: Rene Scharfe <rene.scharfe@lsrfire.ath.cx>
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
  • Loading branch information...
commit c743c21591f9433fe784ac38902872701ce2e850 1 parent 2158f88
René Scharfe authored gitster committed

Showing 3 changed files with 74 additions and 1 deletion. Show diff stats Hide diff stats

  1. +63 1 archive-zip.c
  2. +4 0 t/t1050-large.sh
  3. +7 0 t/t5000-tar-tree.sh
64 archive-zip.c
@@ -211,7 +211,7 @@ static int write_zip_entry(struct archiver_args *args,
211 211 compressed_size = size;
212 212
213 213 if (S_ISREG(mode) && type == OBJ_BLOB && !args->convert &&
214   - size > big_file_threshold && method == 0) {
  214 + size > big_file_threshold) {
215 215 stream = open_istream(sha1, &type, &size, NULL);
216 216 if (!stream)
217 217 return error("cannot stream blob %s",
@@ -308,6 +308,68 @@ static int write_zip_entry(struct archiver_args *args,
308 308 zip_offset += ZIP_DATA_DESC_SIZE;
309 309
310 310 set_zip_dir_data_desc(&dirent, size, compressed_size, crc);
  311 + } else if (stream && method == 8) {
  312 + unsigned char buf[STREAM_BUFFER_SIZE];
  313 + ssize_t readlen;
  314 + git_zstream zstream;
  315 + int result;
  316 + size_t out_len;
  317 + unsigned char compressed[STREAM_BUFFER_SIZE * 2];
  318 +
  319 + memset(&zstream, 0, sizeof(zstream));
  320 + git_deflate_init(&zstream, args->compression_level);
  321 +
  322 + compressed_size = 0;
  323 + zstream.next_out = compressed;
  324 + zstream.avail_out = sizeof(compressed);
  325 +
  326 + for (;;) {
  327 + readlen = read_istream(stream, buf, sizeof(buf));
  328 + if (readlen <= 0)
  329 + break;
  330 + crc = crc32(crc, buf, readlen);
  331 +
  332 + zstream.next_in = buf;
  333 + zstream.avail_in = readlen;
  334 + result = git_deflate(&zstream, 0);
  335 + if (result != Z_OK)
  336 + die("deflate error (%d)", result);
  337 + out = compressed;
  338 + if (!compressed_size)
  339 + out += 2;
  340 + out_len = zstream.next_out - out;
  341 +
  342 + if (out_len > 0) {
  343 + write_or_die(1, out, out_len);
  344 + compressed_size += out_len;
  345 + zstream.next_out = compressed;
  346 + zstream.avail_out = sizeof(compressed);
  347 + }
  348 +
  349 + }
  350 + close_istream(stream);
  351 + if (readlen)
  352 + return readlen;
  353 +
  354 + zstream.next_in = buf;
  355 + zstream.avail_in = 0;
  356 + result = git_deflate(&zstream, Z_FINISH);
  357 + if (result != Z_STREAM_END)
  358 + die("deflate error (%d)", result);
  359 +
  360 + git_deflate_end(&zstream);
  361 + out = compressed;
  362 + if (!compressed_size)
  363 + out += 2;
  364 + out_len = zstream.next_out - out - 4;
  365 + write_or_die(1, out, out_len);
  366 + compressed_size += out_len;
  367 + zip_offset += compressed_size;
  368 +
  369 + write_zip_data_desc(size, compressed_size, crc);
  370 + zip_offset += ZIP_DATA_DESC_SIZE;
  371 +
  372 + set_zip_dir_data_desc(&dirent, size, compressed_size, crc);
311 373 } else if (compressed_size > 0) {
312 374 write_or_die(1, out, compressed_size);
313 375 zip_offset += compressed_size;
4 t/t1050-large.sh
@@ -142,4 +142,8 @@ test_expect_success 'zip achiving, store only' '
142 142 git archive --format=zip -0 HEAD >/dev/null
143 143 '
144 144
  145 +test_expect_success 'zip achiving, deflate' '
  146 + git archive --format=zip HEAD >/dev/null
  147 +'
  148 +
145 149 test_done
7 t/t5000-tar-tree.sh
@@ -250,6 +250,13 @@ test_expect_success UNZIP 'git archive -0 --format=zip on large files' '
250 250 (mkdir large && cd large && $UNZIP ../large.zip)
251 251 '
252 252
  253 +test_expect_success UNZIP 'git archive --format=zip on large files' '
  254 + test_config core.bigfilethreshold 1 &&
  255 + git archive --format=zip HEAD >large-compressed.zip &&
  256 + (mkdir large-compressed && cd large-compressed && $UNZIP ../large-compressed.zip) &&
  257 + test_cmp large-compressed/a/bin/sh large/a/bin/sh
  258 +'
  259 +
253 260 test_expect_success \
254 261 'git archive --list outside of a git repo' \
255 262 'GIT_DIR=some/non-existing/directory git archive --list'

0 comments on commit c743c21

Please sign in to comment.
Something went wrong with that request. Please try again.