Permalink
Browse files

chunked-object: streaming checkout

Define a new streaming decoder to handle blobs stored in chunked encoding,
so that we do not have to slurp the data for the whole object to check it
out to the working tree.

Signed-off-by: Junio C Hamano <gitster@pobox.com>
  • Loading branch information...
gitster committed Dec 2, 2011
1 parent 01f654e commit 4a1242d558144fd476e2cde8f629251609aaf1f2
Showing with 117 additions and 2 deletions.
  1. +2 −1 cache.h
  2. +3 −0 sha1_file.c
  3. +112 −1 streaming.c
View
@@ -1101,7 +1101,8 @@ struct object_info {
struct {
struct packed_git *pack;
off_t offset;
- unsigned int is_delta;
+ unsigned int is_delta:1;
+ unsigned int is_chunked:1;
} packed;
} u;
};
View
@@ -2216,6 +2216,9 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi)
oi->u.packed.pack = e.p;
oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
rtype == OBJ_OFS_DELTA);
+ oi->u.packed.is_chunked =
+ (OBJ_CHUNKED_BLOB <= rtype &&
+ rtype <= OBJ_CHUNKED_BLOB);
}
return status;
View
@@ -3,12 +3,14 @@
*/
#include "cache.h"
#include "streaming.h"
+#include "pack.h"
enum input_source {
stream_error = -1,
incore = 0,
loose = 1,
- pack_non_delta = 2
+ pack_non_delta = 2,
+ pack_chunked = 3,
};
typedef int (*open_istream_fn)(struct git_istream *,
@@ -41,6 +43,7 @@ struct stream_vtbl {
static open_method_decl(incore);
static open_method_decl(loose);
static open_method_decl(pack_non_delta);
+static open_method_decl(pack_chunked);
static struct git_istream *attach_stream_filter(struct git_istream *st,
struct stream_filter *filter);
@@ -49,6 +52,7 @@ static open_istream_fn open_istream_tbl[] = {
open_istream_incore,
open_istream_loose,
open_istream_pack_non_delta,
+ open_istream_pack_chunked,
};
#define FILTER_BUFFER (1024*16)
@@ -88,6 +92,13 @@ struct git_istream {
off_t pos;
} in_pack;
+ struct {
+ int component_cnt;
+ int component_idx;
+ struct git_istream *current;
+ unsigned char (*component)[20];
+ } chunked;
+
struct filtered_istream filtered;
} u;
};
@@ -121,6 +132,8 @@ static enum input_source istream_source(const unsigned char *sha1,
case OI_LOOSE:
return loose;
case OI_PACKED:
+ if (oi->u.packed.is_chunked)
+ return pack_chunked;
if (!oi->u.packed.is_delta && big_file_threshold <= size)
return pack_non_delta;
/* fallthru */
@@ -450,6 +463,104 @@ static open_method_decl(pack_non_delta)
}
+/*****************************************************************
+ *
+ * Chunked packed object stream
+ *
+ *****************************************************************/
+
+static int prepare_component(struct git_istream *st)
+{
+ if (!st->u.chunked.current) {
+ int idx = ++st->u.chunked.component_idx;
+ enum object_type type;
+ unsigned long size;
+ struct git_istream *component;
+
+ if (st->u.chunked.component_cnt <= idx)
+ return -1; /* EOF */
+ component = open_istream(st->u.chunked.component[idx],
+ &type, &size, NULL);
+ if (!component)
+ return -1; /* I/O Error */
+ st->u.chunked.current = component;
+ }
+ return 0;
+}
+
+static read_method_decl(pack_chunked)
+{
+ size_t total = 0;
+
+ while (sz) {
+ ssize_t filled;
+
+ if (prepare_component(st))
+ break;
+ filled = read_istream(st->u.chunked.current, buf, sz);
+ if (filled < 0) {
+ close_istream(st->u.chunked.current);
+ st->u.chunked.current = NULL;
+ st->u.chunked.component_idx = st->u.chunked.component_cnt;
+ return -1; /* I/O error */
+ } else if (!filled) {
+ close_istream(st->u.chunked.current);
+ st->u.chunked.current = NULL;
+ } else {
+ sz -= filled;
+ buf += filled;
+ total += filled;
+ }
+ }
+ return total;
+}
+
+static close_method_decl(pack_chunked)
+{
+ if (st->u.chunked.current)
+ close_istream(st->u.chunked.current);
+ free(st->u.chunked.component);
+ return 0;
+}
+
+static struct stream_vtbl pack_chunked_vtbl = {
+ close_istream_pack_chunked,
+ read_istream_pack_chunked,
+};
+
+static open_method_decl(pack_chunked)
+{
+ struct pack_window *window;
+ struct packed_git *pack = oi->u.packed.pack;
+ off_t offset = oi->u.packed.offset;
+ const unsigned char *in, *ptr;
+ unsigned long avail;
+ int cnt;
+
+ window = NULL;
+ unpack_object_header(pack, &window, &offset, &st->size);
+ in = use_pack(pack, &window, offset, &avail);
+ ptr = in;
+ cnt = decode_in_pack_varint(&ptr);
+ offset += ptr - in;
+ st->u.chunked.component_cnt = cnt;
+ st->u.chunked.component = xcalloc(20, cnt);
+
+ for (cnt = 0; cnt < st->u.chunked.component_cnt; cnt++) {
+ in = use_pack(pack, &window, offset, &avail);
+ memcpy(st->u.chunked.component + cnt, in, 20);
+ offset += 20;
+ }
+ unuse_pack(&window);
+
+ st->u.chunked.component_idx = -1;
+ st->u.chunked.current = NULL;
+ st->z_state = z_unused;
+ st->vtbl = &pack_chunked_vtbl;
+ return 0;
+}
+
+
/*****************************************************************
*
* In-core stream

0 comments on commit 4a1242d

Please sign in to comment.