Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge branch 'sp/maint-dumb-http-pack-reidx' into maint
* sp/maint-dumb-http-pack-reidx:
  http.c::new_http_pack_request: do away with the temp variable filename
  http-fetch: Use temporary files for pack-*.idx until verified
  http-fetch: Use index-pack rather than verify-pack to check packs
  Allow parse_pack_index on temporary files
  Extract verify_pack_index for reuse from verify_pack
  Introduce close_pack_index to permit replacement
  http.c: Remove unnecessary strdup of sha1_to_hex result
  http.c: Don't store destination name in request structures
  http.c: Drop useless != NULL test in finish_http_pack_request
  http.c: Tiny refactoring of finish_http_pack_request
  t5550-http-fetch: Use subshell for repository operations
  http.c: Remove bad free of static block
  • Loading branch information
gitster committed Jun 16, 2010
2 parents d6bf0cf + 90d0571 commit 161cbf0
Show file tree
Hide file tree
Showing 8 changed files with 149 additions and 60 deletions.
3 changes: 2 additions & 1 deletion cache.h
Expand Up @@ -905,7 +905,7 @@ struct extra_have_objects {
extern struct ref **get_remote_heads(int in, struct ref **list, int nr_match, char **match, unsigned int flags, struct extra_have_objects *);
extern int server_supports(const char *feature);

extern struct packed_git *parse_pack_index(unsigned char *sha1);
extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path);

extern void prepare_packed_git(void);
extern void reprepare_packed_git(void);
Expand All @@ -916,6 +916,7 @@ extern struct packed_git *find_sha1_pack(const unsigned char *sha1,

extern void pack_report(void);
extern int open_pack_index(struct packed_git *);
extern void close_pack_index(struct packed_git *);
extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
extern void close_pack_windows(struct packed_git *);
extern void unuse_pack(struct pack_window **);
Expand Down
2 changes: 1 addition & 1 deletion http-walker.c
Expand Up @@ -510,7 +510,7 @@ static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned c
ret = error("File %s has bad hash", hex);
} else if (req->rename < 0) {
ret = error("unable to write sha1 filename %s",
req->filename);
sha1_file_name(req->sha1));
}

release_http_object_request(req);
Expand Down
135 changes: 89 additions & 46 deletions http.c
@@ -1,6 +1,7 @@
#include "http.h"
#include "pack.h"
#include "sideband.h"
#include "run-command.h"

int data_received;
int active_requests;
Expand Down Expand Up @@ -896,47 +897,67 @@ int http_fetch_ref(const char *base, struct ref *ref)
}

/* Helpers for fetching packs */
static int fetch_pack_index(unsigned char *sha1, const char *base_url)
static char *fetch_pack_index(unsigned char *sha1, const char *base_url)
{
int ret = 0;
char *hex = xstrdup(sha1_to_hex(sha1));
char *filename;
char *url = NULL;
char *url, *tmp;
struct strbuf buf = STRBUF_INIT;

if (has_pack_index(sha1)) {
ret = 0;
goto cleanup;
}

if (http_is_verbose)
fprintf(stderr, "Getting index for pack %s\n", hex);
fprintf(stderr, "Getting index for pack %s\n", sha1_to_hex(sha1));

end_url_with_slash(&buf, base_url);
strbuf_addf(&buf, "objects/pack/pack-%s.idx", hex);
strbuf_addf(&buf, "objects/pack/pack-%s.idx", sha1_to_hex(sha1));
url = strbuf_detach(&buf, NULL);

filename = sha1_pack_index_name(sha1);
if (http_get_file(url, filename, 0) != HTTP_OK)
ret = error("Unable to get pack index %s\n", url);
strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(sha1));
tmp = strbuf_detach(&buf, NULL);

if (http_get_file(url, tmp, 0) != HTTP_OK) {
error("Unable to get pack index %s\n", url);
free(tmp);
tmp = NULL;
}

cleanup:
free(hex);
free(url);
return ret;
return tmp;
}

static int fetch_and_setup_pack_index(struct packed_git **packs_head,
unsigned char *sha1, const char *base_url)
{
struct packed_git *new_pack;
char *tmp_idx = NULL;
int ret;

if (fetch_pack_index(sha1, base_url))
if (has_pack_index(sha1)) {
new_pack = parse_pack_index(sha1, NULL);
if (!new_pack)
return -1; /* parse_pack_index() already issued error message */
goto add_pack;
}

tmp_idx = fetch_pack_index(sha1, base_url);
if (!tmp_idx)
return -1;

new_pack = parse_pack_index(sha1);
if (!new_pack)
new_pack = parse_pack_index(sha1, tmp_idx);
if (!new_pack) {
unlink(tmp_idx);
free(tmp_idx);

return -1; /* parse_pack_index() already issued error message */
}

ret = verify_pack_index(new_pack);
if (!ret) {
close_pack_index(new_pack);
ret = move_temp_to_file(tmp_idx, sha1_pack_index_name(sha1));
}
free(tmp_idx);
if (ret)
return -1;

add_pack:
new_pack->next = *packs_head;
*packs_head = new_pack;
return 0;
Expand Down Expand Up @@ -1000,37 +1021,62 @@ void release_http_pack_request(struct http_pack_request *preq)

int finish_http_pack_request(struct http_pack_request *preq)
{
int ret;
struct packed_git **lst;
struct packed_git *p = preq->target;
char *tmp_idx;
struct child_process ip;
const char *ip_argv[8];

preq->target->pack_size = ftell(preq->packfile);
close_pack_index(p);

if (preq->packfile != NULL) {
fclose(preq->packfile);
preq->packfile = NULL;
preq->slot->local = NULL;
}

ret = move_temp_to_file(preq->tmpfile, preq->filename);
if (ret)
return ret;
fclose(preq->packfile);
preq->packfile = NULL;
preq->slot->local = NULL;

lst = preq->lst;
while (*lst != preq->target)
while (*lst != p)
lst = &((*lst)->next);
*lst = (*lst)->next;

if (verify_pack(preq->target))
tmp_idx = xstrdup(preq->tmpfile);
strcpy(tmp_idx + strlen(tmp_idx) - strlen(".pack.temp"),
".idx.temp");

ip_argv[0] = "index-pack";
ip_argv[1] = "-o";
ip_argv[2] = tmp_idx;
ip_argv[3] = preq->tmpfile;
ip_argv[4] = NULL;

memset(&ip, 0, sizeof(ip));
ip.argv = ip_argv;
ip.git_cmd = 1;
ip.no_stdin = 1;
ip.no_stdout = 1;

if (run_command(&ip)) {
unlink(preq->tmpfile);
unlink(tmp_idx);
free(tmp_idx);
return -1;
}

unlink(sha1_pack_index_name(p->sha1));

if (move_temp_to_file(preq->tmpfile, sha1_pack_name(p->sha1))
|| move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
free(tmp_idx);
return -1;
install_packed_git(preq->target);
}

install_packed_git(p);
free(tmp_idx);
return 0;
}

struct http_pack_request *new_http_pack_request(
struct packed_git *target, const char *base_url)
{
char *filename;
long prev_posn = 0;
char range[RANGE_HEADER_SIZE];
struct strbuf buf = STRBUF_INIT;
Expand All @@ -1045,9 +1091,8 @@ struct http_pack_request *new_http_pack_request(
sha1_to_hex(target->sha1));
preq->url = strbuf_detach(&buf, NULL);

filename = sha1_pack_name(target->sha1);
snprintf(preq->filename, sizeof(preq->filename), "%s", filename);
snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp", filename);
snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp",
sha1_pack_name(target->sha1));
preq->packfile = fopen(preq->tmpfile, "a");
if (!preq->packfile) {
error("Unable to open local file %s for pack",
Expand Down Expand Up @@ -1082,7 +1127,6 @@ struct http_pack_request *new_http_pack_request(
return preq;

abort:
free(filename);
free(preq->url);
free(preq);
return NULL;
Expand Down Expand Up @@ -1137,7 +1181,6 @@ struct http_object_request *new_http_object_request(const char *base_url,
freq->localfile = -1;

filename = sha1_file_name(sha1);
snprintf(freq->filename, sizeof(freq->filename), "%s", filename);
snprintf(freq->tmpfile, sizeof(freq->tmpfile),
"%s.temp", filename);

Expand Down Expand Up @@ -1166,8 +1209,8 @@ struct http_object_request *new_http_object_request(const char *base_url,
}

if (freq->localfile < 0) {
error("Couldn't create temporary file %s for %s: %s",
freq->tmpfile, freq->filename, strerror(errno));
error("Couldn't create temporary file %s: %s",
freq->tmpfile, strerror(errno));
goto abort;
}

Expand Down Expand Up @@ -1214,8 +1257,8 @@ struct http_object_request *new_http_object_request(const char *base_url,
prev_posn = 0;
lseek(freq->localfile, 0, SEEK_SET);
if (ftruncate(freq->localfile, 0) < 0) {
error("Couldn't truncate temporary file %s for %s: %s",
freq->tmpfile, freq->filename, strerror(errno));
error("Couldn't truncate temporary file %s: %s",
freq->tmpfile, strerror(errno));
goto abort;
}
}
Expand Down Expand Up @@ -1291,7 +1334,7 @@ int finish_http_object_request(struct http_object_request *freq)
return -1;
}
freq->rename =
move_temp_to_file(freq->tmpfile, freq->filename);
move_temp_to_file(freq->tmpfile, sha1_file_name(freq->sha1));

return freq->rename;
}
Expand Down
2 changes: 0 additions & 2 deletions http.h
Expand Up @@ -153,7 +153,6 @@ struct http_pack_request
struct packed_git *target;
struct packed_git **lst;
FILE *packfile;
char filename[PATH_MAX];
char tmpfile[PATH_MAX];
struct curl_slist *range_header;
struct active_request_slot *slot;
Expand All @@ -168,7 +167,6 @@ extern void release_http_pack_request(struct http_pack_request *preq);
struct http_object_request
{
char *url;
char filename[PATH_MAX];
char tmpfile[PATH_MAX];
int localfile;
CURLcode curl_result;
Expand Down
15 changes: 12 additions & 3 deletions pack-check.c
Expand Up @@ -133,14 +133,13 @@ static int verify_packfile(struct packed_git *p,
return err;
}

int verify_pack(struct packed_git *p)
int verify_pack_index(struct packed_git *p)
{
off_t index_size;
const unsigned char *index_base;
git_SHA_CTX ctx;
unsigned char sha1[20];
int err = 0;
struct pack_window *w_curs = NULL;

if (open_pack_index(p))
return error("packfile %s index not opened", p->pack_name);
Expand All @@ -154,8 +153,18 @@ int verify_pack(struct packed_git *p)
if (hashcmp(sha1, index_base + index_size - 20))
err = error("Packfile index for %s SHA1 mismatch",
p->pack_name);
return err;
}

int verify_pack(struct packed_git *p)
{
int err = 0;
struct pack_window *w_curs = NULL;

err |= verify_pack_index(p);
if (!p->index_data)
return -1;

/* Verify pack file */
err |= verify_packfile(p, &w_curs);
unuse_pack(&w_curs);

Expand Down
1 change: 1 addition & 0 deletions pack.h
Expand Up @@ -57,6 +57,7 @@ struct pack_idx_entry {

extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
extern int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, off_t offset, off_t len, unsigned int nr);
extern int verify_pack_index(struct packed_git *);
extern int verify_pack(struct packed_git *);
extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t, unsigned char *, off_t);
extern char *index_pack_lockfile(int fd);
Expand Down
14 changes: 10 additions & 4 deletions sha1_file.c
Expand Up @@ -599,6 +599,14 @@ void unuse_pack(struct pack_window **w_cursor)
}
}

void close_pack_index(struct packed_git *p)
{
if (p->index_data) {
munmap((void *)p->index_data, p->index_size);
p->index_data = NULL;
}
}

/*
* This is used by git-repack in case a newly created pack happens to
* contain the same set of objects as an existing one. In that case
Expand All @@ -620,8 +628,7 @@ void free_pack_by_name(const char *pack_name)
close_pack_windows(p);
if (p->pack_fd != -1)
close(p->pack_fd);
if (p->index_data)
munmap((void *)p->index_data, p->index_size);
close_pack_index(p);
free(p->bad_object_sha1);
*pp = p->next;
free(p);
Expand Down Expand Up @@ -831,9 +838,8 @@ struct packed_git *add_packed_git(const char *path, int path_len, int local)
return p;
}

struct packed_git *parse_pack_index(unsigned char *sha1)
struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
{
const char *idx_path = sha1_pack_index_name(sha1);
const char *path = sha1_pack_name(sha1);
struct packed_git *p = alloc_packed_git(strlen(path) + 1);

Expand Down

0 comments on commit 161cbf0

Please sign in to comment.