Permalink
Browse files

Merge pull request #641 from carlosmn/networking

More Networking updates
  • Loading branch information...
vmg committed Apr 25, 2012
2 parents 19dd4e2 + f184836 commit f50087c03b08230ba7d912e827a72e857128a7a8
Showing with 254 additions and 334 deletions.
  1. +1 −1 examples/network/Makefile
  2. +85 −72 examples/network/fetch.c
  3. +1 −1 include/git2/indexer.h
  4. +4 −5 include/git2/remote.h
  5. +63 −32 src/fetch.c
  6. +4 −3 src/fetch.h
  7. +1 −1 src/indexer.c
  8. +0 −68 src/pkt.c
  9. +0 −3 src/pkt.h
  10. +26 −6 src/remote.c
  11. +7 −0 src/revwalk.c
  12. +2 −9 src/transport.h
  13. +26 −50 src/transports/git.c
  14. +34 −83 src/transports/http.c
@@ -2,7 +2,7 @@ default: all
CC = gcc
CFLAGS += -g
-CFLAGS += -I../../include -L../../ -lgit2
+CFLAGS += -I../../include -L../../ -lgit2 -lpthread
OBJECTS = \
git2.o \
View
@@ -3,95 +3,108 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <pthread.h>
-static int rename_packfile(char *packname, git_indexer *idx)
+struct dl_data {
+ git_remote *remote;
+ git_off_t *bytes;
+ git_indexer_stats *stats;
+ int ret;
+ int finished;
+};
+
+static void *download(void *ptr)
{
- char path[GIT_PATH_MAX], oid[GIT_OID_HEXSZ + 1], *slash;
- int ret;
-
- strcpy(path, packname);
- slash = strrchr(path, '/');
-
- if (!slash)
- return GIT_EINVALIDARGS;
-
- memset(oid, 0x0, sizeof(oid));
- // The name of the packfile is given by it's hash which you can get
- // with git_indexer_hash after the index has been written out to
- // disk. Rename the packfile to its "real" name in the same
- // directory as it was originally (libgit2 stores it in the folder
- // where the packs go, so a rename in place is the right thing to do here
- git_oid_fmt(oid, git_indexer_hash(idx));
- ret = sprintf(slash + 1, "pack-%s.pack", oid);
- if(ret < 0)
- return GIT_EOSERR;
-
- printf("Renaming pack to %s\n", path);
- return rename(packname, path);
+ struct dl_data *data = (struct dl_data *)ptr;
+
+ // Connect to the remote end specifying that we want to fetch
+ // information from it.
+ if (git_remote_connect(data->remote, GIT_DIR_FETCH) < 0) {
+ data->ret = -1;
+ goto exit;
+ }
+
+ // Download the packfile and index it. This function updates the
+ // amount of received data and the indexer stats which lets you
+ // inform the user about progress.
+ if (git_remote_download(data->remote, data->bytes, data->stats) < 0) {
+ data->ret = -1;
+ goto exit;
+ }
+
+ data->ret = 0;
+
+exit:
+ data->finished = 1;
+ pthread_exit(&data->ret);
+}
+
+int update_cb(const char *refname, const git_oid *a, const git_oid *b)
+{
+ const char *action;
+ char a_str[GIT_OID_HEXSZ+1], b_str[GIT_OID_HEXSZ+1];
+
+ git_oid_fmt(b_str, b);
+ b_str[GIT_OID_HEXSZ] = '\0';
+
+ if (git_oid_iszero(a)) {
+ printf("[new] %.20s %s\n", b_str, refname);
+ } else {
+ git_oid_fmt(a_str, a);
+ a_str[GIT_OID_HEXSZ] = '\0';
+ printf("[updated] %.10s..%.10s %s\n", a_str, b_str, refname);
+ }
+
+ return 0;
}
int fetch(git_repository *repo, int argc, char **argv)
{
git_remote *remote = NULL;
- git_indexer *idx = NULL;
+ git_off_t bytes = 0;
git_indexer_stats stats;
- int error;
- char *packname = NULL;
+ pthread_t worker;
+ struct dl_data data;
- // Get the remote and connect to it
+ // Figure out whether it's a named remote or a URL
printf("Fetching %s\n", argv[1]);
- error = git_remote_new(&remote, repo, argv[1], NULL);
- if (error < GIT_SUCCESS)
- return error;
-
- error = git_remote_connect(remote, GIT_DIR_FETCH);
- if (error < GIT_SUCCESS)
- return error;
-
- // Download the packfile from the server. As we don't know its hash
- // yet, it will get a temporary filename
- error = git_remote_download(&packname, remote);
- if (error < GIT_SUCCESS)
- return error;
-
- // No error and a NULL packname means no packfile was needed
- if (packname != NULL) {
- printf("The packname is %s\n", packname);
-
- // Create a new instance indexer
- error = git_indexer_new(&idx, packname);
- if (error < GIT_SUCCESS)
- return error;
-
- // This should be run in paralel, but it'd be too complicated for the example
- error = git_indexer_run(idx, &stats);
- if (error < GIT_SUCCESS)
- return error;
-
- printf("Received %d objects\n", stats.total);
-
- // Write the index file. The index will be stored with the
- // correct filename
- error = git_indexer_write(idx);
- if (error < GIT_SUCCESS)
- return error;
-
- error = rename_packfile(packname, idx);
- if (error < GIT_SUCCESS)
- return error;
+ if (git_remote_load(&remote, repo, argv[1]) < 0) {
+ if (git_remote_new(&remote, repo, argv[1], NULL) < 0)
+ return -1;
}
+ // Set up the information for the background worker thread
+ data.remote = remote;
+ data.bytes = &bytes;
+ data.stats = &stats;
+ data.ret = 0;
+ data.finished = 0;
+ memset(&stats, 0, sizeof(stats));
+
+ pthread_create(&worker, NULL, download, &data);
+
+ // Loop while the worker thread is still running. Here we show processed
+ // and total objects in the pack and the amount of received
+ // data. Most frontends will probably want to show a percentage and
+ // the download rate.
+ do {
+ usleep(10000);
+ printf("\rReceived %d/%d objects in %d bytes", stats.processed, stats.total, bytes);
+ } while (!data.finished);
+ printf("\rReceived %d/%d objects in %d bytes\n", stats.processed, stats.total, bytes);
+
// Update the references in the remote's namespace to point to the
// right commits. This may be needed even if there was no packfile
// to download, which can happen e.g. when the branches have been
// changed but all the neede objects are available locally.
- error = git_remote_update_tips(remote);
- if (error < GIT_SUCCESS)
- return error;
+ if (git_remote_update_tips(remote, update_cb) < 0)
+ return -1;
- free(packname);
- git_indexer_free(idx);
git_remote_free(remote);
- return GIT_SUCCESS;
+ return 0;
+
+on_error:
+ git_remote_free(remote);
+ return -1;
}
View
@@ -41,7 +41,7 @@ GIT_EXTERN(int) git_indexer_stream_new(git_indexer_stream **out, const char *git
* @param size the size of the data
* @param stats stat storage
*/
-GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, void *data, size_t size, git_indexer_stats *stats);
+GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats);
/**
* Finalize the pack and index
View
@@ -11,6 +11,7 @@
#include "repository.h"
#include "refspec.h"
#include "net.h"
+#include "indexer.h"
/**
* @file git2/remote.h
@@ -150,7 +151,7 @@ GIT_EXTERN(int) git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void
* @param filename where to store the temproray filename
* @return GIT_SUCCESS or an error code
*/
-GIT_EXTERN(int) git_remote_download(char **filename, git_remote *remote);
+GIT_EXTERN(int) git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
/**
* Check whether the remote is connected
@@ -182,12 +183,10 @@ GIT_EXTERN(void) git_remote_free(git_remote *remote);
/**
* Update the tips to the new state
*
- * Make sure that you only call this once you've successfully indexed
- * or expanded the packfile.
- *
* @param remote the remote to update
+ * @param cb callback to run on each ref update. 'a' is the old value, 'b' is then new value
*/
-GIT_EXTERN(int) git_remote_update_tips(git_remote *remote);
+GIT_EXTERN(int) git_remote_update_tips(git_remote *remote, int (*cb)(const char *refname, const git_oid *a, const git_oid *b));
/**
* Return whether a string is a valid remote URL
View
@@ -9,6 +9,7 @@
#include "git2/oid.h"
#include "git2/refs.h"
#include "git2/revwalk.h"
+#include "git2/indexer.h"
#include "common.h"
#include "transport.h"
@@ -101,30 +102,27 @@ int git_fetch_negotiate(git_remote *remote)
return t->negotiate_fetch(t, remote->repo, &remote->refs);
}
-int git_fetch_download_pack(char **out, git_remote *remote)
+int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
{
- if(!remote->need_pack) {
- *out = NULL;
+ if(!remote->need_pack)
return 0;
- }
- return remote->transport->download_pack(out, remote->transport, remote->repo);
+ return remote->transport->download_pack(remote->transport, remote->repo, bytes, stats);
}
/* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
int git_fetch__download_pack(
- char **out,
const char *buffered,
size_t buffered_size,
GIT_SOCKET fd,
- git_repository *repo)
+ git_repository *repo,
+ git_off_t *bytes,
+ git_indexer_stats *stats)
{
- git_filebuf file = GIT_FILEBUF_INIT;
- int error;
+ int recvd;
char buff[1024];
- git_buf path = GIT_BUF_INIT;
- static const char suff[] = "/objects/pack/pack-received";
gitno_buffer buf;
+ git_indexer_stream *idx;
gitno_buffer_setup(&buf, buff, sizeof(buff), fd);
@@ -133,41 +131,74 @@ int git_fetch__download_pack(
return -1;
}
- if (git_buf_joinpath(&path, repo->path_repository, suff) < 0)
- goto on_error;
+ if (git_indexer_stream_new(&idx, git_repository_path(repo)) < 0)
+ return -1;
- if (git_filebuf_open(&file, path.ptr, GIT_FILEBUF_TEMPORARY) < 0)
+ memset(stats, 0, sizeof(git_indexer_stats));
+ if (git_indexer_stream_add(idx, buffered, buffered_size, stats) < 0)
goto on_error;
- /* Part of the packfile has been received, don't loose it */
- if (git_filebuf_write(&file, buffered, buffered_size) < 0)
- goto on_error;
+ *bytes = buffered_size;
- while (1) {
- if (git_filebuf_write(&file, buf.data, buf.offset) < 0)
+ do {
+ if (git_indexer_stream_add(idx, buf.data, buf.offset, stats) < 0)
goto on_error;
gitno_consume_n(&buf, buf.offset);
- error = gitno_recv(&buf);
- if (error < GIT_SUCCESS)
+ if ((recvd = gitno_recv(&buf)) < 0)
goto on_error;
- if (error == 0) /* Orderly shutdown */
- break;
- }
- *out = git__strdup(file.path_lock);
- if (*out == NULL)
- goto on_error;
+ *bytes += recvd;
+ } while(recvd > 0);
- /* A bit dodgy, but we need to keep the pack at the temporary path */
- if (git_filebuf_commit_at(&file, file.path_lock, GIT_PACK_FILE_MODE) < 0)
+ if (git_indexer_stream_finalize(idx, stats))
goto on_error;
- git_buf_free(&path);
+ git_indexer_stream_free(idx);
+ return 0;
+
+on_error:
+ git_indexer_stream_free(idx);
+ return -1;
+}
+
+int git_fetch_setup_walk(git_revwalk **out, git_repository *repo)
+{
+ git_revwalk *walk;
+ git_strarray refs;
+ unsigned int i;
+ git_reference *ref;
+
+ if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0)
+ return -1;
+
+ if (git_revwalk_new(&walk, repo) < 0)
+ return -1;
+ git_revwalk_sorting(walk, GIT_SORT_TIME);
+
+ for (i = 0; i < refs.count; ++i) {
+ /* No tags */
+ if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
+ continue;
+
+ if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
+ goto on_error;
+
+ if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
+ continue;
+ if (git_revwalk_push(walk, git_reference_oid(ref)) < 0)
+ goto on_error;
+
+ git_reference_free(ref);
+ }
+
+ git_strarray_free(&refs);
+ *out = walk;
return 0;
+
on_error:
- git_buf_free(&path);
- git_filebuf_cleanup(&file);
+ git_reference_free(ref);
+ git_strarray_free(&refs);
return -1;
}
View
@@ -10,9 +10,10 @@
#include "netops.h"
int git_fetch_negotiate(git_remote *remote);
-int git_fetch_download_pack(char **out, git_remote *remote);
+int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
-int git_fetch__download_pack(char **out, const char *buffered, size_t buffered_size,
- GIT_SOCKET fd, git_repository *repo);
+int git_fetch__download_pack(const char *buffered, size_t buffered_size, GIT_SOCKET fd,
+ git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
+int git_fetch_setup_walk(git_revwalk **out, git_repository *repo);
#endif
View
@@ -288,7 +288,7 @@ static int hash_and_save(git_indexer_stream *idx, git_rawobj *obj, git_off_t ent
return -1;
}
-int git_indexer_stream_add(git_indexer_stream *idx, void *data, size_t size, git_indexer_stats *stats)
+int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats)
{
int error;
struct git_pack_header hdr;
Oops, something went wrong.

0 comments on commit f50087c

Please sign in to comment.