From c3b99239b4f5f76c01017deabf17134d707a67d8 Mon Sep 17 00:00:00 2001 From: lhchavez Date: Sat, 12 Jan 2019 00:31:03 +0000 Subject: [PATCH] Add support for custom smart transports This change adds support for git smart transports. This will be then used to implement http, https, and ssh transports that don't rely on the libgit2 library. --- clone.go | 1 + git.go | 3 + git_test.go | 10 ++ remote.go | 110 +++++++++++++- repository.go | 2 + transport.go | 372 ++++++++++++++++++++++++++++++++++++++++++++++ transport_test.go | 72 +++++++++ wrapper.c | 113 ++++++++++++++ 8 files changed, 677 insertions(+), 6 deletions(-) create mode 100644 transport.go create mode 100644 transport_test.go diff --git a/clone.go b/clone.go index 4921e126..b02a43e1 100644 --- a/clone.go +++ b/clone.go @@ -85,6 +85,7 @@ func remoteCreateCallback( // clear finalizer as the calling C function will // free the remote itself runtime.SetFinalizer(remote, nil) + remote.repo.Remotes.untrackRemote(remote) return C.int(ErrorCodeOK) } diff --git a/git.go b/git.go index cdcf6bb5..1d686bbf 100644 --- a/git.go +++ b/git.go @@ -124,6 +124,7 @@ var ( ) var pointerHandles *HandleList +var remotePointers *remotePointerList func init() { initLibGit2() @@ -131,6 +132,7 @@ func init() { func initLibGit2() { pointerHandles = NewHandleList() + remotePointers = newRemotePointerList() C.git_libgit2_init() @@ -157,6 +159,7 @@ func initLibGit2() { // undefined behavior, so make sure this is called carefully. func Shutdown() { pointerHandles.Clear() + remotePointers.clear() C.git_libgit2_shutdown() } diff --git a/git_test.go b/git_test.go index 1c57f791..8a12229d 100644 --- a/git_test.go +++ b/git_test.go @@ -23,6 +23,16 @@ func TestMain(m *testing.M) { } pointerHandles.Unlock() + // Or remote pointers. + remotePointers.Lock() + if len(remotePointers.pointers) > 0 { + for ptr, remote := range remotePointers.pointers { + fmt.Printf("%016p: %+v\n", ptr, remote) + } + panic("remote pointer list not empty") + } + remotePointers.Unlock() + Shutdown() os.Exit(ret) diff --git a/remote.go b/remote.go index 1887f790..a8cf3e96 100644 --- a/remote.go +++ b/remote.go @@ -15,6 +15,7 @@ import ( "reflect" "runtime" "strings" + "sync" "unsafe" ) @@ -156,6 +157,64 @@ type Remote struct { repo *Repository } +type remotePointerList struct { + sync.RWMutex + // stores the Go pointers + pointers map[*C.git_remote]*Remote +} + +func newRemotePointerList() *remotePointerList { + return &remotePointerList{ + pointers: make(map[*C.git_remote]*Remote), + } +} + +// track adds the given pointer to the list of pointers to track and +// returns a pointer value which can be passed to C as an opaque +// pointer. +func (v *remotePointerList) track(remote *Remote) { + v.Lock() + v.pointers[remote.ptr] = remote + v.Unlock() + + runtime.SetFinalizer(remote, (*Remote).Free) +} + +// untrack stops tracking the git_remote pointer. +func (v *remotePointerList) untrack(remote *Remote) { + v.Lock() + delete(v.pointers, remote.ptr) + v.Unlock() +} + +// clear stops tracking all the git_remote pointers. +func (v *remotePointerList) clear() { + v.Lock() + var remotes []*Remote + for remotePtr, remote := range v.pointers { + remotes = append(remotes, remote) + delete(v.pointers, remotePtr) + } + v.Unlock() + + for _, remote := range remotes { + remote.free() + } +} + +// get retrieves the pointer from the given *git_remote. +func (v *remotePointerList) get(ptr *C.git_remote) (*Remote, bool) { + v.RLock() + defer v.RUnlock() + + r, ok := v.pointers[ptr] + if !ok { + return nil, false + } + + return r, true +} + type CertificateKind uint const ( @@ -491,16 +550,41 @@ func RemoteIsValidName(name string) bool { return C.git_remote_is_valid_name(cname) == 1 } -// Free releases the resources of the Remote. -func (r *Remote) Free() { +// free releases the resources of the Remote. +func (r *Remote) free() { runtime.SetFinalizer(r, nil) C.git_remote_free(r.ptr) r.ptr = nil r.repo = nil } +// Free releases the resources of the Remote. +func (r *Remote) Free() { + r.repo.Remotes.untrackRemote(r) + r.free() +} + type RemoteCollection struct { repo *Repository + + sync.RWMutex + remotes map[*C.git_remote]*Remote +} + +func (c *RemoteCollection) trackRemote(r *Remote) { + c.Lock() + c.remotes[r.ptr] = r + c.Unlock() + + remotePointers.track(r) +} + +func (c *RemoteCollection) untrackRemote(r *Remote) { + c.Lock() + delete(c.remotes, r.ptr) + c.Unlock() + + remotePointers.untrack(r) } func (c *RemoteCollection) List() ([]string, error) { @@ -535,7 +619,7 @@ func (c *RemoteCollection) Create(name string, url string) (*Remote, error) { if ret < 0 { return nil, MakeGitError(ret) } - runtime.SetFinalizer(remote, (*Remote).Free) + c.trackRemote(remote) return remote, nil } @@ -571,7 +655,7 @@ func (c *RemoteCollection) CreateWithFetchspec(name string, url string, fetch st if ret < 0 { return nil, MakeGitError(ret) } - runtime.SetFinalizer(remote, (*Remote).Free) + c.trackRemote(remote) return remote, nil } @@ -588,7 +672,7 @@ func (c *RemoteCollection) CreateAnonymous(url string) (*Remote, error) { if ret < 0 { return nil, MakeGitError(ret) } - runtime.SetFinalizer(remote, (*Remote).Free) + c.trackRemote(remote) return remote, nil } @@ -605,10 +689,24 @@ func (c *RemoteCollection) Lookup(name string) (*Remote, error) { if ret < 0 { return nil, MakeGitError(ret) } - runtime.SetFinalizer(remote, (*Remote).Free) + c.trackRemote(remote) return remote, nil } +func (c *RemoteCollection) Free() { + var remotes []*Remote + c.Lock() + for remotePtr, remote := range c.remotes { + remotes = append(remotes, remote) + delete(c.remotes, remotePtr) + } + c.Unlock() + + for _, remote := range remotes { + remotePointers.untrack(remote) + } +} + func (o *Remote) Name() string { s := C.git_remote_name(o.ptr) runtime.KeepAlive(o) diff --git a/repository.go b/repository.go index d2b99610..548114aa 100644 --- a/repository.go +++ b/repository.go @@ -45,6 +45,7 @@ func newRepositoryFromC(ptr *C.git_repository) *Repository { repo := &Repository{ptr: ptr} repo.Remotes.repo = repo + repo.Remotes.remotes = make(map[*C.git_remote]*Remote) repo.Submodules.repo = repo repo.References.repo = repo repo.Notes.repo = repo @@ -143,6 +144,7 @@ func (v *Repository) Free() { ptr := v.ptr v.ptr = nil runtime.SetFinalizer(v, nil) + v.Remotes.Free() if v.weak { return } diff --git a/transport.go b/transport.go new file mode 100644 index 00000000..846737fb --- /dev/null +++ b/transport.go @@ -0,0 +1,372 @@ +package git + +/* +#include +#include + +typedef struct { + git_smart_subtransport parent; + void *handle; +} _go_managed_smart_subtransport; + +typedef struct { + git_smart_subtransport_stream parent; + void *handle; +} _go_managed_smart_subtransport_stream; + +int _go_git_transport_register(const char *prefix, void *handle); +int _go_git_transport_smart(git_transport **out, git_remote *owner, int stateless, _go_managed_smart_subtransport *subtransport_payload); +void _go_git_setup_smart_subtransport_stream(_go_managed_smart_subtransport_stream *stream); +*/ +import "C" +import ( + "errors" + "io" + "reflect" + "runtime" + "unsafe" +) + +// SmartServiceAction is an action that the smart transport can ask a +// subtransport to perform. +type SmartServiceAction int + +const ( + // SmartServiceActionUploadpackLs is used upon connecting to a remote, and is + // used to perform reference discovery prior to performing a pull operation. + SmartServiceActionUploadpackLs SmartServiceAction = C.GIT_SERVICE_UPLOADPACK_LS + + // SmartServiceActionUploadpack is used when performing a pull operation. + SmartServiceActionUploadpack SmartServiceAction = C.GIT_SERVICE_UPLOADPACK + + // SmartServiceActionReceivepackLs is used upon connecting to a remote, and is + // used to perform reference discovery prior to performing a push operation. + SmartServiceActionReceivepackLs SmartServiceAction = C.GIT_SERVICE_RECEIVEPACK_LS + + // SmartServiceActionReceivepack is used when performing a push operation. + SmartServiceActionReceivepack SmartServiceAction = C.GIT_SERVICE_RECEIVEPACK +) + +// Transport encapsulates a way to communicate with a Remote. +type Transport struct { + ptr *C.git_transport +} + +// SmartCredentials calls the credentials callback for this transport. +func (t *Transport) SmartCredentials(user string, methods CredentialType) (*Credential, error) { + cred := newCredential() + var cstr *C.char + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + if user != "" { + cstr = C.CString(user) + defer C.free(unsafe.Pointer(cstr)) + } + ret := C.git_transport_smart_credentials(&cred.ptr, t.ptr, cstr, C.int(methods)) + if ret != 0 { + cred.Free() + return nil, MakeGitError(ret) + } + + return cred, nil +} + +// SmartSubtransport is the interface for custom subtransports which carry data +// for the smart transport. +type SmartSubtransport interface { + // Action creates a SmartSubtransportStream for the provided url and + // requested action. + Action(url string, action SmartServiceAction) (SmartSubtransportStream, error) + + // Close closes the SmartSubtransport. + // + // Subtransports are guaranteed a call to Close between + // calls to Action, except for the following two "natural" progressions + // of actions against a constant URL. + // + // 1. UPLOADPACK_LS -> UPLOADPACK + // 2. RECEIVEPACK_LS -> RECEIVEPACK + Close() error + + // Free releases the resources of the SmartSubtransport. + Free() +} + +// SmartSubtransportStream is the interface for streams used by the smart +// transport to read and write data from a subtransport. +type SmartSubtransportStream interface { + io.Reader + io.Writer + + // Free releases the resources of the SmartSubtransportStream. + Free() +} + +// SmartSubtransportCallback is a function which creates a new subtransport for +// the smart transport. +type SmartSubtransportCallback func(remote *Remote, transport *Transport) (SmartSubtransport, error) + +// RegisteredSmartTransport represents a transport that has been registered. +type RegisteredSmartTransport struct { + name string + stateless bool + callback SmartSubtransportCallback + handle unsafe.Pointer +} + +// NewRegisteredSmartTransport adds a custom transport definition, to be used +// in addition to the built-in set of transports that come with libgit2. +func NewRegisteredSmartTransport( + name string, + stateless bool, + callback SmartSubtransportCallback, +) (*RegisteredSmartTransport, error) { + cstr := C.CString(name) + defer C.free(unsafe.Pointer(cstr)) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + registeredSmartTransport := &RegisteredSmartTransport{ + name: name, + stateless: stateless, + callback: callback, + } + registeredSmartTransport.handle = pointerHandles.Track(registeredSmartTransport) + + ret := C._go_git_transport_register(cstr, registeredSmartTransport.handle) + if ret != 0 { + pointerHandles.Untrack(registeredSmartTransport.handle) + return nil, MakeGitError(ret) + } + + runtime.SetFinalizer(registeredSmartTransport, (*RegisteredSmartTransport).Free) + return registeredSmartTransport, nil +} + +// Free releases all resources used by the RegisteredSmartTransport and +// unregisters the custom transport definition referenced by it. +func (t *RegisteredSmartTransport) Free() error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + cstr := C.CString(t.name) + defer C.free(unsafe.Pointer(cstr)) + + if ret := C.git_transport_unregister(cstr); ret < 0 { + return MakeGitError(ret) + } + + pointerHandles.Untrack(t.handle) + runtime.SetFinalizer(t, nil) + t.handle = nil + return nil +} + +//export smartTransportCallback +func smartTransportCallback( + errorMessage **C.char, + out **C.git_transport, + owner *C.git_remote, + handle unsafe.Pointer, +) C.int { + registeredSmartTransport := pointerHandles.Get(handle).(*RegisteredSmartTransport) + remote, ok := remotePointers.get(owner) + if !ok { + err := errors.New("remote pointer not found") + return setCallbackError(errorMessage, err) + } + + managed := &managedSmartSubtransport{ + remote: remote, + callback: registeredSmartTransport.callback, + subtransport: (*C._go_managed_smart_subtransport)(C.calloc(1, C.size_t(unsafe.Sizeof(C._go_managed_smart_subtransport{})))), + } + managedHandle := pointerHandles.Track(managed) + managed.handle = managedHandle + managed.subtransport.handle = managedHandle + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C._go_git_transport_smart(out, owner, cbool(registeredSmartTransport.stateless), managed.subtransport) + if ret != 0 { + pointerHandles.Untrack(managedHandle) + } + return ret +} + +//export smartTransportSubtransportCallback +func smartTransportSubtransportCallback( + errorMessage **C.char, + wrapperPtr *C._go_managed_smart_subtransport, + owner *C.git_transport, +) C.int { + subtransport := pointerHandles.Get(wrapperPtr.handle).(*managedSmartSubtransport) + + underlyingSmartSubtransport, err := subtransport.callback(subtransport.remote, &Transport{ptr: owner}) + if err != nil { + return setCallbackError(errorMessage, err) + } + subtransport.underlying = underlyingSmartSubtransport + return C.int(ErrorCodeOK) +} + +type managedSmartSubtransport struct { + owner *C.git_transport + callback SmartSubtransportCallback + remote *Remote + subtransport *C._go_managed_smart_subtransport + underlying SmartSubtransport + handle unsafe.Pointer + currentManagedStream *managedSmartSubtransportStream +} + +func getSmartSubtransportInterface(subtransport *C.git_smart_subtransport) *managedSmartSubtransport { + wrapperPtr := (*C._go_managed_smart_subtransport)(unsafe.Pointer(subtransport)) + return pointerHandles.Get(wrapperPtr.handle).(*managedSmartSubtransport) +} + +//export smartSubtransportActionCallback +func smartSubtransportActionCallback( + errorMessage **C.char, + out **C.git_smart_subtransport_stream, + t *C.git_smart_subtransport, + url *C.char, + action C.git_smart_service_t, +) C.int { + subtransport := getSmartSubtransportInterface(t) + + underlyingStream, err := subtransport.underlying.Action(C.GoString(url), SmartServiceAction(action)) + if err != nil { + return setCallbackError(errorMessage, err) + } + + // It's okay to do strict equality here: we expect both to be identical. + if subtransport.currentManagedStream == nil || subtransport.currentManagedStream.underlying != underlyingStream { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + stream := (*C._go_managed_smart_subtransport_stream)(C.calloc(1, C.size_t(unsafe.Sizeof(C._go_managed_smart_subtransport_stream{})))) + managed := &managedSmartSubtransportStream{ + underlying: underlyingStream, + streamPtr: stream, + } + managedHandle := pointerHandles.Track(managed) + managed.handle = managedHandle + stream.handle = managedHandle + + C._go_git_setup_smart_subtransport_stream(stream) + + subtransport.currentManagedStream = managed + } + + *out = &subtransport.currentManagedStream.streamPtr.parent + return C.int(ErrorCodeOK) +} + +//export smartSubtransportCloseCallback +func smartSubtransportCloseCallback(errorMessage **C.char, t *C.git_smart_subtransport) C.int { + subtransport := getSmartSubtransportInterface(t) + + subtransport.currentManagedStream = nil + + if subtransport.underlying != nil { + err := subtransport.underlying.Close() + if err != nil { + return setCallbackError(errorMessage, err) + } + } + + return C.int(ErrorCodeOK) +} + +//export smartSubtransportFreeCallback +func smartSubtransportFreeCallback(t *C.git_smart_subtransport) { + subtransport := getSmartSubtransportInterface(t) + + if subtransport.underlying != nil { + subtransport.underlying.Free() + subtransport.underlying = nil + } + pointerHandles.Untrack(subtransport.handle) + C.free(unsafe.Pointer(subtransport.subtransport)) + subtransport.handle = nil + subtransport.subtransport = nil +} + +type managedSmartSubtransportStream struct { + owner *C.git_smart_subtransport_stream + streamPtr *C._go_managed_smart_subtransport_stream + underlying SmartSubtransportStream + handle unsafe.Pointer +} + +func getSmartSubtransportStreamInterface(subtransportStream *C.git_smart_subtransport_stream) *managedSmartSubtransportStream { + managedSubtransportStream := (*C._go_managed_smart_subtransport_stream)(unsafe.Pointer(subtransportStream)) + return pointerHandles.Get(managedSubtransportStream.handle).(*managedSmartSubtransportStream) +} + +//export smartSubtransportStreamReadCallback +func smartSubtransportStreamReadCallback( + errorMessage **C.char, + s *C.git_smart_subtransport_stream, + buffer *C.char, + bufSize C.size_t, + bytesRead *C.size_t, +) C.int { + stream := getSmartSubtransportStreamInterface(s) + + var p []byte + header := (*reflect.SliceHeader)(unsafe.Pointer(&p)) + header.Cap = int(bufSize) + header.Len = int(bufSize) + header.Data = uintptr(unsafe.Pointer(buffer)) + + n, err := stream.underlying.Read(p) + *bytesRead = C.size_t(n) + if n == 0 && err != nil { + if err == io.EOF { + return C.int(ErrorCodeOK) + } + + return setCallbackError(errorMessage, err) + } + + return C.int(ErrorCodeOK) +} + +//export smartSubtransportStreamWriteCallback +func smartSubtransportStreamWriteCallback( + errorMessage **C.char, + s *C.git_smart_subtransport_stream, + buffer *C.char, + bufLen C.size_t, +) C.int { + stream := getSmartSubtransportStreamInterface(s) + + var p []byte + header := (*reflect.SliceHeader)(unsafe.Pointer(&p)) + header.Cap = int(bufLen) + header.Len = int(bufLen) + header.Data = uintptr(unsafe.Pointer(buffer)) + + if _, err := stream.underlying.Write(p); err != nil { + return setCallbackError(errorMessage, err) + } + + return C.int(ErrorCodeOK) +} + +//export smartSubtransportStreamFreeCallback +func smartSubtransportStreamFreeCallback(s *C.git_smart_subtransport_stream) { + stream := getSmartSubtransportStreamInterface(s) + + stream.underlying.Free() + pointerHandles.Untrack(stream.handle) + C.free(unsafe.Pointer(stream.streamPtr)) + stream.handle = nil + stream.streamPtr = nil +} diff --git a/transport_test.go b/transport_test.go new file mode 100644 index 00000000..9f50047c --- /dev/null +++ b/transport_test.go @@ -0,0 +1,72 @@ +package git + +import ( + "io" + "reflect" + "testing" +) + +type testSmartSubtransport struct { +} + +func (t *testSmartSubtransport) Action(url string, action SmartServiceAction) (SmartSubtransportStream, error) { + return &testSmartSubtransportStream{}, nil +} + +func (t *testSmartSubtransport) Close() error { + return nil +} + +func (t *testSmartSubtransport) Free() { +} + +type testSmartSubtransportStream struct { +} + +func (s *testSmartSubtransportStream) Read(buf []byte) (int, error) { + payload := "" + + "001e# service=git-upload-pack\n" + + "0000005d0000000000000000000000000000000000000000 HEAD\x00symref=HEAD:refs/heads/master agent=libgit\n" + + "003f0000000000000000000000000000000000000000 refs/heads/master\n" + + "0000" + + return copy(buf, []byte(payload)), io.EOF +} + +func (s *testSmartSubtransportStream) Write(buf []byte) (int, error) { + return 0, io.EOF +} + +func (s *testSmartSubtransportStream) Free() { +} + +func TestTransport(t *testing.T) { + t.Parallel() + repo := createTestRepo(t) + defer cleanupTestRepo(t, repo) + + callback := func(remote *Remote, transport *Transport) (SmartSubtransport, error) { + return &testSmartSubtransport{}, nil + } + registeredSmartTransport, err := NewRegisteredSmartTransport("foo", true, callback) + checkFatal(t, err) + defer registeredSmartTransport.Free() + + remote, err := repo.Remotes.Create("test", "foo://bar") + checkFatal(t, err) + defer remote.Free() + + err = remote.ConnectFetch(nil, nil, nil) + checkFatal(t, err) + + remoteHeads, err := remote.Ls() + checkFatal(t, err) + + expectedRemoteHeads := []RemoteHead{ + {&Oid{}, "HEAD"}, + {&Oid{}, "refs/heads/master"}, + } + if !reflect.DeepEqual(expectedRemoteHeads, remoteHeads) { + t.Errorf("mismatched remote heads. expected %v, got %v", expectedRemoteHeads, remoteHeads) + } +} diff --git a/wrapper.c b/wrapper.c index 282d9a8d..6f65f715 100644 --- a/wrapper.c +++ b/wrapper.c @@ -504,3 +504,116 @@ int _go_git_indexer_new( indexer_options.progress_cb_payload = progress_cb_payload; return git_indexer_new(out, path, mode, odb, &indexer_options); } + +static int smart_transport_callback( + git_transport **out, + git_remote *owner, + void *param) +{ + char *error_message = NULL; + const int ret = smartTransportCallback( + &error_message, + out, + owner, + param); + return set_callback_error(error_message, ret); +} + +int _go_git_transport_register(const char *prefix, void *param) +{ + return git_transport_register(prefix, smart_transport_callback, param); +} + +static int smart_subtransport_action_callback( + git_smart_subtransport_stream **out, + git_smart_subtransport *transport, + const char *url, + git_smart_service_t action) +{ + char *error_message = NULL; + const int ret = smartSubtransportActionCallback( + &error_message, + out, + transport, + (char *)url, + action); + return set_callback_error(error_message, ret); +} + +static int smart_subtransport_close_callback(git_smart_subtransport *transport) +{ + char *error_message = NULL; + const int ret = smartSubtransportCloseCallback( + &error_message, + transport); + return set_callback_error(error_message, ret); +} + +static int smart_subtransport_callback( + git_smart_subtransport **out, + git_transport *owner, + void *param) +{ + _go_managed_smart_subtransport *subtransport = (_go_managed_smart_subtransport *)param; + subtransport->parent.action = smart_subtransport_action_callback; + subtransport->parent.close = smart_subtransport_close_callback; + subtransport->parent.free = smartSubtransportFreeCallback; + + *out = &subtransport->parent; + char *error_message = NULL; + const int ret = smartTransportSubtransportCallback(&error_message, subtransport, owner); + return set_callback_error(error_message, ret); +} + +int _go_git_transport_smart( + git_transport **out, + git_remote *owner, + int stateless, + _go_managed_smart_subtransport *subtransport_payload) +{ + git_smart_subtransport_definition definition = { + smart_subtransport_callback, + stateless, + subtransport_payload, + }; + + return git_transport_smart(out, owner, &definition); +} + +static int smart_subtransport_stream_read_callback( + git_smart_subtransport_stream *stream, + char *buffer, + size_t buf_size, + size_t *bytes_read) +{ + char *error_message = NULL; + const int ret = smartSubtransportStreamReadCallback( + &error_message, + stream, + buffer, + buf_size, + bytes_read); + return set_callback_error(error_message, ret); +} + +static int smart_subtransport_stream_write_callback( + git_smart_subtransport_stream *stream, + const char *buffer, + size_t len) +{ + char *error_message = NULL; + const int ret = smartSubtransportStreamWriteCallback( + &error_message, + stream, + (char *)buffer, + len); + return set_callback_error(error_message, ret); +} + +void _go_git_setup_smart_subtransport_stream(_go_managed_smart_subtransport_stream *stream) +{ + _go_managed_smart_subtransport_stream *managed_stream = (_go_managed_smart_subtransport_stream *)stream; + managed_stream->parent.read = smart_subtransport_stream_read_callback; + managed_stream->parent.write = smart_subtransport_stream_write_callback; + managed_stream->parent.free = smartSubtransportStreamFreeCallback; +}