docs(architecture): flesh out architecture docs

Provide a detailed guide to the architecture of go-graphsync

Co-Authored-By: MollyM <momack2@users.noreply.github.com>

docs/async-loading.puml

@@ -0,0 +1,75 @@
+@startuml async loading
+participant IPLD
+participant "Intercepted Loader" as ILoader
+participant RequestManager
+participant AsyncLoader
+participant LoadAttemptQueue
+participant ResponseCache
+participant Loader
+participant Storer
+IPLD -> ILoader: Load Link
+activate ILoader
+ILoader -> AsyncLoader: Load Link Asynchronously
+activate AsyncLoader
+ILoader <-- AsyncLoader: Channel For future response
+... transfer to internal process ...
+AsyncLoader -> LoadAttemptQueue: Try Loading This Link
+deactivate AsyncLoader
+
+LoadAttemptQueue -> ResponseCache: Try Loading This Link
+alt response cache has block
+ResponseCache -> Storer: Store the block for later
+ResponseCache -> ResponseCache: Remove the block from cache
+LoadAttemptQueue <-- ResponseCache: "Here's the block"
+note over LoadAttemptQueue: Response = Block
+else response cache told block is missing by remote peer
+LoadAttemptQueue <-- ResponseCache: "We're missing this link"
+note over LoadAttemptQueue: Response = Error Missing Link
+else local store has block
+LoadAttemptQueue <-- ResponseCache: "I Don't Have it!"
+LoadAttemptQueue -> Loader: Try Loading This Link From Local Cache
+LoadAttemptQueue <-- Loader: "Here's the block"
+note over LoadAttemptQueue: Response = Block
+else no block or known missing link yet
+LoadAttemptQueue <-- ResponseCache: "I Don't Have it!"
+LoadAttemptQueue -> Loader: Try Loading This Link From Local Cache
+LoadAttemptQueue <-- Loader: "I Don't Have it!"
+LoadAttemptQueue -> LoadAttemptQueue: Store load request to try later
+end
+loop 0 or more times till I have a response for the link
+...
+opt new responses comes in
+RequestManager -> AsyncLoader: New Responses Present
+activate AsyncLoader
+AsyncLoader -> ResponseCache: New Responses Present
+... transfer to internal process ...
+AsyncLoader -> LoadAttemptQueue: Try Loading Again
+deactivate AsyncLoader
+LoadAttemptQueue -> ResponseCache: Try Loading Stored Links
+alt response cache now has block
+ResponseCache -> Storer: Store the block for later
+ResponseCache -> ResponseCache: Remove the block from cache
+LoadAttemptQueue <-- ResponseCache: "Here's the block"
+note over LoadAttemptQueue: Response = Block
+else response cache now knows link is missing by remote peer
+LoadAttemptQueue <-- ResponseCache: "We're missing this link"
+note over LoadAttemptQueue: Response = Error Missing Link
+else still no response
+LoadAttemptQueue <-- ResponseCache: "I don't have it"
+LoadAttemptQueue -> LoadAttemptQueue: Store load request to try later
+end
+end
+opt no more responses
+RequestManager -> AsyncLoader: No more responses
+activate AsyncLoader
+... transfer to internal process ...
+AsyncLoader -> LoadAttemptQueue: Cancel attempts to load
+note over LoadAttemptQueue: Response = Error Request Finished
+deactivate AsyncLoader
+end
+end
+ILoader <-- LoadAttemptQueue: Response Sent Over Channel
+IPLD <-- ILoader : "Here's the stream of block\n data or an error"
+deactivate ILoader
+
+@enduml

docs/go-graphsync.puml

@@ -29,20 +29,15 @@ package "go-ipld-prime" {
     TraversalProgress *-- TraversalConfig
   }
 
+  interface Storer {
+
+  }
   interface Loader {
 
   }
 }
 
 package "go-graphsync" {
-
-    class ResponseProgress {
-      Node ipld.Node
-      Path ipld.Path
-      LastBlock struct {
-        ipld.Node
-        ipld.Link }
-    }
 
     interface Cid2BlockFn {
 
@@ -56,6 +51,7 @@ package "go-graphsync" {
 
     GraphSync *-- Loader
 
+
   package network {
 
     interface Receiver {
@@ -126,20 +122,110 @@ package "go-graphsync" {
     PeerMessageManager *-- MessageQueue
   }
 
+    package linktracker {
+      class LinkTracker {
+        ShouldSendBlockFor(Link) bool
+        RecordLinkTraversal(GraphSyncRequestID, Link, bool)
+        FinishRequest(GraphSyncRequestID) bool
+      }
+      object "Package Public Functions" as goLinkTrackerPF {
+        New() *LinkTracker
+      }
+    }
+
   package requestmanager {
-    interface ResponseProgress {
+    package types {
+      interface ResponseProgress {
+      }
+      interface AsyncLoadResult {
+      }
+    }
 
+    package "loader" as reqLoader {
+      interface AsyncLoadFn {
+      }
+      object "Package Public Functions" as goRequestLoaderPF {
+        WrapAsyncLoader(context.Context, AsyncLoadFn, GraphSyncRequestID, chan error)
+      }
     }
-    interface ResponseError {
 
+    package asyncloader {
+      package loadattempqueue {
+        interface LoadRequest {
+
+        }
+        interface LoadAttempter {
+          func(GraphSyncRequestID, ipld.Link) ([]byte, error)
+        }
+
+        class LoadAttemptQueue {
+          AttemptLoad(LoadRequest, bool)
+          ClearRequest(GraphSyncRequestID)
+          RetryLoads()
+        }
+        object "Package Public Functions" as goLoadAttemptQueuePF {
+          NewLoadRequest(GraphSyncRequestID, ipld.Link, chan AsyncLoadResult) LoadRequest
+          New(LoadAttempter) *LoadAttemptQueue
+        }
+      }
+      package unverifiedblockstore {
+        class UnverifiedBlockStore {
+          AddUnverifiedBlock(ipld.Link, []byte)
+          PruneBlocks(func(ipld.Link) bool)
+          VerifyBlock(ipld.Link) ([]byte, error)
+        }
+        object "Package Public Functions" as goUnverifiedBlockStore {
+          New(Storer) *UnverifiedBlockStore
+        }
+      }
+      package responsecache {
+        class ResponseCache {
+          FinishRequest(GraphSyncRequestID)
+          AttemptLoad(GraphSyncRequestID, ipld.Link) ([]byte, error)
+          ProcessResponse(map[GraphSyncRequestID]Metadata, []blocks.Block)       
+        }
+        object "Package Public Functions" as goResponseCachePF {
+          New(UnverifiedBlockStore) *ResponseCache
+        }
+        ResponseCache *-- LinkTracker
+        ResponseCache *-- UnverifiedBlockStore
+        ResponseCache .. goLinkTrackerPF
+      }
+
+      class AsyncLoader {
+        StartRequest(GraphSyncRequestID)
+        ProcessResponse(map[gsmsg.GraphSyncRequestID]metadata.Metadata, []blocks.Block)
+        AsyncLoad(requestID gsmsg.GraphSyncRequestID, link ipld.Link) AsyncLoadResult
+        CompleteResponsesFor(GraphSyncRequestID)
+        CleanupRequest(GraphSyncRequestID)
+      }
+
+      object "Package Public Functions" as goAsyncLoaderPF {
+        New(context.Context, ipld.Loader, ipld.Storer) *AsyncLoader
+      }
+      AsyncLoader *-- LoadAttemptQueue
+      AsyncLoader *-- ResponseCache
+      AsyncLoader *-- Loader
+      AsyncLoader *-- Storer
+      AsyncLoader .. goUnverifiedBlockStore
+      AsyncLoader .. goResponseCachePF
+      AsyncLoader .. goLoadAttemptQueuePF
     }
+
+
     class RequestManager {
       SetDelegate(peerHandler PeerMessageManager)
-      SendRequest(ctx context.Context, p peer.ID, cidRootedSelector Node) chan Block
-      ProcessResponses(message GraphSyncMessage)
+      SendRequest(ctx context.Context, p peer.ID, cidRootedSelector Node) chan ResponseProgress, chan error
+      ProcessResponses(peer.ID, []GraphSyncResponse, []blocks.Block)
+    }
+    object "Package Public Functions" as goRequestManagerPF {
+      New(ctx context.Context, asyncLoader AsyncLoader, ipldBridge ipldbridge.IPLDBridge) *RequestManager
     }
-    RequestManager *-- PeerMessageManager
+    RequestManager *-- AsyncLoader
+    RequestManager *-- PeerManager
+    RequestManager .. goRequestLoaderPF
     GraphSync *-- RequestManager
+    GraphSync .. goRequestManagerPF
   }
 
 
@@ -203,21 +289,11 @@ package "go-graphsync" {
       GraphSync .. goPeerTaskQueuePF     
     }
 
-    package loader {
+    package "loader" as resLoader {
       object "Package Public Functions" as goResponseLoaderPF {
         WrapLoader(Loader,GraphSyncRequestID, PeerResponseSender) Loader
       }
     }
-    package linktracker {
-      class LinkTracker {
-        ShouldSendBlockFor(Link) bool
-        RecordLinkTraversal(GraphSyncRequestID, Link, bool)
-        FinishRequest(GraphSyncRequestID) bool
-      }
-      object "Package Public Functions" as goLinkTrackerPF {
-        New() *LinkTracker
-      }
-    }
 
     package responsebuilder {
       class ResponseBuilder {
@@ -253,7 +329,6 @@ package "go-graphsync" {
       PeerResponseSender *-- LinkTracker
       PeerResponseSender *-- ResponseBuilder
       PeerResponseSender *-- PeerMessageManager
-      PeerResponseSender *-- IPLDBridge
       PeerResponseSender .. goLinkTrackerPF
       PeerResponseSender .. goResponseBuilderPF
       GraphSync .. goPeerResponseManagerPF     
@@ -268,7 +343,6 @@ package "go-graphsync" {
     }
     GraphSync *-- ResponseManager
     ResponseManager *-- Loader
-    ResponseManager *-- IPLDBridge
     ResponseManager *-- PeerResponseManager
     ResponseManager *-- PeerTaskQueue
     ResponseManager .. goResponseLoaderPF
@@ -332,7 +406,8 @@ package "go-graphsync" {
     GraphSync *-- IPLDBridge
     RequestManager *-- IPLDBridge
     ResponseManager *-- IPLDBridge
-
+    PeerResponseSender *-- IPLDBridge
+
     class ipldBridge {
     }
 

docs/processes.puml

@@ -0,0 +1,79 @@
+@startuml Overview
+start
+if ()
+:Graphsync Client Makes Request;
+else
+:Incoming Network Traffic;
+:Message Decoding;
+endif
+partition "Top Level Interface" {
+:GraphSync;
+}
+if (operation type) then (outgoing request or incoming response)
+partition "Graphsync Requestor Implementation" {
+:RequestManager;
+if (operation type) then (incoming response)
+:AsyncLoader;
+partition "Verifying Queries" {
+fork
+:ipld.Traverse;
+fork again
+:ipld.Traverse;
+fork again
+:ipld.Traverse;
+end fork
+}
+partition "Collecting Responses" {
+fork
+:Response Collector;
+fork again
+:Response Collector;
+fork again
+:Response Collector;
+end fork
+}
+:Responses returned to client;
+stop
+else (outgoing request)
+:Send Request To Network;
+endif
+}
+else (incoming request)
+partition "Graphsync Responder Implementation" {
+:ResponseManager;
+partition "Performing Queries" {
+:PeerTaskQueue;
+fork
+:ipld.Traverse;
+fork again
+:ipld.Traverse;
+fork again
+:ipld.Traverse;
+end fork
+}
+partition "Aggregating Responses" {
+:PeerResponseManager;
+fork
+:PeerResponseSender;
+fork again
+:PeerResponseSender;
+fork again
+:PeerResponseSender;
+end fork
+}
+}
+endif
+partition "Message Sending Layer" {
+:PeerManager;
+fork
+:MessageQueue;
+fork again
+:MessageQueue;
+fork again
+:MessageQueue;
+end fork
+:Message Encoding;
+}
+:Outgoing Network Traffic;
+stop
+@enduml