Add HTTP integegration tests (#227)

* test: add itests for http

* test: add peer http server, minor refactors and fixes

pkg/internal/itest/http_fetch_test.go

@@ -56,6 +56,7 @@ func TestHttpFetch(t *testing.T) {
 		name             string
 		graphsyncRemotes int
 		bitswapRemotes   int
+		httpRemotes      int
 		disableGraphsync bool
 		expectFail       bool
 		modifyHttpConfig func(httpserver.HttpServerConfig) httpserver.HttpServerConfig
@@ -78,6 +79,13 @@ func TestHttpFetch(t *testing.T) {
 				return []unixfs.DirEntry{unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)}
 			},
 		},
+		{
+			name:        "http large sharded file",
+			httpRemotes: 1,
+			generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
+				return []unixfs.DirEntry{unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20)}
+			},
+		},
 		{
 			name:             "graphsync large directory",
 			graphsyncRemotes: 1,
@@ -92,6 +100,13 @@ func TestHttpFetch(t *testing.T) {
 				return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false)}
 			},
 		},
+		{
+			name:        "http large directory",
+			httpRemotes: 1,
+			generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
+				return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, false)}
+			},
+		},
 		{
 			name:             "graphsync large sharded directory",
 			graphsyncRemotes: 1,
@@ -106,6 +121,13 @@ func TestHttpFetch(t *testing.T) {
 				return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true)}
 			},
 		},
+		{
+			name:        "http large sharded directory",
+			httpRemotes: 1,
+			generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
+				return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, true)}
+			},
+		},
 		{
 			name:             "graphsync max block limit",
 			graphsyncRemotes: 1,
@@ -636,6 +658,8 @@ func TestHttpFetch(t *testing.T) {
 				finishedChans = append(finishedChans, mocknet.SetupRetrieval(t, r))
 			}
 			mrn.AddBitswapPeers(testCase.bitswapRemotes)
+			mrn.AddHttpPeers(testCase.httpRemotes)
+
 			require.NoError(t, mrn.MN.LinkAll())
 
 			carFiles := debugRemotes(t, ctx, testCase.name, mrn.Remotes)

pkg/internal/itest/mocknet/mocknet.go

@@ -22,6 +22,7 @@ import (
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/peer"
 	lpmock "github.com/libp2p/go-libp2p/p2p/net/mock"
+	"github.com/multiformats/go-multicodec"
 	"github.com/stretchr/testify/require"
 )
 
@@ -62,22 +63,25 @@ func NewMockRetrievalNet(ctx context.Context, t *testing.T) *MockRetrievalNet {
 }
 
 func (mrn *MockRetrievalNet) AddBitswapPeers(n int) {
-	peers := mrn.testPeerGenerator.BitswapPeers(n)
-	for i := 0; i < n; i++ {
-		mrn.Remotes = append(mrn.Remotes, peers[i])
-		mrn.RemoteEvents = append(mrn.RemoteEvents, make([]datatransfer.Event, 0)) // not used for bitswap
-		mrn.FinishedChan = append(mrn.FinishedChan, make(chan struct{}, 1))        // not used for bitswap
-	}
+	mrn.addPeers(mrn.testPeerGenerator.BitswapPeers(n))
 }
 
 func (mrn *MockRetrievalNet) AddGraphsyncPeers(n int) {
-	peers := mrn.testPeerGenerator.GraphsyncPeers(n)
-	for i := 0; i < n; i++ {
+	mrn.addPeers(mrn.testPeerGenerator.GraphsyncPeers(n))
+}
+
+func (mrn *MockRetrievalNet) AddHttpPeers(n int) {
+	mrn.addPeers(mrn.testPeerGenerator.HttpPeers(n))
+}
+
+func (mrn *MockRetrievalNet) addPeers(peers []testpeer.TestPeer) {
+	for i := 0; i < len(peers); i++ {
 		mrn.Remotes = append(mrn.Remotes, peers[i])
 		mrn.RemoteEvents = append(mrn.RemoteEvents, make([]datatransfer.Event, 0))
 		mrn.FinishedChan = append(mrn.FinishedChan, make(chan struct{}, 1))
 	}
 }
+
 func SetupRetrieval(t *testing.T, remote testpeer.TestPeer) chan []datatransfer.Event {
 	// Register DealProposal voucher type with automatic Pull acceptance
 	remoteDealValidator := &mockDealValidator{t: t, acceptPull: true}
@@ -129,6 +133,9 @@ func (mrn *MockRetrievalNet) TearDown() error {
 			if h.BitswapNetwork != nil {
 				h.BitswapNetwork.Stop()
 			}
+			if h.HttpServer != nil {
+				h.HttpServer.Close()
+			}
 		}(h)
 	}
 	wg.Wait()
@@ -144,10 +151,13 @@ func (mcf *mockCandidateFinder) FindCandidates(ctx context.Context, cid cid.Cid)
 	for _, h := range mcf.mrn.Remotes {
 		if _, has := h.Cids[cid]; has {
 			var md metadata.Metadata
-			if h.BitswapServer != nil {
+			switch h.Protocol {
+			case multicodec.TransportBitswap:
 				md = metadata.Default.New(metadata.Bitswap{})
-			} else {
+			case multicodec.TransportGraphsyncFilecoinv1:
 				md = metadata.Default.New(&metadata.GraphsyncFilecoinV1{PieceCID: cid})
+			case multicodec.TransportIpfsGatewayHttp:
+				md = metadata.Default.New(&metadata.IpfsGatewayHttp{})
 			}
 			candidates = append(candidates, types.RetrievalCandidate{MinerPeer: *h.AddrInfo(), RootCid: cid, Metadata: md})
 		}