Merge branch 'master' of github.com:ipfs-search/ipfs-search into master

ipfs-search · Oct 22, 2020 · e652a49 · e652a49
2 parents 811d59c + 5abd70f
commit e652a49
Show file tree

Hide file tree

Showing 10 changed files with 106 additions and 52 deletions.
diff --git a/deployment/README.md b/deployment/README.md
@@ -6,6 +6,12 @@ The default Ansible configuration we're using, `ansible.cfg` in the current dire
 
 In order to use Ansible deployment, `backend` and `frontend` host groups need to be configured here. An example is provided in [`inventory_example.yml`](inventory_example.yml).
 
+## Ansible Galaxy requirements
+May be installed with:
+```bash
+$ ansible-galaxy install -r requirements.yml
+```
+
 ## Vault
 In addition to the inventory, a succesful deployment requires several secrets to be configured in `vault/secrets.yml`. An example file is provided in [`vault/secrets_example.yml`](vault/secrets_example.yml), which you can encrypt with `ansible-vault encrypt` and later edited with `ansible-vault edit vault/secrets.yml`.
 

diff --git a/deployment/backend.yml b/deployment/backend.yml
@@ -5,7 +5,9 @@
     - es_heap_size: 16g
     - ipfs_memlimit: 12G
     - crawler_memlimit: 4G
+    - golang_version: 1.14.4
   roles:
+    - { name: golang, tags: ['golang']}
     - { name: disable-eth-offloading, tags: ['disable-eth-offloading']}
     - { name: cgconfig, tags: ['cgconfig']}
     - { name: elasticsearch, tags: ['elasticsearch']}

diff --git a/deployment/requirements.yml b/deployment/requirements.yml
@@ -1,3 +1,5 @@
 ---
 - name: wireguard
   src: adamruzicka.wireguard
+- name: golang
+  src: gantsign.golang
diff --git a/deployment/roles/crawler/templates/config.yml b/deployment/roles/crawler/templates/config.yml
@@ -14,8 +14,8 @@ crawler:
   hash_wait: 100ms  # Time between launching workers
   file_wait: 100ms
   partial_size: 256KB  # Size for partial items - this is the default chunker block size
-  hash_workers: 160
-  file_workers: 60
+  hash_workers: 360
+  file_workers: 30
 sniffer:
   lastseen_expiration: 60m
   lastseen_prunelen: 16383

diff --git a/deployment/roles/ipfs/defaults/main.yml b/deployment/roles/ipfs/defaults/main.yml
@@ -1,7 +1,7 @@
 ---
-binary_url: "https://dist.ipfs.io/go-ipfs/v0.4.23/go-ipfs_v0.4.23_linux-amd64.tar.gz"
-binary_sha512: "5eebebd4d4628a01c3b6615d96123a5c744f64da18fc0950e00d99a36abb02eee694c1bb67549341a645ebb99f30de9198c33b556cdee2609013409a510d1d2b"
+ipfs_version: v0.7.0
 ipfs_path: "/var/lib/ipfs"
+ipfs_build: "/tmp/go-ipfs"
 ipfs_binary: "/usr/local/bin/ipfs"
 ipfs_user: "ipfs"
 ipfs_group: "ipfs"

diff --git a/deployment/roles/ipfs/tasks/main.yml b/deployment/roles/ipfs/tasks/main.yml
@@ -1,16 +1,41 @@
 ---
-- name: download go-ipfs binary
-  get_url: url={{ binary_url }} checksum=sha512:{{ binary_sha512 }} dest=/tmp/go-ipfs.tar.gz
-  register: binary
+- name: build requirements
+  apt:
+    pkg:
+      - build-essential
+      - libssl-dev
+      - pkg-config
+    state: present
 - block:
-    - name: extract go-ipfs archive
-      unarchive: copy=no src=/tmp/go-ipfs.tar.gz dest=/tmp/
-    - name: install go-ipfs
-      copy: remote_src=True src=/tmp/go-ipfs/ipfs dest="{{ ipfs_binary }}" mode=755
-      notify: restart ipfs
-    - name: remove temporary files
-      file: state=absent path=/tmp/go-ipfs
-  when: binary.changed
+  - name: clone go-ipfs
+    git:
+      dest: "{{ ipfs_build }}"
+      repo: https://github.com/ipfs/go-ipfs.git
+      version: "{{ ipfs_version }}"
+    register: source
+  - block:
+    - name: clean go-ipfs
+      make:
+        chdir: "{{ ipfs_build }}"
+        target: clean
+    - name: make go-ipfs
+      make:
+        chdir: "{{ ipfs_build }}"
+        params:
+          GOTAGS=openssl
+        target: build
+  environment:
+    PATH: "{{ golang_install_dir }}/bin:{{ ansible_env.PATH }}"
+  become: false
+- name: Install ipfs binary
+  copy:
+    src: "{{ ipfs_build }}/cmd/ipfs/ipfs"
+    dest: "{{ ipfs_binary }}"
+    remote_src: True
+    mode: 755
+  become: true
+  notify: restart ipfs
+  # when: source.changed
 - group: name=ipfs system=true state=present
 - user: name=ipfs comment="IPFS daemon" group=ipfs system=true state=present home="{{ ipfs_path }}" shell=/usr/sbin/nologin
 - file: path="{{ ipfs_path }}" owner=ipfs group=ipfs mode=750 state=directory

diff --git a/deployment/roles/ipfs/templates/ipfs_config.json b/deployment/roles/ipfs/templates/ipfs_config.json
@@ -6,7 +6,9 @@
     "API": "/ip4/127.0.0.1/tcp/5001",
     "Announce": [
       "/ip4/{{ ipify_public_ip }}/tcp/4001",
-      "/ip6/{{ ansible_default_ipv6.address }}/tcp/4001"
+      "/ip6/{{ ansible_default_ipv6.address }}/tcp/4001",
+      "/ip4/{{ ipify_public_ip }}/udp/4001/quic",
+      "/ip6/{{ ansible_default_ipv6.address }}/udp/4001/quic"
     ],
     "Gateway": "/ip4/127.0.0.1/tcp/8080",
     "NoAnnounce": [
@@ -33,23 +35,26 @@
     ],
     "Swarm": [
       "/ip4/0.0.0.0/tcp/4001",
-      "/ip6/::/tcp/4001"
+      "/ip6/::/tcp/4001",
+      "/ip4/0.0.0.0/udp/4001/quic",
+      "/ip6/::/udp/4001/quic"
     ]
   },
   "Bootstrap": [
-    "/dnsaddr/bootstrap.libp2p.io/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
-    "/dnsaddr/bootstrap.libp2p.io/ipfs/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
-    "/dnsaddr/bootstrap.libp2p.io/ipfs/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
-    "/dnsaddr/bootstrap.libp2p.io/ipfs/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
-    "/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
-    "/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
-    "/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
-    "/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
-    "/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
-    "/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
-    "/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
-    "/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
-    "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd"
+    "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
+    "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
+    "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
+    "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
+    "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
+    "/ip4/104.236.179.241/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
+    "/ip4/128.199.219.111/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
+    "/ip4/104.236.76.40/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
+    "/ip4/178.62.158.247/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
+    "/ip6/2604:a880:1:20::203:d001/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
+    "/ip6/2400:6180:0:d0::151:6001/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
+    "/ip6/2604:a880:800:10::4a:5001/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
+    "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
+    "/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
   ],
   "Datastore": {
     "BloomFilterSize": 0,
@@ -92,18 +97,19 @@
   },
   "Experimental": {
     "FilestoreEnabled": true,
-    "UrlstoreEnabled": false,
-    "ShardingEnabled": true,
+    "GraphsyncEnabled": false,
     "Libp2pStreamMounting": false,
     "P2pHttpProxy": false,
-    "QUIC": false,
-    "PreferTLS": false
+    "ShardingEnabled": true,
+    "StrategicProviding": false,
+    "UrlstoreEnabled": false
   },
   "Gateway": {
     "HTTPHeaders": {
       "Access-Control-Allow-Headers": [
         "X-Requested-With",
-        "Range"
+        "Range",
+        "User-Agent"
       ],
       "Access-Control-Allow-Methods": [
         "GET"
@@ -112,7 +118,10 @@
         "*"
       ]
     },
+    "NoDNSLink": false,
+    "NoFetch": false,
     "PathPrefixes": [],
+    "PublicGateways": null,
     "RootRedirect": "",
     "Writable": false
   },
@@ -130,17 +139,25 @@
     "IPFS": "/ipfs",
     "IPNS": "/ipns"
   },
+  "Peering": {
+    "Peers": null
+  },
+  "Plugins": {
+    "Plugins": null
+  },
+  "Provider": {
+    "Strategy": ""
+  },
   "Pubsub": {
-    "Router": "",
     "DisableSigning": false,
-    "StrictSignatureVerification": false
+    "Router": ""
   },
   "Reprovider": {
-    "Interval": "0",
+    "Interval": "12h",
     "Strategy": "all"
   },
-  "SupernodeRouting": {
-    "Servers": null
+  "Routing": {
+    "Type": "dht"
   },
   "Swarm": {
     "AddrFilters": [
@@ -172,10 +189,11 @@
     "EnableAutoRelay": false,
     "EnableAutoNATService": false,
     "ConnMgr": {
-      "Type": "basic",
-      "LowWater": 9000,
-      "HighWater": 10000,
-      "GracePeriod": "6m"
+      "GracePeriod": "20s",
+      "HighWater": 900,
+      "LowWater": 600,
+      "Type": "basic"
     }
-  }
+  },
+  "apply": "server"
 }
diff --git a/queue/mock.go b/queue/mock.go
@@ -10,13 +10,13 @@ type Mock struct {
 	mock.Mock
 }
 
-func (m *Mock) Publish(pub interface{}, priority uint8) error {
-	args := m.Called(pub, priority)
+func (m *Mock) Publish(ctx context.Context, pub interface{}, priority uint8) error {
+	args := m.Called(ctx, pub, priority)
 	return args.Error(0)
 }
 
-func (m *Mock) Consume() (<-chan amqp.Delivery, error) {
-	args := m.Called()
+func (m *Mock) Consume(ctx context.Context) (<-chan amqp.Delivery, error) {
+	args := m.Called(ctx)
 	return args.Get(0).(<-chan amqp.Delivery), args.Error(1)
 }
 

diff --git a/sniffer/queuer/queuer_test.go b/sniffer/queuer/queuer_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/ipfs-search/ipfs-search/queue"
 	"github.com/ipfs-search/ipfs-search/types"
 
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 )
 
@@ -49,7 +50,7 @@ func (s *QueuerTestSuite) TestQueueContextCancel() {
 
 // TestQueuePublish tests whether a queued provider gets published.
 func (s *QueuerTestSuite) TestQueuePublish() {
-	s.q.On("Publish", &crawler.Args{Hash: s.p.ID}, uint8(9)).Return(nil)
+	s.q.On("Publish", mock.AnythingOfType("*context.valueCtx"), &crawler.Args{Hash: s.p.ID}, uint8(9)).Return(nil)
 
 	ch := make(chan types.Provider)
 
@@ -73,7 +74,7 @@ func (s *QueuerTestSuite) TestQueuePublish() {
 func (s *QueuerTestSuite) TestQueueError() {
 	mockErr := errors.New("mock")
 
-	s.q.On("Publish", &crawler.Args{Hash: s.p.ID}, uint8(9)).Return(mockErr)
+	s.q.On("Publish", mock.AnythingOfType("*context.valueCtx"), &crawler.Args{Hash: s.p.ID}, uint8(9)).Return(mockErr)
 
 	ch := make(chan types.Provider)
 

diff --git a/sniffer/sniffer_test.go b/sniffer/sniffer_test.go
@@ -128,7 +128,7 @@ func (s *SnifferTestSuite) TestHandleToPublish() {
 
 	// Setup Mock Queue
 	qMock := &queue.Mock{}
-	qMock.On("Publish", &crawler.Args{
+	qMock.On("Publish", mock.AnythingOfType("*context.valueCtx"), &crawler.Args{
 		Hash: cidStr,
 	}, uint8(9)).Return(nil).Run(func(args mock.Arguments) {
 		fmt.Println("Publish() called, closing context")