DEVOP-1: Replace bridge-redis container with redis container

README.md

@@ -214,7 +214,29 @@ streamr-docker-dev start --wait
   - The Data Union Server manages the state of the 
   - The Data Union Server will be retired when Data Unions 2.0 is launched in Q1 of 2021
 - 1 x [ethereum-watcher](https://github.com/streamr-dev/streamr-ethereum-watcher)
-
+- 1 x [Chainlink node](https://github.com/streamr-dev/network-contracts)
+  - accessible at http://localhost:6688, user:a@a.com, password:testpass
+- 1 x [TheGraph node](https://github.com/streamr-dev/network-contracts)
+  - GraphQL queries at http://localhost:8000/subgraphs/name/githubname/subgraphname
+  - GUI to past GraphQL queries: http://192.168.0.8:8000/subgraphs/name/githubname/subgraphname/graphql
+  - An example query is:
+```
+{
+   streams {
+    id,
+    metadata,
+    permissions {
+      id,
+  		user,
+  		edit,
+      canDelete,
+      publish,
+      subscribed,
+      share,
+    }
+  }
+}
+```
 ### Supporting services
 - 1 x MySQL instance with databases `core_dev` and `core_test`
 - 1 x Redis instance
@@ -223,6 +245,9 @@ streamr-docker-dev start --wait
 - 1 x [Ethereum Parity node ("mainchain")](https://github.com/streamr-dev/open-ethereum-poa)
 - 1 x [Ethereum Parity node ("sidechain")](https://github.com/streamr-dev/open-ethereum-poa)
 - 1 x nginx
+- 1 x Postgres DB for TheGraph and Chainlink
+- 1 x ipfs for TheGraph
+- 1 x external adapter for ENSqueries from chainlink to mainchain
 
 ## Troubleshooting
 

chainlink_config/.api

@@ -0,0 +1,2 @@
+a@a.com
+testpass

chainlink_config/.env_streamr_sidechain

@@ -0,0 +1,11 @@
+ROOT=/chainlink
+LOG_LEVEL=debug
+ETH_CHAIN_ID=8997
+MIN_OUTGOING_CONFIRMATIONS=2
+LINK_CONTRACT_ADDRESS=0x3387F44140ea19100232873a5aAf9E46608c791E
+CHAINLINK_TLS_PORT=0
+SECURE_COOKIES=false
+GAS_UPDATER_ENABLED=true
+ALLOW_ORIGINS=*
+ETH_URL=ws://streamr-dev-parity-sidechain-node0:8450
+DATABASE_URL=postgresql://streamr:let-me-in@streamr-dev-postgres:5432/streamr?sslmode=disable

chainlink_config/.gitignore

@@ -0,0 +1 @@
+/log.jsonl

chainlink_config/.password

@@ -0,0 +1 @@
+AugustinerQW123!"§

chainlink_config/secret

@@ -0,0 +1 @@
+rrFLy5P2HqF/LG6K+xrsNMqgZJqCIm8SYVIuAgU+WhY=

chainlink_config/tempkeys/UTC--2021-06-06T15-05-54.430149681Z--7b5f1610920d5baf00d684929272213baf962efe

@@ -0,0 +1 @@
+{"address":"7b5f1610920d5baf00d684929272213baf962efe","crypto":{"cipher":"aes-128-ctr","ciphertext":"85271a6da2f5310a3b7120d3a2401565b03b83fc8c40e3d6fac0e4343eef33cf","cipherparams":{"iv":"43e3915ee0e323a0b47e37ac17a63ccd"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"5c6f877db46bec876ca303abc4a1ba12bfbbc52903fb081ab1e92a4a3a14de6f"},"mac":"bd84013d0ba6dde0a848674ee2871ccd6061d571234f2e119b2230df56312e3e"},"id":"c90a93de-913a-42ba-830c-80761e4b2873","version":3}

chainlink_config/tempkeys/UTC--2021-06-06T15-05-58.631255272Z--1dcaf21e385bbfafa02b0cc936eb2df7da155e64

@@ -0,0 +1 @@
+{"address":"1dcaf21e385bbfafa02b0cc936eb2df7da155e64","crypto":{"cipher":"aes-128-ctr","ciphertext":"a02212a07c826d75db11c82dbfe250a5d4e02a2b9c06eeb720c7794a9d739392","cipherparams":{"iv":"9e0dfb63c4a5ff5f9ba6e6e1af3b00bd"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"87504cff6c33dec55616c2953bf8373494ab2e1e43235dfb33665bac4b1e221b"},"mac":"fb64389116c49254e91e647a464b8830edc6f0834d99d74fc4fb18ef1d22a50e"},"id":"8363bfd3-6d24-4af6-84f6-821fec3312bf","version":3}

custom-nginx-reverse-proxy.conf

@@ -3,9 +3,12 @@ worker_processes 1;
 events { worker_connections 1024; }
 
 http {
-
+    include mime.types;
     server_names_hash_bucket_size 128;
     sendfile on;
+
+    proxy_buffers 8 1024k;  
+    proxy_buffer_size 1024k;
 
     upstream eae {
         server 10.200.10.1:8081;
@@ -83,7 +86,7 @@ http {
         }
 
         # Data REST endpoints
-        location ~ /api/v1/streams/(.*)/(data|data/partitions/.*)$ {
+        location ~ /api/v1/streams/(.*)/(data|metadata/partitions/.*|storage/partitions/.*|data/partitions/.*)$ {
             add_header X-debug "/api/v1/streams";
             proxy_pass http://storage_http;
             proxy_read_timeout 240s;
@@ -115,8 +118,6 @@ http {
             client_max_body_size 512m;
             proxy_read_timeout 600;
 
-            proxy_buffers 8 32k;
-
             proxy_set_header X-Forwarded-Proto $scheme;
             proxy_set_header X-Forwarded-Host $host;
             proxy_set_header X-Forwarded-Server $host;

docker-compose.yml

@@ -38,6 +38,14 @@ services:
         image: streamr/redis:v0.0.1
         ports:
             - "6379:6379"
+        networks:
+            - default
+            - net-db-bridge-request
+            - net-db-bridge-affirmation
+            - net-db-bridge-senderhome
+            - net-db-bridge-senderforeign
+        restart: unless-stopped
+        volumes: ['data-bridge-data-redis:/data']
     smtp:
         container_name: streamr-dev-smtp
         image: streamr/smtp:v0.0.1
@@ -63,7 +71,7 @@ services:
           - ./certs:/certs
         command: node tracker.js 0xa4fc7b912ed4588080ac9f2044cb3d6cab961bcf0d96de6fa9e773860bf59e1c tracker-1 --port=30301 ${TRACKER_ARGS}
         healthcheck:
-            test: ["CMD", "curl", "-sS", "${STREAMR_BASE_URL}:30301/topology/"]
+            test: ["CMD", "curl", "-sS", "--insecure", "${STREAMR_BASE_URL}:30301/topology/"]
             interval: 30s
             timeout: 10s
             retries: 20
@@ -77,7 +85,7 @@ services:
           - ./certs:/certs
         command: node tracker.js 0x3b328103c0d043cd1b34f1203c0313eb70c98443444dfdf7125f2d4479b89eb9 tracker-2 --port=30302 ${TRACKER_ARGS}
         healthcheck:
-            test: ["CMD", "curl", "-sS", "${STREAMR_BASE_URL}:30302/topology/"]
+            test: ["CMD", "curl", "-sS", "--insecure", "${STREAMR_BASE_URL}:30302/topology/"]
             interval: 30s
             timeout: 10s
             retries: 20
@@ -91,7 +99,7 @@ services:
           - ./certs:/certs
         command: node tracker.js 0xf456bb2a958574416f4b9e1b1b642cb3092953aa6131aba6e36f7529d1137795 tracker-3 --port=30303 ${TRACKER_ARGS}
         healthcheck:
-            test: ["CMD", "curl", "-sS", "${STREAMR_BASE_URL}:30303/topology/"]
+            test: ["CMD", "curl", "-sS", "--insecure", "${STREAMR_BASE_URL}:30303/topology/"]
             interval: 30s
             timeout: 10s
             retries: 20
@@ -194,10 +202,10 @@ services:
             - core-api
         environment:
             STREAMR_API_URL: "${STREAMR_BASE_URL}/api/v1"
-            METRICS: null
-            ETHEREUM_SERVER_URL: "http://10.200.10.1:8545"
+            DEVOPS_KEY: "0x628acb12df34bb30a0b2f95ec2e6a743b386c5d4f63aa9f338bec6f613160e78"
+            NETWORK_ID: "8995"
             MARKETPLACE_ADDRESS: "0xf1371c0f40528406dc4f4caf89924ea9da49e866"
-            DEVOPS_KEY: "devops-user-key"
+            ETHEREUM_SERVER_URL: "http://10.200.10.1:8545"
         volumes:
             - data-ethereum-watcher/:/app/logs
         healthcheck:
@@ -309,23 +317,6 @@ services:
             interval: 10s
             timeout: 10s
             retries: 60
-    bridge-redis:
-        container_name: streamr-dev-bridge-redis
-        command: [redis-server, --appendonly, 'yes']
-        hostname: bridge-redis
-        image: redis:4
-        networks:
-            - net-db-bridge-request
-            - net-db-bridge-affirmation
-            - net-db-bridge-senderhome
-            - net-db-bridge-senderforeign
-        restart: unless-stopped
-        volumes: ['data-bridge-data-redis:/data']
-        healthcheck:
-            test: ["CMD", "echo"] # TODO: health check
-            interval: 10s
-            timeout: 10s
-            retries: 60
     bridge-request:
         container_name: streamr-dev-bridge-request
         image: poanetwork/tokenbridge-oracle:latest
@@ -394,8 +385,109 @@ services:
             - bridge-senderhome
             - bridge-affirmation
             - bridge-rabbitmq
-            - bridge-redis
+            - redis
             - bridge-request
+    graph-node:
+        container_name: streamr-dev-thegraph-node
+        image: graphprotocol/graph-node
+        ports:
+            - '8000:8000'
+            - '8001:8001'
+            - '8020:8020'
+            - '8030:8030'
+            - '8040:8040'
+        depends_on:
+            - ipfs
+            - postgres
+            - parity-sidechain-node0
+        environment:
+            postgres_host: postgres
+            postgres_user: streamr
+            postgres_pass: let-me-in
+            postgres_db: streamr
+            ipfs: 'streamr-dev-ipfs:5001'
+            ethereum: 'xDai:http://streamr-dev-parity-sidechain-node0:8540'
+            RUST_LOG: info
+        healthcheck:
+            test: ["CMD", "echo"] # TODO: health check
+            interval: 10s
+            timeout: 10s
+            retries: 60
+    graph-deploy-streamregistry-subgraph:
+        container_name: streamr-dev-graph-deploy-streamreg-subgraph
+        image: streamr/graph-deploy-streamregistry-subgraph:dev
+        depends_on:
+            - graph-node
+        volumes:
+            - data-graph-deploy:/firstrun
+    ipfs:
+        container_name: streamr-dev-ipfs
+        image: ipfs/go-ipfs:v0.4.23
+        ports:
+            - '5001:5001'
+        volumes:
+            - data-ipfs:/data/ipfs
+        healthcheck:
+            test: ["CMD", "echo"] # TODO: health check
+            interval: 10s
+            timeout: 10s
+            retries: 60
+    postgres:
+        container_name: streamr-dev-postgres
+        image: postgres
+        ports:
+            - '5432:5432'
+        command: ["postgres", "-cshared_preload_libraries=pg_stat_statements"]
+        environment:
+            POSTGRES_USER: streamr
+            POSTGRES_PASSWORD: let-me-in
+            POSTGRES_DB: streamr
+        volumes:
+            - data-postgres:/var/lib/postgresql/data
+            - ./postgres_init_scripts:/docker-entrypoint-initdb.d
+        healthcheck:
+            test: ["CMD", "echo"] # TODO: health check
+            interval: 10s
+            timeout: 10s
+            retries: 60
+    chainlink:
+        container_name: streamr-dev-chainlink-node
+        image: smartcontract/chainlink:0.10.5
+        ports:
+            - '6688:6688'
+        command: ["local", "n", "-p", "/chainlink/.password", "-a", "/chainlink/.api"]
+        env_file:
+            - ./chainlink_config/.env_streamr_sidechain
+        volumes:
+            - ./chainlink_config:/chainlink
+        user: ${CURRENT_UID}
+        depends_on:
+            - parity-sidechain-node0
+            - postgres
+            - chainlink-external-adapter
+        deploy:
+            restart_policy:
+                condition: on-failure
+                delay: 3s
+                max_attempts: 5
+                window: 60s
+        healthcheck:
+            test: ["CMD", "echo"] # TODO: health check
+            interval: 10s
+            timeout: 10s
+            retries: 60
+    chainlink-external-adapter:
+        container_name: streamr-dev-chainlink-adapter
+        image: streamr/chainlink-external-adapter:dev
+        ports:
+            - 6691:8080
+        depends_on:
+            - parity-node0
+        healthcheck:
+            test: ["CMD", "echo"] # TODO: health check
+            interval: 10s
+            timeout: 10s
+            retries: 60
 
 networks:
     net-db-bridge-request:
@@ -426,3 +518,6 @@ volumes:
     data-parity-node0:
     data-parity-sidechain-node0:
     data-ethereum-watcher:
+    data-ipfs:
+    data-postgres:
+    data-graph-deploy:

oracles.env

@@ -25,7 +25,7 @@ ORACLE_HOME_GAS_PRICE_UPDATE_INTERVAL=600000
 # COMMON_FOREIGN_GAS_PRICE_SPEED_TYPE="fast"
 ORACLE_ALLOW_HTTP_FOR_RPC=yes
 ORACLE_QUEUE_URL=amqp://bridge-rabbitmq
-ORACLE_REDIS_URL=redis://bridge-redis
+ORACLE_REDIS_URL=redis://redis
 
 COMMON_FOREIGN_GAS_PRICE_FALLBACK=10000000000
 COMMON_FOREIGN_GAS_PRICE_FACTOR=1