Upgrade solana.py to 0.21.0, web3.py to 6.0.0b1 (#2751)

* wip * fix ipfshttpclient version * Fix tests * Upgrade discovery test img to python 3.9 * Fix to checksum address discovery test * Update .python-version in discovery * Upgrade jsonschema version in discovery * Update circle test venv script * Update discovery construct dep * fix circle discovery venv * Fix lint in cache balance * Update discovyer circle test scrit * Updates alpine discovery version so python 3.9 is used * Remove .python-version * Remove .python-version file * Update discovery six dependency for python 3.10 compat * Update discovery docker build base img * Add libseccomp * Test different base * Test different base * Test update docker file base img * Test different base * add install pkg before docker build * Update WIP * Test different base * I hope it works, update docker ci * Cleanup unused ci install * Update flask restx for compat * Fix get signature and flaskrestx json * Remove pinned werkzeug - pinned in flask-restx * add comment clarifying web3 upgrade Co-authored-by: jowlee <joeylee0925@gmail.com>
AudiusProject · Mar 30, 2022 · 3b4716a · 3b4716a
1 parent 113b289
commit 3b4716a
Show file tree

Hide file tree

Showing 19 changed files with 129 additions and 83 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -425,7 +425,7 @@ jobs:
 
   test-discovery-provider:
     docker:
-      - image: circleci/python:3.8-node-browsers
+      - image: cimg/python:3.9-node
       # Discovery provider dependencies
       - image: circleci/postgres:11.1
         environment:
@@ -450,19 +450,20 @@ jobs:
           name: python-setup
           command: |
             cd discovery-provider
-            python3 -m venv venv
-            source venv/bin/activate
-            pip install -r requirements.txt
+            python3 -m pip install --user virtualenv
+            python3 -m venv env
+            source env/bin/activate
+            pip3 install -r requirements.txt
       - save_cache:
           paths:
-            - discovery-provider/venv/
+            - discovery-provider/env/
           key: disc-prov-1-{{ checksum "discovery-provider/requirements.txt" }}
       - run:
           name: python-lint
           command: |
             export PROTOCOL_DIR=/home/circleci/project
             cd discovery-provider
-            source venv/bin/activate
+            source env/bin/activate
             sh scripts/lint.sh
       - restore_cache:
           keys:
@@ -685,6 +686,40 @@ jobs:
             docker push audius/<< parameters.repo >>:$IMAGE_TAG
             docker push audius/<< parameters.repo >>:$(git rev-parse HEAD)
 
+  docker-build-and-push-updated:
+    docker:
+      - image: cimg/base:2022.03
+    parameters:
+      repo:
+        type: string
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+          version: 20.10.11
+      - run:
+          name: Get tag or latest
+          command: |
+            # order of precendence for IMAGE tag is:
+            # 1. CIRCLE_TAG if defined,
+            # 2. Branch name if 'hotfix' in branch name (branch name passed in via $CIRCLE_BRANCH)
+            # 3. 'latest', which is the default
+            echo "export IMAGE_TAG=`[ $CIRCLE_TAG ] && echo $(echo $CIRCLE_TAG | cut -d@ -f3) || [[ "$CIRCLE_BRANCH" =~ (hotfix) ]] && echo $CIRCLE_BRANCH || echo "latest" `" | tee -a $BASH_ENV
+      - run:
+          name: Docker login
+          command: |
+            echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+      - run:
+          name: Docker build << parameters.repo >>
+          command: |
+            cd << parameters.repo >>
+            docker build -t audius/<< parameters.repo >>:$IMAGE_TAG -t audius/<< parameters.repo>>:$(git rev-parse HEAD) --build-arg git_sha=$(git rev-parse HEAD) --build-arg audius_loggly_disable=$audius_loggly_disable --build-arg audius_loggly_token=$audius_loggly_token --build-arg audius_loggly_tags=$audius_loggly_tags .
+      - run:
+          name: Docker push << parameters.repo >>
+          command: |
+            docker push audius/<< parameters.repo >>:$IMAGE_TAG
+            docker push audius/<< parameters.repo >>:$(git rev-parse HEAD)
+
 workflows:
   # test, build and push all commits
   test-build-and-push:
@@ -715,7 +750,7 @@ workflows:
 
       - test-discovery-provider:
           name: test-discovery-provider
-      - docker-build-and-push:
+      - docker-build-and-push-updated:
           name: build-discovery-provider
           repo: discovery-provider
 

diff --git a/discovery-provider/.gitignore b/discovery-provider/.gitignore
@@ -51,3 +51,5 @@ server.log
 
 # build in service-commands up
 compose/env/tmp*/
+
+.python-version
diff --git a/discovery-provider/.python-version b/discovery-provider/.python-version
diff --git a/discovery-provider/Dockerfile b/discovery-provider/Dockerfile
@@ -1,15 +1,13 @@
 # requires authentication to Dockerhub
 FROM audius/contracts:1.0.0 as contracts
 
-FROM alpine:3.13
+FROM alpine:3.15
 LABEL maintainer="Hareesh Nagaraj <hareesh@audius.co>"
 
 ENV INSTALL_PATH /audius-discovery-provider
 RUN mkdir -p $INSTALL_PATH
 WORKDIR $INSTALL_PATH
 
-COPY requirements.txt requirements.txt
-
 RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.10/main' >> /etc/apk/repositories && \
     apk update && \
     apk add alpine-sdk && \
@@ -30,7 +28,11 @@ RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.10/main' >> /etc/apk/repositor
     apk add py3-wheel && \
     apk add rsyslog && \
     apk add docker && \
-    apk add --virtual .build-deps gcc musl-dev postgresql-dev=11.12-r0 && \
+    apk add --virtual .build-deps gcc musl-dev postgresql-dev=11.12-r0
+
+COPY requirements.txt requirements.txt
+
+RUN apk add libseccomp-dev && \
     python3 -m pip install -r requirements.txt --no-cache-dir && \
     apk --purge del .build-deps
 

diff --git a/discovery-provider/integration_tests/conftest.py b/discovery-provider/integration_tests/conftest.py
@@ -144,7 +144,7 @@ def init_contracts(config):
     web3 = Web3(HTTPProvider(web3endpoint))
 
     # set pre-funded account as sender
-    web3.eth.defaultAccount = web3.eth.accounts[0]
+    web3.eth.default_account = web3.eth.accounts[0]
 
     registry_address = web3.toChecksumAddress(config["contracts"]["registry"])
 

diff --git a/discovery-provider/integration_tests/tasks/test_index_operations.py b/discovery-provider/integration_tests/tasks/test_index_operations.py
@@ -6,7 +6,7 @@
 # import pytest
 # import src.utils.multihash
 # from chance import chance
-# from integration_tests.utils import to_bytes
+# from integration_tests.utils import toBytes
 # from src.models.models import Block, Track, User
 # from src.queries.get_skipped_transactions import get_indexing_error
 # from src.utils.helpers import remove_test_file
@@ -56,20 +56,20 @@
 #         "primaryType": "AddUserRequest",
 #         "message": {"handle": new_user_handle, "nonce": new_user_nonce},
 #     }
-#     new_user_signature = web3.eth.signTypedData(
-#         web3.eth.defaultAccount, new_user_signature_data
+#     new_user_signature = web3.eth.sign_typed_data(
+#         web3.eth.default_account, new_user_signature_data
 #     )
 
 #     # Add creator to blockchain
 #     new_user_tx_hash = user_factory_contract.functions.addUser(
-#         web3.eth.defaultAccount,
-#         to_bytes(new_user_handle, 16),
+#         web3.eth.default_account,
+#         toBytes(new_user_handle, 16),
 #         new_user_nonce,
 #         new_user_signature,
 #     ).transact()
 
 #     # parse chain transaction results
-#     tx_receipt = web3.eth.waitForTransactionReceipt(new_user_tx_hash)
+#     tx_receipt = web3.eth.wait_for_transaction_receipt(new_user_tx_hash)
 #     tx_new_user_info = user_factory_contract.events.AddUser().processReceipt(tx_receipt)
 #     new_user_args = tx_new_user_info[0].args
 #     user_id_from_event = int(new_user_args._userId)
@@ -144,8 +144,8 @@
 #         },
 #     }
 
-#     new_track_signature = web3.eth.signTypedData(
-#         web3.eth.defaultAccount, new_track_signature_data
+#     new_track_signature = web3.eth.sign_typed_data(
+#         web3.eth.default_account, new_track_signature_data
 #     )
 
 #     # add track to blockchain
@@ -185,7 +185,7 @@
 #         current_block = (
 #             current_block_query[0].number if len(current_block_query) > 0 else 0
 #         )
-#         latest_block = web3.eth.getBlock("latest", True)
+#         latest_block = web3.eth.get_block("latest", True)
 #         while current_block < latest_block.number:
 #             # Process a bunch of blocks to make sure we covered everything
 #             task.run()
@@ -232,7 +232,7 @@
 #             current_block = (
 #                 current_block_query[0].number if len(current_block_query) > 0 else 0
 #             )
-#             latest_block = web3.eth.getBlock("latest", True)
+#             latest_block = web3.eth.get_block("latest", True)
 #             while current_block < latest_block.number:
 #                 # Process a bunch of blocks to make sure we covered everything
 #                 task.run()
@@ -293,7 +293,7 @@
 #             current_block = (
 #                 current_block_query[0].number if len(current_block_query) > 0 else 0
 #             )
-#             latest_block = web3.eth.getBlock("latest", True)
+#             latest_block = web3.eth.get_block("latest", True)
 #             while current_block < latest_block.number:
 #                 # Process a bunch of blocks to make sure we covered everything
 #                 task.run()
@@ -346,7 +346,7 @@
 #             current_block = (
 #                 current_block_query[0].number if len(current_block_query) > 0 else 0
 #             )
-#             latest_block = web3.eth.getBlock("latest", True)
+#             latest_block = web3.eth.get_block("latest", True)
 #             while current_block < latest_block.number:
 #                 # Process a bunch of blocks to make sure we covered everything
 #                 task.run()
@@ -415,7 +415,7 @@
 #             current_block = (
 #                 current_block_query[0].number if len(current_block_query) > 0 else 0
 #             )
-#             latest_block = web3.eth.getBlock("latest", True)
+#             latest_block = web3.eth.get_block("latest", True)
 #             while current_block < latest_block.number:
 #                 # Process a bunch of blocks to make sure we covered everything
 #                 task.run()
@@ -475,7 +475,7 @@
 #         current_block = (
 #             current_block_query[0].number if len(current_block_query) > 0 else 0
 #         )
-#         latest_block = web3.eth.getBlock("latest", True)
+#         latest_block = web3.eth.get_block("latest", True)
 #         while current_block < latest_block.number:
 #             # Process a bunch of blocks to make sure we covered everything
 #             task.run()

diff --git a/discovery-provider/integration_tests/tasks/test_index_trending.py b/discovery-provider/integration_tests/tasks/test_index_trending.py
@@ -18,7 +18,7 @@ class MockEth:
     def __init__(self, return_block: BlockData):
         self.return_block = return_block
 
-    def getBlock(self, *args):
+    def get_block(self, *args):
         return self.return_block
 
 

diff --git a/discovery-provider/integration_tests/utils.py b/discovery-provider/integration_tests/utils.py
@@ -31,7 +31,7 @@ def query_creator_by_name(app, creator_name=None):
             return return_list
 
 
-def to_bytes(val, length=32):
+def toBytes(val, length=32):
     val = val[:length]
     return bytes(val, "utf-8")
 

diff --git a/discovery-provider/requirements.txt b/discovery-provider/requirements.txt
@@ -9,8 +9,8 @@ types-redis==3.5.4
 types-requests==2.25.0
 types-pytz==2021.1.2
 
-# web3 used to be 5.11.0 but due to a build issue in docker, downgraded to 5.8.0
-web3==5.8.0
+# the only version that supports websockets v10.1 (needed for solana py + anchor py)
+web3==6.0.0b1
 Flask==1.0.4
 # pin itsdangerous and markupsafe https://github.com/pallets/flask/issues/4455
 itsdangerous==1.1.0
@@ -23,13 +23,13 @@ redis==3.2.0
 pytest==6.0.1
 SQLAlchemy-Utils==0.37.6
 chance==0.110
-ipfshttpclient==0.8.0a1
+ipfshttpclient==0.8.0a2
 pytest-cov==2.6.0
 pytest-dotenv==0.5.2
 flask-cors==3.0.6
 gunicorn==20.0.4
-jsonschema==3.2.0
-flask-restx==0.2.0
+jsonschema==4.4.0
+flask-restx==0.4.0
 hashids==1.2.0
 fakeredis==1.4.2
 jsonformatter==0.3.0
@@ -38,23 +38,23 @@ eventlet==0.28.0
 psutil==5.8.0
 pytz==2021.1
 prometheus-client==0.13.1
-werkzeug==2.0.3
 click==8.0.4
 
 # Solana support
 base58==2.1.0
 certifi==2020.12.5
 cffi==1.14.5
 chardet==4.0.0
-construct==2.10.60
+construct==2.10.67
 idna==2.10
 pycparser==2.20
 PyNaCl==1.4.0
 requests==2.25.1
-six==1.15.0
-solana==0.10.0
+six==1.16.0
+solana==0.21.0
 typing-extensions==3.10.0.0
 urllib3==1.26.3
+websockets==10.1
 
 # test
 pytest-mock==3.6.1
diff --git a/discovery-provider/scripts/circle-ci.sh b/discovery-provider/scripts/circle-ci.sh
@@ -3,7 +3,7 @@ source ./scripts/utilities.sh
 
 function main {
   set -e
-  source venv/bin/activate
+  source env/bin/activate
 
   # run contract migrations
   cd_contracts_repo

diff --git a/discovery-provider/src/eth_indexing/event_scanner.py b/discovery-provider/src/eth_indexing/event_scanner.py
@@ -86,7 +86,7 @@ def __init__(
         self.event_type = event_type
         self.filters = filters
         self.last_scanned_block = MIN_SCAN_START_BLOCK
-        self.latest_chain_block = self.web3.eth.blockNumber
+        self.latest_chain_block = self.web3.eth.block_number
 
     def restore(self):
         """Restore the last scan state from redis.
@@ -124,7 +124,7 @@ def save(self, block_number: int):
     def get_block_timestamp(self, block_num) -> Union[datetime.datetime, None]:
         """Get Ethereum block timestamp"""
         try:
-            block_info = self.web3.eth.getBlock(block_num)
+            block_info = self.web3.eth.get_block(block_num)
         except BlockNotFound:
             # Block was not mined yet,
             # minor chain reorganisation?
@@ -446,7 +446,7 @@ def _fetch_events_for_all_contracts(
 
     # Call JSON-RPC API on your Ethereum node.
     # get_logs() returns raw AttributedDict entries
-    logs = web3.eth.getLogs(event_filter_params)
+    logs = web3.eth.get_logs(event_filter_params)
 
     # Convert raw binary data to Python proxy objects as described by ABI
     all_events = []

diff --git a/discovery-provider/src/queries/get_health.py b/discovery-provider/src/queries/get_health.py
@@ -216,7 +216,7 @@ def get_health(args: GetHealthArgs, use_redis_cache: bool = True) -> Tuple[Dict,
     # value from redis cache is None
     if not use_redis_cache or latest_block_num is None or latest_block_hash is None:
         # get latest blockchain state from web3
-        latest_block = web3.eth.getBlock("latest", True)
+        latest_block = web3.eth.get_block("latest", True)
         latest_block_num = latest_block.number
         latest_block_hash = latest_block.hash.hex()
 
@@ -556,7 +556,7 @@ def get_latest_chain_block_set_if_nx(redis=None, web3=None):
         latest_block_hash = stored_latest_blockhash.decode("utf-8")
 
     if latest_block_num is None or latest_block_hash is None:
-        latest_block = web3.eth.getBlock("latest", True)
+        latest_block = web3.eth.get_block("latest", True)
         latest_block_num = latest_block.number
         latest_block_hash = latest_block.hash.hex()