Skip to content

Commit

Permalink
Fix the testenv, add curve to k8s demos (#857)
Browse files Browse the repository at this point in the history
  • Loading branch information
hhio618 committed Sep 20, 2023
1 parent a75b52e commit 3d70df6
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 14 deletions.
10 changes: 10 additions & 0 deletions config/Curvefi.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"Coins": [
{
"Symbol": "Curve Coin",
"ForeignName": "CRV",
"Exchange": "Curvefi",
"Ignore": false
}
]
}
85 changes: 85 additions & 0 deletions deployments/k8s-yaml/scraper-dex-curve.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
apiVersion: "v1"
kind: Pod
metadata:
name: scraper-exchange-curve
spec:
containers:
- name: scraper-exchange-curve
image: dia.genericcollector.dev:latest
imagePullPolicy: Never
command: ["collector"]
args: ["-exchange=Curvefi", "-mode=current", "-pairsfile=true"]
env:
- name: USE_ENV
value: "true"
- name: POSTGRES_USER
value: "postgres"
- name: POSTGRES_PASSWORD
value: "password"
- name: POSTGRES_DB
value: "postgres"
- name: POSTGRES_HOST
value: "data-postgres.default.svc.cluster.local"
- name: INFLUXURL
value: "http://data-influx.default.svc.cluster.local:8086"
- name: INFLUXUSER
value: "test"
- name: INFLUXPASSWORD
value: "testtest"
- name: REDISURL
value: "data-redis.default.svc.cluster.local:6379"
- name: KAFKAURL
value: "data-kafka.default.svc.cluster.local:9094"
- name: ETHEREUM_URI_REST
value: "https://mainnet.infura.io/v3/fc2e181454424779b1f09e1f47304bc3"
- name: ETHEREUM_URI_WS
value: "wss://mainnet.infura.io/ws/v3/fc2e181454424779b1f09e1f47304bc3"
volumeMounts:
- name: env-context
mountPath: /mnt/env-context
readOnly: true
initContainers:
- name: check-postgres-connection
image: public.ecr.aws/docker/library/postgres:15-alpine
env:
- name: PGHOST
value: "data-postgres.default.svc.cluster.local"
- name: PGUSER
value: "postgres"
- name: PGPASSWORD
value: "password"
command:
["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"]
- name: check-redis-connection
image: public.ecr.aws/docker/library/redis:7
env:
- name: REDISCLI_URL
value: "redis://data-redis.default.svc.cluster.local:6379/0"
command:
["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"]
- name: check-influxdb-connection
image: public.ecr.aws/docker/library/influxdb:1.8
env:
- name: INFLUXDB_URL
value: "data-influx.default.svc.cluster.local"
- name: INFLUXDB_USERNAME
value: "test"
- name: INFLUXDB_PASSWORD
value: "testtest"
command:
[
"sh",
"-c",
"until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done",
]
- name: check-kafka-connection
image: confluentinc/cp-kafkacat:6.2.1
env:
- name: KAFKA_BROKER_URL
value: "data-kafka.default.svc.cluster.local:9094"
command:
["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"]
volumes:
- name: env-context
hostPath:
path: /mnt/diadata
41 changes: 27 additions & 14 deletions testenv.sh
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ function main() {
esac
done


# We should update the command variable after the shift operations!
command=("$@")
# Check dependencies
if ! hash git 2> /dev/null; then echo "Git not found" >&2; return 1; fi
if ! hash docker 2> /dev/null; then echo "Docker not found" >&2; return 1; fi
Expand Down Expand Up @@ -226,7 +227,7 @@ function main() {
local version_detected
version_detected=$(git describe --tags --abbrev=0)
declare -a demos_scraper_cex=("bitfinex" "bittrex" "coinbase" "mexc")
declare -a demos_scraper_dex=("platypus" "orca")
declare -a demos_scraper_dex=("platypus" "orca", "curve")
declare -a demos_scraper_liquidity=("platypus" "orca")
declare -a demos_scraper_foreign=("yahoofinance")

Expand Down Expand Up @@ -290,8 +291,9 @@ function main() {
_build_ifnotexist Dockerfile-tradesBlockService-Dev dia.tradesblockservice.dev
_build_ifnotexist Dockerfile-pairDiscoveryService-Dev dia.pairdiscoveryservice.dev
_build_ifnotexist Dockerfile-genericCollector-Dev dia.genericcollector.dev
_build_ifnotexist Dockerfile-genericForeignScraper-Dev dia.genericforeignscraper.dev
_build_ifnotexist Dockerfile-liquidityScraper-Dev dia.liquidityscraper.dev
# FIXME: missing docker files!
# _build_ifnotexist Dockerfile-genericForeignScraper-Dev dia.genericforeignscraper.dev
# _build_ifnotexist Dockerfile-liquidityScraper-Dev dia.liquidityscraper.dev
_build_ifnotexist Dockerfile-assetCollectionService-Dev dia.assetcollectionservice.dev
if [ "$arg_full_mode" = true ]; then
_build_ifnotexist Dockerfile-blockchainservice-Dev dia.blockchainservice.dev
Expand All @@ -311,8 +313,9 @@ function main() {
_image_exist dia.tradesblockservice.dev || exit 1
_image_exist dia.pairdiscoveryservice.dev || exit 1
_image_exist dia.genericcollector.dev || exit 1
_image_exist dia.genericforeignscraper.dev || exit 1
_image_exist dia.liquidityscraper.dev || exit 1
# FIXME: missing docker files!
# _image_exist dia.genericforeignscraper.dev || exit 1
# _image_exist dia.liquidityscraper.dev || exit 1
_image_exist dia.assetcollectionservice.dev || exit 1
if [ "$arg_full_mode" = true ]; then
_image_exist dia.blockchainservice.dev || exit 1
Expand Down Expand Up @@ -399,15 +402,25 @@ function main() {
unzip -o snapshot.zip

# Run the psql commands
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-blockchain.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-exchange.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-asset.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-pool.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-poolasset.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-exchangepair.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table blockchain cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table asset cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchange cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchangepair cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table pool cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table poolasset cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy asset FROM 'output_assets.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy blockchain FROM 'output_blockchain.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchangepair FROM 'output_exchangepair.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_cex.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_dex.csv' DELIMITER ';' CSV"
# FIXME: this will raise: ERROR: invalid input syntax for type numeric: "2023-09-07 09:05:45.012443"
# psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy poolasset FROM 'output_poolasset.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy pool FROM 'output_pool.csv' DELIMITER ';' CSV"


# Delete the output-*.sql files
rm output-blockchain.sql
# Delete the output-*.sql and snapshot.zip files
rm output_*.csv
rm snapshot.zip

echo "Data import and processing completed successfully."
fi
Expand Down

0 comments on commit 3d70df6

Please sign in to comment.