Automatize sync

packages/node/chain-setup/btc-mainnet/docker-compose-local-btc-mainnet.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     env_file: .env

packages/node/chain-setup/btc-regtest/docker-compose-local-btc-regtest.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     env_file: .env

packages/node/chain-setup/btc-testnet/docker-compose-local-btc-testnet.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     env_file: .env

packages/node/chain-setup/ltc-mainnet/docker-compose-local-ltc-mainnet.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     env_file: .env
@@ -71,7 +70,7 @@ services:
       - WORKER_ID=${WORKER_ID}
       - NUM_WORKERS=${NUM_WORKERS}
       - SYNC_NON_STANDARD=${SYNC_NON_STANDARD}
-      - BC_START_HEIGHT=2413791
+      - BC_START_HEIGHT=${BC_START_HEIGHT}
     depends_on:
       - db
       - node

packages/node/chain-setup/ltc-regtest/docker-compose-local-ltc-regtest.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     env_file: .env

packages/node/chain-setup/ltc-testnet/docker-compose-local-ltc-testnet.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     env_file: .env
@@ -74,6 +73,8 @@ services:
       - NUM_WORKERS=${NUM_WORKERS}
       - SYNC_NON_STANDARD=${SYNC_NON_STANDARD}
       - BC_START_HEIGHT=${BC_START_HEIGHT}
+    volumes:
+      - ./logs:/dist/packages/node/logs
     depends_on:
       - db
       - node

packages/node/db/db_schema.sql

@@ -73,8 +73,9 @@ CREATE TABLE IF NOT EXISTS
 
 CREATE TABLE IF NOT EXISTS
   "SyncStatus" (
-    "syncedHeight" INTEGER NOT NULL,
-    "workerId" INTEGER NOT NULL PRIMARY KEY
+    "blockToSync" INTEGER NOT NULL,
+    "workerId" INTEGER NOT NULL PRIMARY KEY,
+    "nonStandard" BOOLEAN NOT NULL
   );
 
 CREATE VIEW "Utxos" AS

packages/node/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     image: postgres

packages/node/scripts/reset.sh

@@ -18,8 +18,6 @@ TRUNCATE TABLE "Output";
 TRUNCATE TABLE "User";
 TRUNCATE TABLE "OffChain";
 TRUNCATE TABLE "SyncStatus";
-
-INSERT INTO "SyncStatus" ("syncedHeight", "workerId") VALUES (-1, 1);
 EOF
 
 # stop all docker containers and networks

packages/node/scripts/up.py

@@ -9,27 +9,25 @@
 
 def runSync(args, commandLine):
     if(args.cpus is not None):
-        # numWorkers = cpus - db, node + nonSt
-        numWorkers = args.cpus - 3 if args.cpus - 3 > 0 else 1
+        # numWorkers = #cpus - dbService + nodeSercice + nonStandardWorker
+        numStandardWorkers = args.cpus - 3 if args.cpus - 3 > 0 else 1
     else:
-        numWorkers = multiprocessing.cpu_count() - 3 if multiprocessing.cpu_count() - 3 > 0 else 1
+        numStandardWorkers = multiprocessing.cpu_count() - 3 if multiprocessing.cpu_count() - 3 > 0 else 1
 
     standardTxCommand = ''
-    print('Launching '+str(numWorkers+1)+' workers.')
-    for worker in range(numWorkers):  
+    numWorkers = numStandardWorkers + 1
+    print('Launching '+str(numStandardWorkers)+' standard workers and 1 non-standard worker. ')
+    for worker in range(numStandardWorkers):  
         standardTxCommand+=syncCommandLine('false', str(worker+1), str(numWorkers), commandLine)+ ' && '
 
-    nonStandardTxCommand = syncCommandLine('true', str(numWorkers+1), '1', commandLine)
+    nonStandardTxCommand = syncCommandLine('true', str(numWorkers), str(numWorkers), commandLine)
 
     parallelCommand = standardTxCommand + nonStandardTxCommand
     p = subprocess.run(
         ['sh', '-c', parallelCommand], ) 
 
 def syncCommandLine(nonStandard, workerId, numWorkers, commandLine):
-    if (nonStandard == 'true'):
-        return 'export SYNC_NON_STANDARD='+nonStandard+' WORKER_ID='+str(workerId)+' ; '+ commandLine+' run -d sync'
-    else:
-        return 'export SYNC_NON_STANDARD='+nonStandard+' WORKER_ID='+str(workerId)+' NUM_WORKERS='+str(numWorkers)+'; '+ commandLine+' run -d sync'
+    return 'export SYNC_NON_STANDARD='+nonStandard+' WORKER_ID='+str(workerId)+' NUM_WORKERS='+str(numWorkers)+'; '+ commandLine+' run -d sync'
 
 def main():
 
@@ -53,7 +51,6 @@ def main():
     parser.set_defaults(service='')
 
     parser.add_argument('-cpus', dest="cpus",type=int)
-    parser.add_argument('-optimize', action="store_true")
 
     args = parser.parse_args()
 
@@ -74,19 +71,12 @@ def main():
         # testnet or mainnet
         url = subprocess.check_output("grep BCN_URL .env | cut -d '=' -f2", shell=True).decode("utf-8").strip()
         bcnUrl = url if url != '' else 'https://rltc.node.bitcoincomputer.io'
-        if(args.optimize):
-            # Optimize for speed: skip launching bcn service (no port binding)
-            subprocess.run(
-                ['sh', '-c', commandLine+' run -d -e BCN_URL='+bcnUrl+' bcn']) 
-            # Launch sync in automatic parallel mode. If any service is specified, don't launch sync services
-            if(args.service.strip() == ''):
-                runSync(args, commandLine)
-        else:
-            subprocess.run(
-                ['sh', '-c', commandLine+' run -d -e BCN_URL='+bcnUrl+' -p {0}:{0} bcn'.format(bcnPort)]) 
-            # If any service is specified, don't launch sync services
-            if(args.service.strip() == ''):
-                runSync(args, commandLine)
+
+        subprocess.run(
+            ['sh', '-c', commandLine+' run -d -e BCN_URL='+bcnUrl+' -p {0}:{0} bcn'.format(bcnPort)]) 
+        # If any service is specified, don't launch sync services
+        if(args.service.strip() == ''):
+            runSync(args, commandLine)
 if __name__ == '__main__':
     main()