Simplify sync process

packages/docs/Lib/query.md

@@ -16,7 +16,6 @@ Returns the latest revisions of smart objects. Conditions can be passed in to de
         limit?: number;
         offset?: number;
         order?: "ASC" | "DEC";
-        contract?: { class: T; args?: ConstructorParameters<T> };
       }
 ) => Promise<string[]>;
 ```
@@ -27,8 +26,7 @@ Returns the latest revisions of smart objects. Conditions can be passed in to de
 await computer.query({ publicKey })
 await computer.query({ ids })
 await computer.query({ mod })
-await computer.query({ contract })
-await computer.query({ publicKey, contract, order })
+await computer.query({ publicKey, order })
 ...
 ```
 
@@ -47,7 +45,6 @@ An object with the query parameters.
 | limit | number | Return only limited number of revisions |
 | offset | number | Return results starting from offset |
 | order | 'ASC' \| 'DEC' | Order results in ascending or descending order |
-| contract | { class: T, args?: ConstructorParameters\<T\> } | Return latest revisions of smart objects from a class |
 
 ### Return value
 
@@ -75,16 +72,11 @@ const b = await computer.new(A, [], mod)
 const [rev3] = await computer.query({ mod })
 expect(rev3).to.equal(b._rev)
 
-// query by class
-const contract = { class: A }
-const [rev4] = await computer.query({ contract })
-expect(rev4).to.equal(a._rev)
-
 // query by multiple parameters
 const [revs5] = await computer.query({
   limit: 10,
   order: 'ASC',
-  contract: { class: A }
+  publicKey: { computer.getPublicKey() }
 })
 expect(rev5).to.equal(a._rev)
 ```

packages/node/.env.example

@@ -45,7 +45,17 @@ LOG_MAX_FILES='14d'
 # size is specified the log is not rotated.
 LOG_MAX_SIZE='20m'
 # A boolean to define whether or not to gzip archived log files.
-LOG_ZIP=false
+LOG_ZIP='false'
+
+# Show logs attached to the Console transport
+SHOW_CONSOLE_LOGS='true'
+
+# Rate Limiting Settings
+RATE_LIMIT_ENABLED='false'
+RATE_LIMIT_WINDOW='900000'
+RATE_LIMIT_MAX='300'
+RATE_LIMIT_STANDARD_HEADERS='true'
+RATE_LIMIT_LEGACY_HEADERS='false'
 
 # Comma separated list of banned countries encoded as ISO-3166 alpha2 country
 # codes (see https://www.geonames.org/countries/)

packages/node/README.md

@@ -180,10 +180,9 @@ You can configure several options by editing the `.env` file.
 <font size=1>
 
 ```bash
-# BTC or LTC
+# Chain: BTC or LTC
 CHAIN='LTC'
-
-# mainnet, testnet, or regtest
+# Network: mainnet, testnet, or regtest
 NETWORK='regtest'
 
 # Port for Bitcoin Computer Node
@@ -201,7 +200,7 @@ POSTGRES_PORT='5432'
 RPC_USER='bcn-admin'
 RPC_PASSWORD='kH4nU5Okm6-uyC0_mA5ztVNacJqZbYd_KGLl6mx722A='
 RPC_AUTH='bcn-admin:c71460f0f08e4eeec90e033c04f7bb82$c36e8561d46abbf3bf13da6b88612c19d758d46d02c45cd2716f06a13ec407af'
-RPC_HOST='127.0.0.1'  # should this be node?
+RPC_HOST='127.0.0.1'
 RPC_PORT='19332'
 RPC_PROTOCOL='http'
 ZMQ_URL='tcp://node:28332'
@@ -228,7 +227,17 @@ LOG_MAX_FILES='14d'
 # size is specified the log is not rotated.
 LOG_MAX_SIZE='20m'
 # A boolean to define whether or not to gzip archived log files.
-LOG_ZIP=false
+LOG_ZIP='false'
+
+# Show logs attached to the Console transport
+SHOW_CONSOLE_LOGS='true'
+
+# Rate Limiting Settings
+RATE_LIMIT_ENABLED='false'
+RATE_LIMIT_WINDOW='900000'
+RATE_LIMIT_MAX='300'
+RATE_LIMIT_STANDARD_HEADERS='true'
+RATE_LIMIT_LEGACY_HEADERS='false'
 
 # Comma separated list of banned countries encoded as ISO-3166 alpha2 country
 # codes (see https://www.geonames.org/countries/)

packages/node/chain-setup/btc-mainnet/docker-compose-local-btc-mainnet.yml

@@ -42,14 +42,13 @@ services:
       - db
       - node
     environment:
-      - BCN_ENV=prod
+      - BCN_ENV=dev
       - RPC_HOST=node
       - RPC_PORT=8332
       - ZMQ_URL=tcp://node:28332
       - NETWORK=mainnet
       - CHAIN=BTC
       - BCN_URL=https://btc.node.bitcoincomputer.io
-      - BC_START_HEIGHT=${BC_START_HEIGHT}
   sync:
     command: npm run sync
     build: .
@@ -60,7 +59,7 @@ services:
       - db
       - node
     environment:
-      - BCN_ENV=prod
+      - BCN_ENV=dev
       - RPC_HOST=node
       - RPC_PORT=8332
       - NETWORK=mainnet
@@ -74,8 +73,8 @@ services:
       - RPC_PASSWORD=${RPC_PASSWORD}
       - WORKER_ID=${WORKER_ID}
       - NUM_WORKERS=${NUM_WORKERS}
-      - SYNC_NON_STANDARD=${SYNC_NON_STANDARD}
-      - BC_START_HEIGHT=${BC_START_HEIGHT}
+    volumes:
+      - ./logs:/dist/packages/node/logs
     networks:
       - bitcoin
       - bcn

packages/node/chain-setup/btc-testnet/docker-compose-local-btc-testnet.yml

@@ -45,14 +45,13 @@ services:
       - db
       - node
     environment:
-      - BCN_ENV=prod
+      - BCN_ENV=dev
       - RPC_HOST=node
       - RPC_PORT=18332
       - ZMQ_URL=tcp://node:28332
       - NETWORK=testnet
       - CHAIN=BTC
       - BCN_URL=https://tbtc.node.bitcoincomputer.io
-      - BC_START_HEIGHT=${BC_START_HEIGHT}
   sync:
     command: npm run sync
     image: bitcoin-computer-node
@@ -62,7 +61,7 @@ services:
       - db
       - node
     environment:
-      - BCN_ENV=prod
+      - BCN_ENV=dev
       - RPC_HOST=node
       - RPC_PORT=18332
       - NETWORK=testnet
@@ -76,8 +75,8 @@ services:
       - RPC_PASSWORD=${RPC_PASSWORD}
       - WORKER_ID=${WORKER_ID}
       - NUM_WORKERS=${NUM_WORKERS}
-      - SYNC_NON_STANDARD=${SYNC_NON_STANDARD}
-      - BC_START_HEIGHT=${BC_START_HEIGHT}
+    volumes:
+      - ./logs:/dist/packages/node/logs
     networks:
       - bitcoin
       - bcn

packages/node/chain-setup/ltc-mainnet/docker-compose-local-ltc-mainnet.yml

@@ -38,7 +38,7 @@ services:
     env_file: .env
     restart: always
     environment:
-      - BCN_ENV=prod
+      - BCN_ENV=dev
       - RPC_HOST=node
       - RPC_PORT=9332
       - ZMQ_URL=tcp://node:28332
@@ -55,7 +55,7 @@ services:
     env_file: .env
     restart: always
     environment:
-      - BCN_ENV=prod
+      - BCN_ENV=dev
       - RPC_HOST=node
       - RPC_PORT=9332
       - NETWORK=mainnet
@@ -69,8 +69,8 @@ services:
       - RPC_PASSWORD=${RPC_PASSWORD}
       - WORKER_ID=${WORKER_ID}
       - NUM_WORKERS=${NUM_WORKERS}
-      - SYNC_NON_STANDARD=${SYNC_NON_STANDARD}
-      - BC_START_HEIGHT=${BC_START_HEIGHT}
+    volumes:
+      - ./logs:/dist/packages/node/logs
     depends_on:
       - db
       - node

packages/node/chain-setup/ltc-testnet/docker-compose-local-ltc-testnet.yml

@@ -39,14 +39,13 @@ services:
     env_file: .env
     restart: always
     environment:
-      - BCN_ENV=prod
+      - BCN_ENV=dev
       - RPC_HOST=node
       - RPC_PORT=19332
       - ZMQ_URL=tcp://node:28332
       - NETWORK=testnet
       - CHAIN=LTC
       - BCN_URL=https://tltc.node.bitcoincomputer.io
-      - BC_START_HEIGHT=${BC_START_HEIGHT}
     depends_on:
       - db
       - node
@@ -57,7 +56,7 @@ services:
     env_file: .env
     restart: always
     environment:
-      - BCN_ENV=prod
+      - BCN_ENV=dev
       - RPC_HOST=node
       - RPC_PORT=19332
       - NETWORK=testnet
@@ -71,8 +70,6 @@ services:
       - RPC_PASSWORD=${RPC_PASSWORD}
       - WORKER_ID=${WORKER_ID}
       - NUM_WORKERS=${NUM_WORKERS}
-      - SYNC_NON_STANDARD=${SYNC_NON_STANDARD}
-      - BC_START_HEIGHT=${BC_START_HEIGHT}
     volumes:
       - ./logs:/dist/packages/node/logs
     depends_on:

packages/node/db/db_schema.sql

@@ -4,7 +4,12 @@ CREATE TABLE IF NOT EXISTS
     "address" VARCHAR(66),
     "satoshis" BIGINT NOT NULL,
     "scriptPubKey" TEXT NOT NULL,
-    "publicKeys" VARCHAR(66)[]
+    "publicKeys" VARCHAR(66)[],
+    "hash" VARCHAR(64),
+    "mod" VARCHAR(70),
+    "isTbcOutput" BOOLEAN NOT NULL,
+    "previous" VARCHAR(70),
+    "timestamp" timestamp default CURRENT_TIMESTAMP not null
   );
 
 CREATE TABLE IF NOT EXISTS
@@ -13,52 +18,12 @@ CREATE TABLE IF NOT EXISTS
     "spendingInput" VARCHAR(70) NOT NULL
   );
 
-CREATE TABLE IF NOT EXISTS
-  "NonStandard" (
-    "lastUpdated" timestamp default CURRENT_TIMESTAMP not null,
-    "rev" VARCHAR(70) NOT NULL PRIMARY KEY REFERENCES "Output"("rev") ON DELETE RESTRICT,
-    "id" VARCHAR(70) NOT NULL,
-    "publicKeys" VARCHAR(66)[],
-    "hash" VARCHAR(64),
-    "mod" VARCHAR(70)
-  );
-
-CREATE TABLE IF NOT EXISTS
-  "RevToId" (
-    "rev" VARCHAR(70) NOT NULL PRIMARY KEY REFERENCES "Output"("rev") ON DELETE RESTRICT,
-    "id" VARCHAR(70) NOT NULL
-  );
-
-CREATE OR REPLACE FUNCTION triggerSetTimestamp()
-RETURNS TRIGGER AS $$
-BEGIN
-  NEW."lastUpdated" = CURRENT_TIMESTAMP;
-  RETURN NEW;
-END;
-$$ language 'plpgsql';
-
-CREATE OR REPLACE TRIGGER updateNonStandardLastUpdated
-  BEFORE UPDATE
-  ON
-    "NonStandard"
-  FOR EACH ROW
-EXECUTE PROCEDURE triggerSetTimestamp();
-
-CREATE UNIQUE INDEX "NonStandardUniqueIndex"
-ON "NonStandard"("id");
-
-CREATE INDEX "NonStandardPublicKeysIndex"
-ON "NonStandard"("publicKeys");
-
-CREATE INDEX "NonStandardHashIndex"
-ON "NonStandard"("hash");
-
-CREATE INDEX "NonStandardModIndex"
-ON "NonStandard"("mod");
-
 CREATE INDEX "OutputAddressIndex"
 ON "Output"("address");
 
+CREATE INDEX "OutputPreviousIndex"
+ON "Output"("previous");
+
 CREATE TABLE IF NOT EXISTS
   "User" (
     "publicKey" VARCHAR(66) NOT NULL PRIMARY KEY,
@@ -74,14 +39,11 @@ CREATE TABLE IF NOT EXISTS
 CREATE TABLE IF NOT EXISTS
   "SyncStatus" (
     "blockToSync" INTEGER NOT NULL,
-    "workerId" INTEGER NOT NULL PRIMARY KEY,
-    "nonStandard" BOOLEAN NOT NULL
+    "workerId" INTEGER NOT NULL PRIMARY KEY
   );
 
 CREATE VIEW "Utxos" AS
-SELECT "rev", "address", "satoshis", "scriptPubKey", "publicKeys"
-FROM "Output" WHERE NOT EXISTS
-(SELECT 1 FROM "Input" ip WHERE "ip"."outputSpent" = "Output".rev
-union
-select 1 from "NonStandard" ns where "ns"."rev" = "Output".rev)
+SELECT "rev", "address", "satoshis", "scriptPubKey", "publicKeys", "timestamp"
+FROM "Output" WHERE "isTbcOutput" = false AND NOT EXISTS
+(SELECT 1 FROM "Input" ip WHERE "ip"."outputSpent" = "Output"."rev")
 

packages/node/scripts/reset.sh

@@ -12,7 +12,6 @@ DATABASE=${ENV_DATABASE:-$ENV_DATABASE:-bcn}
 HOSTNAME=localhost
 
 psql "postgresql://$USERNAME:$PASSWORD@$HOSTNAME/$DATABASE" << EOF
-TRUNCATE TABLE "NonStandard";
 TRUNCATE TABLE "Input";
 TRUNCATE TABLE "Output";
 TRUNCATE TABLE "User";

packages/node/scripts/up.py

@@ -7,27 +7,26 @@
 import multiprocessing
 
 def runSync(args, commandLine):
-    if(args.cpus is not None):
-        # We reserve 3 cpus, one for the db-service, one for the node-service, and one for the non-standard-worker
-        numStandardWorkers = args.cpus - 3 if args.cpus - 3 > 0 else 1
+    numWorkers = 1
+    if(args.cpus is None):
+        cpus = multiprocessing.cpu_count()
     else:
-        numStandardWorkers = multiprocessing.cpu_count() - 3 if multiprocessing.cpu_count() - 3 > 0 else 1
+        cpus = args.cpus
 
-    standardTxCommand = ''
-    numWorkers = numStandardWorkers + 1
-    print('Launching '+str(numStandardWorkers)+' standard workers and 1 non-standard worker. ')
-    for worker in range(numStandardWorkers):  
-        standardTxCommand+=syncCommandLine('false', str(worker+1), str(numWorkers), commandLine)+ ' && '
-
-    nonStandardTxCommand = syncCommandLine('true', str(numWorkers), str(numWorkers), commandLine)
+    # We reserve 2 cpus, one for the db-service, one for the node-service
+    if cpus - 2 > 0:
+        numWorkers = cpus - 2   
 
-    parallelCommand = standardTxCommand + nonStandardTxCommand
+    standardTxCommand = 'true '
+    print('Launching '+str(numWorkers)+' workers. ')
+    for workerId in range(numWorkers):  
+        standardTxCommand+=' && export WORKER_ID='+str(workerId+1)+' NUM_WORKERS='+str(numWorkers)+'; '+ commandLine+' run -d sync'
+
+    print(standardTxCommand)
+    parallelCommand = standardTxCommand 
     p = subprocess.run(
         ['sh', '-c', parallelCommand], ) 
 
-def syncCommandLine(nonStandard, workerId, numWorkers, commandLine):
-    return 'export SYNC_NON_STANDARD='+nonStandard+' WORKER_ID='+str(workerId)+' NUM_WORKERS='+str(numWorkers)+'; '+ commandLine+' run -d sync'
-
 def main():
 
     parser = argparse.ArgumentParser()