cleanup: npm scripts, circleci, docker (#397)

- Use build, lint and install as standard scripts for every package
- rename lint-checks to lint
- remove old dockerfile
- rename dockerfiles to use `Dockerfile` name
- Simplify circleci build
- add tslint to utils
- normalize script names
    - rename compile-typescript to build:ts
    - create build:gen for generated files
- don't compile typescript on lint. (build will do that)

- Fix linting problems

* Revisit celotool docker file
  - Build projects
  - fix missing dependencies
  - improve docker caching & size

* circleci: lint & prettify:diff as 2 steps

.circleci/config.yml

@@ -7,6 +7,9 @@ defaults: &defaults
   working_directory: ~/app
   docker:
     - image: celohq/node8:gcloud-deps
+  environment:
+    # To avoid ENOMEM problem when running node
+    NODE_OPTIONS: "--max-old-space-size=4096"
 
 android-defaults: &android-defaults
   <<: *defaults
@@ -52,96 +55,59 @@ jobs:
           at: ~/app
 
       - restore_cache:
-          key: yarn-v2-{{ checksum "yarn.lock" }}-{{ arch }}
+          keys: 
+            - yarn-v2-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}
+            - yarn-v2-{{ arch }}-{{ .Branch }}-
+            - yarn-v2-{{ arch }}-
+            - yarn-v2-
 
       - run:
           name: Delete @celo dir from node_modules (if its there)
           command: rm -rf ~/app/node_modules/@celo
 
       - run:
-          name: yarn
+          name: Install dependencies
           command: |
             # Deals with yarn install flakiness which can come due to yarnpkg.com being
             # unreliable. For example, https://circleci.com/gh/celo-org/celo-monorepo/82685
             yarn install || yarn install
-
       - run:
           name: Fail if someone forgot to commit "yarn.lock"
           command: |
             if [[ $(git status --porcelain) ]]; then
               echo "There are git differences after running yarn install"
               exit 1
             fi
-
       - run: npm rebuild scrypt
 
       - save_cache:
-          key: yarn-v2-{{ checksum "yarn.lock" }}-{{ arch }}
+          key: yarn-v2-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}
           paths:
             - node_modules
             - packages/*/node_modules
+
+      - run:
+          name: Build packages
+          command: |
+            # separate build to avoid ENOMEN in CI :(
+            yarn build --scope @celo/protocol
+            yarn build --scope docs
+            yarn build --scope @celo/contractkit
+            yarn build --ignore @celo/protocol --ignore docs --ignore @celo/contractkit
+
 
       - persist_to_workspace:
           root: .
-          paths: .
+          paths: 
+            - .
 
   lint-checks:
     <<: *defaults
     steps:
       - attach_workspace:
           at: ~/app
-
-      - run: yarn run ci-lint
-
-  build-all-packages:
-     <<: *defaults
-     steps:
-       - attach_workspace:
-          at: ~/app
-
-       - run:
-           name: Build SDK
-           command: |
-              set -e
-              yarn --cwd=packages/contractkit build alfajores
-
-       - run:
-          name: Build CLI
-          command: |
-            yarn --cwd=packages/cli build-sdk alfajores
-            yarn --cwd=packages/cli build
-            yarn --cwd=packages/cli docs
-
-       - run:
-          name: Fail if someone forgot to commit CLI docs
-          command: |
-            if [[ $(git status packages/docs/command-line-interface --porcelain) ]]; then
-              git --no-pager diff packages/docs/command-line-interface
-              echo "There are git differences after generating CLI docs"
-              exit 1
-            fi
-
-       - run:
-          name: Build mobile
-          command: yarn --cwd=packages/mobile build
-
-       - run:
-          name: Build Verification Pool
-          command: yarn --cwd=packages/verification-pool-api compile-typescript
-
-       - run:
-          name: Build Verifier
-          command: yarn --cwd=packages/verifier build:typescript
-
-       - run:
-          name: Build Web
-          command: |
-              set -e
-              yarn --cwd=packages/web build
-
-       - run:
-          name: Build Docs
-          command: yarn --cwd=packages/docs build
+      - run: yarn run prettify:diff
+      - run: yarn run lint
 
   general-test:
     <<: *defaults
@@ -269,6 +235,15 @@ jobs:
           command: |
             set -euo pipefail
             yarn --cwd=packages/cli test
+      - run:
+          name: Fail if someone forgot to commit CLI docs
+          command: |
+            yarn --cwd=packages/cli docs
+            if [[ $(git status packages/docs/command-line-interface --porcelain) ]]; then
+              git --no-pager diff packages/docs/command-line-interface
+              echo "There are git differences after generating CLI docs"
+              exit 1
+            fi            
 
   end-to-end-geth-transfer-test:
     <<: *defaults
@@ -456,54 +431,44 @@ workflows:
       - lint-checks:
           requires:
             - install_dependencies
-      - build-all-packages:
-           requires:
-            - install_dependencies
-            - lint-checks
       - general-test:
           requires:
             - install_dependencies
-            - lint-checks
       - contractkit-test:
           requires:
             - install_dependencies
-            - lint-checks
       - cli-test:
           requires:
             - install_dependencies
-            - lint-checks
       - mobile-test:
           requires:
-            - install_dependencies
             - lint-checks
       - mobile-test-build-app:
           requires:
-            - install_dependencies
-            - lint-checks
+            - mobile-test
       - verification-pool-api:
           requires:
-            - install_dependencies
             - lint-checks
       - protocol-test:
           requires:
-            - install_dependencies
             - lint-checks
+            - contractkit-test
       - end-to-end-geth-transfer-test:
           requires:
-            - install_dependencies
             - lint-checks
+            - contractkit-test
       - end-to-end-geth-governance-test:
           requires:
-            - install_dependencies
             - lint-checks
+            - contractkit-test
       - end-to-end-geth-sync-test:
           requires:
-            - install_dependencies
             - lint-checks
+            - contractkit-test
       - end-to-end-geth-integration-sync-test:
           requires:
-            - install_dependencies
             - lint-checks
+            - contractkit-test
   nightly:
     triggers:
       - schedule:

.dockerignore

@@ -1,4 +1,20 @@
-packages/mobile
 node_modules
 **/node_modules
-*node_modules
+*node_modules
+
+# root folders
+.github
+dockerfiles
+.vscode
+
+# not docker packages
+packages/analytics
+packages/blockchain-api
+packages/docs
+packages/faucet
+packages/helm-charts
+packages/mobile
+packages/notification-service
+# packages/verification-pool-api
+packages/verifier
+packages/web

.mergify.yml

@@ -11,9 +11,9 @@ pull_request_rules:
       # Only enable this when the pull request is being merged into master
       - "base=master"
       # List of all the tests that should pass.
-      - "status-success=ci/circleci: general-test"
       - "status-success=ci/circleci: install_dependencies"
       - "status-success=ci/circleci: lint-checks"
+      - "status-success=ci/circleci: general-test"
       - "status-success=ci/circleci: mobile-test"
       - "status-success=ci/circleci: protocol-test"
       - "status-success=ci/circleci: verification-pool-api"

.prettierignore

@@ -1,21 +1,38 @@
 package.json
 **/.next
 **/coverage
-packages/functions/lib
-packages/protocol/types/typechain
+**/node_modules
+**/privacy/*
+.git/
+
+packages/blockchain-api/dist
+
+packages/cli/lib
+
+packages/contractkit/lib/
+packages/contractkit/.artifacts/
+packages/contractkit/contracts/
+packages/contractkit/types/
+packages/dappkit/lib/
+packages/docs/_book
+packages/faucet/dist/
+packages/mobile/src/geth/*.json
+packages/notification-service/dist/
+
+packages/protocol/build/
+packages/protocol/types/
 packages/protocol/lib/**/*.js
 packages/protocol/scripts/**/*.js
 packages/protocol/migrations/**/*.js
 packages/protocol/tests/**/*.js
-packages/protocol/types/**/*.js
-**/privacy/*
-packages/mobile/src/geth/*.json
+
 packages/transaction-metrics-exporter/src/contracts/*.ts
-packages/verification-pool-api/lib/**/*.js
+packages/dappkit/lib/
+packages/utils/lib/
+
+packages/verification-pool-api/lib/
 packages/verification-pool-api/contracts/*.ts
-packages/contractkit/.artifacts/
-packages/contractkit/contracts/
-packages/contractkit/lib/
-packages/contractkit/types/
+
 packages/web/dist
-.git/
+packages/web/.next
+

cloudbuild.yaml

@@ -3,33 +3,33 @@
 
 steps:
 
-- name: gcr.io/kaniko-project/executor:latest
+- id: "docker:celotool"
+  name: gcr.io/kaniko-project/executor:latest
   args: [
-    "--dockerfile=dockerfiles/monorepo/Dockerfile.celotool",
+    "--dockerfile=dockerfiles/celotool/Dockerfile",
     "--cache=true",
     "--destination=gcr.io/$PROJECT_ID/celo-monorepo:celotool-$COMMIT_SHA"
   ]
-  id: Build celotool docker image
   waitFor: ['-']
 
-- name: gcr.io/kaniko-project/executor:latest
+- id: "docker:transaction-metrics-exporter"
+  name: gcr.io/kaniko-project/executor:latest
   args: [
-    "--dockerfile=dockerfiles/monorepo/Dockerfile.transaction-metrics-exporter",
+    "--dockerfile=dockerfiles/transaction-metrics-exporter/Dockerfile",
     "--cache=true",
     "--destination=gcr.io/$PROJECT_ID/celo-monorepo:transaction-metrics-exporter-$COMMIT_SHA"
   ]
-  id: Build transaction metrics exporter docker image
   waitFor: ['-']
 
-- name: gcr.io/kaniko-project/executor:latest
+- id: "docker:cli"
+  name: gcr.io/kaniko-project/executor:latest
   args: [
-    "--dockerfile=dockerfiles/cli/Dockerfile.cli",
+    "--dockerfile=dockerfiles/cli/Dockerfile",
     "--cache=true",
     "--destination=gcr.io/$PROJECT_ID/celocli:$COMMIT_SHA",
     "--build-arg",
     "celo_env=alfajores"
   ]
-  id: Build CLI docker image
   waitFor: ['-']
 
 timeout: 3000s

dockerfiles/celotool/Dockerfile

@@ -0,0 +1,36 @@
+FROM node:8
+WORKDIR /celo-monorepo
+
+# Needed for gsutil
+RUN apt-get update && \
+    apt-get install -y lsb-release && \
+    export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
+    echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
+    apt-get update -y && \
+    apt-get install -y google-cloud-sdk && \
+    rm -rf /var/lib/apt/lists/*
+
+
+# ensure yarn.lock is evaluated by kaniko cache diff
+COPY lerna.json package.json yarn.lock ./
+COPY scripts/ scripts/
+COPY packages/utils/package.json packages/utils/
+COPY packages/typescript/package.json packages/typescript/
+COPY packages/contractkit/package.json packages/contractkit/
+COPY packages/verification-pool-api/package.json packages/verification-pool-api/
+COPY packages/celotool/package.json packages/celotool/
+
+RUN yarn install --frozen-lockfile && yarn cache clean
+
+COPY packages/utils packages/utils/
+COPY packages/typescript packages/typescript/
+COPY packages/contractkit packages/contractkit/
+COPY packages/verification-pool-api packages/verification-pool-api/
+COPY packages/celotool packages/celotool/
+
+# RUN yarn build
+
+ENV PATH="/celo-monorepo/packages/celotool/bin:${PATH}"
+
+CMD ["celotooljs.sh"]