nix: Use nix/build.sh for release-android make target

Signed-off-by: Jakub Sokołowski <jakub@status.im>

.gitignore

@@ -7,6 +7,7 @@
 
 # Xcode
 #
+result/
 build/
 *.pbxuser
 !default.pbxuser
@@ -170,4 +171,4 @@ status-modules/resources
 
 ## coverage
 /.nyc_output
-/coverage-report
+/coverage-report

Makefile

@@ -30,9 +30,15 @@ HOST_OS := $(shell uname | tr '[:upper:]' '[:lower:]')
 # Defines which variables will be kept for Nix pure shell, use semicolon as divider
 export _NIX_KEEP ?= BUILD_ENV
 export NIX_CONF_DIR = $(PWD)/nix
+# We don't want to use /run/user/$UID because it runs out of space too easilly
+export TMPDIR = /tmp
 
 export REACT_SERVER_PORT ?= 5001 # any value different from default 5000 will work; this has to be specified for both the Node.JS server process and the Qt process
 
+#----------------
+# Nix targets
+#----------------
+
 # WARNING: This has to be located right before the targets
 ifdef IN_NIX_SHELL
 SHELL := env bash
@@ -47,6 +53,29 @@ else
 	@echo "${YELLOW}Nix shell is already active$(RESET)"
 endif
 
+nix-clean: ##@nix Remove all status-react build artifacts from /nix/store
+	nix/clean.sh
+
+nix-purge: SHELL := /bin/sh
+nix-purge: ##@nix Completely remove the complete Nix setup
+	sudo rm -rf /nix ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ~/.cache/nix ~/.status .nix-gcroots
+
+nix-add-gcroots: SHELL := /bin/sh
+nix-add-gcroots: ##@nix Add Nix GC roots to avoid status-react expressions being garbage collected
+	scripts/add-nix-gcroots.sh
+
+nix-update-npm: SHELL := /bin/sh
+nix-update-npm: ##@nix Update node2nix expressions based on current package.json
+	nix/desktop/realm-node/generate-nix.sh
+
+nix-update-gradle: SHELL := /bin/sh
+nix-update-gradle: ##@nix Update maven nix expressions based on current gradle setup
+	nix/mobile/android/maven-and-npm-deps/maven/generate-nix.sh
+
+nix-update-lein: SHELL := /bin/sh
+nix-update-lein: ##@nix Update maven nix expressions based on current lein setup
+	nix/tools/lein/generate-nix.sh nix/lein
+
 #----------------
 # General targets
 #----------------
@@ -72,40 +101,22 @@ disable-githooks: ##@prepare Disables lein githooks
 		-e 's|:pre-commit|;; :pre-commit|' project.clj; \
 	rm project.clj~
 
-#----------------
-# Nix targets
-#----------------
-
-nix-clean: SHELL := /bin/sh
-nix-clean: ##@nix Remove complete nix setup
-	sudo rm -rf /nix ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ~/.cache/nix ~/.status .nix-gcroots
-
-nix-add-gcroots: SHELL := /bin/sh
-nix-add-gcroots: ##@nix Add Nix GC roots to avoid status-react expressions being garbage collected
-	scripts/add-nix-gcroots.sh
-
-nix-update-npm: SHELL := /bin/sh
-nix-update-npm: ##@nix Update node2nix expressions based on current package.json
-	nix/desktop/realm-node/generate-nix.sh
-
-nix-update-gradle: SHELL := /bin/sh
-nix-update-gradle: ##@nix Update maven nix expressions based on current gradle setup
-	nix/mobile/android/maven-and-npm-deps/maven/generate-nix.sh
-
-nix-update-lein: SHELL := /bin/sh
-nix-update-lein: ##@nix Update maven nix expressions based on current lein setup
-	nix/tools/lein/generate-nix.sh nix/lein
-
 #----------------
 # Release builds
 #----------------
 release: release-android release-ios ##@build build release for Android and iOS
 
-release-android: SHELL := /bin/sh
 release-android: export TARGET_OS ?= android
 release-android: export BUILD_ENV ?= prod
+release-android: export BUILD_TYPE ?= nightly
+release-android: export NDK_ABI_FILTERS ?= armeabi-v7a;arm64-v8a;x86
+release-android: export STORE_FILE ?= ~/.gradle/status-im.keystore
 release-android: ##@build build release for Android
-	scripts/release-$(TARGET_OS).sh
+	nix/build.sh targets.mobile.$(TARGET_OS).release \
+		--arg env '{NDK_ABI_FILTERS="$(NDK_ABI_FILTERS)";}' \
+		--argstr build-type $(BUILD_TYPE) \
+		--argstr keystore-file $(STORE_FILE) \
+		--option extra-sandbox-paths $(STORE_FILE)
 
 release-ios: export TARGET_OS ?= ios
 release-ios: export BUILD_ENV ?= prod
@@ -139,9 +150,9 @@ jsbundle-android: export TARGET_OS ?= android
 jsbundle-android: export BUILD_ENV ?= prod
 jsbundle-android: ##@jsbundle Compile JavaScript and Clojure into index.android.js 
 	# Call nix-build to build the 'targets.mobile.jsbundle' attribute and copy the index.android.js file to the project root
-	@git clean -dxf -f ./index.$(TARGET_OS).js && \
-	_NIX_RESULT_PATH=$(shell . ~/.nix-profile/etc/profile.d/nix.sh && nix-build --argstr target-os $(TARGET_OS) --pure --no-out-link --show-trace -A targets.mobile.jsbundle) && \
-	[ -n "$${_NIX_RESULT_PATH}" ] && cp -av $${_NIX_RESULT_PATH}/* .
+	@git clean -dxf ./index.$(TARGET_OS).js
+	nix/build.sh targets.mobile.jsbundle && \
+	mv result/index.$(TARGET_OS).js ./
 
 prod-build-ios: jsbundle-ios ##@legacy temporary legacy alias for jsbundle-ios
 	@echo "${YELLOW}This a deprecated target name, use jsbundle-ios.$(RESET)"
@@ -238,14 +249,17 @@ coverage: ##@test Run tests once in NodeJS generating coverage
 # Other
 #--------------
 react-native-desktop: export TARGET_OS ?= $(HOST_OS)
+react-native-desktop: export _NIX_PURE ?= true
 react-native-desktop: ##@other Start react native packager
 	@scripts/start-react-native.sh
 
 react-native-android: export TARGET_OS ?= android
+react-native-android: export _NIX_PURE ?= true
 react-native-android: ##@other Start react native packager for Android client
 	@scripts/start-react-native.sh
 
 react-native-ios: export TARGET_OS ?= ios
+react-native-ios: export _NIX_PURE ?= true
 react-native-ios: ##@other Start react native packager for Android client
 	@scripts/start-react-native.sh
 

ci/Jenkinsfile.android

@@ -26,6 +26,7 @@ pipeline {
     LC_ALL   = "en_US.UTF-8"
     LANGUAGE = "en_US.UTF-8"
     TARGET_OS = 'android'
+    BUILD_ENV = 'prod'
     NIX_CONF_DIR = "${env.WORKSPACE}/nix"
     FASTLANE_DISABLE_COLORS = 1
     REALM_DISABLE_ANALYTICS = 1
@@ -83,7 +84,9 @@ pipeline {
         stage('Build') { stages {
           stage('JSBundle') {
             steps {
-              script { cmn.nix.build(attr: 'targets.mobile.jsbundle') }
+              script {
+                cmn.nix.shell('make jsbundle-android', pure: false)
+              }
             }
           }
           stage('Bundle') {

ci/Jenkinsfile.macos

@@ -12,7 +12,7 @@ pipeline {
   options {
     timestamps()
     /* Prevent Jenkins jobs from running forever */
-    timeout(time: 25, unit: 'MINUTES')
+    timeout(time: 30, unit: 'MINUTES')
     /* Limit builds retained */
     buildDiscarder(logRotator(
       numToKeepStr: '10',

ci/Jenkinsfile.nix-cache

@@ -23,73 +23,81 @@ pipeline {
   }
 
   stages {
+    stage('Prep') {
+      steps { script {
+        nix = load('ci/nix.groovy')
+      } }
+    }
     stage('Setup') {
-      steps {
+      steps { script {
         sh 'scripts/setup'
-        sh '''
-          . ~/.nix-profile/etc/profile.d/nix.sh && \
-          nix-env -i openssh
-        '''
-      }
+        nix.shell('nix-env -i openssh', pure: false)
+      } }
     }
     stage('Build status-go') {
       steps { script {
         ['android', 'desktop', 'ios'].each { os ->
-          sh """
-            . ~/.nix-profile/etc/profile.d/nix.sh && \
-            nix-build --argstr target-os all -A targets.status-go.${os}.buildInputs
-          """
+          nix.build(
+            attr: "targets.status-go.${os}.buildInputs",
+            args: ['target-os': 'all'],
+            link: false
+          )
         }
       } }
     }
-    stage('Build jsbundle-android') {
-      steps {
-        // Run a Nix build to build/fetch everything that is necessary to produce a jsbundle for TARGET_OS=android (e.g. maven and node repos)
-        sh '''
-          args="--argstr target-os android --show-trace -A targets.mobile.jsbundle"
-
-          . ~/.nix-profile/etc/profile.d/nix.sh && \
-          nix-build --pure --no-out-link $args
-
-          prodBuildDrv=$(nix-instantiate --quiet $args) && \
-          prodBuildSrcPath=$(nix-store -q --binding src $prodBuildDrv) && \
-          prodBuildOutPath=$(nix-store -q --outputs $prodBuildDrv) && \
-          nix-store --delete $prodBuildDrv $prodBuildSrcPath $prodBuildOutPath
-        '''
-      }
+    stage('Build jsbundle') {
+      steps { script {
+        /* build/fetch things required to produce a js-bundle for android
+         * (e.g. maven and node repos) */
+        nix.build(
+          attr: 'targets.mobile.jsbundle',
+          args: ['target-os': 'android'],
+          pure: false,
+          link: false
+        )
+      } }
+    }
+    stage('Build android deps') {
+      steps { script {
+        /* build/fetch things required to build jsbundle and android */
+        nix.build(
+          attr: 'targets.mobile.android.buildInputs',
+          args: ['target-os': 'android'],
+          pure: false,
+          link: false
+        )
+      } }
     }
     stage('Build nix shell deps') {
-      steps {
-        // Run a Nix build to build/fetch everything that is necessary to instantiate shell.nix for TARGET_OS=all
-        sh '''
-          . ~/.nix-profile/etc/profile.d/nix.sh && \
-          nix-shell --argstr target-os all --pure --show-trace shell.nix
-        '''
-      }
+      steps { script {
+        /* build/fetch things required to instantiate shell.nix for TARGET_OS=all */
+        nix.shell("nix-build ${args.join(' ')}", pure: false)
+        nix.build(
+          attr: 'shell',
+          args: ['target-os': 'all'],
+          link: false
+        )
+      } }
     }
     stage('Upload') {
-      steps {
+      steps { script {
         sshagent(credentials: ['nix-cache-ssh']) {
-          sh """
-            . ~/.nix-profile/etc/profile.d/nix.sh && \
-            find /nix/store/ -mindepth 1 -maxdepth 1 \
-              -not -name '.links' -and -not -name '*.lock' \
-              -and -not -name '*-status-react-jsbundle-source' \
-              -and -not -name '*-status-react-release-android-source' \
-              -and -not -name '*-jsbundle-*' \
-              -and -not -name '*-release-android' | \
-              xargs nix-copy-closure -v --to ${NIX_CACHE_USER}@${NIX_CACHE_HOST}
-          """
+          nix.shell("""
+              find /nix/store/ -mindepth 1 -maxdepth 1 \
+                -not -name '.links' -and -not -name '*.lock' \
+                -and -not -name '*-status-react-*' \
+                | xargs nix-copy-closure -v --to ${NIX_CACHE_USER}@${NIX_CACHE_HOST}
+            """,
+            pure: false
+          )
         }
-      }
+      } }
     }
   }
   post {
-    always {
-      sh '''
-        . ~/.nix-profile/etc/profile.d/nix.sh && \
-        nix-store --optimize
-      '''
-   }
+    always { script {
+      nix.shell('nix-store --optimize', pure: false)
+      nix.shell('nix/clean.sh', pure: false)
+    } }
   }
 }

ci/android.groovy

@@ -54,6 +54,7 @@ def bundle() {
       )
     }
   }
+  /* because nix-build was run in `android` dir that's where `result` is */
   def outApk = "android/result/app.apk"
   def pkg = utils.pkgFilename(btype, 'apk')
   /* rename for upload */

ci/nix.groovy

@@ -7,7 +7,7 @@
 def shell(Map opts = [:], String cmd) {
   def defaults = [
     pure: true,
-    args: ['target-os': env.TARGET_OS],
+    args: ['target-os': env.TARGET_OS ? env.TARGET_OS : 'none'],
     keep: ['LOCALE_ARCHIVE_2_27', 'IN_CI_ENVIRONMENT'],
   ]
   /* merge defaults with received opts */
@@ -18,34 +18,40 @@ def shell(Map opts = [:], String cmd) {
   if (env.TARGET_OS in ['windows', 'ios']) {
     opts.pure = false
   }
-  sh """
-    set +x
-    . ~/.nix-profile/etc/profile.d/nix.sh
-    set -x
-    IN_CI_ENVIRONMENT=1 \\
-    nix-shell --run \'${cmd}\' ${_getNixCommandArgs(opts, true)}
-  """
+  def stdOut = sh(
+    script: """
+      set +x
+      . ~/.nix-profile/etc/profile.d/nix.sh
+      set -x
+      IN_CI_ENVIRONMENT=1 \\
+      nix-shell --run \'${cmd}\' ${_getNixCommandArgs(opts, true)}
+    """,
+    returnStdout: true
+  )
+  return stdOut.trim()
 }
 
 /**
  * Arguments:
  *  - pure - Use --pure mode with Nix for more deterministic behaviour
+ *  - link - Bu default build creates a `result` directory, you can turn that off
  *  - keep - List of env variables to pass through to Nix build
  *  - args - Map of arguments to provide to --argstr
  *  - attr - Name of attribute to use with --attr flag
- *  - safeEnv - Name of env variables to pass securely through to Nix build (they won't get captured in Nix derivation file)
  *  - sbox - List of host file paths to pass to the Nix expression
+ *  - safeEnv - Name of env variables to pass securely through to Nix build (they won't get captured in Nix derivation file)
  **/
 def build(Map opts = [:]) {
   env.IN_CI_ENVIRONMENT = '1'
 
   def defaults = [
     pure: true,
+    link: true,
     args: ['target-os': env.TARGET_OS],
     keep: ['IN_CI_ENVIRONMENT'],
-    safeEnv: [],
     attr: null,
-    sbox: []
+    sbox: [],
+    safeEnv: [],
   ]
   /* merge defaults with received opts */
   opts = defaults + opts
@@ -61,7 +67,7 @@ def build(Map opts = [:]) {
       set -x
       nix-build ${_getNixCommandArgs(opts, false)}
     """
-    ).trim()
+  ).trim()
 }
 
 private makeNixBuildEnvFile(Map opts = [:]) {
@@ -115,6 +121,7 @@ private def _getNixCommandArgs(Map opts = [:], boolean isShell) {
 
   return [
     opts.pure ? "--pure" : "",
+    opts.link ? "" : "--no-out-link",
     keepFlags.join(" "),
     argsFlags.join(" "),
     extraSandboxPathsFlag,