MiLM2 Integration (#43)

.github/workflows/build.yaml

@@ -4,8 +4,10 @@ on:
   push:
     tags: [ '*' ]
     branches: [ '*' ]
+  schedule:
+    - cron: '10 1 * * *'
 
-  workflow_dispatch: {}
+  workflow_dispatch: { }
 
 jobs:
   init:
@@ -49,23 +51,29 @@ jobs:
 
       # Distribution
       dist-docker: false
-      dist-archive: true
+      dist-archive: ${{ github.event_name != 'schedule' }}
       dist-archive-tasks: 'distributionZip'
       dist-archive-paths: './distributions/*.zip'
 
       # Release to S3
-      release-to-s3: true
+      release-to-s3: ${{ github.event_name != 'schedule' }}
       release-s3-add-version: true
       release-s3-add-sha: ${{ fromJSON(needs.init.outputs.is-release) && 'false' || '8' }}
 
       # Send notifications
       notify-telegram: true
 
     secrets:
+      env: |
+        { "MI_LICENSE": ${{ toJSON(secrets.MI_LICENSE) }} }
+
       GRADLE_PROPERTIES: |
         miRepoAccessKeyId=${{ secrets.AWS_CI_ACCESS_KEY_ID }}
         miRepoSecretAccessKey= ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }}
 
+        miGitHubMavenUser=${{ github.actor }}
+        miGitHubMavenToken=${{ secrets.GITHUB_TOKEN }}
+
       AWS_KEY_ID: ${{ secrets.AWS_CI_ACCESS_KEY_ID }}
       AWS_KEY_SECRET: ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }}
 

.gitignore

@@ -10,3 +10,5 @@ test_target
 src/test/resources/sequences/big
 tmp
 pack
+build
+.gradle

.mi-ci/hooks/test-integration-after.sh

@@ -2,5 +2,9 @@
 
 script_dir=$(dirname "${0}")
 
+cd "${script_dir}/../.." || exit
+
+#export MI_LICENSE_DEBUG=MI_LICENSE_DEBUG
+
 # Run integration tests
-"${script_dir}/../../itests.sh"
+./itests.sh test

.mi-ci/hooks/test-integration-before.sh

@@ -2,8 +2,12 @@
 
 script_dir=$(dirname "${0}")
 
-# downloadable test data
-"${script_dir}/../../ensure-test-data.sh"
+cd "${script_dir}/../.." || exit
 
-# builds predprocessed test data
-"${script_dir}/../../prepare-test-data.sh"
+#export MI_LICENSE_DEBUG=MI_LICENSE_DEBUG
+
+# Downloadable test data
+./ensure-test-data.sh
+
+# Builds pre-processed test data
+./prepare-test-data.sh

.mi-ci/hooks/test-unit-before.sh

@@ -2,8 +2,12 @@
 
 script_dir=$(dirname "${0}")
 
-# downloadable test data
-"${script_dir}/../../ensure-test-data.sh"
+cd "${script_dir}/../.." || exit
 
-# builds predprocessed test data
-"${script_dir}/../../prepare-test-data.sh"
+#export MI_LICENSE_DEBUG=MI_LICENSE_DEBUG
+
+# Downloadable test data
+./ensure-test-data.sh
+
+# Builds pre-processed test data
+./prepare-test-data.sh

build.gradle.kts

@@ -3,7 +3,16 @@ import com.palantir.gradle.gitversion.VersionDetails
 import groovy.lang.Closure
 import java.net.InetAddress
 
-gradle.startParameter.excludedTaskNames += listOf("assembleDist", "assembleShadowDist", "distTar", "distZip", "installDist", "installShadowDist", "shadowDistTar", "shadowDistZip")
+gradle.startParameter.excludedTaskNames += listOf(
+    "assembleDist",
+    "assembleShadowDist",
+    "distTar",
+    "distZip",
+    "installDist",
+    "installShadowDist",
+    "shadowDistTar",
+    "shadowDistZip"
+)
 
 plugins {
     `java-library`
@@ -13,6 +22,9 @@ plugins {
     id("com.github.johnrengelman.shadow") version "7.1.2"
 }
 
+// val miGitHubMavenUser: String by project
+// val miGitHubMavenToken: String by project
+
 val miRepoAccessKeyId: String? by project
 val miRepoSecretAccessKey: String? by project
 
@@ -45,10 +57,15 @@ tasks.withType<JavaCompile> {
 }
 
 tasks.withType<Javadoc> {
-    (options as StandardJavadocDocletOptions).addStringOption("Xdoclint:none", "-quiet")
+    options {
+        this as StandardJavadocDocletOptions
+        addStringOption("Xdoclint:none", "-quiet")
+    }
 }
 
 repositories {
+    // mavenLocal()
+
     mavenCentral()
 
     // Snapshot versions of redberry-pipe, milib and repseqio distributed via this repo
@@ -67,6 +84,9 @@ dependencies {
         exclude("com.milaboratory", "milib")
     }
 
+    // implementation("com.milaboratory:milm2-jvm:0.2.0-test-2") { isChanging = true }
+    implementation("com.milaboratory:milm2-jvm:1.1.0")
+
     implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")
     implementation("commons-io:commons-io:2.11.0")
     implementation("org.lz4:lz4-java:1.8.0")
@@ -146,6 +166,10 @@ tasks.test {
     minHeapSize = "1024m"
     maxHeapSize = "2048m"
 
+    testLogging {
+        showStandardStreams = true
+    }
+
     miCiStage?.let {
         if (it == "test") {
             systemProperty("longTests", "true")

itests/case8.sh

@@ -23,9 +23,9 @@ mixcr analyze amplicon \
   --impute-germline-on-export --json-report case8 \
   CD4M1_test_R1.fastq.gz CD4M1_test_R2.fastq.gz case8
 
-assert "cat case8.align.jsonl | head -n 1 | jq -r .chainUsage.chains.TRA" "197562"
-assert "cat case8.assemble.jsonl | head -n 1 | jq -r .readsInClones" "161627"
-assert "cat case8.assembleContigs.jsonl | head -n 1 | jq -r .longestContigLength" "223"
-assert "cat case8.assembleContigs.jsonl | head -n 1 | jq -r .clonesWithAmbiguousLetters" "1378"
+assert "cat case8.align.jsonl | head -n 1 | jq -r .chainUsage.chains.TRA" "197642"
+assert "cat case8.assemble.jsonl | head -n 1 | jq -r .readsInClones" "161771"
+assert "cat case8.assembleContigs.jsonl | head -n 1 | jq -r .longestContigLength" "227"
+assert "cat case8.assembleContigs.jsonl | head -n 1 | jq -r .clonesWithAmbiguousLetters" "978"
 assert "cat case8.assembleContigs.jsonl | head -n 1 | jq -r .assemblePrematureTerminationEvents" "1"
-assert "cat case8.assembleContigs.jsonl | head -n 1 | jq -r .finalCloneCount" "22402"
+assert "cat case8.assembleContigs.jsonl | head -n 1 | jq -r .finalCloneCount" "22384"

itests/case9.sh

@@ -24,9 +24,9 @@ mixcr analyze amplicon \
   --impute-germline-on-export --json-report case9 \
   CD4M1_test_R1.fastq.gz CD4M1_test_R2.fastq.gz case9
 
-assert "cat case9.align.jsonl | head -n 1 | jq -r .chainUsage.chains.TRA" "197562"
-assert "cat case9.assemble.jsonl | head -n 1 | jq -r .readsInClones" "161627"
-assert "cat case9.assembleContigs.jsonl | head -n 1 | jq -r .longestContigLength" "223"
-assert "cat case9.assembleContigs.jsonl | head -n 1 | jq -r .clonesWithAmbiguousLetters" "1378"
+assert "cat case9.align.jsonl | head -n 1 | jq -r .chainUsage.chains.TRA" "197642"
+assert "cat case9.assemble.jsonl | head -n 1 | jq -r .readsInClones" "161771"
+assert "cat case9.assembleContigs.jsonl | head -n 1 | jq -r .longestContigLength" "227"
+assert "cat case9.assembleContigs.jsonl | head -n 1 | jq -r .clonesWithAmbiguousLetters" "978"
 assert "cat case9.assembleContigs.jsonl | head -n 1 | jq -r .assemblePrematureTerminationEvents" "1"
-assert "cat case9.assembleContigs.jsonl | head -n 1 | jq -r .finalCloneCount" "22402"
+assert "cat case9.assembleContigs.jsonl | head -n 1 | jq -r .finalCloneCount" "22384"

mixcr

@@ -193,6 +193,6 @@ if [[ "$jar" == "" ]]; then
   exit 1
 fi
 
-$java -D${app}.path="$dir" -D${app}.command=${app} "${javaArgs[@]}" -jar "$jar" "${appArgs[@]}"
+$java -Dmi.license.file="$dir/mi.license" -D${app}.path="$dir" -D${app}.command=${app} "${javaArgs[@]}" -jar "$jar" "${appArgs[@]}"
 
 exit $?

src/main/java/com/milaboratory/mixcr/cli/Main.java

@@ -30,6 +30,9 @@
 package com.milaboratory.mixcr.cli;
 
 import com.milaboratory.cli.ValidationException;
+import com.milaboratory.milm.LM;
+import com.milaboratory.milm.LicenseError;
+import com.milaboratory.milm.LicenseErrorType;
 import com.milaboratory.util.TempFileManager;
 import com.milaboratory.util.VersionInfo;
 import io.repseq.core.VDJCLibraryRegistry;
@@ -45,17 +48,86 @@
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
-
+import java.util.Scanner;
+import java.util.function.Consumer;
+import java.util.prefs.Preferences;
 
 public final class Main {
 
     private static boolean initialized = false;
+    public static final LM lm = new LM(Main.class);
+
+    public static final String MI_LICENSE_PREFERENCES_KEY = "mi_license";
 
     public static void main(String... args) {
+        Preferences prefs = Preferences.userNodeForPackage(Main.class);
+        if (args.length > 0 && "activate-license".equals(args[0])) {
+            try (Scanner reader = new Scanner(System.in)) {
+                System.out.println("Please enter the license:");
+                String envelope = reader.nextLine();
+                System.out.println("Checking the license...");
+                lm.addStringLicenseSource(envelope);
+                lm.setWarningHandler(warning -> System.err.println("License Warning:\n" + warning));
+                Consumer<LicenseError> errorHandler = err -> System.err.println("License Error:\n" + err.getType() + "\n" + err.getMessage());
+                lm.setAsyncErrorHandler(errorHandler);
+                LicenseError err = lm.init();
+                if (err != null) {
+                    errorHandler.accept(err);
+                    System.exit(LM.LicenseErrorExitCode);
+                }
+                System.out.println("Ok");
+                prefs.put(MI_LICENSE_PREFERENCES_KEY, envelope);
+                System.out.println("License activated successfully.");
+                System.exit(0);
+            }
+        }
+
+        if (args.length > 0 && "deactivate-license".equals(args[0])) {
+            prefs.remove(MI_LICENSE_PREFERENCES_KEY);
+            System.out.println("License successfully deactivated.");
+            System.exit(0);
+        }
+
+        lm.addPreferenceLicenseSource(prefs, MI_LICENSE_PREFERENCES_KEY);
+        lm.addDefaultSources();
+
+        Consumer<LicenseError> licenseErrorHandler = licenseError -> {
+            if (licenseError.getType() == LicenseErrorType.NoLicense) {
+                System.err.println("=== No License ===");
+                System.err.println();
+                System.err.println("To use MiXCR, please, provide a valid license.");
+                System.err.println();
+                System.err.println("If you already have a license, activate it by calling:");
+                System.err.println("  mixcr activate-license");
+                System.err.println();
+                System.err.println("You can also activate the license via a special file, environment");
+                System.err.println("variable or other means, please check the docs."); // TODO provide a link
+                System.err.println();
+                System.err.println("If you don't have a license check https://licensing.milaboratories.com/.");
+                System.err.println("Free license is provided for academic users and non-profit organisations.");
+            } else
+                System.err.println("License error: " + licenseError);
+            System.exit(LM.LicenseErrorExitCode);
+        };
+
+        lm.setAsyncErrorHandler(licenseErrorHandler);
+        lm.setWarningHandler(warning -> System.err.println("License Warning: " + warning));
+        LicenseError lmError = lm.init();
+        if (lmError != null) licenseErrorHandler.accept(lmError);
+
+        VersionInfo versionInfo = VersionInfo.getVersionInfoForArtifact("mixcr");
+        lm.sendGenericStats("mixcr." +
+                        versionInfo.getVersion() + "." +
+                        versionInfo.getBranch() + "." +
+                        versionInfo.getRevision() + "." +
+                        versionInfo.getTimestamp().toInstant().getEpochSecond(),
+                args);
+
         Thread.setDefaultUncaughtExceptionHandler((t, e) -> {
             e.printStackTrace();
             System.exit(2);
         });
+
         handleParseResult(parseArgs(args).getParseResult(), args);
     }