Split docker tests (#3900)

* Split docker tests

* reset settings.gradle

* reset some gradles and ymls

* try optimize yamls

* removed some yaml comments

* restored job versions from  to

* moved container test to extended-it

.github/actions/setup-gradle-cache/action.yaml

@@ -0,0 +1,12 @@
+name: "Setups cache for Gradle"
+description: "Setups cache for Gradle"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}

.github/actions/setup-jdk/action.yaml

@@ -0,0 +1,11 @@
+name: "Setups JDK"
+description: "Setups JDK"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up JDK 17
+      uses: actions/setup-java@v2
+      with:
+        java-version: '17.0.8'
+        distribution: 'temurin'

.github/actions/test-gradle-project/action.yml

@@ -0,0 +1,25 @@
+name: "Test gradle project"
+description: "Test gradle project"
+
+inputs:
+  project-name:
+    description: "Name of gradle project to test"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/actions/setup-gradle-cache
+    - name: Run compile tests 
+      shell: bash
+      run: ./gradlew :${{inputs.project-name}}:compileJava :${{inputs.project-name}}:compileTestJava
+    - name: Run tests
+      shell: bash
+      run: ./gradlew :${{inputs.project-name}}:check --parallel
+    - name: Archive test results
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: ${{inputs.project-name}}-test-results
+        path: |
+          ${{inputs.project-name}}/build/reports/tests/test/

.github/workflows/CI.yaml

@@ -6,57 +6,86 @@ on:
   pull_request:
     branches: [ "dev" ]
 
+env:
+  DOCKER_ENTERPRISE_URL: ${{ secrets.DOCKER_ENTERPRISE_DEV_URL }}
+  DOCKER_COMMUNITY_URL: ${{ secrets.DOCKER_COMMUNITY_DEV_URL }}
+  TEAMCITY_DEV_URL: ${{ secrets.TEAMCITY_DEV_URL }}
+  TEAMCITY_USER: ${{ secrets.TEAMCITY_USER }}
+  TEAMCITY_PASSWORD: ${{ secrets.TEAMCITY_PASSWORD }}
+  ENTERPRISE_TAR: enterprise-docker.tar
+  COMMUNITY_TAR: community-docker.tar
+
+
 jobs:
-  build:
+  code-ql:
     runs-on: ubuntu-latest
-    env:
-      DOCKER_ENTERPRISE_URL: ${{ secrets.DOCKER_ENTERPRISE_DEV_URL }}
-      DOCKER_COMMUNITY_URL: ${{ secrets.DOCKER_COMMUNITY_DEV_URL }}
-      TEAMCITY_DEV_URL: ${{ secrets.TEAMCITY_DEV_URL }}
-      TEAMCITY_USER: ${{ secrets.TEAMCITY_USER }}
-      TEAMCITY_PASSWORD: ${{ secrets.TEAMCITY_PASSWORD }}
-      ENTERPRISE_TAR: enterprise-docker.tar
-      COMMUNITY_TAR: community-docker.tar
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java', 'javascript' ]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-jdk
+      - uses: ./.github/actions/setup-gradle-cache
+
+      - name: Compile Java
+        run: |
+          chmod +x gradlew
+          ./gradlew --no-daemon --init-script init.gradle clean
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+
+      # Autobuild attempts to build any compiled languages 
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
 
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{matrix.language}}"
+
+  tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        project: [ 'extended', 'extended-it' ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
       - name: Set up JDK 17
         uses: actions/setup-java@v2
         with:
           java-version: '17.0.8'
           distribution: 'temurin'
-
-      - name: Download neo4j dev docker container
-        run: |
-          curl -s -L0 -u "${TEAMCITY_USER}:${TEAMCITY_PASSWORD}" -X GET ${DOCKER_ENTERPRISE_URL} -o ${ENTERPRISE_TAR} &
-          curl -s -L0 -u "${TEAMCITY_USER}:${TEAMCITY_PASSWORD}" -X GET ${DOCKER_COMMUNITY_URL} -o ${COMMUNITY_TAR} &
-          wait
-          docker load --input ${ENTERPRISE_TAR}
-          docker load --input ${COMMUNITY_TAR}
-
       - uses: actions/cache@v2
         with:
           path: |
             ~/.gradle/caches
             ~/.gradle/wrapper
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
 
-      - name: Build with Gradle
+      - name: Init gradle
         run: |
           chmod +x gradlew
           ./gradlew --init-script init.gradle
-          ./gradlew build --parallel
 
-      - name: Clean neo4j artifacts
-        if: always() # run this step even if previous step failed
+      - name: Download neo4j dev docker container
+        if: matrix.project == 'extended-it'
         run: |
-          (find ~/.gradle/caches -name "*neo4j*" -exec rm -rf {} \;) || echo "All neo4j files cleaned"
-          rm -rf ${ENTERPRISE_TAR} ${COMMUNITY_TAR}
+          curl -s -L0 -u "${TEAMCITY_USER}:${TEAMCITY_PASSWORD}" -X GET ${DOCKER_ENTERPRISE_URL} -o ${ENTERPRISE_TAR} &
+          curl -s -L0 -u "${TEAMCITY_USER}:${TEAMCITY_PASSWORD}" -X GET ${DOCKER_COMMUNITY_URL} -o ${COMMUNITY_TAR} &
+          wait
+          docker load --input ${ENTERPRISE_TAR}
+          docker load --input ${COMMUNITY_TAR}
 
-      - name: Archive test results
-        uses: actions/upload-artifact@v2
-        if: always()
+      - name: Run ${{ matrix.project }} tests
+        uses: ./.github/actions/test-gradle-project
         with:
-          name: test-results
-          path: |
-            extended/build/reports/tests/test/
+          project-name: ${{ matrix.project }}

build.gradle

@@ -78,7 +78,7 @@ subprojects {
 
         maxHeapSize = "5G"
         forkEvery = 50
-        maxParallelForks = 1
+        maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
         minHeapSize = "128m"
 
         // This would apply only to TeamCity

extended-it/build.gradle

@@ -0,0 +1,62 @@
+plugins {
+    id 'java'
+    id 'com.github.johnrengelman.shadow'
+    id 'maven-publish'
+    id 'antlr'
+    id "org.jetbrains.kotlin.jvm" version "1.6.0"
+    id "com.diffplug.spotless" version "6.7.2"
+}
+
+description = 'APOC :: Apoc Extended Integration Tests Module'
+
+test {
+    maxParallelForks = 1
+}
+
+dependencies {
+    apt project(':processor')
+    apt group: 'org.neo4j', name: 'neo4j', version: neo4jVersionEffective   // mandatory to run @ServiceProvider based META-INF code generation
+
+    def withoutJacksons = {
+        exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations'
+        exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
+    }    
+    def withoutServers = {
+        exclude group: 'org.eclipse.jetty'
+        exclude group: 'org.eclipse.jetty.aggregate'
+        exclude group: 'org.apache.hive', module: 'hive-service'
+    }
+
+    implementation project(":extended")
+
+    testImplementation group: 'us.fatehi', name: 'schemacrawler-mysql', version: '16.20.8'
+    testImplementation group: 'org.postgresql', name: 'postgresql', version: '42.1.4'
+
+    testImplementation project(':core')
+    testImplementation project(':test-utils')
+    testImplementation project(':common')
+    testImplementation project(':extended')
+    testImplementation project(':core').sourceSets.test.output
+    compileOnly project(':extended').sourceSets.main.allJava
+    testImplementation project(':extended').sourceSets.test.output
+    testImplementation files(project(':extended').sourceSets.test.output)
+
+    testImplementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.425'
+    testImplementation group: 'org.xmlunit', name: 'xmlunit-core', version: '2.9.1'
+    testImplementation group: 'com.couchbase.client', name: 'java-client', version: '3.3.0', withoutJacksons
+    testImplementation group: 'io.lettuce', name: 'lettuce-core', version: '6.1.1.RELEASE'
+    testImplementation group: 'org.mongodb', name: 'mongodb-driver-sync', version: '4.11.1', {
+        exclude group: 'io.netty'
+    }
+    testImplementation group: 'org.apache.parquet', name: 'parquet-hadoop', version: '1.13.1', withoutServers
+
+    configurations.all {
+        exclude group: 'org.slf4j', module: 'slf4j-nop'
+        exclude group: 'ch.qos.logback', module: 'logback-classic'
+    }
+}
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_17
+    targetCompatibility = JavaVersion.VERSION_17
+}

extended/src/test/java/apoc/couchbase/CouchbaseTestUtils.java → extended-it/src/test/java/apoc/couchbase/CouchbaseTestUtils.java

@@ -5,6 +5,7 @@
 import com.couchbase.client.core.io.CollectionIdentifier;
 import com.couchbase.client.java.Bucket;
 import com.couchbase.client.java.Cluster;
+import com.couchbase.client.java.ClusterOptions;
 import com.couchbase.client.java.Collection;
 import com.couchbase.client.java.json.JsonArray;
 import com.couchbase.client.java.json.JsonObject;
@@ -25,7 +26,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
-import static com.couchbase.client.java.ClusterOptions.clusterOptions;
 import static com.couchbase.client.java.query.QueryOptions.queryOptions;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertEquals;
@@ -146,7 +146,7 @@ protected static void createCouchbaseContainer() {
                 Optional.of(couchbase.getBootstrapCarrierDirectPort()),
                 Optional.of(couchbase.getBootstrapHttpDirectPort())));
 
-        Cluster cluster = Cluster.connect(seedNodes, clusterOptions(USERNAME, PASSWORD).environment(environment));
+        Cluster cluster = Cluster.connect(seedNodes, ClusterOptions.clusterOptions(USERNAME, PASSWORD).environment(environment));
 
         boolean isFilled = fillDB(cluster);
         HOST = getUrl(couchbase);

extended/src/test/java/apoc/dv/DataVirtualizationCatalogTest.java → extended-it/src/test/java/apoc/dv/DataVirtualizationCatalogTest.java

@@ -1,6 +1,7 @@
 package apoc.dv;
 
 import apoc.create.Create;
+import apoc.dv.DataVirtualizationCatalog;
 import apoc.load.Jdbc;
 import apoc.load.LoadCsv;
 import apoc.util.TestUtil;
@@ -39,8 +40,6 @@
 
 public class DataVirtualizationCatalogTest {
 
-    public static final Label PERSON = Label.label("Person");
-
     public static JdbcDatabaseContainer mysql;
 
     @Rule
@@ -63,11 +62,6 @@ public static void tearDownContainer() {
         mysql.stop();
     }
 
-    @AfterAll
-    public void tearDownDb() {
-        db.shutdown();
-    }
-
     @Test
     public void testVirtualizeCSV() {
         final String name = "csv_vr";

extended/src/test/java/apoc/es/ElasticSearchTest.java → extended-it/src/test/java/apoc/es/ElasticSearchTest.java

@@ -1,5 +1,6 @@
 package apoc.es;
 
+import apoc.es.ElasticSearch;
 import apoc.util.JsonUtil;
 import apoc.util.TestUtil;
 import apoc.util.Util;

extended/src/test/java/apoc/ApocSplitTest.java → extended-it/src/test/java/apoc/neo4j/docker/ApocSplitTest.java

@@ -1,4 +1,4 @@
-package apoc;
+package apoc.neo4j.docker;
 
 import apoc.util.Neo4jContainerExtension;
 import apoc.util.TestContainerUtil;