From dd5589aa1f05f7395630cd94d22b8a3d5feeb1f6 Mon Sep 17 00:00:00 2001 From: eneskuluk <54481799+eneskuluk@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:05:29 -0700 Subject: [PATCH 01/13] antrun order change and maven-jar addition for spark exclusion --- pom.xml | 82 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 24 deletions(-) diff --git a/pom.xml b/pom.xml index e8ef417b3..4e059c016 100644 --- a/pom.xml +++ b/pom.xml @@ -89,11 +89,13 @@ org.apache.spark ${spark.core.artifactid} ${spark.version} + true org.apache.spark ${spark.sql.artifactid} ${spark.version} + true com.google.protobuf @@ -193,29 +195,6 @@ - - maven-antrun-plugin - 3.1.0 - - - unzip-test-artifacts - process-test-resources - - - - - - - - - - - - run - - - - @@ -296,6 +275,25 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + spark-excluded + + jar + + + spark + + org/genomicsdb/spark/** + + + + + org.apache.maven.plugins maven-shade-plugin @@ -308,7 +306,7 @@ true - allinone + allinone-spark *:* @@ -330,6 +328,42 @@ + + maven-antrun-plugin + 3.1.0 + + + unzip-test-artifacts + process-test-resources + + + + + + + + + + + + run + + + + replace-core-jar + package + + + + + + + run + + + + org.apache.maven.plugins maven-surefire-plugin From 896f79d6844f18f1718ce3194f70c05a4acd48ca Mon Sep 17 00:00:00 2001 From: eneskuluk <54481799+eneskuluk@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:06:46 -0700 Subject: [PATCH 02/13] match allione-spark naming change in CMakeLists --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 11bef4c20..f684ecba9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -548,14 +548,14 @@ if(BUILD_JAVA) #Maven build - depends on dynamic library add_custom_command( - OUTPUT ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}.jar ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone.jar + OUTPUT ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}.jar ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone-spark.jar COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/pom.xml ${CMAKE_BINARY_DIR}/pom.xml COMMAND mvn versions:set ${MAVEN_QUIET_ARGS} -DnewVersion=${GENOMICSDB_RELEASE_VERSION} ${MAVEN_PROFILE} COMMAND mvn package -DskipTests ${MAVEN_ARGS} DEPENDS tiledbgenomicsdb ${JAVA_SCALA_SOURCES} pom.xml WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - install(FILES ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}.jar ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone.jar DESTINATION bin) + install(FILES ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}.jar ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone-spark.jar DESTINATION bin) execute_process( COMMAND ln -sf ${CMAKE_SOURCE_DIR}/tests ${CMAKE_BINARY_DIR}) @@ -575,7 +575,7 @@ if(BUILD_JAVA) add_jar(genomicsdb-${GENOMICSDB_RELEASE_VERSION}-examples SOURCES ${GENOMICSDB_EXAMPLE_SOURCES} log4j.properties - INCLUDE_JARS ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone.jar + INCLUDE_JARS ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone-spark.jar OUTPUT_DIR ${GENOMICSDB_MAVEN_BUILD_DIR}) #Deploy to Maven central From ffa384a6c1fbde04801cd61ab20c1f0acae2d428 Mon Sep 17 00:00:00 2001 From: eneskuluk <54481799+eneskuluk@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:07:51 -0700 Subject: [PATCH 03/13] allinone naming change match for tests folder --- tests/common.py | 4 ++-- tests/run_spark_hdfs.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/common.py b/tests/common.py index 6fbe85a22..ee87ad172 100644 --- a/tests/common.py +++ b/tests/common.py @@ -55,7 +55,7 @@ def __find_genomicsdb_jar(target_dir, jar_file_name): def setup_classpath(build_dir): target_dir=os.path.join(build_dir,'target') - allinone_jar=__find_genomicsdb_jar(target_dir,'genomicsdb-*allinone.jar') + allinone_jar=__find_genomicsdb_jar(target_dir,'genomicsdb-*allinone-spark.jar') examples_jar=__find_genomicsdb_jar(target_dir,'genomicsdb-*examples.jar') if 'CLASSPATH' in os.environ: classpath=os.environ['CLASSPATH'] @@ -94,7 +94,7 @@ def setup_jacoco(build_dir, build_type): if e.errno != errno.EEXIST: __error_exit('could not create jacoco-reports dir:'+e.errno+' '+e.filename+' '+e.strerror) genomicsdb_classes_dir = os.path.join(target_dir, 'jacoco-classes') - allinone_archive = zipfile.ZipFile(__find_genomicsdb_jar(target_dir,'genomicsdb-*allinone.jar')) + allinone_archive = zipfile.ZipFile(__find_genomicsdb_jar(target_dir,'genomicsdb-*allinone-spark.jar')) for file in allinone_archive.namelist(): if file.startswith('org/genomicsdb'): allinone_archive.extract(file, genomicsdb_classes_dir) diff --git a/tests/run_spark_hdfs.py b/tests/run_spark_hdfs.py index 9ce17627f..3b4af39b2 100755 --- a/tests/run_spark_hdfs.py +++ b/tests/run_spark_hdfs.py @@ -222,7 +222,7 @@ def sanity_test_spark_bindings(tmpdir, lib_path, jar_dir, jacoco, genomicsdb_ver substitute_placeholders(querypb_json, sanity_test_dir) # Expected exception when run without json files - spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar' + spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar' pid = subprocess.Popen(spark_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_string, stderr_string = pid.communicate() if(pid.returncode == 0): @@ -231,7 +231,7 @@ def sanity_test_spark_bindings(tmpdir, lib_path, jar_dir, jacoco, genomicsdb_ver output_string = "[row=0 col=12140 HG00141 1:12141-12295 {REF=C, DP_FORMAT=2, MIN_DP=0, ALT=[], GQ=0, PL=[0, 0, 0], GT=0/0}, row=1 col=12144 HG01958 1:12145-12277 {REF=C, DP_FORMAT=3, MIN_DP=0, ALT=[], GQ=0, PL=[0, 0, 0], GT=0/0}, row=0 col=17384 HG00141 1:17385-17385 {MQRankSum=-0.329000, AD=[58, 22, 17], MQ=31.719999, DP_FORMAT=80, ALT=[A, ], BaseQRankSum=-2.096000, GQ=99, PID=17385_G_A, ReadPosRankSum=0.005000, MQ0=8, GT=0/1, SB=[58, 0, 22, 0], RAW_MQ=5.500000, REF=G, ClippingRankSum=-1.859000, PL=[504, 0, 9807, 678, 1870, 2548], PGT=0|1}, row=1 col=17384 HG01958 1:17385-17385 {MQRankSum=-1.369000, AD=[0, 120, 37], MQ=29.820000, DP_FORMAT=120, ALT=[T, ], BaseQRankSum=-2.074000, GQ=99, PID=17385_G_T, ReadPosRankSum=-0.101000, DP=120, MQ0=3, GT=1/1, SB=[0, 0, 0, 0], RAW_MQ=2.500000, REF=G, ClippingRankSum=0.555000, PL=[3336, 358, 0, 4536, 958, 7349], PGT=0|1}, row=2 col=17384 HG01530 1:17385-17385 {MQRankSum=-0.432000, AD=[40, 36, 0], MQ=59.369999, DP_FORMAT=76, ALT=[A, ], BaseQRankSum=1.046000, GQ=99, ReadPosRankSum=2.055000, DP=76, MQ0=0, GT=0/1, SB=[9, 31, 13, 23], REF=G, ClippingRankSum=-2.242000, PL=[1018, 0, 1116, 1137, 1224, 2361]}]" - spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar '+loader_json+' '+query_json + spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar '+loader_json+' '+query_json pid = subprocess.Popen(spark_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_string, stderr_string = pid.communicate() if(pid.returncode != 0): @@ -241,7 +241,7 @@ def sanity_test_spark_bindings(tmpdir, lib_path, jar_dir, jacoco, genomicsdb_ver sys.stderr.write('Expected output not found in sanity test with query.json\n') print_error_and_exit(namenode, tmpdir, stdout_string, stderr_string) - spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar '+loader_json+' '+querypb_json+' true' + spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar '+loader_json+' '+querypb_json+' true' pid = subprocess.Popen(spark_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_string, stderr_string = pid.communicate() if(pid.returncode != 0): @@ -475,7 +475,7 @@ def main(): with open(query_json_filename, 'w') as fptr: json.dump(test_query_dict, fptr, indent=4, separators=(',', ': ')); fptr.close(); - spark_cmd = 'spark-submit --class TestGenomicsDBSparkHDFS --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --jars '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-examples.jar --loader '+loader_json_filename+' --query '+query_json_filename+' --template_vcf_header '+template_vcf_header_path+' --spark_master '+spark_master+' --jar_dir '+jar_dir; + spark_cmd = 'spark-submit --class TestGenomicsDBSparkHDFS --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --jars '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-examples.jar --loader '+loader_json_filename+' --query '+query_json_filename+' --template_vcf_header '+template_vcf_header_path+' --spark_master '+spark_master+' --jar_dir '+jar_dir; if (test_name == "t6_7_8"): spark_cmd = spark_cmd + ' --use-query-protobuf'; if (test_name == "t0_1_2_combined"): @@ -512,7 +512,7 @@ def main(): vid_path_final=vid_path+query_param_dict['vid_mapping_file']; else: vid_path_final=vid_path+"inputs"+os.path.sep+"vid.json"; - spark_cmd_v2 = 'spark-submit --class TestGenomicsDBSource --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --jars '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-examples.jar --loader '+loader_json_filename+' --query '+query_json_filename+' --vid '+vid_path_final+' --spark_master '+spark_master; + spark_cmd_v2 = 'spark-submit --class TestGenomicsDBSource --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --jars '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-examples.jar --loader '+loader_json_filename+' --query '+query_json_filename+' --vid '+vid_path_final+' --spark_master '+spark_master; if (gdb_datasource != ""): spark_cmd_v2 += ' --gdb_datasource=' + gdb_datasource if (test_name == "t6_7_8"): From cf42294dfcee62ca820d2c89ec2b16e7f083afcb Mon Sep 17 00:00:00 2001 From: eneskuluk <54481799+eneskuluk@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:11:54 -0700 Subject: [PATCH 04/13] name change in api/sparkbindings --- .../org/genomicsdb/spark/api/GenomicsDBSparkBindings.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/genomicsdb/spark/api/GenomicsDBSparkBindings.java b/src/main/java/org/genomicsdb/spark/api/GenomicsDBSparkBindings.java index a8e7a2767..30c6328aa 100644 --- a/src/main/java/org/genomicsdb/spark/api/GenomicsDBSparkBindings.java +++ b/src/main/java/org/genomicsdb/spark/api/GenomicsDBSparkBindings.java @@ -42,19 +42,19 @@ /** * Example Invocation - * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone.jar loader.json querypb.json true + * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone-spark.jar loader.json querypb.json true * querypb.json should be parseable by GenomicsDBExportConfiguration.ExportConfiguration * OR - * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone.jar loader.json query.json false + * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone-spark.jar loader.json query.json false * OR - * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone.jar loader.json query.json + * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone-spark.jar loader.json query.json */ public class GenomicsDBSparkBindings { List variantCalls; public static void main(String[] args) throws IOException, ClassNotFoundException { if (args.length < 2) { - throw new RuntimeException("Usage: spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb--allinone.jar []"+ + throw new RuntimeException("Usage: spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb--allinone-spark.jar []"+ "Optional Argument 2 - Date: Fri, 24 Mar 2023 10:12:50 -0700 Subject: [PATCH 05/13] update workflows according to naming change from allinone to allinone-spark --- .github/workflows/release_jar.yml | 2 +- .github/workflows/release_publish.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_jar.yml b/.github/workflows/release_jar.yml index d8077482a..26927def7 100644 --- a/.github/workflows/release_jar.yml +++ b/.github/workflows/release_jar.yml @@ -46,7 +46,7 @@ jobs: docker create -it --name genomicsdb ghcr.io/genomicsdb/genomicsdb:release bash docker cp genomicsdb:/build/GenomicsDB/build/src/main/libtiledbgenomicsdb.so . docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}.jar . - docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}-allinone.jar . + docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}-allinone-spark.jar . docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}-sources.jar . docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}-javadoc.jar . docker cp genomicsdb:/build/GenomicsDB/pom.xml genomicsdb-${VERSION_NUMBER}.pom diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 12b694787..b750fe0a9 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -58,17 +58,17 @@ jobs: -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ - -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone.jar \ + -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ -Dtypes=jar \ - -Dclassifiers=allinone + -Dclassifiers=allinone-spark mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}.jar \ -DgroupId=org.genomicsdb -DartifactId=genomicsdb -Dversion=${VERSION_NUMBER} \ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ - -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone.jar \ + -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ -Dtypes=jar \ - -Dclassifiers=allinone + -Dclassifiers=allinone-spark if [[ ${VERSION_NUMBER} != *SNAPSHOT ]]; then mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -f genomicsdb-${VERSION_NUMBER}.pom stagingRepoId=$(mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh | grep orggenomicsdb|cut -f2 -d' ') From 694dfe1c76eacec73700594f48217a71c888acbf Mon Sep 17 00:00:00 2001 From: eneskuluk <54481799+eneskuluk@users.noreply.github.com> Date: Tue, 28 Mar 2023 15:24:31 -0700 Subject: [PATCH 06/13] changes in publish.yml --- .github/workflows/release_publish.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index b750fe0a9..b45b9d20b 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -58,15 +58,25 @@ jobs: -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ - -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ -Dtypes=jar \ - -Dclassifiers=allinone-spark mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}.jar \ -DgroupId=org.genomicsdb -DartifactId=genomicsdb -Dversion=${VERSION_NUMBER} \ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ - -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ + -Dtypes=jar \ + echo mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ + -DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \ + -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ + -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ + -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ + -Dtypes=jar \ + -Dclassifiers=allinone-spark + mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ + -DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \ + -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ + -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ + -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ -Dtypes=jar \ -Dclassifiers=allinone-spark if [[ ${VERSION_NUMBER} != *SNAPSHOT ]]; then From 5fcd3188a1943c243e06d0c543bc3d4caea001fc Mon Sep 17 00:00:00 2001 From: eneskuluk <54481799+eneskuluk@users.noreply.github.com> Date: Thu, 30 Mar 2023 10:27:39 -0700 Subject: [PATCH 07/13] workflow update, publish/test --- .github/workflows/release_publish.yml | 8 +++++--- .github/workflows/release_test.yml | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index b45b9d20b..5aaca94f1 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -65,16 +65,17 @@ jobs: -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ -Dtypes=jar \ + cp genomicsdb-${VERSION_NUMBER}.pom genomicsdb-${VERSION_NUMBER}-spark.pom echo mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ -DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \ - -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ + -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}-spark.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ -Dtypes=jar \ -Dclassifiers=allinone-spark mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ -DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \ - -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ + -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}-spark.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ -Dtypes=jar \ @@ -83,7 +84,8 @@ jobs: mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -f genomicsdb-${VERSION_NUMBER}.pom stagingRepoId=$(mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh | grep orggenomicsdb|cut -f2 -d' ') echo $stagingRepoId - mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom + mvn -X nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom + mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}-spark.pom fi env: OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }} diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml index 3c24a9100..61b1abf67 100644 --- a/.github/workflows/release_test.yml +++ b/.github/workflows/release_test.yml @@ -28,6 +28,7 @@ jobs: - name: Set version number run: | + echo ${{github.ref_name}} echo VERSION_NUMBER=${GITHUB_REF_NAME:1} >> $GITHUB_ENV echo GENOMICSDB_VERSION=${GITHUB_REF_NAME:1} >> $GITHUB_ENV @@ -50,7 +51,8 @@ jobs: -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom ./test_genomicsdbjar.sh # -# - name: Checkout GATK +# - name: Checkout GATK +# if: ${{github.ref_name}} != *" skip-gatk-it"* # uses: actions/checkout@v3 # with: # repository: broadinstitute/gatk From fef29231f1bb437a8b0b61eab5cfede1b1fbabce Mon Sep 17 00:00:00 2001 From: eneskuluk <54481799+eneskuluk@users.noreply.github.com> Date: Fri, 14 Apr 2023 21:16:07 -0700 Subject: [PATCH 08/13] spark publish fix --- .github/workflows/release_publish.yml | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 5aaca94f1..6ba1ca1e4 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -41,7 +41,7 @@ jobs: uses: actions/download-artifact@v3 with: name: ${{ inputs.release_artifact }} - + - name: Deploy Maven Central shell: bash run: | @@ -57,35 +57,28 @@ jobs: -DgroupId=org.genomicsdb -DartifactId=genomicsdb -Dversion=${VERSION_NUMBER} \ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ - -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ - -Dtypes=jar \ + -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}.jar \ -DgroupId=org.genomicsdb -DartifactId=genomicsdb -Dversion=${VERSION_NUMBER} \ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ - -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ - -Dtypes=jar \ - cp genomicsdb-${VERSION_NUMBER}.pom genomicsdb-${VERSION_NUMBER}-spark.pom + -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar echo mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ -DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \ - -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}-spark.pom -DrepositoryId=$REPO_ID \ + -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ - -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ - -Dtypes=jar \ - -Dclassifiers=allinone-spark + -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \ -DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \ - -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}-spark.pom -DrepositoryId=$REPO_ID \ + -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \ - -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \ - -Dtypes=jar \ - -Dclassifiers=allinone-spark + -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar if [[ ${VERSION_NUMBER} != *SNAPSHOT ]]; then mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -f genomicsdb-${VERSION_NUMBER}.pom stagingRepoId=$(mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh | grep orggenomicsdb|cut -f2 -d' ') echo $stagingRepoId - mvn -X nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom - mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}-spark.pom + mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom + mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom fi env: OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }} From cda67c3f4030c0a034b426bb1a10bf792895ce71 Mon Sep 17 00:00:00 2001 From: eneskuluk <54481799+eneskuluk@users.noreply.github.com> Date: Fri, 14 Apr 2023 22:08:46 -0700 Subject: [PATCH 09/13] capture tag message, enable gatk tests according to tag message --- .github/workflows/release.yml | 11 +++++++++- .github/workflows/release_test.yml | 35 +++++++++++++++++++----------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bddde8315..73b83388b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,14 @@ jobs: - name: Set version number run: echo VERSION_NUMBER=${GITHUB_REF_NAME:1} >> $GITHUB_ENV + + - name: Set tag messsage + run: | + git fetch --tags --force + echo "TAG_MESSAGE=$(git tag -l --sort=-taggerdate --format='%(contents)' $(git describe --tags $(git branch --show-current) ))" >> $GITHUB_ENV + + - name: Check tag messsage + run: echo ${TAG_MESSAGE} - name: Install Prerequisites shell: bash @@ -62,10 +70,11 @@ jobs: dylib_artifact: libtiledbgenomicsdb.dylib.${{ github.ref_name }} test: - needs: [release-jar] + needs: [build-and-push-mac-dylib, release-jar] uses: ./.github/workflows/release_test.yml with: release_artifact: release.${{ github.ref_name }} + tag_message: ${{ needs.build-and-push-mac-dylib.outputs.tag_message }} publish: needs: [test] diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml index 177b973ce..ff4cff92d 100644 --- a/.github/workflows/release_test.yml +++ b/.github/workflows/release_test.yml @@ -6,6 +6,9 @@ on: release_artifact: required: true type: string + tag_message: + required: false + type: string jobs: test-jar: @@ -33,6 +36,11 @@ jobs: echo GENOMICSDB_VERSION=${GITHUB_REF_NAME:1} >> $GITHUB_ENV echo GENOMICSDB_MIN_TAG=v1.5.0-SNAPSHOT >> $GITHUB_ENV + - name: Check tag messsage + run: | + echo "tag message is" + echo ${{inputs.tag_message}} + - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -55,16 +63,17 @@ jobs: shell: bash run: GENOMICSDB_TAG=${GENOMICSDB_MIN_TAG} ./test_genomicsdbjar.sh -# -# - name: Checkout GATK -# if: ${{github.ref_name}} != *" skip-gatk-it"* -# uses: actions/checkout@v3 -# with: -# repository: broadinstitute/gatk -# lfs: 'true' -# -# - name: Try GATK integration test -# shell: bash -# run: | -# ./gradlew installDist -Dgenomicsdb.version=${VERSION_NUMBER} -# ./gradlew test --tests *GenomicsDB* + + - name: Checkout GATK + if: ${{inputs.tag_message}} != *"skip-gatk-it"* + uses: actions/checkout@v3 + with: + repository: broadinstitute/gatk + lfs: 'true' + + - name: Try GATK integration test + if: ${{inputs.tag_message}} != *"skip-gatk-it"* + shell: bash + run: | + ./gradlew installDist -Dgenomicsdb.version=${VERSION_NUMBER} + ./gradlew test --tests *GenomicsDB* From 042e0138b35dc5a085291e149c145b57c6ea23ba Mon Sep 17 00:00:00 2001 From: Enes Kuluk <54481799+eneskuluk@users.noreply.github.com> Date: Sat, 15 Apr 2023 00:09:07 -0700 Subject: [PATCH 10/13] indentation fix --- .github/workflows/release_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml index ff4cff92d..5fb4dadb7 100644 --- a/.github/workflows/release_test.yml +++ b/.github/workflows/release_test.yml @@ -64,7 +64,7 @@ jobs: run: GENOMICSDB_TAG=${GENOMICSDB_MIN_TAG} ./test_genomicsdbjar.sh - - name: Checkout GATK + - name: Checkout GATK if: ${{inputs.tag_message}} != *"skip-gatk-it"* uses: actions/checkout@v3 with: From 9066625a6e12b1743c26490c02de04c650c87208 Mon Sep 17 00:00:00 2001 From: Enes Kuluk <54481799+eneskuluk@users.noreply.github.com> Date: Sat, 15 Apr 2023 01:42:19 -0700 Subject: [PATCH 11/13] fix output add output for build --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73b83388b..6af382a31 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,8 @@ env: jobs: build-and-push-mac-dylib: runs-on: macos-11 + outputs: + tag_message: ${{env.TAG_MESSAGE}} permissions: contents: read packages: write From ae2d82a7d1d819a68fcc8e6e69f4dcd004a1f4a8 Mon Sep 17 00:00:00 2001 From: eneskuluk <54481799+eneskuluk@users.noreply.github.com> Date: Fri, 14 Apr 2023 22:40:01 -0700 Subject: [PATCH 12/13] omit tag message from workflow, gatk if check --- .github/workflows/release.yml | 2 -- .github/workflows/release_test.yml | 8 ++------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6af382a31..143d9339d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,8 +34,6 @@ jobs: git fetch --tags --force echo "TAG_MESSAGE=$(git tag -l --sort=-taggerdate --format='%(contents)' $(git describe --tags $(git branch --show-current) ))" >> $GITHUB_ENV - - name: Check tag messsage - run: echo ${TAG_MESSAGE} - name: Install Prerequisites shell: bash diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml index 5fb4dadb7..7ceaed784 100644 --- a/.github/workflows/release_test.yml +++ b/.github/workflows/release_test.yml @@ -36,10 +36,6 @@ jobs: echo GENOMICSDB_VERSION=${GITHUB_REF_NAME:1} >> $GITHUB_ENV echo GENOMICSDB_MIN_TAG=v1.5.0-SNAPSHOT >> $GITHUB_ENV - - name: Check tag messsage - run: | - echo "tag message is" - echo ${{inputs.tag_message}} - uses: actions/setup-java@v3 with: @@ -65,14 +61,14 @@ jobs: - name: Checkout GATK - if: ${{inputs.tag_message}} != *"skip-gatk-it"* + if: ${{ !contains(inputs.tag_message,'skip-gatk-it') }} uses: actions/checkout@v3 with: repository: broadinstitute/gatk lfs: 'true' - name: Try GATK integration test - if: ${{inputs.tag_message}} != *"skip-gatk-it"* + if: ${{ !contains(inputs.tag_message,'skip-gatk-it') }} shell: bash run: | ./gradlew installDist -Dgenomicsdb.version=${VERSION_NUMBER} From 22f65ca340dca9ee32f13ae19a85e2d3be243793 Mon Sep 17 00:00:00 2001 From: Enes Kuluk <54481799+eneskuluk@users.noreply.github.com> Date: Mon, 17 Apr 2023 21:43:38 -0700 Subject: [PATCH 13/13] fixing duplicate line of code --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 6ba1ca1e4..63221c3c9 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -78,7 +78,7 @@ jobs: stagingRepoId=$(mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh | grep orggenomicsdb|cut -f2 -d' ') echo $stagingRepoId mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom - mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom + mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-spark-${VERSION_NUMBER}.pom fi env: OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}