From dd5589aa1f05f7395630cd94d22b8a3d5feeb1f6 Mon Sep 17 00:00:00 2001
From: eneskuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Fri, 24 Mar 2023 10:05:29 -0700
Subject: [PATCH 01/13] antrun order change and maven-jar addition for spark
exclusion
---
pom.xml | 82 ++++++++++++++++++++++++++++++++++++++++-----------------
1 file changed, 58 insertions(+), 24 deletions(-)
diff --git a/pom.xml b/pom.xml
index e8ef417b3..4e059c016 100644
--- a/pom.xml
+++ b/pom.xml
@@ -89,11 +89,13 @@
org.apache.spark
${spark.core.artifactid}
${spark.version}
+ true
org.apache.spark
${spark.sql.artifactid}
${spark.version}
+ true
com.google.protobuf
@@ -193,29 +195,6 @@
-
- maven-antrun-plugin
- 3.1.0
-
-
- unzip-test-artifacts
- process-test-resources
-
-
-
-
-
-
-
-
-
-
-
- run
-
-
-
-
@@ -296,6 +275,25 @@
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.3.0
+
+
+ spark-excluded
+
+ jar
+
+
+ spark
+
+ org/genomicsdb/spark/**
+
+
+
+
+
org.apache.maven.plugins
maven-shade-plugin
@@ -308,7 +306,7 @@
true
- allinone
+ allinone-spark
*:*
@@ -330,6 +328,42 @@
+
+ maven-antrun-plugin
+ 3.1.0
+
+
+ unzip-test-artifacts
+ process-test-resources
+
+
+
+
+
+
+
+
+
+
+
+ run
+
+
+
+ replace-core-jar
+ package
+
+
+
+
+
+
+ run
+
+
+
+
org.apache.maven.plugins
maven-surefire-plugin
From 896f79d6844f18f1718ce3194f70c05a4acd48ca Mon Sep 17 00:00:00 2001
From: eneskuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Fri, 24 Mar 2023 10:06:46 -0700
Subject: [PATCH 02/13] match allione-spark naming change in CMakeLists
---
CMakeLists.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 11bef4c20..f684ecba9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -548,14 +548,14 @@ if(BUILD_JAVA)
#Maven build - depends on dynamic library
add_custom_command(
- OUTPUT ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}.jar ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone.jar
+ OUTPUT ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}.jar ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone-spark.jar
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/pom.xml ${CMAKE_BINARY_DIR}/pom.xml
COMMAND mvn versions:set ${MAVEN_QUIET_ARGS} -DnewVersion=${GENOMICSDB_RELEASE_VERSION} ${MAVEN_PROFILE}
COMMAND mvn package -DskipTests ${MAVEN_ARGS}
DEPENDS tiledbgenomicsdb ${JAVA_SCALA_SOURCES} pom.xml
WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
- install(FILES ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}.jar ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone.jar DESTINATION bin)
+ install(FILES ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}.jar ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone-spark.jar DESTINATION bin)
execute_process(
COMMAND ln -sf ${CMAKE_SOURCE_DIR}/tests ${CMAKE_BINARY_DIR})
@@ -575,7 +575,7 @@ if(BUILD_JAVA)
add_jar(genomicsdb-${GENOMICSDB_RELEASE_VERSION}-examples
SOURCES ${GENOMICSDB_EXAMPLE_SOURCES}
log4j.properties
- INCLUDE_JARS ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone.jar
+ INCLUDE_JARS ${GENOMICSDB_MAVEN_BUILD_DIR}/genomicsdb-${GENOMICSDB_RELEASE_VERSION}-allinone-spark.jar
OUTPUT_DIR ${GENOMICSDB_MAVEN_BUILD_DIR})
#Deploy to Maven central
From ffa384a6c1fbde04801cd61ab20c1f0acae2d428 Mon Sep 17 00:00:00 2001
From: eneskuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Fri, 24 Mar 2023 10:07:51 -0700
Subject: [PATCH 03/13] allinone naming change match for tests folder
---
tests/common.py | 4 ++--
tests/run_spark_hdfs.py | 10 +++++-----
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/tests/common.py b/tests/common.py
index 6fbe85a22..ee87ad172 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -55,7 +55,7 @@ def __find_genomicsdb_jar(target_dir, jar_file_name):
def setup_classpath(build_dir):
target_dir=os.path.join(build_dir,'target')
- allinone_jar=__find_genomicsdb_jar(target_dir,'genomicsdb-*allinone.jar')
+ allinone_jar=__find_genomicsdb_jar(target_dir,'genomicsdb-*allinone-spark.jar')
examples_jar=__find_genomicsdb_jar(target_dir,'genomicsdb-*examples.jar')
if 'CLASSPATH' in os.environ:
classpath=os.environ['CLASSPATH']
@@ -94,7 +94,7 @@ def setup_jacoco(build_dir, build_type):
if e.errno != errno.EEXIST:
__error_exit('could not create jacoco-reports dir:'+e.errno+' '+e.filename+' '+e.strerror)
genomicsdb_classes_dir = os.path.join(target_dir, 'jacoco-classes')
- allinone_archive = zipfile.ZipFile(__find_genomicsdb_jar(target_dir,'genomicsdb-*allinone.jar'))
+ allinone_archive = zipfile.ZipFile(__find_genomicsdb_jar(target_dir,'genomicsdb-*allinone-spark.jar'))
for file in allinone_archive.namelist():
if file.startswith('org/genomicsdb'):
allinone_archive.extract(file, genomicsdb_classes_dir)
diff --git a/tests/run_spark_hdfs.py b/tests/run_spark_hdfs.py
index 9ce17627f..3b4af39b2 100755
--- a/tests/run_spark_hdfs.py
+++ b/tests/run_spark_hdfs.py
@@ -222,7 +222,7 @@ def sanity_test_spark_bindings(tmpdir, lib_path, jar_dir, jacoco, genomicsdb_ver
substitute_placeholders(querypb_json, sanity_test_dir)
# Expected exception when run without json files
- spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar'
+ spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar'
pid = subprocess.Popen(spark_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_string, stderr_string = pid.communicate()
if(pid.returncode == 0):
@@ -231,7 +231,7 @@ def sanity_test_spark_bindings(tmpdir, lib_path, jar_dir, jacoco, genomicsdb_ver
output_string = "[row=0 col=12140 HG00141 1:12141-12295 {REF=C, DP_FORMAT=2, MIN_DP=0, ALT=[], GQ=0, PL=[0, 0, 0], GT=0/0}, row=1 col=12144 HG01958 1:12145-12277 {REF=C, DP_FORMAT=3, MIN_DP=0, ALT=[], GQ=0, PL=[0, 0, 0], GT=0/0}, row=0 col=17384 HG00141 1:17385-17385 {MQRankSum=-0.329000, AD=[58, 22, 17], MQ=31.719999, DP_FORMAT=80, ALT=[A, ], BaseQRankSum=-2.096000, GQ=99, PID=17385_G_A, ReadPosRankSum=0.005000, MQ0=8, GT=0/1, SB=[58, 0, 22, 0], RAW_MQ=5.500000, REF=G, ClippingRankSum=-1.859000, PL=[504, 0, 9807, 678, 1870, 2548], PGT=0|1}, row=1 col=17384 HG01958 1:17385-17385 {MQRankSum=-1.369000, AD=[0, 120, 37], MQ=29.820000, DP_FORMAT=120, ALT=[T, ], BaseQRankSum=-2.074000, GQ=99, PID=17385_G_T, ReadPosRankSum=-0.101000, DP=120, MQ0=3, GT=1/1, SB=[0, 0, 0, 0], RAW_MQ=2.500000, REF=G, ClippingRankSum=0.555000, PL=[3336, 358, 0, 4536, 958, 7349], PGT=0|1}, row=2 col=17384 HG01530 1:17385-17385 {MQRankSum=-0.432000, AD=[40, 36, 0], MQ=59.369999, DP_FORMAT=76, ALT=[A, ], BaseQRankSum=1.046000, GQ=99, ReadPosRankSum=2.055000, DP=76, MQ0=0, GT=0/1, SB=[9, 31, 13, 23], REF=G, ClippingRankSum=-2.242000, PL=[1018, 0, 1116, 1137, 1224, 2361]}]"
- spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar '+loader_json+' '+query_json
+ spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar '+loader_json+' '+query_json
pid = subprocess.Popen(spark_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_string, stderr_string = pid.communicate()
if(pid.returncode != 0):
@@ -241,7 +241,7 @@ def sanity_test_spark_bindings(tmpdir, lib_path, jar_dir, jacoco, genomicsdb_ver
sys.stderr.write('Expected output not found in sanity test with query.json\n')
print_error_and_exit(namenode, tmpdir, stdout_string, stderr_string)
- spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar '+loader_json+' '+querypb_json+' true'
+ spark_cmd = 'spark-submit --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --class org.genomicsdb.spark.api.GenomicsDBSparkBindings '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar '+loader_json+' '+querypb_json+' true'
pid = subprocess.Popen(spark_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_string, stderr_string = pid.communicate()
if(pid.returncode != 0):
@@ -475,7 +475,7 @@ def main():
with open(query_json_filename, 'w') as fptr:
json.dump(test_query_dict, fptr, indent=4, separators=(',', ': '));
fptr.close();
- spark_cmd = 'spark-submit --class TestGenomicsDBSparkHDFS --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --jars '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-examples.jar --loader '+loader_json_filename+' --query '+query_json_filename+' --template_vcf_header '+template_vcf_header_path+' --spark_master '+spark_master+' --jar_dir '+jar_dir;
+ spark_cmd = 'spark-submit --class TestGenomicsDBSparkHDFS --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --jars '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-examples.jar --loader '+loader_json_filename+' --query '+query_json_filename+' --template_vcf_header '+template_vcf_header_path+' --spark_master '+spark_master+' --jar_dir '+jar_dir;
if (test_name == "t6_7_8"):
spark_cmd = spark_cmd + ' --use-query-protobuf';
if (test_name == "t0_1_2_combined"):
@@ -512,7 +512,7 @@ def main():
vid_path_final=vid_path+query_param_dict['vid_mapping_file'];
else:
vid_path_final=vid_path+"inputs"+os.path.sep+"vid.json";
- spark_cmd_v2 = 'spark-submit --class TestGenomicsDBSource --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --jars '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone.jar '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-examples.jar --loader '+loader_json_filename+' --query '+query_json_filename+' --vid '+vid_path_final+' --spark_master '+spark_master;
+ spark_cmd_v2 = 'spark-submit --class TestGenomicsDBSource --master '+spark_master+' --deploy-mode '+spark_deploy+' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions='+jacoco+'" --conf "spark.driver.extraJavaOptions='+jacoco+'" --jars '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-allinone-spark.jar '+jar_dir+'/genomicsdb-'+genomicsdb_version+'-examples.jar --loader '+loader_json_filename+' --query '+query_json_filename+' --vid '+vid_path_final+' --spark_master '+spark_master;
if (gdb_datasource != ""):
spark_cmd_v2 += ' --gdb_datasource=' + gdb_datasource
if (test_name == "t6_7_8"):
From cf42294dfcee62ca820d2c89ec2b16e7f083afcb Mon Sep 17 00:00:00 2001
From: eneskuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Fri, 24 Mar 2023 10:11:54 -0700
Subject: [PATCH 04/13] name change in api/sparkbindings
---
.../org/genomicsdb/spark/api/GenomicsDBSparkBindings.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/main/java/org/genomicsdb/spark/api/GenomicsDBSparkBindings.java b/src/main/java/org/genomicsdb/spark/api/GenomicsDBSparkBindings.java
index a8e7a2767..30c6328aa 100644
--- a/src/main/java/org/genomicsdb/spark/api/GenomicsDBSparkBindings.java
+++ b/src/main/java/org/genomicsdb/spark/api/GenomicsDBSparkBindings.java
@@ -42,19 +42,19 @@
/**
* Example Invocation
- * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone.jar loader.json querypb.json true
+ * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone-spark.jar loader.json querypb.json true
* querypb.json should be parseable by GenomicsDBExportConfiguration.ExportConfiguration
* OR
- * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone.jar loader.json query.json false
+ * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone-spark.jar loader.json query.json false
* OR
- * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone.jar loader.json query.json
+ * spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb-1.3.1-SNAPSHOT-allinone-spark.jar loader.json query.json
*/
public class GenomicsDBSparkBindings {
List variantCalls;
public static void main(String[] args) throws IOException, ClassNotFoundException {
if (args.length < 2) {
- throw new RuntimeException("Usage: spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb--allinone.jar []"+
+ throw new RuntimeException("Usage: spark-submit --class org.genomicsdb.spark.api.GenomicsDBSparkBindings genomicsdb--allinone-spark.jar []"+
"Optional Argument 2 -
Date: Fri, 24 Mar 2023 10:12:50 -0700
Subject: [PATCH 05/13] update workflows according to naming change from
allinone to allinone-spark
---
.github/workflows/release_jar.yml | 2 +-
.github/workflows/release_publish.yml | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/release_jar.yml b/.github/workflows/release_jar.yml
index d8077482a..26927def7 100644
--- a/.github/workflows/release_jar.yml
+++ b/.github/workflows/release_jar.yml
@@ -46,7 +46,7 @@ jobs:
docker create -it --name genomicsdb ghcr.io/genomicsdb/genomicsdb:release bash
docker cp genomicsdb:/build/GenomicsDB/build/src/main/libtiledbgenomicsdb.so .
docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}.jar .
- docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}-allinone.jar .
+ docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}-allinone-spark.jar .
docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}-sources.jar .
docker cp genomicsdb:/build/GenomicsDB/build/target/genomicsdb-${VERSION_NUMBER}-javadoc.jar .
docker cp genomicsdb:/build/GenomicsDB/pom.xml genomicsdb-${VERSION_NUMBER}.pom
diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml
index 12b694787..b750fe0a9 100644
--- a/.github/workflows/release_publish.yml
+++ b/.github/workflows/release_publish.yml
@@ -58,17 +58,17 @@ jobs:
-Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
-Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
- -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone.jar \
+ -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
-Dtypes=jar \
- -Dclassifiers=allinone
+ -Dclassifiers=allinone-spark
mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}.jar \
-DgroupId=org.genomicsdb -DartifactId=genomicsdb -Dversion=${VERSION_NUMBER} \
-Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
-Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
- -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone.jar \
+ -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
-Dtypes=jar \
- -Dclassifiers=allinone
+ -Dclassifiers=allinone-spark
if [[ ${VERSION_NUMBER} != *SNAPSHOT ]]; then
mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -f genomicsdb-${VERSION_NUMBER}.pom
stagingRepoId=$(mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh | grep orggenomicsdb|cut -f2 -d' ')
From 694dfe1c76eacec73700594f48217a71c888acbf Mon Sep 17 00:00:00 2001
From: eneskuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Tue, 28 Mar 2023 15:24:31 -0700
Subject: [PATCH 06/13] changes in publish.yml
---
.github/workflows/release_publish.yml | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml
index b750fe0a9..b45b9d20b 100644
--- a/.github/workflows/release_publish.yml
+++ b/.github/workflows/release_publish.yml
@@ -58,15 +58,25 @@ jobs:
-Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
-Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
- -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
-Dtypes=jar \
- -Dclassifiers=allinone-spark
mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}.jar \
-DgroupId=org.genomicsdb -DartifactId=genomicsdb -Dversion=${VERSION_NUMBER} \
-Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
-Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
- -Dfiles=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
+ -Dtypes=jar \
+ echo mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
+ -DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \
+ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
+ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
+ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
+ -Dtypes=jar \
+ -Dclassifiers=allinone-spark
+ mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
+ -DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \
+ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
+ -Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
+ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
-Dtypes=jar \
-Dclassifiers=allinone-spark
if [[ ${VERSION_NUMBER} != *SNAPSHOT ]]; then
From 5fcd3188a1943c243e06d0c543bc3d4caea001fc Mon Sep 17 00:00:00 2001
From: eneskuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Thu, 30 Mar 2023 10:27:39 -0700
Subject: [PATCH 07/13] workflow update, publish/test
---
.github/workflows/release_publish.yml | 8 +++++---
.github/workflows/release_test.yml | 4 +++-
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml
index b45b9d20b..5aaca94f1 100644
--- a/.github/workflows/release_publish.yml
+++ b/.github/workflows/release_publish.yml
@@ -65,16 +65,17 @@ jobs:
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
-Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
-Dtypes=jar \
+ cp genomicsdb-${VERSION_NUMBER}.pom genomicsdb-${VERSION_NUMBER}-spark.pom
echo mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
-DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \
- -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
+ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}-spark.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
-Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
-Dtypes=jar \
-Dclassifiers=allinone-spark
mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
-DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \
- -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
+ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}-spark.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
-Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
-Dtypes=jar \
@@ -83,7 +84,8 @@ jobs:
mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -f genomicsdb-${VERSION_NUMBER}.pom
stagingRepoId=$(mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh | grep orggenomicsdb|cut -f2 -d' ')
echo $stagingRepoId
- mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom
+ mvn -X nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom
+ mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}-spark.pom
fi
env:
OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml
index 3c24a9100..61b1abf67 100644
--- a/.github/workflows/release_test.yml
+++ b/.github/workflows/release_test.yml
@@ -28,6 +28,7 @@ jobs:
- name: Set version number
run: |
+ echo ${{github.ref_name}}
echo VERSION_NUMBER=${GITHUB_REF_NAME:1} >> $GITHUB_ENV
echo GENOMICSDB_VERSION=${GITHUB_REF_NAME:1} >> $GITHUB_ENV
@@ -50,7 +51,8 @@ jobs:
-Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom
./test_genomicsdbjar.sh
#
-# - name: Checkout GATK
+# - name: Checkout GATK
+# if: ${{github.ref_name}} != *" skip-gatk-it"*
# uses: actions/checkout@v3
# with:
# repository: broadinstitute/gatk
From fef29231f1bb437a8b0b61eab5cfede1b1fbabce Mon Sep 17 00:00:00 2001
From: eneskuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Fri, 14 Apr 2023 21:16:07 -0700
Subject: [PATCH 08/13] spark publish fix
---
.github/workflows/release_publish.yml | 25 +++++++++----------------
1 file changed, 9 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml
index 5aaca94f1..6ba1ca1e4 100644
--- a/.github/workflows/release_publish.yml
+++ b/.github/workflows/release_publish.yml
@@ -41,7 +41,7 @@ jobs:
uses: actions/download-artifact@v3
with:
name: ${{ inputs.release_artifact }}
-
+
- name: Deploy Maven Central
shell: bash
run: |
@@ -57,35 +57,28 @@ jobs:
-DgroupId=org.genomicsdb -DartifactId=genomicsdb -Dversion=${VERSION_NUMBER} \
-Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
- -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
- -Dtypes=jar \
+ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar
mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}.jar \
-DgroupId=org.genomicsdb -DartifactId=genomicsdb -Dversion=${VERSION_NUMBER} \
-Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
- -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
- -Dtypes=jar \
- cp genomicsdb-${VERSION_NUMBER}.pom genomicsdb-${VERSION_NUMBER}-spark.pom
+ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar
echo mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
-DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \
- -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}-spark.pom -DrepositoryId=$REPO_ID \
+ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
- -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
- -Dtypes=jar \
- -Dclassifiers=allinone-spark
+ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar
mvn gpg:sign-and-deploy-file -Durl=$URL -Dfile=genomicsdb-${VERSION_NUMBER}-allinone-spark.jar \
-DgroupId=org.genomicsdb -DartifactId=genomicsdb-spark -Dversion=${VERSION_NUMBER} \
- -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}-spark.pom -DrepositoryId=$REPO_ID \
+ -Dpackaging=jar -DpomFile=genomicsdb-${VERSION_NUMBER}.pom -DrepositoryId=$REPO_ID \
-Djavadoc=genomicsdb-${VERSION_NUMBER}-javadoc.jar \
- -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar \
- -Dtypes=jar \
- -Dclassifiers=allinone-spark
+ -Dsources=genomicsdb-${VERSION_NUMBER}-sources.jar
if [[ ${VERSION_NUMBER} != *SNAPSHOT ]]; then
mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -f genomicsdb-${VERSION_NUMBER}.pom
stagingRepoId=$(mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh | grep orggenomicsdb|cut -f2 -d' ')
echo $stagingRepoId
- mvn -X nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom
- mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}-spark.pom
+ mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom
+ mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom
fi
env:
OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
From cda67c3f4030c0a034b426bb1a10bf792895ce71 Mon Sep 17 00:00:00 2001
From: eneskuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Fri, 14 Apr 2023 22:08:46 -0700
Subject: [PATCH 09/13] capture tag message, enable gatk tests according to tag
message
---
.github/workflows/release.yml | 11 +++++++++-
.github/workflows/release_test.yml | 35 +++++++++++++++++++-----------
2 files changed, 32 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index bddde8315..73b83388b 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -26,6 +26,14 @@ jobs:
- name: Set version number
run: echo VERSION_NUMBER=${GITHUB_REF_NAME:1} >> $GITHUB_ENV
+
+ - name: Set tag messsage
+ run: |
+ git fetch --tags --force
+ echo "TAG_MESSAGE=$(git tag -l --sort=-taggerdate --format='%(contents)' $(git describe --tags $(git branch --show-current) ))" >> $GITHUB_ENV
+
+ - name: Check tag messsage
+ run: echo ${TAG_MESSAGE}
- name: Install Prerequisites
shell: bash
@@ -62,10 +70,11 @@ jobs:
dylib_artifact: libtiledbgenomicsdb.dylib.${{ github.ref_name }}
test:
- needs: [release-jar]
+ needs: [build-and-push-mac-dylib, release-jar]
uses: ./.github/workflows/release_test.yml
with:
release_artifact: release.${{ github.ref_name }}
+ tag_message: ${{ needs.build-and-push-mac-dylib.outputs.tag_message }}
publish:
needs: [test]
diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml
index 177b973ce..ff4cff92d 100644
--- a/.github/workflows/release_test.yml
+++ b/.github/workflows/release_test.yml
@@ -6,6 +6,9 @@ on:
release_artifact:
required: true
type: string
+ tag_message:
+ required: false
+ type: string
jobs:
test-jar:
@@ -33,6 +36,11 @@ jobs:
echo GENOMICSDB_VERSION=${GITHUB_REF_NAME:1} >> $GITHUB_ENV
echo GENOMICSDB_MIN_TAG=v1.5.0-SNAPSHOT >> $GITHUB_ENV
+ - name: Check tag messsage
+ run: |
+ echo "tag message is"
+ echo ${{inputs.tag_message}}
+
- uses: actions/setup-java@v3
with:
distribution: 'temurin'
@@ -55,16 +63,17 @@ jobs:
shell: bash
run: GENOMICSDB_TAG=${GENOMICSDB_MIN_TAG} ./test_genomicsdbjar.sh
-#
-# - name: Checkout GATK
-# if: ${{github.ref_name}} != *" skip-gatk-it"*
-# uses: actions/checkout@v3
-# with:
-# repository: broadinstitute/gatk
-# lfs: 'true'
-#
-# - name: Try GATK integration test
-# shell: bash
-# run: |
-# ./gradlew installDist -Dgenomicsdb.version=${VERSION_NUMBER}
-# ./gradlew test --tests *GenomicsDB*
+
+ - name: Checkout GATK
+ if: ${{inputs.tag_message}} != *"skip-gatk-it"*
+ uses: actions/checkout@v3
+ with:
+ repository: broadinstitute/gatk
+ lfs: 'true'
+
+ - name: Try GATK integration test
+ if: ${{inputs.tag_message}} != *"skip-gatk-it"*
+ shell: bash
+ run: |
+ ./gradlew installDist -Dgenomicsdb.version=${VERSION_NUMBER}
+ ./gradlew test --tests *GenomicsDB*
From 042e0138b35dc5a085291e149c145b57c6ea23ba Mon Sep 17 00:00:00 2001
From: Enes Kuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Sat, 15 Apr 2023 00:09:07 -0700
Subject: [PATCH 10/13] indentation fix
---
.github/workflows/release_test.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml
index ff4cff92d..5fb4dadb7 100644
--- a/.github/workflows/release_test.yml
+++ b/.github/workflows/release_test.yml
@@ -64,7 +64,7 @@ jobs:
run: GENOMICSDB_TAG=${GENOMICSDB_MIN_TAG} ./test_genomicsdbjar.sh
- - name: Checkout GATK
+ - name: Checkout GATK
if: ${{inputs.tag_message}} != *"skip-gatk-it"*
uses: actions/checkout@v3
with:
From 9066625a6e12b1743c26490c02de04c650c87208 Mon Sep 17 00:00:00 2001
From: Enes Kuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Sat, 15 Apr 2023 01:42:19 -0700
Subject: [PATCH 11/13] fix output
add output for build
---
.github/workflows/release.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 73b83388b..6af382a31 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -16,6 +16,8 @@ env:
jobs:
build-and-push-mac-dylib:
runs-on: macos-11
+ outputs:
+ tag_message: ${{env.TAG_MESSAGE}}
permissions:
contents: read
packages: write
From ae2d82a7d1d819a68fcc8e6e69f4dcd004a1f4a8 Mon Sep 17 00:00:00 2001
From: eneskuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Fri, 14 Apr 2023 22:40:01 -0700
Subject: [PATCH 12/13] omit tag message from workflow, gatk if check
---
.github/workflows/release.yml | 2 --
.github/workflows/release_test.yml | 8 ++------
2 files changed, 2 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6af382a31..143d9339d 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -34,8 +34,6 @@ jobs:
git fetch --tags --force
echo "TAG_MESSAGE=$(git tag -l --sort=-taggerdate --format='%(contents)' $(git describe --tags $(git branch --show-current) ))" >> $GITHUB_ENV
- - name: Check tag messsage
- run: echo ${TAG_MESSAGE}
- name: Install Prerequisites
shell: bash
diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml
index 5fb4dadb7..7ceaed784 100644
--- a/.github/workflows/release_test.yml
+++ b/.github/workflows/release_test.yml
@@ -36,10 +36,6 @@ jobs:
echo GENOMICSDB_VERSION=${GITHUB_REF_NAME:1} >> $GITHUB_ENV
echo GENOMICSDB_MIN_TAG=v1.5.0-SNAPSHOT >> $GITHUB_ENV
- - name: Check tag messsage
- run: |
- echo "tag message is"
- echo ${{inputs.tag_message}}
- uses: actions/setup-java@v3
with:
@@ -65,14 +61,14 @@ jobs:
- name: Checkout GATK
- if: ${{inputs.tag_message}} != *"skip-gatk-it"*
+ if: ${{ !contains(inputs.tag_message,'skip-gatk-it') }}
uses: actions/checkout@v3
with:
repository: broadinstitute/gatk
lfs: 'true'
- name: Try GATK integration test
- if: ${{inputs.tag_message}} != *"skip-gatk-it"*
+ if: ${{ !contains(inputs.tag_message,'skip-gatk-it') }}
shell: bash
run: |
./gradlew installDist -Dgenomicsdb.version=${VERSION_NUMBER}
From 22f65ca340dca9ee32f13ae19a85e2d3be243793 Mon Sep 17 00:00:00 2001
From: Enes Kuluk <54481799+eneskuluk@users.noreply.github.com>
Date: Mon, 17 Apr 2023 21:43:38 -0700
Subject: [PATCH 13/13] fixing duplicate line of code
---
.github/workflows/release_publish.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml
index 6ba1ca1e4..63221c3c9 100644
--- a/.github/workflows/release_publish.yml
+++ b/.github/workflows/release_publish.yml
@@ -78,7 +78,7 @@ jobs:
stagingRepoId=$(mvn nexus-staging:rc-list -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh | grep orggenomicsdb|cut -f2 -d' ')
echo $stagingRepoId
mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom
- mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-${VERSION_NUMBER}.pom
+ mvn nexus-staging:rc-close -DserverId=ossrh -DnexusUrl=https://oss.sonatype.org/ -DstagingRepositoryId=$stagingRepoId -f genomicsdb-spark-${VERSION_NUMBER}.pom
fi
env:
OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}