From 27c3b3798a1f06275b7c349d7c341f972fb03924 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 11 May 2021 14:00:57 +0200 Subject: [PATCH 1/3] Resolve conflicts for master (#81) Related to merging master into dev --- drop-boxes/register-omero-metadata/backendinterface.py | 1 - .../register-omero-metadata/image_registration_process.py | 8 ++++++-- drop-boxes/register-omero-metadata/register-omero.py | 8 +++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/drop-boxes/register-omero-metadata/backendinterface.py b/drop-boxes/register-omero-metadata/backendinterface.py index 0e2b0b1..a313a29 100644 --- a/drop-boxes/register-omero-metadata/backendinterface.py +++ b/drop-boxes/register-omero-metadata/backendinterface.py @@ -407,7 +407,6 @@ def get_args(): key_value = pair.split("::") key_value_data.append(key_value) - add_annotations_to_image(conn, str(args.image_id), key_value_data) print "0" diff --git a/drop-boxes/register-omero-metadata/image_registration_process.py b/drop-boxes/register-omero-metadata/image_registration_process.py index a2ab062..2a800d4 100644 --- a/drop-boxes/register-omero-metadata/image_registration_process.py +++ b/drop-boxes/register-omero-metadata/image_registration_process.py @@ -24,6 +24,7 @@ def __init__(self, transaction, env_name="omero_env_0", project_code="", sample_ self._sample_code = sample_code ### set exec. env + self._conda_path = conda_home_path if not conda_path is None: self._conda_path = conda_path @@ -47,6 +48,7 @@ def __init__(self, transaction, env_name="omero_env_0", project_code="", sample_ self._init_cmd_list.append('export PATH=$PATH:' + self._conda_path + 'envs/' + env_name + '/bin') # move to the dir where backendinterface.py lives for exec. + self._init_cmd_list.append('cd ' + self._etl_path) def fetchOpenBisSampleCode(self): @@ -63,6 +65,7 @@ def fetchOpenBisSampleCode(self): def searchOpenBisSample(self, sample_code): # find specific sample + sc = SearchCriteria() sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sample_code)) foundSamples = self._search_service.searchForSamples(sc) @@ -100,7 +103,7 @@ def requestOmeroDatasetId(self, project_code=None, sample_code=None): return ds_id def registerImageFileInOmero(self, file_path, dataset_id): - + cmd_list = list(self._init_cmd_list) cmd_list.append( "python backendinterface.py -f " + file_path + " -d " + str(dataset_id) ) @@ -123,6 +126,7 @@ def triggerOMETiffConversion(self): pass #ToDo Check if Metadata file is provided as defined + def extractMetadataFromTSV(self, tsv_file_path): tsvFileMap = {} try: @@ -147,6 +151,7 @@ def registerOmeroKeyValuePairs(self, image_id, property_map): cmd_list = list(self._init_cmd_list) # string format: key1::value1//key2::value2//key3::value3//... + key_value_str = "" for key in property_map.keys(): key_value_str = key_value_str + str(key) + "::" + str(property_map[key]) + "//" @@ -161,7 +166,6 @@ def registerOmeroKeyValuePairs(self, image_id, property_map): process = Popen( "/bin/bash", shell=False, universal_newlines=True, stdin=PIPE, stdout=PIPE, stderr=PIPE ) out, err = process.communicate( commands ) - return 0 diff --git a/drop-boxes/register-omero-metadata/register-omero.py b/drop-boxes/register-omero-metadata/register-omero.py index b5fbaaf..6872776 100755 --- a/drop-boxes/register-omero-metadata/register-omero.py +++ b/drop-boxes/register-omero-metadata/register-omero.py @@ -55,13 +55,11 @@ # and delete the data! ##### - INCOMING_DATE_FORMAT = '%d.%m.%Y' OPENBIS_DATE_FORMAT = '%Y-%m-%d' PROPPERTY_FILTER_LIST = ["IMAGE_FILE_NAME", "INSTRUMENT_USER", "IMAGING_DATE"] - def mapDateString(date_string): return datetime.datetime.strptime(date_string, INCOMING_DATE_FORMAT).strftime(OPENBIS_DATE_FORMAT) @@ -245,6 +243,8 @@ def process(transaction): # 1. Initialize the image registration process registrationProcess = irp.ImageRegistrationProcess(transaction) + + print "started reg. process" # 2. We want to get the openBIS sample code from the incoming data # This tells us to which biological sample the image data was aquired from. @@ -253,13 +253,14 @@ def process(transaction): #find specific sample tissueSample = registrationProcess.searchOpenBisSample(sample_code) space = tissueSample.getSpace() - + # 3. We now request the associated omero dataset id for the openBIS sample code. # Each dataset in OMERO contains the associated openBIS biological sample id, which # happened during the experimental design registration with the projectwizard. # Starts omero registration # returns -1 if fetching dataset-id operation failed + omero_dataset_id = registrationProcess.requestOmeroDatasetId(project_code=project_code, sample_code=sample_code) print "omero dataset id:" @@ -342,3 +343,4 @@ def process(transaction): # 7. Last but not least we create the open science file format for images which is # OMERO-Tiff and store it in OMERO next to the proprierary vendor format. + From b86b358abbebed68505602093b136b9919d58a6a Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 11 May 2021 14:31:06 +0200 Subject: [PATCH 2/3] Release/1.8.0 (#82) * Sync development with master (#61) * Provide java example dropbox Co-authored-by: Sven F Co-authored-by: luiskuhn <38211686+luiskuhn@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ README.md | 5 +++++ .../register-example-java-dropbox/README.md | 16 ++++++++++++++++ .../register-example-java-dropbox/lib/README.md | 2 ++ .../plugin.properties | 11 +++++++++++ 5 files changed, 38 insertions(+) create mode 100644 drop-boxes/register-example-java-dropbox/README.md create mode 100644 drop-boxes/register-example-java-dropbox/lib/README.md create mode 100644 drop-boxes/register-example-java-dropbox/plugin.properties diff --git a/CHANGELOG.md b/CHANGELOG.md index 7039bf9..9c57935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.8.0 2021-05-11 + +* Add example Java dropbox + ## 1.7.0 2021-03-19 * Provides fully tested functionality to register generic imaging data, with OMERO server support (v5.4.10). [Link to PR](https://github.com/qbicsoftware/etl-scripts/pull/78) diff --git a/README.md b/README.md index 860ad62..e3395ed 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,11 @@ OpenBIS loads Java libararies on startup, if they are provided in a `lib` folder We decoupled some shared functionality in the [data-model-lib](https://github.com/qbicsoftware/data-model-lib) and the [core-utils-lib](https://github.com/qbicsoftware/core-utils-lib). Please make sure to deploy them as well in of the lib folders, such that the classes are loaded by the etlserver class loader and available during runtime. +##4. Dependencies for the example dropbox written in pure Java/Groovy + +Just deploy the compiled JAR of the [Java openBIS dropbox](https://github +.com/qbicsoftware/java-openbis-dropboxes) in the `lib` folder of the dropbox (` +./register-example-java-dropbox/lib`). ## Data format guidelines diff --git a/drop-boxes/register-example-java-dropbox/README.md b/drop-boxes/register-example-java-dropbox/README.md new file mode 100644 index 0000000..6b59fc1 --- /dev/null +++ b/drop-boxes/register-example-java-dropbox/README.md @@ -0,0 +1,16 @@ +# Example openBIS dropbox written in Java + +Please find the source code of the ETL routine that this article is referring to in the +[Java openBIS dropboxes](https://github.com/qbicsoftware/java-openbis-dropboxes) Github repository. + +## Installation + +Please provide the Java binaries as JAR from the [Java openBIS dropbox](https://github.com/qbicsoftware/java-openbis-dropboxes) in this directories +folder `./lib`. + +The DSS needs to be restarted in order to activate this dropbox. + +## ETL routine + +This dropbox expects a file of any type and creates a new openBIS dataset from it. This dataset +is then attached to a fixed sample with id `/TEST28/QXEGD018AW` for demonstration purposes. \ No newline at end of file diff --git a/drop-boxes/register-example-java-dropbox/lib/README.md b/drop-boxes/register-example-java-dropbox/lib/README.md new file mode 100644 index 0000000..8b06845 --- /dev/null +++ b/drop-boxes/register-example-java-dropbox/lib/README.md @@ -0,0 +1,2 @@ +Put the compiled Java binaries as JARs in this directory in order +to be loaded by the openBIS DSS class loader on DSS startup. \ No newline at end of file diff --git a/drop-boxes/register-example-java-dropbox/plugin.properties b/drop-boxes/register-example-java-dropbox/plugin.properties new file mode 100644 index 0000000..0a31c61 --- /dev/null +++ b/drop-boxes/register-example-java-dropbox/plugin.properties @@ -0,0 +1,11 @@ +# +# Drop box for registering a fastq file as a data set +# +# Variables: +# incoming-root-dir +# Path to the directory which contains incoming directories for drop boxes. +incoming-dir = ${incoming-root-dir}/QBiC-register-example-java-data +incoming-data-completeness-condition = marker-file +top-level-data-set-handler = ch.systemsx.cisd.etlserver.registrator.api.v2.JavaTopLevelDataSetHandlerV2 +program-class = life.qbic.registration.MainETL +storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor From 1f6b8b326207a0ea56c548765858faf3e24d6b25 Mon Sep 17 00:00:00 2001 From: Sven F Date: Mon, 28 Jun 2021 14:21:54 +0200 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Tobias Koch --- CHANGELOG.md | 6 +++--- drop-boxes/register-all-dropbox/README.md | 4 ++-- drop-boxes/register-all-dropbox/plugin.properties | 9 +++++---- drop-boxes/register-omero-metadata/register-omero.py | 2 -- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7da806..d7b4f9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ # 1.9.0 2021-06-28 -* Provides new ETL routine written in Java, that will replace all Jython scripts at some point [PR](https://github.com/qbicsoftware/etl-scripts/pull/85) -* Support for nf-core pipeline result registration [PR](https://github.com/qbicsoftware/etl-scripts/pull/85) -* Provides metadata validation for imaging data (OMERO etl). [PR](https://github.com/qbicsoftware/etl-scripts/pull/83) +* Provides new ETL routine written in Java, that will replace all Jython scripts at some point [(#85)](https://github.com/qbicsoftware/etl-scripts/pull/85) +* Support for nf-core pipeline result registration [(#85)](https://github.com/qbicsoftware/etl-scripts/pull/85) +* Provides metadata validation for imaging data (OMERO etl). [(#85)](https://github.com/qbicsoftware/etl-scripts/pull/83) ## 1.8.0 2021-05-11 diff --git a/drop-boxes/register-all-dropbox/README.md b/drop-boxes/register-all-dropbox/README.md index 22ba0ed..20dbdf3 100644 --- a/drop-boxes/register-all-dropbox/README.md +++ b/drop-boxes/register-all-dropbox/README.md @@ -5,7 +5,7 @@ Please find the source code of the ETL routine that this article is referring to ## Installation -Please provide the Java binaries as JAR from the [Java openBIS dropbox](https://github.com/qbicsoftware/java-openbis-dropboxes) in this directories +Please provide the Java binaries as JAR from the [Java openBIS dropbox](https://github.com/qbicsoftware/java-openbis-dropboxes) in this repositories folder `./lib`. The DSS needs to be restarted in order to activate this dropbox. @@ -13,4 +13,4 @@ The DSS needs to be restarted in order to activate this dropbox. ## ETL routine This dropbox expects a folder containing data and creates new openBIS dataset from it. For more information -please visit [Java openBIS dropbox](https://github.com/qbicsoftware/java-openbis-dropboxes). \ No newline at end of file +please visit [Java openBIS dropbox](https://github.com/qbicsoftware/java-openbis-dropboxes). diff --git a/drop-boxes/register-all-dropbox/plugin.properties b/drop-boxes/register-all-dropbox/plugin.properties index 3b9572c..f49d390 100644 --- a/drop-boxes/register-all-dropbox/plugin.properties +++ b/drop-boxes/register-all-dropbox/plugin.properties @@ -1,11 +1,12 @@ # # Drop box for registering a fastq file as a data set # -# Variables: -# incoming-root-dir -# Path to the directory which contains incoming directories for drop boxes. -incoming-dir = ${incoming-root-dir}/QBiC-register-all-data + incoming-data-completeness-condition = marker-file top-level-data-set-handler = ch.systemsx.cisd.etlserver.registrator.api.v2.JavaTopLevelDataSetHandlerV2 program-class = life.qbic.registration.MainETL storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor +# Variables: +# incoming-root-dir +# Path to the directory which contains incoming directories for drop boxes. +incoming-dir = ${incoming-root-dir}/QBiC-register-all-data diff --git a/drop-boxes/register-omero-metadata/register-omero.py b/drop-boxes/register-omero-metadata/register-omero.py index 99e2df6..ea51a45 100755 --- a/drop-boxes/register-omero-metadata/register-omero.py +++ b/drop-boxes/register-omero-metadata/register-omero.py @@ -61,7 +61,6 @@ PROPPERTY_FILTER_LIST = ["IMAGE_FILENAME", "INSTRUMENT_USER", "IMAGING_DATE"] - def mapDateString(date_string): return datetime.datetime.strptime(date_string, INCOMING_DATE_FORMAT).strftime(OPENBIS_DATE_FORMAT) @@ -380,4 +379,3 @@ def process(transaction): # 7. Last but not least we create the open science file format for images which is # OMERO-Tiff and store it in OMERO next to the proprierary vendor format. -