Autocreate Zenodo DOIs and create shared access links

dockstore-integration-testing/src/test/java/io/dockstore/webservice/ZenodoIT.java

@@ -31,6 +31,7 @@
 import static io.dockstore.webservice.helpers.ZenodoHelper.NO_DOCKSTORE_DOI;
 import static io.dockstore.webservice.helpers.ZenodoHelper.NO_ZENODO_USER_TOKEN;
 import static io.dockstore.webservice.helpers.ZenodoHelper.PUBLISHED_ENTRY_REQUIRED;
+import static io.dockstore.webservice.helpers.ZenodoHelper.UNHIDDEN_VERSION_REQUIRED;
 import static io.dockstore.webservice.resources.WorkflowResource.A_WORKFLOW_MUST_BE_UNPUBLISHED_TO_RESTUB;
 import static io.dockstore.webservice.resources.WorkflowResource.A_WORKFLOW_MUST_HAVE_NO_DOI_TO_RESTUB;
 import static io.dockstore.webservice.resources.WorkflowResource.A_WORKFLOW_MUST_HAVE_NO_SNAPSHOT_TO_RESTUB;
@@ -285,11 +286,18 @@ void testGenerateDOIFrozenVersion(Hoverfly hoverfly) throws ApiException {
         exception = assertThrows(ApiException.class, () -> workflowsApi.restub(workflowId));
         assertTrue(exception.getMessage().contains(A_WORKFLOW_MUST_BE_UNPUBLISHED_TO_RESTUB));
 
+        // should not be able to request a DOI for a hidden version
+        testingPostgres.runUpdateStatement("update version_metadata set hidden = true");
+        master = workflowsApi.getWorkflowVersionById(workflowId, versionId, "");
+        assertTrue(master.isHidden());
+        exception = assertThrows(ApiException.class, () -> workflowsApi.requestDOIForWorkflowVersion(workflowId, versionId, ""));
+        assertTrue(exception.getMessage().contains(UNHIDDEN_VERSION_REQUIRED));
+        testingPostgres.runUpdateStatement("update version_metadata set hidden = false");
+
         // Unpublish workflow
         workflowsApi.publish1(workflowId, CommonTestUtilities.createOpenAPIPublishRequest(false));
 
         // don't die horribly when stubbing something with snapshots, explain the error
-        testingPostgres.runUpdateStatement("update workflow set conceptdoi = null");
         exception = assertThrows(ApiException.class, () -> workflowsApi.restub(workflowId));
         assertTrue(exception.getMessage().contains(A_WORKFLOW_MUST_HAVE_NO_SNAPSHOT_TO_RESTUB));
 

dockstore-webservice/src/main/java/io/dockstore/webservice/DockstoreWebserviceApplication.java

@@ -471,7 +471,7 @@ public void run(DockstoreWebserviceConfiguration configuration, Environment envi
 
         MetadataResourceHelper.init(configuration);
         ORCIDHelper.init(configuration);
-        ZenodoHelper.init(configuration, httpClient, getHibernate().getSessionFactory(), tokenDAO, workflowDAO, workflowVersionDAO);
+        ZenodoHelper.init(configuration, httpClient, getHibernate().getSessionFactory());
         environment.jersey().register(new UserResourceDockerRegistries(getHibernate().getSessionFactory()));
         final MetadataResource metadataResource = new MetadataResource(getHibernate().getSessionFactory(), configuration);
         environment.jersey().register(metadataResource);

dockstore-webservice/src/main/java/io/dockstore/webservice/core/Doi.java

@@ -27,6 +27,8 @@
 import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
+import jakarta.persistence.NamedQueries;
+import jakarta.persistence.NamedQuery;
 import jakarta.persistence.Table;
 import jakarta.persistence.UniqueConstraint;
 import java.sql.Timestamp;
@@ -38,6 +40,9 @@
 @Entity
 @Schema(description = "A Digital Object Identifier (DOI)")
 @Table(name = "doi", uniqueConstraints = @UniqueConstraint(name = "unique_doi_name", columnNames = { "name" }))
+@NamedQueries({
+    @NamedQuery(name = "io.dockstore.webservice.core.Doi.findByName", query = "SELECT d FROM Doi d WHERE :doiName = d.name")
+})
 public class Doi {
     // DOI order of precedence from greatest to least
     public static final List<DoiInitiator> DOI_ORDER_OF_PRECEDENCE = List.of(DoiInitiator.USER, DoiInitiator.GITHUB, DoiInitiator.DOCKSTORE);

dockstore-webservice/src/main/java/io/dockstore/webservice/core/Entry.java

@@ -34,7 +34,6 @@
 import io.dockstore.webservice.helpers.EntryStarredSerializer;
 import io.swagger.annotations.ApiModelProperty;
 import io.swagger.v3.oas.annotations.media.Schema;
-import jakarta.persistence.CascadeType;
 import jakarta.persistence.CollectionTable;
 import jakarta.persistence.Column;
 import jakarta.persistence.ElementCollection;
@@ -262,7 +261,7 @@ public abstract class Entry<S extends Entry, T extends Version> implements Compa
     @Deprecated(since = "1.16")
     private String conceptDoi;
 
-    @ManyToMany(fetch = FetchType.EAGER, cascade = CascadeType.ALL)
+    @ManyToMany(fetch = FetchType.EAGER)
     @JoinTable(name = "entry_concept_doi", joinColumns = @JoinColumn(name = "entryid", referencedColumnName = "id", columnDefinition = "bigint"), inverseJoinColumns = @JoinColumn(name = "doiid", referencedColumnName = "id", columnDefinition = "bigint"))
     @MapKey(name = "initiator")
     @MapKeyEnumerated(EnumType.STRING)

dockstore-webservice/src/main/java/io/dockstore/webservice/core/VersionMetadata.java

@@ -21,7 +21,6 @@
 import io.dockstore.webservice.core.Doi.DoiInitiator;
 import io.swagger.annotations.ApiModelProperty;
 import io.swagger.v3.oas.annotations.media.Schema;
-import jakarta.persistence.CascadeType;
 import jakarta.persistence.CollectionTable;
 import jakarta.persistence.Column;
 import jakarta.persistence.Convert;
@@ -81,7 +80,7 @@ public class VersionMetadata {
     @Schema(deprecated = true)
     protected String doiURL;
 
-    @ManyToMany(fetch = FetchType.EAGER, cascade = CascadeType.ALL)
+    @ManyToMany(fetch = FetchType.EAGER)
     @JoinTable(name = "version_metadata_doi", joinColumns = @JoinColumn(name = "versionmetadataid", referencedColumnName = "id", columnDefinition = "bigint"), inverseJoinColumns = @JoinColumn(name = "doiid", referencedColumnName = "id", columnDefinition = "bigint"))
     @MapKey(name = "initiator")
     @MapKeyEnumerated(EnumType.STRING)

dockstore-webservice/src/main/java/io/dockstore/webservice/core/WorkflowVersion.java

@@ -59,7 +59,7 @@
     @NamedQuery(name = "io.dockstore.webservice.core.WorkflowVersion.getByAlias", query = "SELECT e from WorkflowVersion e JOIN e.aliases a WHERE KEY(a) IN :alias"),
     @NamedQuery(name = "io.dockstore.webservice.core.WorkflowVersion.getByWorkflowIdAndVersionName", query = "select v FROM WorkflowVersion v WHERE v.parent.id = :id And v.name = :name"),
     @NamedQuery(name = "io.dockstore.webservice.core.WorkflowVersion.getByWorkflowId", query = "FROM WorkflowVersion v WHERE v.parent.id = :id ORDER by lastModified DESC"),
-    @NamedQuery(name = "io.dockstore.webservice.core.WorkflowVersion.getTagsByWorkflowIdOrderedByLastModified", query = "FROM WorkflowVersion v WHERE v.parent.id = :id AND v.referenceType = 'TAG' ORDER by lastModified DESC")
+    @NamedQuery(name = "io.dockstore.webservice.core.WorkflowVersion.getTagsByWorkflowIdOrderedByLastModified", query = "SELECT v FROM WorkflowVersion v WHERE v.parent.id = :id AND v.referenceType = 'TAG' ORDER BY v.lastModified DESC")
 })
 
 @SuppressWarnings("checkstyle:magicnumber")

dockstore-webservice/src/main/java/io/dockstore/webservice/helpers/ZenodoHelper.java

@@ -17,6 +17,7 @@
 import io.dockstore.webservice.core.Version.ReferenceType;
 import io.dockstore.webservice.core.Workflow;
 import io.dockstore.webservice.core.WorkflowVersion;
+import io.dockstore.webservice.jdbi.DoiDAO;
 import io.dockstore.webservice.jdbi.TokenDAO;
 import io.dockstore.webservice.jdbi.WorkflowDAO;
 import io.dockstore.webservice.jdbi.WorkflowVersionDAO;
@@ -74,6 +75,7 @@ public final class ZenodoHelper {
     public static final String NO_ZENODO_USER_TOKEN = "Could not get Zenodo token for user";
     public static final String AT_LEAST_ONE_AUTHOR_IS_REQUIRED_TO_PUBLISH_TO_ZENODO = "At least one author is required to publish to Zenodo";
     public static final String FROZEN_VERSION_REQUIRED = "Frozen version required to generate DOI";
+    public static final String UNHIDDEN_VERSION_REQUIRED = "Unhidden version required to generate DOI";
     public static final String PUBLISHED_ENTRY_REQUIRED = "Published entry required to generate DOI";
     public static final String VERSION_ALREADY_HAS_DOI = "Version already has DOI. Dockstore can only create one DOI per version.";
     public static final String NO_DOCKSTORE_DOI = "The entry does not have DOIs created by Dockstore's Zenodo account.";
@@ -86,6 +88,7 @@ public final class ZenodoHelper {
     private static String dockstoreZenodoCommunityId;
     private static HttpClient httpClient;
     private static SessionFactory sessionFactory;
+    private static DoiDAO doiDAO;
     private static TokenDAO tokenDAO;
     private static WorkflowDAO workflowDAO;
     private static WorkflowVersionDAO workflowVersionDAO;
@@ -97,16 +100,16 @@ public final class ZenodoHelper {
     private ZenodoHelper() {
     }
 
-    public static void init(DockstoreWebserviceConfiguration configuration, HttpClient initHttpClient, SessionFactory initSessionFactory,
-            TokenDAO initTokenDAO, WorkflowDAO initWorkflowDAO, WorkflowVersionDAO initWorkflowVersionDAO) {
+    public static void init(DockstoreWebserviceConfiguration configuration, HttpClient initHttpClient, SessionFactory initSessionFactory) {
         dockstoreUrl = configuration.getExternalConfig().computeBaseUrl();
         dockstoreZenodoAccessToken = configuration.getDockstoreZenodoAccessToken();
         dockstoreZenodoCommunityId = configuration.getDockstoreZenodoCommunityId();
         httpClient = initHttpClient;
         sessionFactory = initSessionFactory;
-        tokenDAO = initTokenDAO;
-        workflowDAO = initWorkflowDAO;
-        workflowVersionDAO = initWorkflowVersionDAO;
+        doiDAO = new DoiDAO(sessionFactory);
+        tokenDAO = new TokenDAO(sessionFactory);
+        workflowDAO = new WorkflowDAO(sessionFactory);
+        workflowVersionDAO = new WorkflowVersionDAO(sessionFactory);
         zenodoUrl = configuration.getZenodoUrl();
         zenodoClientID = configuration.getZenodoClientID();
         zenodoClientSecret = configuration.getZenodoClientSecret();
@@ -267,22 +270,15 @@ public static ZenodoDoiResult registerZenodoDOI(ApiClient zenodoClient, Workflow
             try {
                 // No DOI has been assigned to any version of the workflow yet
                 // So create a new deposit which will enable creation of a new
-                // concept DOI and new version DOI
-                returnDeposit = depositApi.createDeposit(deposit);
-                depositionID = returnDeposit.getId();
-                depositMetadata = returnDeposit.getMetadata();
-
-                fillInMetadata(depositMetadata, workflow, workflowVersion);
+                // concept DOI and new version DOI.
                 // The returned deposit will contain
                 // the reserved DOI which we can use to create a workflow alias
                 // Later on we will update the Zenodo deposit (put the deposit on
                 // Zenodo again  in the call to putDepositionOnZenodo) so it contains the workflow version alias
                 // constructed with the DOI
-                // Retrieve the DOI so we can use it to create a Dockstore alias
-                // to the workflow; we will add that alias as a Zenodo related identifier
-                String doi = returnDeposit.getMetadata().getPrereserveDoi().getDoi();
-                doiAlias = createAliasUsingDoi(doi);
-                setMetadataRelatedIdentifiers(depositMetadata, workflowUrl, workflow, workflowVersion, doiAlias);
+                returnDeposit = depositApi.createDeposit(deposit);
+                depositionID = returnDeposit.getId();
+                depositMetadata = returnDeposit.getMetadata();
             } catch (ApiException e) {
                 LOG.error("Could not create deposition on Zenodo. Error is {}", e.getMessage(), e);
                 throw new CustomWebApplicationException("Could not create deposition on Zenodo. "
@@ -305,12 +301,6 @@ public static ZenodoDoiResult registerZenodoDOI(ApiClient zenodoClient, Workflow
                 depositionID = Integer.parseInt(depositionIDStr);
                 returnDeposit = depositApi.getDeposit(depositionID);
                 depositMetadata = returnDeposit.getMetadata();
-                // Retrieve the DOI so we can use it to create a Dockstore alias
-                // to the workflow; we will add that alias as a Zenodo related identifier
-                String doi = depositMetadata.getPrereserveDoi().getDoi();
-                doiAlias = createAliasUsingDoi(doi);
-                setMetadataRelatedIdentifiers(depositMetadata, workflowUrl, workflow, workflowVersion, doiAlias);
-                fillInMetadata(depositMetadata, workflow, workflowVersion);
             } catch (ApiException e) {
                 LOG.error("Could not create new deposition version on Zenodo. Error is {}", e.getMessage(), e);
                 if (e.getCode() == HttpStatus.SC_FORBIDDEN) {
@@ -331,6 +321,13 @@ public static ZenodoDoiResult registerZenodoDOI(ApiClient zenodoClient, Workflow
             }
         }
 
+        // Retrieve the DOI so we can use it to create a Dockstore alias
+        // to the workflow; we will add that alias as a Zenodo related identifier
+        String doi = depositMetadata.getPrereserveDoi().getDoi();
+        doiAlias = createAliasUsingDoi(doi);
+        setMetadataRelatedIdentifiers(depositMetadata, workflowUrl, workflow, workflowVersion, doiAlias);
+        fillInMetadata(depositMetadata, workflow, workflowVersion);
+
         provisionWorkflowVersionUploadFiles(zenodoClient, returnDeposit, depositionID,
                 workflow, workflowVersion);
 
@@ -343,10 +340,9 @@ public static ZenodoDoiResult registerZenodoDOI(ApiClient zenodoClient, Workflow
         String conceptDoi = extractDoiFromDoiUrl(conceptDoiUrl);
 
         ZenodoDoiResult zenodoDoiResult = new ZenodoDoiResult(doiAlias, publishedDeposit.getMetadata().getDoi(), conceptDoi);
-        workflowVersion.getDois().put(doiInitiator, new Doi(DoiType.VERSION, doiInitiator, zenodoDoiResult.doiUrl()));
-        if (!workflow.getConceptDois().containsKey(doiInitiator)) {
-            workflow.getConceptDois().put(doiInitiator, new Doi(DoiType.CONCEPT, doiInitiator, zenodoDoiResult.conceptDoi()));
-        }
+        workflowVersion.getDois().put(doiInitiator, getDoiFromDatabase(DoiType.VERSION, doiInitiator, zenodoDoiResult.doiUrl()));
+        workflow.getConceptDois().put(doiInitiator, getDoiFromDatabase(DoiType.CONCEPT, doiInitiator, zenodoDoiResult.conceptDoi()));
+
         // Only add the alias to the workflow version after publishing the DOI succeeds
         // Otherwise if the publish call fails we will have added an alias
         // that will not be used and cannot be deleted
@@ -359,6 +355,15 @@ public static ZenodoDoiResult registerZenodoDOI(ApiClient zenodoClient, Workflow
         return zenodoDoiResult;
     }
 
+    public static Doi getDoiFromDatabase(DoiType doiType, DoiInitiator doiInitiator, String doiName) {
+        Doi doi = doiDAO.findByName(doiName);
+        if (doi == null) {
+            long doiId = doiDAO.create(new Doi(doiType, doiInitiator, doiName));
+            return doiDAO.findById(doiId);
+        }
+        return doi;
+    }
+
     /**
      * extract a digital object identifier (DOI) from a DOI target URL
      * @param doiUrl digital object identifier
@@ -839,6 +844,11 @@ public static void checkCanRegisterDoi(Workflow workflow, WorkflowVersion workfl
             throw new CustomWebApplicationException(String.format("Could not generate DOI for %s. %s", workflowNameAndVersion, FROZEN_VERSION_REQUIRED), HttpStatus.SC_BAD_REQUEST);
         }
 
+        if (workflowVersion.isHidden()) {
+            LOG.error("{}: Could not generate DOI for {}. {}", user.getUsername(), workflowNameAndVersion, UNHIDDEN_VERSION_REQUIRED);
+            throw new CustomWebApplicationException(String.format("Could not generate DOI for %s. %s", workflowNameAndVersion, UNHIDDEN_VERSION_REQUIRED), HttpStatus.SC_BAD_REQUEST);
+        }
+
         checkForExistingDOIForWorkflowVersion(workflowVersion, doiInitiator);
         checkHasSourceFiles(workflowVersion);
         getAndCheckAuthorsForMetadataCreator(workflow, workflowVersion);

dockstore-webservice/src/main/java/io/dockstore/webservice/jdbi/DoiDAO.java

@@ -0,0 +1,41 @@
+/*
+ *    Copyright 2024 OICR and UCSC
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package io.dockstore.webservice.jdbi;
+
+import io.dockstore.webservice.core.Doi;
+import io.dropwizard.hibernate.AbstractDAO;
+import org.hibernate.SessionFactory;
+
+public class DoiDAO extends AbstractDAO<Doi> {
+
+    public DoiDAO(SessionFactory sessionFactory) {
+        super(sessionFactory);
+    }
+
+    public Doi findById(Long id) {
+        return get(id);
+    }
+
+    public Doi findByName(String doiName) {
+        return uniqueResult(namedTypedQuery("io.dockstore.webservice.core.Doi.findByName").setParameter("doiName", doiName));
+    }
+
+    public long create(Doi doi) {
+        return persist(doi).getId();
+    }
+
+}

dockstore-webservice/src/main/java/io/dockstore/webservice/jdbi/WorkflowVersionDAO.java

@@ -50,7 +50,7 @@ public WorkflowVersion getWorkflowVersionByWorkflowIdAndVersionName(long workflo
     }
 
     public List<WorkflowVersion> getTagsByWorkflowIdOrderedByLastModified(long workflowId, int limit) {
-        Query<WorkflowVersion> query = namedTypedQuery("io.dockstore.webservice.core.WorkflowVersion.getByWorkflowId");
+        Query<WorkflowVersion> query = namedTypedQuery("io.dockstore.webservice.core.WorkflowVersion.getTagsByWorkflowIdOrderedByLastModified");
         query.setParameter("id", workflowId);
         query.setMaxResults(limit);
         return query.getResultList();

dockstore-webservice/src/main/java/io/dockstore/webservice/resources/WorkflowResource.java

@@ -295,6 +295,7 @@ public Workflow refresh(@ApiParam(hidden = true) @Parameter(hidden = true, name
         @ApiParam(value = "workflow ID", required = true) @PathParam("workflowId") Long workflowId,
         @ApiParam(value = "completely refresh all versions, even if they have not changed", defaultValue = "true") @QueryParam("hardRefresh") @DefaultValue("true") Boolean hardRefresh) {
         Workflow workflow = refreshWorkflow(user, workflowId, Optional.empty(), hardRefresh);
+        automaticallyRegisterDockstoreDOIForRecentTags(workflow, user, this);
         EntryVersionHelper.removeSourceFilesFromEntry(workflow, sessionFactory);
         return workflow;
     }
@@ -374,8 +375,6 @@ private Workflow refreshWorkflow(User user, Long workflowId, Optional<String> ve
         // Update file formats in each version and then the entry
         FileFormatHelper.updateFileFormats(existingWorkflow, newWorkflow.getWorkflowVersions(), fileFormatDAO, true);
 
-        automaticallyRegisterDockstoreDOIForRecentTags(existingWorkflow, user, this);
-
         // Keep this code that updates the existing workflow BEFORE refreshing its checker workflow below. Refreshing the checker workflow will eventually call
         // EntryVersionHelper.removeSourceFilesFromEntry() which performs a session.flush and commits to the db. It's important the parent workflow is updated completely before committing to the db..
         existingWorkflow.getWorkflowVersions().forEach(Version::updateVerified);
@@ -755,7 +754,9 @@ public Workflow publish(@ApiParam(hidden = true) @Parameter(hidden = true, name
         checkNotArchived(workflow);
 
         Workflow publishedWorkflow = publishWorkflow(workflow, request.getPublish(), userDAO.findById(user.getId()));
-        automaticallyRegisterDockstoreDOIForRecentTags(workflow, user, this);
+        if (request.getPublish()) {
+            automaticallyRegisterDockstoreDOIForRecentTags(workflow, user, this);
+        }
         Hibernate.initialize(publishedWorkflow.getWorkflowVersions());
         return publishedWorkflow;
     }