Allow studies to prevent new timepoint creation on data import.

api/schemas/study.xsd

@@ -469,6 +469,13 @@
                     </xs:documentation>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="failForUndefinedTimepoints" type="xs:boolean">
+                <xs:annotation>
+                    <xs:documentation>
+                        Indicates whether new timepoints can be created automatically when data is imported into the study.
+                    </xs:documentation>
+                </xs:annotation>
+            </xs:attribute>
             <xs:anyAttribute processContents="lax"/>
         </xs:complexType>
 	</xs:element>

api/src/org/labkey/api/study/Study.java

@@ -152,6 +152,7 @@ public interface Study extends StudyEntity
     boolean isAllowReqLocClinic();
     boolean isAllowReqLocSal();
     boolean isAllowReqLocEndpoint();
+    boolean isFailForUndefinedTimepoints();
 
     // "is" prefix doesn't work with "Boolean", use get
     /**

pipeline/src/org/labkey/pipeline/startPipelineImport.jsp

@@ -93,7 +93,7 @@ Ext4.onReady(function()
                 isApplyToMultipleFolders: <%=bean.isApplyToMultipleFolders()%>,
                 isFailForUndefinedVisits: <%=bean.isFailForUndefinedVisits()%>,
                 isCloudRoot: <%=bean.isCloudRoot()%>,   // Remove as part of Issue #43835
-                showFailForUndefinedVisits: <%=timepointType == null || timepointType == TimepointType.VISIT%>
+                showFailForUndefinedVisits: <%=(timepointType == null || timepointType == TimepointType.VISIT) && ((study == null) || !study.isFailForUndefinedTimepoints())%>
             });
         },
         failure: function(response)

specimen/src/org/labkey/specimen/SpecimenModule.java

@@ -38,6 +38,7 @@
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.QueryParam;
 import org.labkey.api.query.QueryView;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.security.User;
 import org.labkey.api.security.roles.RoleManager;
 import org.labkey.api.specimen.SpecimenMigrationService;
@@ -189,7 +190,7 @@ public ActionURL getUpdateSpecimenQueryRowURL(Container c, String schemaName, Ta
             }
 
             @Override
-            public void importSpecimenArchive(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge, boolean syncParticipantVisit) throws PipelineJobException
+            public void importSpecimenArchive(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge, boolean syncParticipantVisit) throws PipelineJobException, ValidationException
             {
                 AbstractSpecimenTask.doImport(inputFile, job, ctx, merge, syncParticipantVisit);
             }

specimen/src/org/labkey/specimen/actions/ShowUploadSpecimensAction.java

@@ -25,6 +25,7 @@
 import org.labkey.api.data.RuntimeSQLException;
 import org.labkey.api.pipeline.PipelineUrls;
 import org.labkey.api.portal.ProjectUrls;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.reader.ColumnDescriptor;
 import org.labkey.api.reader.TabLoader;
 import org.labkey.api.security.RequiresPermission;
@@ -262,7 +263,7 @@ else if (study.getTimepointType() == TimepointType.VISIT && null == row.get(Simp
             if (!errors.hasErrors())
                 importer.process(specimenRows, form.isMerge());
         }
-        catch (IllegalStateException e)
+        catch (IllegalStateException | ValidationException e)
         {
             errors.reject(SpringActionController.ERROR_MSG, e.getMessage());
         }

specimen/src/org/labkey/specimen/importer/AbstractSpecimenTask.java

@@ -25,6 +25,7 @@
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.RecordedActionSet;
 import org.labkey.api.pipeline.TaskFactory;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.specimen.importer.SpecimenImporter;
 import org.labkey.api.study.SpecimenService;
 import org.labkey.api.study.SpecimenTransform;
@@ -98,7 +99,7 @@ protected SimpleStudyImportContext getImportContext(PipelineJob job)
     }
 
     public static void doImport(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge,
-            boolean syncParticipantVisit) throws PipelineJobException
+            boolean syncParticipantVisit) throws PipelineJobException, ValidationException
     {
         doImport(inputFile, job, ctx, merge, syncParticipantVisit, new DefaultImportHelper());
     }
@@ -115,7 +116,7 @@ private static void doPostTransform(SpecimenTransform transformer, Path inputFil
     }
 
     public static void doImport(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge,
-                                boolean syncParticipantVisit, ImportHelper importHelper) throws PipelineJobException
+                                boolean syncParticipantVisit, ImportHelper importHelper) throws PipelineJobException, ValidationException
     {
         // do nothing if we've specified data types and specimen is not one of them
         if (!ctx.isDataTypeSelected(SpecimenArchiveDataTypes.SPECIMENS))
@@ -172,7 +173,11 @@ public static void doImport(@Nullable Path inputFile, PipelineJob job, SimpleStu
             // Since changing specimens in this study will impact specimens in ancillary studies dependent on this study,
             // we need to force a participant/visit refresh in those study containers (if any):
             for (Study dependentStudy : StudyService.get().getAncillaryStudies(ctx.getContainer()))
-                VisitService.get().updateParticipantVisits(dependentStudy, ctx.getUser());
+            {
+                ValidationException errors = VisitService.get().updateParticipantVisits(dependentStudy, ctx.getUser());
+                if (errors.hasErrors())
+                    throw errors;
+            }
         }
     }
 

study/api-src/org/labkey/api/specimen/SpecimenMigrationService.java

@@ -5,6 +5,7 @@
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.security.User;
 import org.labkey.api.services.ServiceRegistry;
 import org.labkey.api.study.importer.SimpleStudyImportContext;
@@ -36,7 +37,7 @@ static void setInstance(SpecimenMigrationService impl)
     ActionURL getUpdateSpecimenQueryRowURL(Container c, String schemaName, TableInfo table);
 
     void importSpecimenArchive(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge,
-                               boolean syncParticipantVisit) throws PipelineJobException;
+                               boolean syncParticipantVisit) throws PipelineJobException, ValidationException;
 
     void clearRequestCaches(Container c);
     void clearGroupedValuesForColumn(Container container);

study/api-src/org/labkey/api/specimen/importer/SpecimenImporter.java

@@ -256,7 +256,7 @@ public SpecimenImporter(Container container, User user)
                 SpecimenSchema.get().getTableInfoSpecimenPrimaryType(getContainer()).getSelectName(), SpecimenColumns.PRIMARYTYPE_COLUMNS);
     }
 
-    private void resyncStudy(boolean syncParticipantVisit)
+    private void resyncStudy(boolean syncParticipantVisit, boolean failForUndefinedVisits) throws ValidationException
     {
         TableInfo tableParticipant = SpecimenSchema.get().getTableInfoParticipant();
         TableInfo tableSpecimen = getTableInfoSpecimen();
@@ -271,7 +271,9 @@ private void resyncStudy(boolean syncParticipantVisit)
         {
             Study study = StudyService.get().getStudy(getContainer());
             info("Updating study-wide subject/visit information...");
-            VisitService.get().updateParticipantVisitsWithCohortUpdate(study, getUser(), _logger);
+            ValidationException errors = VisitService.get().updateParticipantVisitsWithCohortUpdate(study, getUser(), failForUndefinedVisits, _logger);
+            if (errors.hasErrors())
+                throw errors;
             info("Subject/visit update complete.");
         }
 
@@ -302,14 +304,16 @@ public void process(VirtualFile specimensDir, boolean merge, SimpleStudyImportCo
             throws IOException, ValidationException
     {
         Map<SpecimenTableType, SpecimenImportFile> sifMap = populateFileMap(specimensDir, new HashMap<>());
-        process(sifMap, merge, ctx.getLogger(), job, syncParticipantVisit, false, ctx.isFailForUndefinedVisits());
+        Study study = StudyService.get().getStudy(getContainer());
+        process(sifMap, merge, ctx.getLogger(), job, syncParticipantVisit, false, ctx.isFailForUndefinedVisits() || study.isFailForUndefinedTimepoints());
     }
 
     protected void process(Map<SpecimenTableType, SpecimenImportFile> sifMap, boolean merge, Logger logger, @Nullable PipelineJob job,
                            boolean syncParticipantVisit, boolean editingSpecimens)
             throws IOException, ValidationException
     {
-        process(sifMap, merge, logger, job, syncParticipantVisit, editingSpecimens, false);
+        Study study = StudyService.get().getStudy(getContainer());
+        process(sifMap, merge, logger, job, syncParticipantVisit, editingSpecimens, study.isFailForUndefinedTimepoints());
     }
 
     private void process(Map<SpecimenTableType, SpecimenImportFile> sifMap, boolean merge, Logger logger, @Nullable PipelineJob job,
@@ -367,10 +371,6 @@ private void process(Map<SpecimenTableType, SpecimenImportFile> sifMap, boolean
             SpecimenImportFile specimenFile = sifMap.get(_specimensTableType);
             SpecimenLoadInfo loadInfo = populateTempSpecimensTable(specimenFile, merge);
 
-            Study study = StudyService.get().getStudy(getContainer());
-            if (loadInfo.getRowCount() > 0 && failForUndefinedVisits && study.getTimepointType() == TimepointType.VISIT)
-                checkForUndefinedVisits(loadInfo, study);
-
             // NOTE: if no rows were loaded in the temp table, don't remove existing materials/specimens/vials/events.
             if (loadInfo.getRowCount() > 0)
                 populateSpecimenTables(loadInfo, merge);
@@ -390,7 +390,7 @@ private void process(Map<SpecimenTableType, SpecimenImportFile> sifMap, boolean
 
             setStatus(GENERAL_JOB_STATUS_MSG + " (update study)");
             _iTimer.setPhase(ImportPhases.ResyncStudy);
-            resyncStudy(syncParticipantVisit);
+            resyncStudy(syncParticipantVisit, failForUndefinedVisits);
 
             ensureNotCanceled();
             _iTimer.setPhase(ImportPhases.SetLastSpecimenLoad);
@@ -452,29 +452,6 @@ private SpecimenLoadInfo populateTempSpecimensTable(SpecimenImportFile file, boo
         return new SpecimenLoadInfo(getUser(), getContainer(), DbSchema.getTemp(), columns, rowCount, tempTablesHolder.getTempTableInfo());
     }
 
-    private void checkForUndefinedVisits(SpecimenLoadInfo info, Study study) throws ValidationException
-    {
-        SQLFragment sql = new SQLFragment()
-            .append("SELECT DISTINCT VisitValue FROM ")
-            .append(info.getTempTableName())
-            .append(" tt ")
-            .append("\nLEFT JOIN study.Visit v")
-            .append("\nON tt.VisitValue >= v.SequenceNumMin AND tt.VisitValue <=v.SequenceNumMax AND v.Container = ?")
-            .append("\nWHERE tt.VisitValue IS NOT NULL AND v.RowId IS NULL");
-
-        // shared visit container
-        Study visitStudy = StudyService.get().getStudyForVisits(study);
-        sql.add(visitStudy.getContainer().getId());
-
-        SqlSelector selector = new SqlSelector(SpecimenSchema.get().getSchema(), sql);
-        List<Double> undefinedVisits = selector.getArrayList(Double.class);
-        if (!undefinedVisits.isEmpty())
-        {
-            Collections.sort(undefinedVisits);
-            throw new ValidationException("The following undefined visits exist in the specimen data: " + StringUtils.join(undefinedVisits, ", "));
-        }
-    }
-
     private void populateSpecimenTables(SpecimenLoadInfo info, boolean merge) throws ValidationException
     {
         setStatus(GENERAL_JOB_STATUS_MSG + " (populate tables)");

study/api-src/org/labkey/api/study/model/VisitService.java

@@ -2,6 +2,7 @@
 
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.security.User;
 import org.labkey.api.services.ServiceRegistry;
 import org.labkey.api.study.Study;
@@ -27,10 +28,10 @@ static void setInstance(VisitService impl)
 
     List<? extends Visit> getVisits(Study study, Visit.Order order);
 
-    void updateParticipantVisitsWithCohortUpdate(Study study, User user, @Nullable Logger logger);
+    ValidationException updateParticipantVisitsWithCohortUpdate(Study study, User user, boolean failForUndefinedVisits, @Nullable Logger logger);
 
     /**
      * Updates this study's participant, visit, and participant visit tables. Also updates automatic cohort assignments.
      */
-    void updateParticipantVisits(Study study, User user);
+    ValidationException updateParticipantVisits(Study study, User user);
 }

study/resources/schemas/dbscripts/postgresql/study-23.001-23.002.sql

@@ -0,0 +1 @@
+ALTER TABLE study.study ADD COLUMN FailForUndefinedTimepoints BOOLEAN NOT NULL DEFAULT FALSE;

study/resources/schemas/dbscripts/sqlserver/study-23.001-23.002.sql

@@ -0,0 +1 @@
+ALTER TABLE study.study ADD FailForUndefinedTimepoints BIT NOT NULL DEFAULT 0;

study/resources/schemas/study.xml

@@ -159,6 +159,9 @@
       <column columnName="ValidateQueriesAfterImport">
         <description>Flag to perform query validation after import.</description>
       </column>
+      <column columnName="FailForUndefinedTimepoints">
+        <description>Flag to prevent the creation of new timepoints during data import.</description>
+      </column>
     </columns>
   </table>
   <table tableName="Site" tableDbType="NOT_IN_DB">

study/src/org/labkey/study/StudyModule.java

@@ -209,7 +209,7 @@ public String getName()
     @Override
     public Double getSchemaVersion()
     {
-        return 23.001;
+        return 23.002;
     }
 
     @Override

study/src/org/labkey/study/VisitServiceImpl.java

@@ -1,7 +1,9 @@
 package org.labkey.study;
 
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.security.User;
 import org.labkey.api.study.Study;
 import org.labkey.api.study.Visit;
@@ -20,14 +22,14 @@ public List<? extends Visit> getVisits(Study study, Visit.Order order)
     }
 
     @Override
-    public void updateParticipantVisitsWithCohortUpdate(Study study, User user, @Nullable Logger logger)
+    public @NotNull ValidationException updateParticipantVisitsWithCohortUpdate(Study study, User user, boolean failForUndefinedVisits, @Nullable Logger logger)
     {
-        StudyManager.getInstance().getVisitManager(study).updateParticipantVisitsWithCohortUpdate(user, logger);
+        return StudyManager.getInstance().getVisitManager(study).updateParticipantVisitsWithCohortUpdate(user, failForUndefinedVisits, logger);
     }
 
     @Override
-    public void updateParticipantVisits(Study study, User user)
+    public @NotNull ValidationException updateParticipantVisits(Study study, User user)
     {
-        StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.emptySet());
+        return StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.emptySet());
     }
 }

study/src/org/labkey/study/assay/StudyPublishManager.java

@@ -456,16 +456,21 @@ else if (!datasetPublishSourceId.equals(publishSource.second) || dataset.getPubl
                 logPublishEvent(publishSource.first, source, dataMaps, user, sourceContainer, targetContainer, dataset);
             }
 
+            //Make sure that the study is updated with the correct timepoints.
+            ValidationException validationErrors = StudyManager.getInstance().getVisitManager(targetStudy).updateParticipantVisits(user, Collections.singleton(dataset));
+            if (validationErrors.hasErrors())
+            {
+                errors.add(validationErrors.getMessage());
+                return null;
+            }
+
             transaction.commit();
         }
         catch (ChangePropertyDescriptorException | IOException e)
         {
             throw UnexpectedException.wrap(e);
         }
 
-        //Make sure that the study is updated with the correct timepoints.
-        StudyManager.getInstance().getVisitManager(targetStudy).updateParticipantVisits(user, Collections.singleton(dataset));
-
         return PageFlowUtil.urlProvider(StudyUrls.class).getDatasetURL(targetContainer, dataset.getRowId());
     }
 
@@ -978,12 +983,17 @@ public Pair<List<String>, UploadLog> importDatasetTSV(User user, StudyImpl study
                     defaultQCState = DataStateManager.getInstance().getStateForRowId(study.getContainer(), defaultQCStateId.intValue());
                 lsids = StudyManager.getInstance().importDatasetData(user, dsd, dl, columnMap, errors, DatasetDefinition.CheckForDuplicates.sourceOnly,
                         defaultQCState, insertOption, null, importLookupByAlternateKey, auditBehaviorType);
+
+                if (!errors.hasErrors())
+                {
+                    ValidationException validationException = StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.singleton(dsd));
+                    if (validationException.hasErrors())
+                        errors.addRowError(validationException);
+                }
+
                 if (!errors.hasErrors())
                     transaction.commit();
             }
-
-            if (!errors.hasErrors())
-                StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.singleton(dsd));
         }
         catch (IOException x)
         {
@@ -1272,6 +1282,9 @@ public void autoLinkSamples(ExpSampleType sampleType, List<Map<FieldKey, Object>
                         ExpMaterialTable.Column.RowId.toString(),
                         publishErrors
                     );
+
+                    if (!publishErrors.isEmpty())
+                        publishErrors.forEach(LOG::error);
                 }
                 else
                 {