Adjust fetch size. Purge exp.* rows.

api/src/org/labkey/api/data/AggregateColumnInfo.java

@@ -108,7 +108,7 @@ public StringExpression getURL(ColumnInfo parent)
                 }
 
                 @Override
-                public NamedObjectList getSelectList(RenderContext ctx)
+                public @NotNull NamedObjectList getSelectList(RenderContext ctx)
                 {
                     return fk.getSelectList(ctx);
                 }

api/src/org/labkey/api/data/ForeignKey.java

@@ -32,12 +32,12 @@
 
 /**
  * Interface describing a ColumnInfo's foreign key relationship, which might be a "real" FK in the underlying
- * database of a "soft" FK, making it a lookup to the foreign key's target.
+ * database or a "soft" FK, making it a lookup to the foreign key's target.
  */
 public interface ForeignKey
 {
     /**
-     * Return a new lookup column with the specified displayField.  If displayField is null, then this method
+     * Return a new lookup column with the specified displayField. If displayField is null, then this method
      * should return the title column (default display field) of the foreign table.
      * It should return null if there is no default display field.
      * The ColumnInfo parent is a column which has the value of the foreign key.
@@ -47,8 +47,8 @@ public interface ForeignKey
     ColumnInfo createLookupColumn(ColumnInfo parent, String displayField);
 
     /**
-     * Return the TableInfo for the foreign table.  This TableInfo can be used to discover the names of available
-     * columns in a UI.  The returned TableInfo will not necessarily be one that can be used for querying (e.g. passing
+     * Return the TableInfo for the foreign table. This TableInfo can be used to discover the names of available
+     * columns in a UI. The returned TableInfo will not necessarily be one that can be used for querying (e.g. passing
      * to Table.select...
      */
     @Nullable
@@ -61,7 +61,7 @@ default TableDescription getLookupTableDescription()
     }
 
     /**
-     * Return an URL expression for what the hyperlink for this column should be.  The hyperlink must be able to be
+     * Return a URL expression for what the hyperlink for this column should be. The hyperlink must be able to be
      * constructed knowing only the foreign key value, as other columns may not be available in the ResultSet.
      */
     @Nullable StringExpression getURL(ColumnInfo parent);
@@ -72,7 +72,7 @@ default TableDescription getLookupTableDescription()
     @NotNull NamedObjectList getSelectList(RenderContext ctx);
 
     /**
-     * @return The container id of the foreign user schema table.  Null means current container.
+     * @return The container id of the foreign user schema table. Null means current container.
      */
     Container getLookupContainer();
 
@@ -111,7 +111,7 @@ default String getLookupSchemaName()
 
     /**
      * Fixup any references fo FieldKeys that may have been reparented or renamed by Query and
-     * generate a new ForeignKey.  If fixup is not needed, return null.
+     * generate a new ForeignKey. If fixup is not needed, return null.
      *
      * @param parent A new parent FieldKey to inject, e.g. "title" becomes "parent/title".
      * @param mapping Rename FieldKeys, e.g. "foo" becomes "bar".

api/src/org/labkey/api/data/SqlExecutingSelector.java

@@ -16,7 +16,6 @@
 
 package org.labkey.api.data;
 
-import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -25,6 +24,7 @@
 import org.labkey.api.data.dialect.StatementWrapper;
 import org.labkey.api.util.ExceptionUtil;
 import org.labkey.api.util.MemTracker;
+import org.labkey.api.util.logging.LogHelper;
 import org.springframework.dao.ConcurrencyFailureException;
 import org.springframework.jdbc.BadSqlGrammarException;
 
@@ -40,19 +40,21 @@
 import static org.labkey.api.util.ExceptionUtil.CALCULATED_COLUMN_SQL_TAG;
 
 /**
- * Selector that is driven by SQL, which subclasses can control how it's interpreted (LabKey SQL, raw DB SQL, etc)
+ * Selector that is driven by SQL, which subclasses can control how it's interpreted (LabKey SQL, raw DB SQL, etc.)
  */
 public abstract class SqlExecutingSelector<FACTORY extends SqlFactory, SELECTOR extends SqlExecutingSelector<FACTORY, SELECTOR>> extends BaseSelector<SELECTOR>
 {
+    private static final Logger LOGGER = LogHelper.getLogger(SqlExecutingSelector.class, "Log warnings about SQL exceptions");
+
     int _maxRows = Table.ALL_ROWS;
     protected long _offset = Table.NO_OFFSET;
     @Nullable Map<String, Object> _namedParameters = null;
     private ConnectionFactory _connectionFactory = super::getConnection;
+    private Integer _fetchSize = null; // By default, use the standard fetch size
 
-    private @Nullable AsyncQueryRequest _asyncRequest = null;
+    private @Nullable AsyncQueryRequest<?> _asyncRequest = null;
     private @Nullable StackTraceElement[] _loggingStacktrace = null;
     private final QueryLogging _queryLogging;
-    private static final Logger LOGGER = LogManager.getLogger(SqlExecutingSelector.class);
 
     // SQL factory used for the duration of a single query execution. This allows reuse of instances, since query-specific
     // optimizations won't mutate the ExecutingSelector's externally set state.
@@ -111,6 +113,16 @@ public SELECTOR setJdbcCaching(boolean cache)
         return getThis();
     }
 
+    /**
+     * Set a ResultSet fetch size that differs from the default value (1,000 rows on PostgreSQL). This is normally a
+     * fine fetch size, but not when dealing with rows containing large TEXT or BYTEA columns.
+     */
+    public SELECTOR setFetchSize(int fetchSize)
+    {
+        _fetchSize = fetchSize;
+        return getThis();
+    }
+
     @Override
     protected ResultSetFactory getStandardResultSetFactory()
     {
@@ -282,7 +294,7 @@ protected boolean exists(FACTORY factory)
     }
 
 
-    void setAsyncRequest(@Nullable AsyncQueryRequest asyncRequest)
+    void setAsyncRequest(@Nullable AsyncQueryRequest<?> asyncRequest)
     {
         _asyncRequest = asyncRequest;
 
@@ -291,7 +303,7 @@ void setAsyncRequest(@Nullable AsyncQueryRequest asyncRequest)
     }
 
     @Nullable
-    private AsyncQueryRequest getAsyncRequest()
+    private AsyncQueryRequest<?> getAsyncRequest()
     {
         return _asyncRequest;
     }
@@ -466,7 +478,7 @@ public <T> T handleResultSet(ResultSetHandler<T> handler)
             }
         }
 
-        private ResultSet executeQuery(Connection conn, SQLFragment sqlFragment, boolean scrollable, @Nullable AsyncQueryRequest asyncRequest, @Nullable Integer statementMaxRows) throws SQLException
+        private ResultSet executeQuery(Connection conn, SQLFragment sqlFragment, boolean scrollable, @Nullable AsyncQueryRequest<?> asyncRequest, @Nullable Integer statementMaxRows) throws SQLException
         {
             List<Object> parameters = sqlFragment.getParams();
             String sql = sqlFragment.getSQL();
@@ -475,13 +487,13 @@ private ResultSet executeQuery(Connection conn, SQLFragment sqlFragment, boolean
             if (null == parameters || parameters.isEmpty())
             {
                 Statement stmt = conn.createStatement(scrollable ? ResultSet.TYPE_SCROLL_INSENSITIVE : ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
-                initializeStatement(conn, stmt, asyncRequest, statementMaxRows);
+                initializeStatement(stmt, asyncRequest, statementMaxRows);
                 rs = stmt.executeQuery(sql);
             }
             else
             {
                 PreparedStatement stmt = conn.prepareStatement(sql, scrollable ? ResultSet.TYPE_SCROLL_INSENSITIVE : ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
-                initializeStatement(conn, stmt, asyncRequest, statementMaxRows);
+                initializeStatement(stmt, asyncRequest, statementMaxRows);
 
                 try (Parameter.ParameterList jdbcParameters = new Parameter.ParameterList())
                 {
@@ -499,14 +511,19 @@ private ResultSet executeQuery(Connection conn, SQLFragment sqlFragment, boolean
             return rs;
         }
 
-        private void initializeStatement(Connection conn, Statement stmt, @Nullable AsyncQueryRequest asyncRequest, @Nullable Integer statementMaxRows) throws SQLException
+        private void initializeStatement(Statement stmt, @Nullable AsyncQueryRequest<?> asyncRequest, @Nullable Integer statementMaxRows) throws SQLException
         {
             // Don't set max rows if null or special ALL_ROWS value (we're assuming statement.getMaxRows() defaults to 0, though this isn't actually documented...)
             if (null != statementMaxRows && Table.ALL_ROWS != statementMaxRows)
             {
                 stmt.setMaxRows(statementMaxRows == Table.NO_ROWS ? 1 : statementMaxRows);
             }
 
+            if (null != _fetchSize)
+            {
+                stmt.setFetchSize(_fetchSize);
+            }
+
             if (asyncRequest != null)
             {
                 asyncRequest.setStatement(stmt);

api/src/org/labkey/api/data/SqlFactory.java

@@ -20,11 +20,6 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
-/**
- * User: adam
- * Date: 1/22/12
- * Time: 2:33 PM
- */
 public interface SqlFactory
 {
     /** Returns the SQL to execute. If null, execution is skipped and handler is called with null parameters, allowing

api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java

@@ -1810,7 +1810,7 @@ public ConnectionFactory getConnectionFactory(boolean useJdbcCaching, DbScope sc
 
     private Closer configureToDisableJdbcCaching(ConnectionWrapper connection, DbScope scope) throws SQLException
     {
-        assert connection.getAutoCommit(); // We just got a new connection... it better  be set to auto commit
+        assert connection.getAutoCommit(); // We just got a new connection... it better be set to auto commit
 
         try
         {

api/src/org/labkey/api/util/StringUtilsLabKey.java

@@ -26,7 +26,6 @@
 import org.junit.Test;
 import org.labkey.api.data.Container;
 import org.labkey.api.exp.Identifiable;
-import org.labkey.api.security.Encryption;
 import org.labkey.api.view.ViewServlet;
 
 import java.nio.charset.Charset;

api/src/org/labkey/api/view/FolderManagement.java

@@ -47,7 +47,7 @@ public enum TYPE
         FolderManagement
         {
             @Override
-            void addNavTrail(BaseViewAction action, NavTree root, Container c, User user)
+            void addNavTrail(BaseViewAction<?> action, NavTree root, Container c, User user)
             {
                 // In the root, view is rendered as a standalone page (no tab strip). Create an appropriate nav trail.
                 if (c.isRoot())
@@ -76,7 +76,7 @@ boolean shouldRenderTabStrip(Container container)
         ProjectSettings // Used for project settings
         {
             @Override
-            void addNavTrail(BaseViewAction action, NavTree root, Container c, User user)
+            void addNavTrail(BaseViewAction<?> action, NavTree root, Container c, User user)
             {
                 action.setHelpTopic("customizeLook");
                 root.addChild("Project Settings");
@@ -91,7 +91,7 @@ boolean shouldRenderTabStrip(Container container)
         LookAndFeelSettings // Used for the admin console actions -- allows for troubleshooter permissions
         {
             @Override
-            void addNavTrail(BaseViewAction action, NavTree root, Container c, User user)
+            void addNavTrail(BaseViewAction<?> action, NavTree root, Container c, User user)
             {
                 action.setHelpTopic("customizeLook");
                 PageFlowUtil.urlProvider(AdminUrls.class).addAdminNavTrail(root, "Look and Feel Settings", action.getClass(), c);
@@ -104,7 +104,7 @@ boolean shouldRenderTabStrip(Container container)
             }
         };
 
-        abstract void addNavTrail(BaseViewAction action, NavTree root, Container c, User user);
+        abstract void addNavTrail(BaseViewAction<?> action, NavTree root, Container c, User user);
 
         abstract boolean shouldRenderTabStrip(Container container);
     }
@@ -170,14 +170,14 @@ public List<NavTree> getTabList()
         protected abstract List<TabProvider> getTabProviders();
 
         @Override
-        public abstract HttpView getTabView(String tabId) throws Exception;
+        public abstract HttpView<?> getTabView(String tabId) throws Exception;
     }
 
 
     /**
      * Marker interface for actions that register themselves with a management page
      */
-    interface ManagementAction extends Controller
+    public interface ManagementAction extends Controller
     {
     }
 
@@ -208,7 +208,7 @@ public final ModelAndView getView(Object form, BindException errors) throws Exce
         }
 
         protected abstract TYPE getType();
-        protected abstract HttpView getTabView() throws Exception;
+        protected abstract HttpView<?> getTabView() throws Exception;
 
         @Override
         public void addNavTrail(NavTree root)
@@ -244,7 +244,7 @@ public URLHelper getSuccessURL(FORM form)
 
         protected abstract TYPE getType();
 
-        protected abstract HttpView getTabView(FORM form, boolean reshow, BindException errors) throws Exception;
+        protected abstract HttpView<?> getTabView(FORM form, boolean reshow, BindException errors) throws Exception;
 
         @Override
         public void addNavTrail(NavTree root)
@@ -255,7 +255,7 @@ public void addNavTrail(NavTree root)
 
 
     /** Wrap the provided view in a tab strip, if that's what the type desires **/
-    private static HttpView wrapViewInTabStrip(BaseViewAction action, TYPE type, HttpView view, BindException errors)
+    private static HttpView<?> wrapViewInTabStrip(BaseViewAction<?> action, TYPE type, HttpView<?> view, BindException errors)
     {
         ViewContext ctx = action.getViewContext();
         Container c = ctx.getContainer();
@@ -268,7 +268,7 @@ private static HttpView wrapViewInTabStrip(BaseViewAction action, TYPE type, Htt
             return new ManagementTabStrip(c, tabId, errors)
             {
                 @Override
-                public HttpView getTabView(String tabId)
+                public HttpView<?> getTabView(String tabId)
                 {
                     return view;
                 }

experiment/resources/schemas/dbscripts/postgresql/exp-25.011-25.012.sql

@@ -0,0 +1,3 @@
+-- These tables have FKs to exp.Data without corresponding indices. Add indices to speed up exp.Data delete.
+CREATE INDEX IX_DataInput_DataId ON exp.DataInput (DataId);
+CREATE INDEX IX_DataAncestors_RowId ON exp.DataAncestors (RowId);

experiment/resources/schemas/dbscripts/postgresql/exp-25.012-25.013.sql

@@ -0,0 +1,4 @@
+-- Not needed since it's redundant with exp.DataInput's PK
+DROP INDEX exp.IX_DataInput_DataId;
+
+CREATE INDEX IX_MaterialAncestors_RowId ON exp.MaterialAncestors (RowId);

experiment/resources/schemas/dbscripts/sqlserver/exp-25.011-25.012.sql

@@ -0,0 +1,3 @@
+-- These tables have FKs to exp.Data without corresponding indices. Add indices to speed up exp.Data delete.
+CREATE INDEX IX_DataInput_DataId ON exp.DataInput (DataId);
+CREATE INDEX IX_DataAncestors_RowId ON exp.DataAncestors (RowId);

experiment/resources/schemas/dbscripts/sqlserver/exp-25.012-25.013.sql

@@ -0,0 +1,4 @@
+-- Not needed since it's redundant with exp.DataInput's PK
+DROP INDEX IX_DataInput_DataId ON exp.DataInput;
+
+CREATE INDEX IX_MaterialAncestors_RowId ON exp.MaterialAncestors (RowId);

experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java

@@ -16,8 +16,11 @@
 import org.labkey.api.data.SimpleFilter.InClause;
 import org.labkey.api.data.SimpleFilter.OrClause;
 import org.labkey.api.data.SimpleFilter.SQLClause;
+import org.labkey.api.data.SqlExecutor;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
+import org.labkey.api.data.dialect.SqlDialect;
+import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.util.Formats;
 import org.labkey.api.util.GUID;
@@ -82,10 +85,42 @@ public void addDomainDataFilter(OrClause orClause, DomainFilter filter, TableInf
     @Override
     public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilter notCopiedFilter)
     {
-        // TODO: delete orphaned rows in exp.Data and exp.Object
+        // exp.Data has an index on ObjectId, so use ObjectIds to delete from exp.Data tables as well as exp.Object.
+        // Our notCopiedFilter works on the data class provisioned table, so we need to select LSIDs from that table
+        // and then select from exp.Data to map those LSIDs to ObjectIds.
         Collection<String> notCopiedLsids = new TableSelector(sourceTable, Collections.singleton("LSID"), notCopiedFilter, null).getCollection(String.class);
+
         if (!notCopiedLsids.isEmpty())
-            LOG.info("   {} rows not copied", Formats.commaf0.format(notCopiedLsids.size()));
+        {
+            SimpleFilter dataFilter = new SimpleFilter(new InClause(FieldKey.fromParts("LSID"), notCopiedLsids));
+            Collection<Integer> notCopiedObjectIds = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("ObjectId"), dataFilter, null).getCollection(Integer.class);
+
+            LOG.info("   {} rows not copied -- deleting associated rows from exp.Data, exp.Object, etc.", Formats.commaf0.format(notCopiedObjectIds.size()));
+            SqlExecutor executor = new SqlExecutor(ExperimentService.get().getSchema());
+            SqlDialect dialect = ExperimentService.get().getSchema().getSqlDialect();
+            SQLFragment objectIdClause = new SQLFragment()
+                .appendInClause(notCopiedObjectIds, dialect);
+
+            // Delete from exp.Data (and associated tables)
+            LOG.info("   exp.DataInput");
+            executor.execute(
+                new SQLFragment("DELETE FROM exp.DataInput WHERE DataId IN (SELECT RowId FROM exp.Data WHERE ObjectId")
+                    .append(objectIdClause)
+                    .append(")")
+            );
+            LOG.info("   exp.DataAliasMap");
+            executor.execute(
+                new SQLFragment("DELETE FROM exp.DataAliasMap WHERE LSID")
+                    .appendInClause(notCopiedLsids, dialect)
+            );
+            LOG.info("   exp.Data");
+            executor.execute(
+                new SQLFragment("DELETE FROM exp.Data WHERE ObjectId")
+                    .append(objectIdClause)
+            );
+
+            ExperimentMigrationSchemaHandler.deleteObjectIds(objectIdClause);
+        }
 
         String name = sourceTable.getName();
         int idx = name.indexOf('_');