issue #262 modify SQLQueryBuilder to improve performance.

fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java

@@ -75,7 +75,7 @@ List<Resource> history(String resourceType, String logicalId, Timestamp fromDate
      */
     int historyCount(String resourceType, String logicalId, Timestamp fromDateTime)
             throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException;
-    
+
     /**
      * Executes the search contained in the passed SqlQueryData, using it's encapsulated search string and bind variables.
      * @param queryData - Contains a search string and (optionally) bind variables.
@@ -84,6 +84,19 @@ int historyCount(String resourceType, String logicalId, Timestamp fromDateTime)
      * @throws FHIRPersistenceDBConnectException
      */
     List<Resource> search(SqlQueryData queryData) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException;
+
+    /**
+     * Executes the search contained in the passed SqlQueryData, using it's encapsulated search string and bind variables.
+     * @param queryData - Contains a search string and (optionally) bind variables.
+     * @return List<String> A list of strings satisfying the passed search.
+     * @throws FHIRPersistenceDataAccessException
+     * @throws FHIRPersistenceDBConnectException
+     * @implNote This method is used within searches which have _include or _revinclude parameters 
+     *           in order to return a list of Reference values (e.g. {@code "Patient/<UUID>"}) 
+     *           to use for filtering the list of resources to be included with the response.
+     */
+    List<String> searchStringValues(SqlQueryData queryData) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException;
+
 
     /**
      * Executes the passed fully-formed SQL Select statement and returns the results

fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FHIRDbDAOImpl.java

@@ -525,4 +525,59 @@ protected FHIRPersistenceDataAccessException buildExceptionWithIssue(String msg,
         Issue ooi = FHIRUtil.buildOperationOutcomeIssue(msg, issueType);
         return new FHIRPersistenceDataAccessException(msg).withIssue(ooi);
     }
+
+    /**
+     * Creates and executes a PreparedStatement using the passed parameters that returns a collection of String values.
+     * @param sql - The SQL template to execute.
+     * @param searchArgs - An array of arguments to be substituted into the SQL template.
+     * @return List<String> - A List of strings resulting from the executed query.
+     * @throws FHIRPersistenceDataAccessException
+     * @throws FHIRPersistenceDBConnectException 
+     */
+    protected List<String> runQuery_STR_VALUES(String sql, Object... searchArgs) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException {
+        final String METHODNAME = "runQuery_STR_VALUES";
+        log.entering(CLASSNAME, METHODNAME);
+        List<String> strValues = new ArrayList<String>();
+        Connection connection = null;
+        PreparedStatement stmt = null;
+        ResultSet resultSet = null;
+        String errMsg;
+        long dbCallStartTime;
+        double dbCallDuration;
+
+        try {
+            connection = this.getConnection();
+            stmt = connection.prepareStatement(sql);
+            // Inject arguments into the prepared stmt.
+            for (int i = 0; i <searchArgs.length;  i++) {
+                stmt.setObject(i+1, searchArgs[i]);
+            }
+            dbCallStartTime = System.nanoTime();
+            resultSet = stmt.executeQuery();
+            dbCallDuration = (System.nanoTime()-dbCallStartTime)/1e6;  
+
+            while(resultSet.next()) {
+                strValues.add(resultSet.getString(1));
+            }
+
+            if (log.isLoggable(Level.FINE)) {
+                log.fine("Successfully retrieved string values. SQL=" + sql + "  searchArgs=" + Arrays.toString(searchArgs) + 
+                         " executionTime=" + dbCallDuration + "ms");
+            }
+        } 
+        catch(FHIRPersistenceException e) {
+            throw e;
+        }
+        catch (Throwable e) {
+            // avoid leaking SQL because the exception message might be returned to a client
+            FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("Failure retrieving string values");
+            errMsg = "Failure retrieving string values. SQL=" + sql + "  searchArgs=" + Arrays.toString(searchArgs);
+            throw severe(log, fx, errMsg, e);
+        } 
+        finally {
+            this.cleanup(resultSet, stmt, connection);
+            log.exiting(CLASSNAME, METHODNAME);
+        }
+        return strValues;
+    }
 }

fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java

@@ -872,4 +872,19 @@ public int searchCount(String sqlSelectCount) throws FHIRPersistenceDataAccessEx
         return count;
     }
 
+    @Override
+    public List<String> searchStringValues(SqlQueryData queryData)
+            throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException {
+        final String METHODNAME = "searchSTR_VALUES";
+        log.entering(CLASSNAME, METHODNAME);
+
+        String sqlSelect = queryData.getQueryString();
+        Object[] bindVariables = queryData.getBindVariables().toArray();
+        try {
+            return this.runQuery_STR_VALUES(sqlSelect, bindVariables);
+        }
+        finally {
+            log.exiting(CLASSNAME, METHODNAME);
+        }
+    }
 }

fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/InclusionQuerySegmentAggregator.java

@@ -183,10 +183,52 @@ protected SqlQueryData buildQuery() throws Exception {
         return queryData;
     }
 
+    /**
+     * Appends values like ({@code ('Patient/<resource_id>', 'Patient/<resource_id>' ...)}) to the queryString
+     */
+    private void executeIncludeSubQuery(StringBuilder queryString, InclusionParameter includeParm, 
+            List<Object> bindVariables) throws Exception{
+        StringBuilder subQueryString = new StringBuilder();
+        // SELECT P1.STR_VALUE FROM OBSERVATION_STR_VALUES P1 WHERE
+        subQueryString.append("SELECT P1.STR_VALUE FROM ").append(this.resourceType.getSimpleName()).append("_STR_VALUES P1 WHERE ");
+        // P1.PARAMETER_NAME_ID=xx AND 
+        subQueryString.append("P1.PARAMETER_NAME_ID=").append(this.getParameterNameId(includeParm.getSearchParameter())).append(" AND ");
+        // P1.RESOURCE_ID IN 
+        subQueryString.append("P1.LOGICAL_RESOURCE_ID IN ");
+        // (SELECT R.LOGICAL_RESOURCE_ID  
+        subQueryString.append("(SELECT R.LOGICAL_RESOURCE_ID ");
+        // Add FROM clause for "root" resource type
+        subQueryString.append(super.buildFromClause());
+        // Add WHERE clause for "root" resource type
+        subQueryString.append(super.buildWhereClause());
+        subQueryString.append(")");
+
+        queryString.append("(");
+        //The subquery should return a list of strings in the FHIR Reference String value format 
+        //(e.g. {@code "Patient/<resource_id>"})
+        SqlQueryData subQueryData = new SqlQueryData(subQueryString.toString(), bindVariables);
+        boolean isFirstItem = true;
+        for (String strValue: this.resourceDao.searchStringValues(subQueryData)) {
+            if (!isFirstItem) {
+                queryString.append(" , ");
+            }
+            if (strValue != null) {
+                queryString.append("'").append(strValue).append("'");
+                isFirstItem = false;
+            }
+        }
+        // if nothing added so far, then need to add '', otherwise sql will fail. 
+        if (isFirstItem) {
+            queryString.append("''");
+        }
+        queryString.append(")");
+    }
+
+
     private void processIncludeParameters(StringBuilder queryString, List<Object> bindVariables) throws Exception {
         final String METHODNAME = "processIncludeParameters";
         log.entering(CLASSNAME, METHODNAME);
-        
+
         for (InclusionParameter includeParm : this.includeParameters) {
             // UNION ALL
             queryString.append(UNION_ALL);
@@ -199,23 +241,12 @@ private void processIncludeParameters(StringBuilder queryString, List<Object> bi
             // R.RESOURCE_ID = LR.CURRENT_RESOURCE_ID AND
             queryString.append("R.RESOURCE_ID = LR.CURRENT_RESOURCE_ID AND ");
             // ('Organization/' || LR.LOGICAL_ID IN 
-            queryString.append("('").append(includeParm.getSearchParameterTargetType()).append("/' || LR.LOGICAL_ID IN ");
-            // (SELECT P1.STR_VALUE FROM OBSERVATION_STR_VALUES P1 WHERE
-            queryString.append("(SELECT P1.STR_VALUE FROM ").append(this.resourceType.getSimpleName()).append("_STR_VALUES P1 WHERE ");
-            // P1.PARAMETER_NAME_ID=xx AND 
-            queryString.append("P1.PARAMETER_NAME_ID=").append(this.getParameterNameId(includeParm.getSearchParameter())).append(" AND ");
-            // P1.RESOURCE_ID IN 
-            queryString.append("P1.LOGICAL_RESOURCE_ID IN ");
-            // (SELECT R.LOGICAL_RESOURCE_ID  
-            queryString.append("(SELECT R.LOGICAL_RESOURCE_ID ");
-            // Add FROM clause for "root" resource type
-            queryString.append(super.buildFromClause());
-            // Add WHERE clause for "root" resource type
-            queryString.append(super.buildWhereClause());
-
-            queryString.append(")))");
-
-            this.addBindVariables(bindVariables);
+            queryString.append("('").append(includeParm.getSearchParameterTargetType()).append("/' || LR.LOGICAL_ID IN "); 
+
+            // Execute sub query to get the string values for constructing the query string.
+            // This avoids DB engine to run this sub query once for each record in the previously joined tables.
+            executeIncludeSubQuery(queryString, includeParm, bindVariables);
+            queryString.append(")");
         }
         log.exiting(CLASSNAME, METHODNAME);
     }