Skip to content
Permalink
Browse files

0003817: Initial load use estimated counts

  • Loading branch information...
erilong committed Dec 6, 2018
1 parent dbdbb12 commit 29f29b076f8aaf05859c4f1c88e36e91ba39f636
@@ -995,28 +995,34 @@ private void insertLoadBatchesForReload(Node targetNode, long loadId, String cre
}
}

protected int getDataCountForReload(Table table, Node targetNode, String selectSql) {
DatabaseInfo dbInfo = platform.getDatabaseInfo();
String quote = dbInfo.getDelimiterToken();
String catalogSeparator = dbInfo.getCatalogSeparator();
String schemaSeparator = dbInfo.getSchemaSeparator();

String sql = String.format("select count(*) from %s t where %s", table
.getQualifiedTableName(quote, catalogSeparator, schemaSeparator), selectSql);
sql = FormatUtils.replace("groupId", targetNode.getNodeGroupId(), sql);
sql = FormatUtils.replace("externalId", targetNode.getExternalId(), sql);
sql = FormatUtils.replace("nodeId", targetNode.getNodeId(), sql);
for (IReloadVariableFilter filter : extensionService.getExtensionPointList(IReloadVariableFilter.class)) {
sql = filter.filterPurgeSql(sql, targetNode, table);
}

try {
int rowCount = sqlTemplate.queryForInt(sql);
return rowCount;
} catch (Exception ex) {
throw new SymmetricException("Failed to execute row count SQL while starting reload. If this is a syntax error, check your input and check "
+ engine.getTablePrefix() + "_table_reload_request. Statement attempted: \"" + sql + "\"", ex);
protected long getDataCountForReload(Table table, Node targetNode, String selectSql) {
long rowCount = 0;
if (parameterService.is(ParameterConstants.INITIAL_LOAD_USE_ESTIMATED_COUNTS) &&
(selectSql == null || StringUtils.isBlank(selectSql) || selectSql.replace(" ", "").equals("1=1"))) {
rowCount = platform.getEstimatedRowCount(table);
} else {
DatabaseInfo dbInfo = platform.getDatabaseInfo();
String quote = dbInfo.getDelimiterToken();
String catalogSeparator = dbInfo.getCatalogSeparator();
String schemaSeparator = dbInfo.getSchemaSeparator();

String sql = String.format("select count(*) from %s t where %s", table
.getQualifiedTableName(quote, catalogSeparator, schemaSeparator), selectSql);
sql = FormatUtils.replace("groupId", targetNode.getNodeGroupId(), sql);
sql = FormatUtils.replace("externalId", targetNode.getExternalId(), sql);
sql = FormatUtils.replace("nodeId", targetNode.getNodeId(), sql);
for (IReloadVariableFilter filter : extensionService.getExtensionPointList(IReloadVariableFilter.class)) {
sql = filter.filterPurgeSql(sql, targetNode, table);
}

try {
rowCount = sqlTemplate.queryForLong(sql);
} catch (Exception ex) {
throw new SymmetricException("Failed to execute row count SQL while starting reload. If this is a syntax error, check your input and check "
+ engine.getTablePrefix() + "_table_reload_request. Statement attempted: \"" + sql + "\"", ex);
}
}
return rowCount;
}

protected int getTransformMultiplier(Table table, TriggerRouter triggerRouter) {
@@ -1142,4 +1142,16 @@ public boolean supportsTransactions() {
}
return supportsTransactions;
}

public long getEstimatedRowCount(Table table) {
DatabaseInfo dbInfo = getDatabaseInfo();
String quote = dbInfo.getDelimiterToken();
String catalogSeparator = dbInfo.getCatalogSeparator();
String schemaSeparator = dbInfo.getSchemaSeparator();

String sql = String.format("select count(*) from %s", table.getQualifiedTableName(quote, catalogSeparator, schemaSeparator));

return getSqlTemplateDirty().queryForLong(sql);
}

}
@@ -192,4 +192,7 @@ public DmlStatement createDmlStatement(DmlType dmlType, String catalogName, Stri
public boolean isUseMultiThreadSyncTriggers();

public boolean supportsTransactions();

public long getEstimatedRowCount(Table table);

}
@@ -157,4 +157,11 @@ public void makePlatformSpecific(Database database) {
}
super.makePlatformSpecific(database);
}

@Override
public long getEstimatedRowCount(Table table) {
return getSqlTemplateDirty().queryForLong("select table_rows from information_schema.tables where table_name = ? and table_schema = ?",
table.getName(), table.getCatalog());
}

}
@@ -23,6 +23,7 @@

import org.apache.commons.lang.StringUtils;
import org.jumpmind.db.model.Column;
import org.jumpmind.db.model.Table;
import org.jumpmind.db.platform.AbstractJdbcDatabasePlatform;
import org.jumpmind.db.platform.DatabaseNamesConstants;
import org.jumpmind.db.platform.PermissionResult;
@@ -140,5 +141,11 @@ public PermissionResult getExecuteSymPermission() {

return result;
}


@Override
public long getEstimatedRowCount(Table table) {
return getSqlTemplateDirty().queryForLong("select num_rows from all_tables where table_name = ? and owner = ?",
table.getName(), table.getSchema());
}

}
@@ -30,6 +30,7 @@

import org.apache.commons.lang.StringUtils;
import org.jumpmind.db.model.Column;
import org.jumpmind.db.model.Table;
import org.jumpmind.db.platform.AbstractJdbcDatabasePlatform;
import org.jumpmind.db.platform.DatabaseNamesConstants;
import org.jumpmind.db.platform.PermissionResult;
@@ -234,5 +235,12 @@ public PermissionResult getDropSymTriggerPermission() {

return result;
}


@Override
public long getEstimatedRowCount(Table table) {
return getSqlTemplateDirty().queryForLong("select c.reltuples from pg_catalog.pg_class c inner join pg_catalog.pg_namespace n " +
"on n.oid = c.relnamespace where c.relname = ? and n.nspname = ?",
table.getName(), table.getSchema());
}

}

0 comments on commit 29f29b0

Please sign in to comment.
You can’t perform that action at this time.