Skip to content
Permalink
Browse files

0003770: Remove deprecated and unused columns in SYM tables

  • Loading branch information...
erilong committed Oct 29, 2018
1 parent 6b994fd commit 964efd07aada04a5441c475239cdff23aca54fbe
@@ -252,11 +252,10 @@ public static File createSnapshot(ISymmetricEngine engine) {
List<DataGap> gaps = engine.getRouterService().getDataGaps();
SimpleDateFormat dformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
fos = new FileOutputStream(new File(tmpDir, "sym_data_gap_cache.csv"));
fos.write("start_id,end_id,create_time,last_update_time\n".getBytes());
fos.write("start_id,end_id,create_time\n".getBytes());
if (gaps != null) {
for (DataGap gap : gaps) {
fos.write((gap.getStartId() + "," + gap.getEndId() + ",\"" + dformat.format(gap.getCreateTime()) + "\",\""
+ dformat.format(gap.getLastUpdateTime()) + "\"\n").getBytes());
fos.write((gap.getStartId() + "," + gap.getEndId() + ",\"" + dformat.format(gap.getCreateTime()) + "\",\"" + "\"\n").getBytes());
}
}
} catch (IOException e) {
@@ -551,7 +550,7 @@ protected static void writeRuntimeStats(ISymmetricEngine engine, File tmpDir) {
runtimeProperties.setProperty("batch.incoming.errors.count",
Long.toString(engine.getIncomingBatchService().countIncomingBatchesInError()));

List<DataGap> gaps = engine.getDataService().findDataGapsByStatus(DataGap.Status.GP);
List<DataGap> gaps = engine.getDataService().findDataGapsUnchecked();
runtimeProperties.setProperty("data.gap.count", Long.toString(gaps.size()));
if (gaps.size() > 0) {
runtimeProperties.setProperty("data.gap.start.id", Long.toString(gaps.get(0).getStartId()));
@@ -28,25 +28,20 @@

private static final long serialVersionUID = 1L;

public enum Status {GP,SK,OK};

private long startId;
private long endId;
private Date createTime;
private Date lastUpdateTime;

public DataGap(long startId, long endId) {
this.startId = startId;
this.endId = endId;
this.createTime = new Date();
this.lastUpdateTime = createTime;
}

public DataGap(long startId, long endId, Date createTime) {
this.startId = startId;
this.endId = endId;
this.createTime = createTime;
this.lastUpdateTime = createTime;
}

@Override
@@ -66,14 +61,6 @@ public Date getCreateTime() {
return createTime;
}

public Date getLastUpdateTime() {
return lastUpdateTime;
}

public void setLastUpdateTime(Date lastUpdateTime) {
this.lastUpdateTime = lastUpdateTime;
}

public boolean contains(DataGap gap) {
return startId <= gap.startId && endId >= gap.endId;
}
@@ -23,7 +23,6 @@
import org.jumpmind.extension.IBuiltInExtensionPoint;
import org.jumpmind.symmetric.ISymmetricEngine;
import org.jumpmind.symmetric.ext.ISymmetricEngineAware;
import org.jumpmind.symmetric.model.DataGap;
import org.jumpmind.symmetric.model.Monitor;
import org.jumpmind.symmetric.model.MonitorEvent;
import org.jumpmind.symmetric.service.IDataService;
@@ -40,7 +39,7 @@ public String getName() {
@Override
public MonitorEvent check(Monitor monitor) {
MonitorEvent event = new MonitorEvent();
event.setValue(dataService.countDataGapsByStatus(DataGap.Status.GP));
event.setValue(dataService.countDataGaps());
return event;
}

@@ -128,9 +128,9 @@ public void insertScriptEvent(ISqlTransaction transaction, String channelId,
*/
public int countDataInRange(long firstDataId, long secondDataId);

public long countDataGapsByStatus(DataGap.Status status);
public long countDataGaps();

public List<DataGap> findDataGapsByStatus(DataGap.Status status);
public List<DataGap> findDataGapsUnchecked();

public List<DataGap> findDataGaps();

@@ -2074,23 +2074,23 @@ protected String getCsvDataFor(ISqlTransaction transaction, Trigger trigger, Tri
return data;
}

public long countDataGapsByStatus(DataGap.Status status) {
return sqlTemplate.queryForLong(getSql("countDataGapsByStatusSql"), new Object[] { status.name() });
public long countDataGaps() {
return sqlTemplate.queryForLong(getSql("countDataGapsSql"));
}

public List<DataGap> findDataGapsByStatus(DataGap.Status status) {
return sqlTemplate.query(getSql("findDataGapsByStatusSql"), new ISqlRowMapper<DataGap>() {
public List<DataGap> findDataGapsUnchecked() {
return sqlTemplate.query(getSql("findDataGapsSql"), new ISqlRowMapper<DataGap>() {
public DataGap mapRow(Row rs) {
return new DataGap(rs.getLong("start_id"), rs.getLong("end_id"), rs
.getDateTime("create_time"));
}
}, status.name());
});
}

public List<DataGap> findDataGaps() {
final long maxDataToSelect = parameterService
.getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE);
List<DataGap> gaps = findDataGapsByStatus(DataGap.Status.GP);
List<DataGap> gaps = findDataGapsUnchecked();
boolean lastGapExists = false;
long lastGapStartId = 0;
for (DataGap dataGap : gaps) {
@@ -2146,9 +2146,9 @@ public void insertDataGap(DataGap gap) {
public void insertDataGap(ISqlTransaction transaction, DataGap gap) {
log.debug("Inserting data gap: {}", gap);
transaction.prepareAndExecute(getSql("insertDataGapSql"),
new Object[] { DataGap.Status.GP.name(), AppUtils.getHostName(), gap.getStartId(), gap.getEndId(),
gap.getLastUpdateTime(), gap.getCreateTime() }, new int[] {
Types.VARCHAR, Types.VARCHAR, Types.NUMERIC, Types.NUMERIC, Types.TIMESTAMP, Types.TIMESTAMP });
new Object[] { AppUtils.getHostName(), gap.getStartId(), gap.getEndId(),
gap.getCreateTime() }, new int[] {
Types.VARCHAR, Types.NUMERIC, Types.NUMERIC, Types.TIMESTAMP });
}

@Override
@@ -96,15 +96,13 @@ public DataServiceSqlMap(IDatabasePlatform platform, Map<String, String> replace
putSql("findMinDataSql", ""
+ "select min(data_id) from $(data) where data_id >= ?");

putSql("countDataGapsByStatusSql", "select count(*) from $(data_gap) where status = ?");
putSql("countDataGapsSql", "select count(*) from $(data_gap)");

putSql("findDataGapsByStatusSql",
""
+ "select start_id, end_id, create_time from $(data_gap) where status=? order by start_id asc ");
putSql("findDataGapsSql",
"select start_id, end_id, create_time from $(data_gap) order by start_id asc");

putSql("insertDataGapSql",
""
+ "insert into $(data_gap) (status, last_update_hostname, start_id, end_id, last_update_time, create_time) values(?, ?, ?, ?, ?, ?) ");
"insert into $(data_gap) (last_update_hostname, start_id, end_id, create_time) values(?, ?, ?, ?)");

putSql("deleteDataGapSql",
"delete from $(data_gap) where start_id=? and end_id=? ");
@@ -283,15 +283,15 @@ public void save(Node node) {
getSql("insertNodeSql"),
new Object[] { node.getNodeGroupId(), node.getExternalId(), node.getDatabaseType(),
node.getDatabaseVersion(), node.getSchemaVersion(),
node.getSymmetricVersion(), node.getSyncUrl(), new Date(),
node.isSyncEnabled() ? 1 : 0, AppUtils.getTimezoneOffset(),
node.getSymmetricVersion(), node.getSyncUrl(),
node.isSyncEnabled() ? 1 : 0,
node.getBatchToSendCount(), node.getBatchInErrorCount(),
node.getCreatedAtNodeId(), node.getDeploymentType(),
node.getDeploymentSubType(), node.getConfigVersion(),
node.getNodeId() },
new int[] { Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR,
Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.TIMESTAMP,
Types.INTEGER, Types.VARCHAR, Types.INTEGER, Types.INTEGER, Types.VARCHAR,
Types.VARCHAR, Types.VARCHAR, Types.VARCHAR,
Types.INTEGER, Types.INTEGER, Types.INTEGER, Types.VARCHAR,
Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR });

flushNodeGroupCache();
@@ -304,14 +304,14 @@ public boolean updateNode(Node node) {
getSql("updateNodeSql"),
new Object[] { node.getNodeGroupId(), node.getExternalId(), node.getDatabaseType(),
node.getDatabaseVersion(), node.getSchemaVersion(),
node.getSymmetricVersion(), node.getSyncUrl(), new Date(),
node.isSyncEnabled() ? 1 : 0, AppUtils.getTimezoneOffset(),
node.getSymmetricVersion(), node.getSyncUrl(),
node.isSyncEnabled() ? 1 : 0,
node.getBatchToSendCount(), node.getBatchInErrorCount(),
node.getCreatedAtNodeId(), node.getDeploymentType(), node.getDeploymentSubType(),
node.getConfigVersion(), node.getNodeId() },
new int[] { Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR,
Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.TIMESTAMP,
Types.INTEGER, Types.VARCHAR, Types.INTEGER, Types.INTEGER, Types.VARCHAR,
Types.VARCHAR, Types.VARCHAR, Types.VARCHAR,
Types.INTEGER, Types.INTEGER, Types.INTEGER, Types.VARCHAR,
Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR }) == 1;

return updated;
@@ -51,13 +51,13 @@ public NodeServiceSqlMap(IDatabasePlatform platform, Map<String, String> replace

putSql("insertNodeSql",
"insert into $(node) (node_group_id, external_id, database_type, database_version, schema_version, symmetric_version, sync_url," +
"heartbeat_time, sync_enabled, timezone_offset, batch_to_send_count, batch_in_error_count, created_at_node_id, " +
"deployment_type, deployment_sub_type, config_version, node_id) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
"sync_enabled, batch_to_send_count, batch_in_error_count, created_at_node_id, " +
"deployment_type, deployment_sub_type, config_version, node_id) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");

putSql("updateNodeSql",
"update $(node) set node_group_id=?, external_id=?, database_type=?, "
+ " database_version=?, schema_version=?, symmetric_version=?, sync_url=?, heartbeat_time=?, "
+ " sync_enabled=?, timezone_offset=?, batch_to_send_count=?, batch_in_error_count=?, "
+ " database_version=?, schema_version=?, symmetric_version=?, sync_url=?, "
+ " sync_enabled=?, batch_to_send_count=?, batch_in_error_count=?, "
+ " created_at_node_id=?, deployment_type=?, deployment_sub_type=?, config_version = ? where node_id = ?");

putSql("findNodeSql", "where node_id = ? ");
@@ -152,7 +152,7 @@ public NodeServiceSqlMap(IDatabasePlatform platform, Map<String, String> replace

putSql("selectNodePrefixSql",
"select c.node_id, c.node_group_id, c.external_id, c.sync_enabled, c.sync_url, "
+ " c.schema_version, c.database_type, c.database_version, c.symmetric_version, c.created_at_node_id, c.heartbeat_time, c.timezone_offset, c.batch_to_send_count, c.batch_in_error_count, c.deployment_type, c.deployment_sub_type, c.config_version from "
+ " c.schema_version, c.database_type, c.database_version, c.symmetric_version, c.created_at_node_id, c.batch_to_send_count, c.batch_in_error_count, c.deployment_type, c.deployment_sub_type, c.config_version from "
+ " $(node) c ");

putSql("updateNodeSecuritySql",
@@ -113,13 +113,8 @@
<table name="data_gap" description="Used only when routing.data.reader.type is set to 'gap.' Table that tracks gaps in the data table so that they may be processed efficiently, if data shows up. Gaps can show up in the data table if a database transaction is rolled back.">
<column name="start_id" type="BIGINT" required="true" primaryKey="true" description="The first missing data_id from the data table where a gap is detected. This could be the last data_id inserted plus one." />
<column name="end_id" type="BIGINT" required="true" primaryKey="true" description="The last missing data_id from the data table where a gap is detected. If the start_id is the last data_id inserted plus one, then this field is filled in with a -1." />
<column name="status" type="CHAR" size="2" description="GP, SK, or FL. GP means there is a detected gap. FL means that the gap has been filled. SK means that the gap has been skipped either because the gap expired or because no database transaction was detected which means that no data will be committed to fill in the gap." />
<column name="create_time" type="TIMESTAMP" required="true" description="Timestamp when this entry was created." />
<column name="last_update_hostname" type="VARCHAR" size="255" description="The host who last updated this entry." />
<column name="last_update_time" type="TIMESTAMP" required="true" description="Timestamp when a user last updated this entry." />
<index name="idx_dg_status">
<index-column name="status"/>
</index>
</table>

<table name="extension" description="Dynamic extensions stored in the database that plug-in to the running engine and receive callbacks according to their interface.">
@@ -371,8 +366,6 @@
<column name="node_id" type="VARCHAR" size="50" required="true" primaryKey="true" description="A unique identifier for a node." />
<column name="node_group_id" type="VARCHAR" size="50" required="true" description="The node group that this node belongs to, such as 'store'." />
<column name="external_id" type="VARCHAR" size="255" required="true" description="A domain-specific identifier for context within the local system. For example, the retail store number. "/>
<column name="heartbeat_time" type="TIMESTAMP" description="Deprecated. Use node_host.heartbeat_time instead." />
<column name="timezone_offset" type="VARCHAR" size="6" description="Deprecated. Use node_host.timezone_offset instead." />
<column name="sync_enabled" type="BOOLEANINT" size="1" default="0" description="Indicates whether this node should be sent synchronization. Disabled nodes are ignored by the triggers, so no entries are made in data_event for the node."/>
<column name="sync_url" type="VARCHAR" size="255" description="The URL to contact the node for synchronization." />
<column name="schema_version" type="VARCHAR" size="50" description="The version of the database schema this node manages. Useful for specifying synchronization by version. "/>
@@ -831,7 +831,7 @@ public void testGapWithGapAtEnd() {

long startId = getSqlTemplate().queryForLong("select max(start_id) from sym_data_gap");

getSqlTemplate().update("update sym_data_gap set status='OK'");
getSqlTemplate().update("delete from sym_data_gap");
getDataService().insertDataGap(new DataGap(startId, startId + 10));
getDataService().insertDataGap(
new DataGap(startId + 11, startId + 11 + getParameterService().getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE)));
@@ -269,7 +269,7 @@ protected void resetBatches() {
getSqlTemplate().update("update sym_outgoing_batch set status='OK' where status != 'OK'");
long startId = getSqlTemplate().queryForLong("select max(start_id) from sym_data_gap");
getSqlTemplate()
.update("update sym_data_gap set status='OK' where start_id != ?", startId);
.update("delete from sym_data_gap where start_id != ?", startId);
checkForOpenResources();
}

@@ -120,12 +120,12 @@ insert into sym_trigger_router
(trigger_id,router_id,initial_load_order,last_update_time,create_time)
values('sale_return_line_item_corp','corp_2_store', 200, current_timestamp, current_timestamp);

insert into sym_node (node_id,node_group_id,external_id,sync_enabled,sync_url,schema_version,symmetric_version,database_type,database_version,heartbeat_time,timezone_offset,batch_to_send_count,batch_in_error_count,created_at_node_id)
values ('000','corp','000',1,null,null,null,null,null,current_timestamp,null,0,0,'000');
insert into sym_node (node_id,node_group_id,external_id,sync_enabled,sync_url,schema_version,symmetric_version,database_type,database_version,heartbeat_time,timezone_offset,batch_to_send_count,batch_in_error_count,created_at_node_id)
values ('001','store','001',1,null,null,null,null,null,current_timestamp,null,0,0,'000');
insert into sym_node (node_id,node_group_id,external_id,sync_enabled,sync_url,schema_version,symmetric_version,database_type,database_version,heartbeat_time,timezone_offset,batch_to_send_count,batch_in_error_count,created_at_node_id)
values ('002','store','002',1,null,null,null,null,null,current_timestamp,null,0,0,'000');
insert into sym_node (node_id,node_group_id,external_id,sync_enabled,sync_url,schema_version,symmetric_version,database_type,database_version,batch_to_send_count,batch_in_error_count,created_at_node_id)
values ('000','corp','000',1,null,null,null,null,null,0,0,'000');
insert into sym_node (node_id,node_group_id,external_id,sync_enabled,sync_url,schema_version,symmetric_version,database_type,database_version,batch_to_send_count,batch_in_error_count,created_at_node_id)
values ('001','store','001',1,null,null,null,null,null,0,0,'000');
insert into sym_node (node_id,node_group_id,external_id,sync_enabled,sync_url,schema_version,symmetric_version,database_type,database_version,batch_to_send_count,batch_in_error_count,created_at_node_id)
values ('002','store','002',1,null,null,null,null,null,0,0,'000');


insert into sym_node_security (node_id,node_password,registration_enabled,registration_time,initial_load_enabled,initial_load_time,created_at_node_id)
@@ -103,7 +103,7 @@

static final String isRegistrationClosedSql = "select count(*) from sym_node_security where registration_enabled=0 and node_id=?";

static final String makeHeartbeatOld = "update sym_node set heartbeat_time={ts '2000-01-01 01:01:01.000'} where node_id=?";
static final String makeHeartbeatOld = "update sym_node_host set heartbeat_time={ts '2000-01-01 01:01:01.000'} where node_id=?";

static final String deleteNode = "delete from sym_node where node_id=?";

0 comments on commit 964efd0

Please sign in to comment.
You can’t perform that action at this time.