Skip to content

Commit

Permalink
0004909: Added snapshot progress bar (#152)
Browse files Browse the repository at this point in the history
  • Loading branch information
@evan-miller-jumpmind
evan-miller-jumpmind committed Mar 23, 2021
1 parent 0899ffd commit 9dc6b3e
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 62 deletions.
149 changes: 87 additions & 62 deletions symmetric-client/src/main/java/org/jumpmind/symmetric/util/SnapshotUtil.java
View file
Expand Up @@ -87,6 +87,7 @@
import org.jumpmind.symmetric.model.TriggerHistory;
import org.jumpmind.symmetric.monitor.MonitorTypeBlock;
import org.jumpmind.symmetric.service.IClusterService;
import org.jumpmind.symmetric.service.IExtensionService;
import org.jumpmind.symmetric.service.INodeService;
import org.jumpmind.symmetric.service.IParameterService;
import org.jumpmind.symmetric.service.ITriggerRouterService;
Expand All @@ -110,6 +111,11 @@ public static File getSnapshotDirectory(ISymmetricEngine engine) {
}

public static File createSnapshot(ISymmetricEngine engine) {

IExtensionService extensionService = engine.getExtensionService();
for (ISnapshotCreationListener l : extensionService.getExtensionPointList(ISnapshotCreationListener.class)) {
l.snapshotStepCompleted(0, 5);
}

String dirName = engine.getEngineName().replaceAll(" ", "-") + "-" + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());

Expand Down Expand Up @@ -138,6 +144,10 @@ public static File createSnapshot(ISymmetricEngine engine) {
} catch (Exception e) {
log.warn("Failed to copy " + serviceConfFile.getName() + " to the snapshot directory", e);
}

for (ISnapshotCreationListener l : extensionService.getExtensionPointList(ISnapshotCreationListener.class)) {
l.snapshotStepCompleted(1, 5);
}

FileOutputStream fos = null;
try {
Expand Down Expand Up @@ -209,6 +219,10 @@ public static File createSnapshot(ISymmetricEngine engine) {
} catch(IOException e) { }
}
}

for (ISnapshotCreationListener l : extensionService.getExtensionPointList(ISnapshotCreationListener.class)) {
l.snapshotStepCompleted(2, 5);
}

String tablePrefix = engine.getTablePrefix();

Expand Down Expand Up @@ -291,60 +305,7 @@ public static File createSnapshot(ISymmetricEngine engine) {

extract(export, new File(tmpDir, "sym_node_host_channel_stats.csv"),
TableConstants.getTableName(tablePrefix, TableConstants.SYM_NODE_HOST_CHANNEL_STATS));

if (engine.getSymmetricDialect() instanceof FirebirdSymmetricDialect) {
final String[] monTables = { "mon$database", "mon$attachments", "mon$transactions", "mon$statements", "mon$io_stats",
"mon$record_stats", "mon$memory_usage", "mon$call_stack", "mon$context_variables" };
for (String table : monTables) {
extract(export, new File(tmpDir, "firebird-" + table + ".csv"), table);
}
}

if (engine.getSymmetricDialect() instanceof MySqlSymmetricDialect) {
extractQuery(engine.getSqlTemplate(), tmpDir + File.separator + "mysql-processlist.csv",
"show processlist");
extractQuery(engine.getSqlTemplate(), tmpDir + File.separator + "mysql-global-variables.csv",
"show global variables");
extractQuery(engine.getSqlTemplate(), tmpDir + File.separator + "mysql-session-variables.csv",
"show session variables");
}

if (!engine.getParameterService().is(ParameterConstants.CLUSTER_LOCKING_ENABLED)) {
try {
List<DataGap> gaps = engine.getRouterService().getDataGaps();
SimpleDateFormat dformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
fos = new FileOutputStream(new File(tmpDir, "sym_data_gap_cache.csv"));
fos.write("start_id,end_id,create_time\n".getBytes());
if (gaps != null) {
for (DataGap gap : gaps) {
fos.write((gap.getStartId() + "," + gap.getEndId() + ",\"" + dformat.format(gap.getCreateTime()) + "\",\"" + "\"\n").getBytes());
}
}
} catch (Exception e) {
log.warn("Failed to export data gap information", e);
} finally {
if(fos != null) {
try {
fos.close();
} catch(IOException e) { }
}
}
}

createThreadsFile(tmpDir.getPath(), false);
createThreadsFile(tmpDir.getPath(), true);
createThreadStatsFile(tmpDir.getPath());

try {
List<Transaction> transactions = engine.getDatabasePlatform().getTransactions();
if (!transactions.isEmpty()) {
createTransactionsFile(engine, tmpDir.getPath(), transactions);
}
} catch (Exception e) {
log.warn("Failed to create transactions file", e);
}


fos = null;
try {
fos = new FileOutputStream(new File(tmpDir, "parameters.properties"));
Expand Down Expand Up @@ -402,14 +363,7 @@ public static File createSnapshot(ISymmetricEngine engine) {
} catch(IOException e) { }
}
}

writeRuntimeStats(engine, tmpDir);
writeJobsStats(engine, tmpDir);

if ("true".equals(System.getProperty(SystemConstants.SYSPROP_STANDALONE_WEB))) {
writeDirectoryListing(engine, tmpDir);
}


fos = null;
try {
fos = new FileOutputStream(new File(tmpDir, "system.properties"));
Expand All @@ -426,6 +380,69 @@ public static File createSnapshot(ISymmetricEngine engine) {
}
}

if (engine.getSymmetricDialect() instanceof FirebirdSymmetricDialect) {
final String[] monTables = { "mon$database", "mon$attachments", "mon$transactions", "mon$statements", "mon$io_stats",
"mon$record_stats", "mon$memory_usage", "mon$call_stack", "mon$context_variables" };
for (String table : monTables) {
extract(export, new File(tmpDir, "firebird-" + table + ".csv"), table);
}
}

if (engine.getSymmetricDialect() instanceof MySqlSymmetricDialect) {
extractQuery(engine.getSqlTemplate(), tmpDir + File.separator + "mysql-processlist.csv",
"show processlist");
extractQuery(engine.getSqlTemplate(), tmpDir + File.separator + "mysql-global-variables.csv",
"show global variables");
extractQuery(engine.getSqlTemplate(), tmpDir + File.separator + "mysql-session-variables.csv",
"show session variables");
}

for (ISnapshotCreationListener l : extensionService.getExtensionPointList(ISnapshotCreationListener.class)) {
l.snapshotStepCompleted(3, 5);
}

if (!engine.getParameterService().is(ParameterConstants.CLUSTER_LOCKING_ENABLED)) {
try {
List<DataGap> gaps = engine.getRouterService().getDataGaps();
SimpleDateFormat dformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
fos = new FileOutputStream(new File(tmpDir, "sym_data_gap_cache.csv"));
fos.write("start_id,end_id,create_time\n".getBytes());
if (gaps != null) {
for (DataGap gap : gaps) {
fos.write((gap.getStartId() + "," + gap.getEndId() + ",\"" + dformat.format(gap.getCreateTime()) + "\",\"" + "\"\n").getBytes());
}
}
} catch (Exception e) {
log.warn("Failed to export data gap information", e);
} finally {
if(fos != null) {
try {
fos.close();
} catch(IOException e) { }
}
}
}

createThreadsFile(tmpDir.getPath(), false);
createThreadsFile(tmpDir.getPath(), true);
createThreadStatsFile(tmpDir.getPath());

try {
List<Transaction> transactions = engine.getDatabasePlatform().getTransactions();
if (!transactions.isEmpty()) {
createTransactionsFile(engine, tmpDir.getPath(), transactions);
}
} catch (Exception e) {
log.warn("Failed to create transactions file", e);
}

writeRuntimeStats(engine, tmpDir);
writeJobsStats(engine, tmpDir);

if ("true".equals(System.getProperty(SystemConstants.SYSPROP_STANDALONE_WEB))) {
writeDirectoryListing(engine, tmpDir);
}

File logDir = null;

String parameterizedLogDir = parameterService.getString("server.log.dir");
Expand Down Expand Up @@ -472,6 +489,10 @@ public static File createSnapshot(ISymmetricEngine engine) {
}
}
}

for (ISnapshotCreationListener l : extensionService.getExtensionPointList(ISnapshotCreationListener.class)) {
l.snapshotStepCompleted(4, 5);
}

File jarFile = null;
try {
Expand All @@ -486,6 +507,10 @@ public static File createSnapshot(ISymmetricEngine engine) {
throw new IoException("Failed to package snapshot files into archive", e);
}

for (ISnapshotCreationListener l : extensionService.getExtensionPointList(ISnapshotCreationListener.class)) {
l.snapshotStepCompleted(5, 5);
}

log.info("Done creating snapshot file");
return jarFile;
}
Expand Down Expand Up @@ -567,7 +592,7 @@ public int compare(File o1, File o2) {
printDirectoryContents(home, output, fileComparator);
FileUtils.write(new File(tmpDir, "directory-listing.txt"), output, Charset.defaultCharset(), false);
} catch (Exception ex) {
log.warn("Failed to output the direcetory listing", ex);
log.warn("Failed to output the directory listing", ex);
}
}

Expand Down
29 changes: 29 additions & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/util/ISnapshotCreationListener.java
View file
@@ -0,0 +1,29 @@
/**
* Licensed to JumpMind Inc under one or more contributor
* license agreements. See the NOTICE file distributed
* with this work for additional information regarding
* copyright ownership. JumpMind Inc licenses this file
* to you under the GNU General Public License, version 3.0 (GPLv3)
* (the "License"); you may not use this file except in compliance
* with the License.
*
* You should have received a copy of the GNU General Public License,
* version 3.0 (GPLv3) along with this library; if not, see
* <http://www.gnu.org/licenses/>.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jumpmind.symmetric.util;

import org.jumpmind.extension.IExtensionPoint;

public interface ISnapshotCreationListener extends IExtensionPoint {

public void snapshotStepCompleted(int stepNumber, int totalSteps);

}

0 comments on commit 9dc6b3e

Please sign in to comment.