Skip to content

Commit

Permalink
catch exceptions and proceed with experiments
Browse files Browse the repository at this point in the history
  • Loading branch information
@bentoumia
bentoumia committed Aug 31, 2022
1 parent 2c2a9a0 commit 2fd85d4
Showing 1 changed file with 163 additions and 151 deletions.
314 changes: 163 additions & 151 deletions src/orchestrator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,163 +95,175 @@ async function main() {
let clientSettings = e[experimentId].client
let shadowFilePath = path.join(executionDir, process.env.SHADOW_FILE)
let networkFilePath = path.join(executionDir, process.env.NETWORK_FILE)
logger.info('deleting clashing directories ...')
await deleteDirectoryIfExists(path.join(experimentsPath, experimentId))
let shadowTemplate = await yg.makeConfigTemplate(
isNullOrEmpty(process.env.SHADOW_TEMPLATE)
? null
: process.env.SHADOW_TEMPLATE,
process.env.NETWORK_FILE,
path.join(experimentsPath, experimentId),
e[experimentId].misc,
)
await build(protocol, replicaSettings, clientSettings, logger)
let hosts = await configure(
protocol,
replicaSettings,
clientSettings,
logger,
)
shadowTemplate = await createShadowHostConfig(shadowTemplate, hosts)
// Generate Shadow File
await yg.out(shadowFilePath, shadowTemplate)
let myGraph = ''
if (e[experimentId].network.latency.uniform)
myGraph = eg.createGraphSimple(
hosts,
e[experimentId].network.bandwidthUp,
e[experimentId].network.bandwidthDown,
e[experimentId].network.latency.replicas,
e[experimentId].network.latency.clients,
'0.0',
try {
logger.info('deleting clashing directories ...')
await deleteDirectoryIfExists(path.join(experimentsPath, experimentId))
let shadowTemplate = await yg.makeConfigTemplate(
isNullOrEmpty(process.env.SHADOW_TEMPLATE)
? null
: process.env.SHADOW_TEMPLATE,
process.env.NETWORK_FILE,
path.join(experimentsPath, experimentId),
e[experimentId].misc,
)
else {
myGraph = await eg.makeAWSGraph(
hosts,
e[experimentId].network.latency.replicas,
e[experimentId].network.latency.clients,
e[experimentId].network.bandwidthUp,
e[experimentId].network.bandwidthDown,
'0.0',
await build(protocol, replicaSettings, clientSettings, logger)
let hosts = await configure(
protocol,
replicaSettings,
clientSettings,
logger,
)
}
await fs.writeFile(networkFilePath, myGraph)
let experimentStartTime = performance.now()
await Promise.all([
run(executionDir, logger),
(shadowInterval = monitor.register(
process.env.SHADOW_PROCESS,
2000,
logger,
)),
(procInterval = monitor.register(
protocol.getProcessName(),
2000,
logger,
)),
(totalInterval = monitor.registerSI(2000, logger)),
])
let experimentEndTime = performance.now()
let elapsedSeconds = (experimentEndTime - experimentStartTime) / 1000
let resourceUsage = await monitor.unregister(logger)
let perfStats = await getStats(protocol, experimentId, logger)
let statsForCSV = {
experimentId: experimentId,
maxThroughput: perfStats.maxThroughput,
avgThroughput: perfStats.avgThroughput,
latencyAll: perfStats.latencyAll,
latencyOutlierRemoved: perfStats.latencyOutlierRemoved,
cpuShadow: resourceUsage[process.env.SHADOW_PROCESS].medianCPU,
memShadow: resourceUsage[process.env.SHADOW_PROCESS].maxMEM,
cpuApp: resourceUsage[protocol.getProcessName()].medianCPU,
memApp: resourceUsage[protocol.getProcessName()].maxMEM,
hostActive: resourceUsage['total'].maxMEM,
elapsed: elapsedSeconds,
}
csvUtil.values.push(statsForCSV)
await csvUtil.save(path.join(experimentsPath, process.env.STATS_FILE))
if (e[experimentId].plots) {
for (let p of e[experimentId].plots) {
if (p.metric == 'tps') {
plot.pushValue(p.name, p.datasetId, p.label, perfStats.maxThroughput)
continue
}
if (p.metric == 'latency') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
perfStats.latencyOutlierRemoved,
)
continue
}
if (p.metric == 'cpu-shadow') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage[process.env.SHADOW_PROCESS].medianCPU,
)
continue
}
if (p.metric == 'mem-shadow') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage[process.env.SHADOW_PROCESS].maxMEM,
)
continue
}
if (p.metric == 'cpu-app') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage[protocol.getProcessName()].medianCPU,
)
continue
}
if (p.metric == 'mem-app') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage[protocol.getProcessName()].maxMEM,
)
continue
}
if (p.metric == 'mem-host') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage['total'].maxMEM,
)
continue
}
if (p.metric == 'elapsed') {
plot.pushValue(p.name, p.datasetId, p.label, elapsedSeconds)
continue
shadowTemplate = await createShadowHostConfig(shadowTemplate, hosts)
// Generate Shadow File
await yg.out(shadowFilePath, shadowTemplate)
let myGraph = ''
if (e[experimentId].network.latency.uniform)
myGraph = eg.createGraphSimple(
hosts,
e[experimentId].network.bandwidthUp,
e[experimentId].network.bandwidthDown,
e[experimentId].network.latency.replicas,
e[experimentId].network.latency.clients,
'0.0',
)
else {
myGraph = await eg.makeAWSGraph(
hosts,
e[experimentId].network.latency.replicas,
e[experimentId].network.latency.clients,
e[experimentId].network.bandwidthUp,
e[experimentId].network.bandwidthDown,
'0.0',
logger,
)
}
await fs.writeFile(networkFilePath, myGraph)
let experimentStartTime = performance.now()
await Promise.all([
run(executionDir, logger),
(shadowInterval = monitor.register(
process.env.SHADOW_PROCESS,
2000,
logger,
)),
(procInterval = monitor.register(
protocol.getProcessName(),
2000,
logger,
)),
(totalInterval = monitor.registerSI(2000, logger)),
])
let experimentEndTime = performance.now()
let elapsedSeconds = (experimentEndTime - experimentStartTime) / 1000
let resourceUsage = await monitor.unregister(logger)
let perfStats = await getStats(protocol, experimentId, logger)
let statsForCSV = {
experimentId: experimentId,
maxThroughput: perfStats.maxThroughput,
avgThroughput: perfStats.avgThroughput,
latencyAll: perfStats.latencyAll,
latencyOutlierRemoved: perfStats.latencyOutlierRemoved,
cpuShadow: resourceUsage[process.env.SHADOW_PROCESS].medianCPU,
memShadow: resourceUsage[process.env.SHADOW_PROCESS].maxMEM,
cpuApp: resourceUsage[protocol.getProcessName()].medianCPU,
memApp: resourceUsage[protocol.getProcessName()].maxMEM,
hostActive: resourceUsage['total'].maxMEM,
elapsed: elapsedSeconds,
}
csvUtil.values.push(statsForCSV)
await csvUtil.save(path.join(experimentsPath, process.env.STATS_FILE))
if (e[experimentId].plots) {
for (let p of e[experimentId].plots) {
if (p.metric == 'tps') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
perfStats.maxThroughput,
)
continue
}
if (p.metric == 'latency') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
perfStats.latencyOutlierRemoved,
)
continue
}
if (p.metric == 'cpu-shadow') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage[process.env.SHADOW_PROCESS].medianCPU,
)
continue
}
if (p.metric == 'mem-shadow') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage[process.env.SHADOW_PROCESS].maxMEM,
)
continue
}
if (p.metric == 'cpu-app') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage[protocol.getProcessName()].medianCPU,
)
continue
}
if (p.metric == 'mem-app') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage[protocol.getProcessName()].maxMEM,
)
continue
}
if (p.metric == 'mem-host') {
plot.pushValue(
p.name,
p.datasetId,
p.label,
resourceUsage['total'].maxMEM,
)
continue
}
if (p.metric == 'elapsed') {
plot.pushValue(p.name, p.datasetId, p.label, elapsedSeconds)
continue
}
}
}
}

await backUpArtifact(
networkFilePath,
path.join(
experimentsPath,
path.join(experimentId, process.env.NETWORK_FILE),
),
)
await backUpArtifact(
shadowFilePath,
path.join(
experimentsPath,
path.join(experimentId, process.env.SHADOW_FILE),
),
)
await backUpArtifact(
networkFilePath,
path.join(
experimentsPath,
path.join(experimentId, process.env.NETWORK_FILE),
),
)
await backUpArtifact(
shadowFilePath,
path.join(
experimentsPath,
path.join(experimentId, process.env.SHADOW_FILE),
),
)
} catch (err) {
logger.error(
`An error occurred while processing experiment ${experimentId}, error message: ${err.message}`,
)
logger.info('moving on to next experiment ...')
}
}
logger.info('generating plots...')
await plot.generatePlots(experimentsPath)
Expand Down

0 comments on commit 2fd85d4

Please sign in to comment.