This repository has been archived by the owner on Mar 19, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1
/
AzureDataService.js
127 lines (109 loc) · 5.04 KB
/
AzureDataService.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
const requireEnv = require('require-environment-variables');
requireEnv(['AZURE_STORAGE_CONNECTION_STRING']);
const azureService = require('./lib/azureService');
const filters = require('./lib/filters');
const fsHelper = require('./lib/fsHelper');
const getDateFromFilename = require('./lib/getDateFromFilename');
const validateConfig = require('./lib/validateConfig');
const libConfig = require('./lib/config');
class AzureDataService {
constructor(config) {
this.log = config.log;
this.containerName = config.containerName;
this.outputFile = config.outputFile;
this.outputDir = config.outputDir;
this.localFile = `${this.outputDir}/${this.outputFile}.json`;
this.summaryFile = config.summaryFile || 'summary';
this.localSummaryFile = `${this.outputDir}/${this.summaryFile}.json`;
this.seedIdFile = `${config.outputFile}-seed-ids`;
this.localSeedIdFile = `${this.outputDir}/${this.seedIdFile}.json`;
this.version = config.version;
this.dateFormat = libConfig.dateFormat;
validateConfig(this);
}
getSuffix(startMoment) {
return `-${startMoment.format(this.dateFormat)}.json`;
}
getSuffixWithVersion(startMoment) {
return `-${startMoment.format(this.dateFormat)}-${this.version}.json`;
}
async downloadLatest(blobName, filename) {
this.log.info(`Latest version of ${filename} file identified as '${blobName}'`);
await azureService.downloadFromAzure(this.containerName, filename, blobName);
this.log.info(`Remote file '${blobName}' downloaded locally as '${filename}'`);
const data = fsHelper.loadJsonSync(filename);
const date = getDateFromFilename(blobName);
return { data, date };
}
async getLatestIds() {
const filter = b => b.name.startsWith(`${this.seedIdFile}-`);
const blob = await azureService.getLatestBlob(this.containerName, filter);
if (blob) {
return this.downloadLatest(blob.name, this.localSeedIdFile);
}
throw Error('unable to retrieve ID list');
}
async getLatestData() {
const filter = filters.createFileVersionFilter(this.outputFile, this.version);
const lastScan = await azureService.getLatestBlob(this.containerName, filter);
if (lastScan) {
return this.downloadLatest(lastScan.name, this.localFile);
}
this.log.info(`unable to retrieve data, no data available for release ${this.version}?`);
return { data: [] };
}
async uploadData(startMoment) {
this.log.info(`Overwriting '${this.outputFile}.json' in Azure`);
await azureService.uploadToAzure(this.containerName, this.localFile, `${this.outputFile}.json`);
this.log.info(`Saving date stamped version of '${this.outputFile}' in Azure`);
await azureService.uploadToAzure(this.containerName, this.localFile, `${this.outputFile}${this.getSuffixWithVersion(startMoment)}`);
}
async uploadIds(startMoment) {
this.log.info(`Saving date stamped version of '${this.seedIdFile}' in Azure`);
await azureService.uploadToAzure(this.containerName, this.localSeedIdFile, `${this.seedIdFile}${this.getSuffix(startMoment)}`);
}
async uploadSummary(startMoment) {
this.log.info('Saving summary file in Azure');
await azureService.uploadToAzure(this.containerName, this.localSummaryFile, `${this.outputFile}-${this.summaryFile}${this.getSuffixWithVersion(startMoment)}`);
}
async pruneDataFiles(oldestMoment, files) {
const filter = filters.createExpiredDataFilter(this.outputFile, this.version, oldestMoment);
const fileVersionFilter = filters.createFileVersionFilter(this.outputFile, this.version);
await this.pruneExpiredFiles(files, filter, fileVersionFilter);
}
async pruneIdListFiles(oldestMoment, files) {
const filter = filters.createExpiredIdListFilter(this.outputFile, this.version, oldestMoment);
const latestFilter = filters.createIdListFilter(this.seedIdFile);
await this.pruneExpiredFiles(files, filter, latestFilter);
}
async pruneSummaryFiles(oldestMoment, files) {
const filter = filters.createExpiredSummaryFilter(
this.outputFile,
this.summaryFile,
this.version, oldestMoment
);
const latestFilter = filters.createSummaryFileFilter(this.outputFile, this.summaryFile);
await this.pruneExpiredFiles(files, filter, latestFilter);
}
async pruneExpiredFiles(files, filter, latestFilter) {
const expiredFiles = files.filter(filter);
const latest = await azureService.getLatestBlob(this.containerName, latestFilter);
// eslint-disable-next-line no-restricted-syntax
for (const file of expiredFiles) {
// safeguard to stop deleting latest data
if (!latest || file.name !== latest.name) {
// eslint-disable-next-line no-await-in-loop
await azureService.deleteFromAzure(this.containerName, file.name);
}
}
}
async pruneFilesOlderThan(oldestMoment) {
const files = await azureService.listBlobs(this.containerName);
if (files) {
await this.pruneDataFiles(oldestMoment, files);
await this.pruneIdListFiles(oldestMoment, files);
await this.pruneSummaryFiles(oldestMoment, files);
}
}
}
module.exports = AzureDataService;