Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

skip single bad record #8

Merged
merged 2 commits into from
Apr 5, 2022
Merged

skip single bad record #8

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d300fb8
fix: skip single record with error
Kpoke Apr 2, 2022
53ace9f
fix: add boolean flag to confirm whether or not to set processed to true
Kpoke Apr 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions process-bulk-uploads.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ const v1Requests = async (data, endpoint, httpVerb) => {
};
const rval = await pool.query(query);

outerLoop: for (let row of rval.rows) {
for (let row of rval.rows) {
let shouldBeProcessed = true;
console.log("processing key: " + row.key);
const bulkData = row.bulk_data;

Expand All @@ -93,8 +94,9 @@ const v1Requests = async (data, endpoint, httpVerb) => {
try {
await v2Requests(wallet_registration, "wallet_registrations");
} catch (e) {
shouldBeProcessed = false;
errorHandler(e, wallet_registration);
continue outerLoop;
continue;
}
}
console.log("v2 wallet_registrations done");
Expand All @@ -106,8 +108,9 @@ const v1Requests = async (data, endpoint, httpVerb) => {
try {
await v2Requests(device_configuration, "device_configurations");
} catch (e) {
shouldBeProcessed = false;
errorHandler(e, device_configuration);
continue outerLoop;
continue;
}
}
console.log("v2 device_configurations done");
Expand All @@ -119,8 +122,9 @@ const v1Requests = async (data, endpoint, httpVerb) => {
try {
await v2Requests(session, "sessions", true);
} catch (e) {
shouldBeProcessed = false;
errorHandler(e, session);
continue outerLoop;
continue;
}
}
console.log("v2 sessions done");
Expand All @@ -132,8 +136,9 @@ const v1Requests = async (data, endpoint, httpVerb) => {
try {
await v2Requests(capture, "captures");
} catch (e) {
shouldBeProcessed = false;
errorHandler(e, capture);
continue outerLoop;
continue;
}
}
console.log("v2 captures done");
Expand All @@ -145,8 +150,9 @@ const v1Requests = async (data, endpoint, httpVerb) => {
try {
await v2Requests(message, "messages", true);
} catch (e) {
shouldBeProcessed = false;
errorHandler(e, message);
continue outerLoop;
continue;
}
}
console.log("v2 messages done");
Expand All @@ -160,8 +166,9 @@ const v1Requests = async (data, endpoint, httpVerb) => {
try {
await v1Requests(planter, "planter");
} catch (e) {
shouldBeProcessed = false;
errorHandler(e, planter);
continue outerLoop;
continue;
}
}
console.log("v1 registrations done");
Expand All @@ -173,8 +180,9 @@ const v1Requests = async (data, endpoint, httpVerb) => {
try {
await v1Requests(device, "device", "PUT");
} catch (e) {
shouldBeProcessed = false;
errorHandler(e, device);
continue outerLoop;
continue;
}
}
console.log("v1 devices done");
Expand All @@ -186,8 +194,9 @@ const v1Requests = async (data, endpoint, httpVerb) => {
try {
await v1Requests(tree, "tree");
} catch (e) {
shouldBeProcessed = false;
errorHandler(e, tree);
continue outerLoop;
continue;
}
}
console.log("v1 trees done");
Expand All @@ -202,7 +211,9 @@ const v1Requests = async (data, endpoint, httpVerb) => {
values: [row.id],
};
console.log("update");
await pool.query(update);
if (shouldBeProcessed) {
await pool.query(update);
}
console.log(`Processed bulk tree upload ${row.id}`);
}
console.log("done");
Expand Down