Skip to content

Commit

Permalink
fixes flickr-scrape updating db
Browse files Browse the repository at this point in the history
  • Loading branch information
davelandry committed Apr 13, 2021
1 parent 033cb30 commit c92896b
Showing 1 changed file with 21 additions and 10 deletions.
31 changes: 21 additions & 10 deletions scripts/flickr-scrape.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,15 @@ const Flickr = require("flickr-sdk"),

const creds = require("../google_auth.json");

const slugMap = {
cip: "CIP",
geo: "Geography",
naics: "PUMS Industry",
napcs: "NAPCS",
soc: "PUMS Occupation",
university: "University"
};

// Query to generate CSV for google sheet
//
// select s.id, s.zvalue, s.display as name, i.url as image_link, i.meta as image_meta
Expand Down Expand Up @@ -125,15 +134,15 @@ function fetchImage(row) {
const thumbPath = path.join(process.cwd(), `static/images/profile/thumb/${imageId}.jpg`);

if (!created && shell.test("-e", splashPath) && shell.test("-e", thumbPath)) {
return db.search.update({imageId}, {where: {id: row.id, dimension}})
return db.search.update({imageId}, {where: {id: row.id, dimension: slugMap[dimension]}})
.then(printProgress);
}
else {

console.log(`\nNew Image: ${imageId}`);

return db.search
.update({imageId}, {where: {id: row.id, dimension}})
.update({imageId}, {where: {id: row.id, dimension: slugMap[dimension]}})
.then(() => flickr.photos.getSizes({photo_id: photoId}))
.then(res => {
let image = res.body.sizes.size.find(d => parseInt(d.width, 10) >= 1600);
Expand Down Expand Up @@ -204,14 +213,16 @@ async function run() {
console.log(`Images found: ${total}`);
await Promise.all(fetches);

fetches = [];
updates.forEach(row => {
fetches.push(googleThrottle.add(saveRow.bind(this, row)));
});
checked = 0;
total = fetches.length;
console.log("\n\nUpdating Spreadsheet Error Column");
await Promise.all(fetches);
if (updates.length) {
fetches = [];
updates.forEach(row => {
fetches.push(googleThrottle.add(saveRow.bind(this, row)));
});
checked = 0;
total = fetches.length;
console.log("\n\nUpdating Spreadsheet Error Column");
await Promise.all(fetches);
}

console.log("\n");
shell.exit(0);
Expand Down

0 comments on commit c92896b

Please sign in to comment.