Skip to content
Permalink
Browse files

react-window, import cleanup, caching

  • Loading branch information...
staylor committed Dec 10, 2018
1 parent 43adff8 commit d9bff69e332ad9de8351c67f4848fc7968209eff
Showing with 284 additions and 221 deletions.
  1. BIN database.sqlite
  2. +2 −1 package.json
  3. +54 −28 pitchfork.js
  4. +0 −168 src/components/App.js
  5. +167 −0 src/components/App/index.js
  6. +42 −0 src/components/App/styled.js
  7. +19 −24 yarn.lock
BIN +1.38 MB (130%) database.sqlite
Binary file not shown.
@@ -28,7 +28,8 @@
"react": "^16.6.3",
"react-apollo": "^2.3.2",
"react-dom": "^16.6.3",
"react-virtualized": "9.9.0",
"react-window": "^1.3.1",
"react-window-infinite-loader": "^1.0.3",
"sql-template-strings": "^2.2.2",
"sqlite": "^3.0.0"
},
@@ -8,7 +8,8 @@ import Promise from 'bluebird';

/* eslint-disable no-console */

const maxPages = 2000;
const MAX_PAGES = 2000;
const MAX_SCORES_PER_PAGE = 25;

const parseReview = (db, $) => async (i, review) => {
const $r = $(review);
@@ -28,51 +29,75 @@ const parseReview = (db, $) => async (i, review) => {

return db
.run(
SQL`INSERT OR REPLACE INTO albums (name, date, image, url) VALUES (${title}, ${date}, ${image}, ${url})`
SQL`INSERT OR IGNORE INTO albums (name, date, image, url) VALUES (${title}, ${date}, ${image}, ${url})`
)
.then(async ({ stmt }) => {
console.log('Inserted:', stmt.lastID);
const albumId = stmt.lastID;

// console.log(title, artists);
let albumId;
if (stmt.changes === 0) {
albumId = await db
.get(SQL`SELECT album_id as id FROM albums WHERE name = ${title} AND url = ${url}`)
.then(({ id }) => id);
} else {
albumId = stmt.lastID;
}

await Promise.all(
const artistIds = await Promise.all(
artists.map(artist =>
db
.run(SQL`INSERT OR REPLACE INTO artists (name) VALUES (${artist})`)
.then(({ stmt: st }) => st.lastID)
.run(SQL`INSERT OR IGNORE INTO artists (name) VALUES (${artist})`)
.then(({ stmt: st }) => {
if (st.changes === 0) {
return db
.get(SQL`SELECT artist_id as id FROM artists WHERE name = ${artist}`)
.then(({ id }) => id);
}
return st.lastID;
})
)
).then(artistIds =>
Promise.all(
artistIds.map(artistId =>
db.run(
SQL`INSERT OR REPLACE INTO album_artists (album_id, artist_id) VALUES (${albumId}, ${artistId})`
);

console.log(albumId, title, artists, artistIds);

await Promise.all(
artistIds.map(artistId =>
db
.run(
SQL`INSERT OR IGNORE INTO album_artists (album_id, artist_id) VALUES (${albumId}, ${artistId})`
)
)
.then(({ stmt: st }) => st.lastID)
)
);

await Promise.all(
const genreIds = await Promise.all(
genres.map(genre =>
db
.run(SQL`INSERT OR REPLACE INTO genres (name) VALUES (${genre})`)
.then(({ stmt: st }) => st.lastID)
.run(SQL`INSERT OR IGNORE INTO genres (name) VALUES (${genre})`)
.then(({ stmt: st }) => {
if (st.changes === 0) {
return db
.get(SQL`SELECT genre_id as id FROM genres WHERE name = ${genre}`)
.then(({ id }) => id);
}
return st.lastID;
})
)
).then(genreIds =>
Promise.all(
genreIds.map(genreId =>
db.run(
SQL`INSERT OR REPLACE INTO album_genres (album_id, genre_id) VALUES (${albumId}, ${genreId})`
);

await Promise.all(
genreIds.map(genreId =>
db
.run(
SQL`INSERT OR IGNORE INTO album_genres (album_id, genre_id) VALUES (${albumId}, ${genreId})`
)
)
.then(({ stmt: st }) => st.lastID)
)
);

return Promise.resolve();
});
};

const request = (db, base, i = 1) => {
const request = ({ db, base, maxPages, i = 1 }) => {
const url = `${base}?page=${i}`;

console.log('Fetching:', url);
@@ -88,15 +113,15 @@ const request = (db, base, i = 1) => {
reviews.each(iter);

if (i < maxPages) {
return request(db, base, i + 1);
return request({ db, base, maxPages, i: i + 1 });
}
return Promise.resolve();
});
};

const pageRows = async (db, rows) => {
const queries = [];
const promises = rows.splice(0, 25).map(row =>
const promises = rows.splice(0, MAX_SCORES_PER_PAGE).map(row =>
fetch(row.url)
.then(r => r.text())
.then(review => {
@@ -125,6 +150,7 @@ const pageRows = async (db, rows) => {
let doRequest = true;
let offset = 0;
let limit = -1;
const maxPages = argv.pages || MAX_PAGES;

if (argv.reviews) {
base = 'https://pitchfork.com/reviews/albums/';
@@ -145,7 +171,7 @@ const pageRows = async (db, rows) => {
}

if (doRequest) {
await request(db, base);
await request({ db, base, maxPages });
}

const rows = await db.all(

This file was deleted.

Oops, something went wrong.
Oops, something went wrong.

0 comments on commit d9bff69

Please sign in to comment.
You can’t perform that action at this time.