Skip to content
Permalink
Browse files

reduce scrapes

- turned off broken sites
- reduce refresh rate for less popular item types (highest cost is dyanmo db writes...)
  • Loading branch information
nexus-uw committed Jan 20, 2020
1 parent a4d137e commit 79e762dd766cb04f55b68cdf8915a35fbd27ffed
Showing with 41 additions and 23 deletions.
  1. +24 −21 src/constants.ts
  2. +17 −2 src/refresher/lambda.ts
@@ -25,13 +25,14 @@ export const VENDORS: IVendor[] = [
provinces: [Province.ON, Province.QC],
location: 'all over the place',
},
{
name: 'Firearms Outlet Canada',
link: 'firearmsoutletcanada.com',
logo: 'foc-logo.jpg',
provinces: [Province.ON],
location: 'Ajax',
},
// 20200119 cert issues
// {
// name: 'Firearms Outlet Canada',
// link: 'firearmsoutletcanada.com',
// logo: 'foc-logo.jpg',
// provinces: [Province.ON],
// location: 'Ajax',
// },
{
name: `Al Flaherty's`,
link: 'alflahertys.com',
@@ -66,27 +67,29 @@ export const VENDORS: IVendor[] = [
],
location: 'all over the place',
},
{
name: 'Reliable Gun',
link: 'reliablegun.com',
logo: 'reliable-gun-logo.jpg',
provinces: [Province.BC],
location: 'Vancover',
},
// 20200119: all pages fail with 500
// {
// name: 'Reliable Gun',
// link: 'reliablegun.com',
// logo: 'reliable-gun-logo.jpg',
// provinces: [Province.BC],
// location: 'Vancover',
// },
{
name: 'Tenda',
link: 'gotenda.com',
logo: 'tenda-logo.png',
provinces: [Province.ON],
location: 'Richmond Hill',
},
{
name: 'Canada Ammo',
link: 'canadaammo.com',
logo: 'canada-ammo-logo.jpg',
provinces: [Province.BC, Province.ON],
location: 'all over the place',
},
// 20200119: cert issues
// {
// name: 'Canada Ammo',
// link: 'canadaammo.com',
// logo: 'canada-ammo-logo.jpg',
// provinces: [Province.BC, Province.ON],
// location: 'all over the place',
// },
{
name: 'Wolverine Supplies',
link: 'wolverinesupplies.com',
@@ -3,7 +3,7 @@
*/
import { SQS } from 'aws-sdk'
import { ScheduledEvent } from 'aws-lambda'
import { SOURCES, TYPES } from '../constants'
import { SOURCES, TYPES, AMMO_TYPES } from '../constants'
import { logger } from '../logger'
import { ItemType } from '../graphql-types'
/**
@@ -47,8 +47,23 @@ export async function handler(e: ScheduledEvent) {
type: 'refresh-cache',
roundType: 'all',
})

/**
* 20200119 most of AWS bill is DynamoDB, and 97% of the cost is due to writes.
* since most people are only looking at centerfire, reduce refresh rate for others...
*/
const today = new Date()
let types: ItemType[]
if (today.getUTCDay() === 1) {
types = TYPES
} else if (today.getUTCDay() % 3 === 0) {
types = AMMO_TYPES
} else {
types = [ItemType.centerfire]
}

return Promise.all(
TYPES.reduce(
types.reduce(
(lst, t) =>
lst.concat(
SOURCES.map(source =>

0 comments on commit 79e762d

Please sign in to comment.
You can’t perform that action at this time.