From 2ee650c7c7bfb9e8d63f4dc02ae67485e11ff499 Mon Sep 17 00:00:00 2001 From: Emerick Rogul Date: Mon, 18 Jan 2021 11:00:59 -0500 Subject: [PATCH 1/3] Limit calls to Twitter API --- scripts/brave_rewards/publisher/twitter/api.ts | 15 ++++++++++++--- scripts/brave_rewards/publisher/twitter/auth.ts | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/brave_rewards/publisher/twitter/api.ts b/scripts/brave_rewards/publisher/twitter/api.ts index 753b6e3..8719829 100644 --- a/scripts/brave_rewards/publisher/twitter/api.ts +++ b/scripts/brave_rewards/publisher/twitter/api.ts @@ -3,10 +3,12 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ import { getPort } from '../common/messaging' -import { getAuthHeaders } from './auth' +import { getAuthHeaders, hasRequiredAuthHeaders } from './auth' import * as types from './types' +let lastRequestTime = 0 + const sendAPIRequest = (name: string, url: string) => { return new Promise((resolve, reject) => { if (!name || !url) { @@ -14,8 +16,7 @@ const sendAPIRequest = (name: string, url: string) => { return } - const authHeaders = getAuthHeaders() - if (Object.keys(authHeaders).length === 0) { + if (!hasRequiredAuthHeaders()) { reject(new Error('Missing auth headers')) return } @@ -26,6 +27,14 @@ const sendAPIRequest = (name: string, url: string) => { return } + if ((lastRequestTime !== 0) && (Date.now() - lastRequestTime < 3000)) { + reject(new Error('Ignoring API request due to network throttle')) + return + } + + lastRequestTime = Date.now() + + const authHeaders = getAuthHeaders() port.postMessage({ type: 'OnAPIRequest', mediaType: types.mediaType, diff --git a/scripts/brave_rewards/publisher/twitter/auth.ts b/scripts/brave_rewards/publisher/twitter/auth.ts index 13bacff..2371dc8 100644 --- a/scripts/brave_rewards/publisher/twitter/auth.ts +++ b/scripts/brave_rewards/publisher/twitter/auth.ts @@ -35,6 +35,11 @@ export const getAuthHeaders = () => { return authHeaders } +export const hasRequiredAuthHeaders = () => { + return authHeaders['authorization'] && + (authHeaders['x-csrf-token'] || authHeaders['x-guest-token']) +} + export const processRequestHeaders = (requestHeaders: any[]) => { if (!requestHeaders) { return false From 675bc918d3da3cd7c19cff487c27430bfd0494c7 Mon Sep 17 00:00:00 2001 From: Emerick Rogul Date: Mon, 18 Jan 2021 11:01:39 -0500 Subject: [PATCH 2/3] X-Twitter-User should always be true --- scripts/brave_rewards/publisher/twitter/auth.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/brave_rewards/publisher/twitter/auth.ts b/scripts/brave_rewards/publisher/twitter/auth.ts index 2371dc8..360a8e1 100644 --- a/scripts/brave_rewards/publisher/twitter/auth.ts +++ b/scripts/brave_rewards/publisher/twitter/auth.ts @@ -63,6 +63,11 @@ export const processRequestHeaders = (requestHeaders: any[]) => { } } + // For our purposes (authentication), we want this always to be 'yes' + if (headers['x-twitter-active-user'] !== 'yes') { + headers['x-twitter-active-user'] = 'yes' + } + if (commonUtils.areObjectsEqualShallow(authHeaders, headers)) { return false } From 8113e27844382a1b9325cc8291cbe0cfd47f8cef Mon Sep 17 00:00:00 2001 From: Emerick Rogul Date: Mon, 18 Jan 2021 11:02:27 -0500 Subject: [PATCH 3/3] Implement simple LRU cache for publisher info lookups --- .../publisher/common/lruCache.ts | 38 ++++++++++ .../publisher/twitter/publisherInfo.ts | 70 ++++++++++++------- 2 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 scripts/brave_rewards/publisher/common/lruCache.ts diff --git a/scripts/brave_rewards/publisher/common/lruCache.ts b/scripts/brave_rewards/publisher/common/lruCache.ts new file mode 100644 index 0000000..643ce36 --- /dev/null +++ b/scripts/brave_rewards/publisher/common/lruCache.ts @@ -0,0 +1,38 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +export class LruCache { + + private values: Map = new Map() + private maxEntries: number + + constructor (maxEntries: number) { + this.maxEntries = maxEntries + } + + get (key: string): T | null { + if (!key) { + return null + } + + const entry = this.values.get(key) + if (!entry) { + return null + } + + this.values.delete(key) + this.values.set(key, entry) + + return entry + } + + put (key: string, value: T) { + if (this.values.size >= this.maxEntries) { + const key = this.values.keys().next().value + this.values.delete(key) + } + + this.values.set(key, value) + } +} diff --git a/scripts/brave_rewards/publisher/twitter/publisherInfo.ts b/scripts/brave_rewards/publisher/twitter/publisherInfo.ts index bb86bf0..ada03ba 100644 --- a/scripts/brave_rewards/publisher/twitter/publisherInfo.ts +++ b/scripts/brave_rewards/publisher/twitter/publisherInfo.ts @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ +import { LruCache } from '../common/lruCache' import { getPort } from '../common/messaging' import * as commonUtils from '../common/utils' @@ -10,6 +11,41 @@ import * as api from './api' import * as types from './types' import * as utils from './utils' +const userCache = new LruCache(128) + +const savePublisherVisit = (screenName: string, userDetails: any) => { + if (!screenName || !userDetails) { + return + } + + const userId = userDetails.id_str + const publisherKey = + commonUtils.buildPublisherKey(types.mediaType, userId) + const publisherName = screenName + const mediaKey = commonUtils.buildMediaKey(types.mediaType, screenName) + const favIconUrl = + userDetails.profile_image_url_https.replace('_normal', '') + + const profileUrl = utils.buildProfileUrl(screenName, userId) + + const port = getPort() + if (!port) { + return + } + + port.postMessage({ + type: 'SavePublisherVisit', + mediaType: types.mediaType, + data: { + url: profileUrl, + publisherKey, + publisherName, + mediaKey, + favIconUrl + } + }) +} + const sendForExcludedPage = () => { const url = `https://${types.mediaDomain}` const publisherKey = types.mediaDomain @@ -41,34 +77,16 @@ const sendForStandardPage = (url: URL) => { return } + const userDetails = userCache.get(screenName) + if (userDetails) { + savePublisherVisit(screenName, userDetails) + return + } + api.getUserDetails(screenName) .then((userDetails: any) => { - const userId = userDetails.id_str - const publisherKey = - commonUtils.buildPublisherKey(types.mediaType, userId) - const publisherName = screenName - const mediaKey = commonUtils.buildMediaKey(types.mediaType, screenName) - const favIconUrl = - userDetails.profile_image_url_https.replace('_normal', '') - - const profileUrl = utils.buildProfileUrl(screenName, userId) - - const port = getPort() - if (!port) { - return - } - - port.postMessage({ - type: 'SavePublisherVisit', - mediaType: types.mediaType, - data: { - url: profileUrl, - publisherKey, - publisherName, - mediaKey, - favIconUrl - } - }) + userCache.put(screenName, userDetails) + savePublisherVisit(screenName, userDetails) }) .catch(error => { console.error(`Failed to fetch user details for ${screenName}: ${error.message}`)