Skip to content
Permalink
Browse files

feat(apps/user-access): aggressively cache user access

* cache user access by login
* invalidate user access based on received webhook events

Related to #80
  • Loading branch information
nikku committed Nov 17, 2019
1 parent ad782de commit de5403da7eb7dc5dde71ded3add6f08e1ed7ff27
Showing with 262 additions and 33 deletions.
  1. +117 −0 packages/app/lib/apps/user-access/TreeCache.js
  2. +145 −33 packages/app/lib/apps/user-access/UserAccess.js
@@ -0,0 +1,117 @@
/**
* A cache that allows tree-like keys
*/
class TreeCache {

constructor(ttl = -1) {
this._cache = {};
this._accessed = {};

this._ttl = ttl;
}

async get(key, defaultValue) {

this._access(key);

if (key in this._cache) {
return this._cache[key];
}

if (typeof defaultValue === 'function') {
defaultValue = await defaultValue(key);
}

this.set(key, defaultValue);

return defaultValue;
}

set(key, value) {
this._cache[key] = value;
}

_access(key) {
this._accessed[key] = Date.now();
}

invalidate(key) {

const matches = this.match(key);

for (const match of matches) {
this.remove(match.key);
}
}

match(key) {

let test;

if (key.includes('*')) {
const pattern = new RegExp('^' + key.split('*').map(regexEscape).join('([^:]+)') + '$');

test = (str) => pattern.exec(str);
} else {
test = (str) => str === key;
}

const matches = [];

for (const cacheKey of Object.keys(this._cache)) {

const match = test(cacheKey);

if (!match) {
continue;
}

if (match === true) {
matches.push({
key: cacheKey,
value: this._cache[cacheKey],
match: []
});
} else {

const [ _, ...matchedPatterns ] = match;

matches.push({
key: cacheKey,
value: this._cache[cacheKey],
match: matchedPatterns
});
}
}

return matches;
}

remove(key) {
delete this._cache[key];
delete this._accessed[key];
}

evict(now = Date.now()) {

const evictBefore = now - this._ttl;

for (const [ key, accessed ] of Object.entries(this._accessed)) {

if (accessed < evictBefore) {
this.remove(key);
}
}

}

}

module.exports = TreeCache;


// helpers ///////////////

function regexEscape(s) {
return s.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
}
@@ -1,9 +1,7 @@
const {
Cache
} = require('../../util');
const TreeCache = require('./TreeCache');

// 30 minutes
const TTL = 1000 * 60 * 30;
// 9 days
const TTL = 1000 * 60 * 60 * 24 * 9;


/**
@@ -13,16 +11,17 @@ const TTL = 1000 * 60 * 30;
* @param {Logger} logger
* @param {GitHubClient} githubClient
* @param {Events} events
* @param {WebhookEvents} webhookEvents
*/
function UserAccess(logger, githubClient, events) {
function UserAccess(logger, githubClient, events, webhookEvents) {

const log = logger.child({
name: 'wuffle:user-access'
});

const cache = new Cache(TTL);
const cache = new TreeCache(TTL);

function getRepository(issue) {
function getIssueRepository(issue) {
const {
key,
repository
@@ -35,32 +34,62 @@ function UserAccess(logger, githubClient, events) {
return repository;
}

async function fetchUserRepositories(token) {
async function fetchUserInstallations(user) {

const github = await githubClient.getUserScoped(token);
const github = await githubClient.getUserScoped(user);

const installations = await github.paginate(
return github.paginate(
github.apps.listInstallationsForAuthenticatedUser.endpoint.merge({}),
res => res.data
);
}

async function fetchUserRepositories(user) {
const installations = await getUserInstallations(user);

const repositoriesByInstallation = await Promise.all(installations.map(
installation => github.paginate(
github.apps.listInstallationReposForAuthenticatedUser.endpoint.merge({
installation_id: installation.id
}),
res => res.data
)
));
const repositoriesByInstallation = await Promise.all(
installations.map(installation => getUserRepositoriesForInstallation(user, installation))
);

return [].concat(...repositoriesByInstallation);
}

function getUserVisibleRepositoryNames(token) {
async function fetchUserRepositoriesForInstallation(user, installation) {

const github = await githubClient.getUserScoped(user);

return await github.paginate(
github.apps.listInstallationReposForAuthenticatedUser.endpoint.merge({
installation_id: installation.id
}),
res => res.data
);
}

function getUserRepositoriesForInstallation(user, installation) {

return cache.get(`login=${user.login}:installation_repositories=${installation.id}`, () => {
return fetchUserRepositoriesForInstallation(user, installation);
});
}

function getUserInstallations(user) {

return cache.get(`login=${user.login}:installations`, () => {
return fetchUserInstallations(user);
});

return cache.get(`user-repositories:${token}`, () => {
return fetchUserRepositories(token);
}).then(repositories => repositories.map(repo => {
}

function getUserRepositories(user) {

return cache.get(`login=${user.login}:repositories`, () => {
return fetchUserRepositories(user);
});
}

function getUserVisibleRepositoryNames(user) {
return getUserRepositories(user).then(repositories => repositories.map(repo => {
return repo.full_name;
}));
}
@@ -69,7 +98,7 @@ function UserAccess(logger, githubClient, events) {
* Show publicly accessible issues only.
*/
function filterPublic(issue) {
return !getRepository(issue).private;
return !getIssueRepository(issue).private;
}

/**
@@ -91,24 +120,23 @@ function UserAccess(logger, githubClient, events) {
return true;
}

const repository = getRepository(issue);
const repository = getIssueRepository(issue);

return fullName(repository) in repositoryMap;
};
}

function createReadFilter(user) {

const t = Date.now();

const {
login,
access_token
login
} = user;

const t = Date.now();

log.debug({ login }, 'creating read filter');

return getUserVisibleRepositoryNames(access_token).then(repositoryNames => {
return getUserVisibleRepositoryNames(user).then(repositoryNames => {

log.debug({
login,
@@ -132,8 +160,8 @@ function UserAccess(logger, githubClient, events) {
login
} = user;

return cache.get(`${login}:read-filter`, () => createReadFilter(user)).catch(err => {
log.warn({ login }, 'failed to retrieve token-based access filter, defaulting to public read', err);
return cache.get(`login=${login}:read_filter`, () => createReadFilter(user)).catch(err => {
log.warn({ login }, 'failed to create read filter, defaulting to public read', err);

return filterPublic;
});
@@ -188,10 +216,94 @@ function UserAccess(logger, githubClient, events) {
events.once('wuffle.start', function() {
setInterval(() => {
cache.evict();
}, 1000 * 10);
}, 1000 * 60 * 15);
});
}


// https://developer.github.com/v3/activity/events/types/#githubappauthorizationevent

webhookEvents.on([
'github_app_authorization.revoked'
], (context) => {

const {
sender: {
login
}
} = context.payload;

cache.invalidate(`login=${login}:*`);
});


// https://developer.github.com/v3/activity/events/types/#installationrepositoriesevent

webhookEvents.on([
'installation_repositories.added',
'installation_repositories.removed'
], function(context) {

const {
installation: {
id: installation_id
}
} = context.payload;

const installationMatches = cache.match('login=*:installations');

for (const match of installationMatches) {

const {
match: [ login ],
value: installations
} = match;

const isInstallationMember = installations.find(
installation => installation.id === installation_id
);

if (isInstallationMember) {
cache.invalidate(`login=${login}:installation_repositories=${installation_id}`);
cache.invalidate(`login=${login}:repositories`);
cache.invalidate(`login=${login}:read_filter`);
}
}
});


// https://developer.github.com/v3/activity/events/types/#installationevent

webhookEvents.on([
'installation.created',
'installation.deleted'
], function(context) {

cache.invalidate('login=*:installations');
cache.invalidate('login=*:installation_repositories=*');
cache.invalidate('login=*:repositories');
cache.invalidate('login=*:read_filter');
});


// https://developer.github.com/v3/activity/events/types/#memberevent

webhookEvents.on([
'member'
], function(context) {

const {
member: {
login
}
} = context.payload;

cache.invalidate(`login=${login}:installations`);
cache.invalidate(`login=${login}:installation_repositories=*`);
cache.invalidate(`login=${login}:repositories`);
cache.invalidate(`login=${login}:read_filter`);
});

}

module.exports = UserAccess;

0 comments on commit de5403d

Please sign in to comment.
You can’t perform that action at this time.