Skip to content

Commit

Permalink
fix: Merge window and browser queries
Browse files Browse the repository at this point in the history 
Improves performance as events only needs to be fetched, flooded and
afk-filtered once
  • Loading branch information
@johan-bjareholt
johan-bjareholt committed May 31, 2020
1 parent df5a625 commit f9bcaed
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 104 deletions.
117 changes: 52 additions & 65 deletions src/queries.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,42 +39,6 @@ function canonicalEvents(params: DesktopQueryParams): string {

const default_limit = 100; // Hardcoded limit per group

export function windowQuery(
windowbucket,
afkbucket,
filterAFK,
classes,
filterCategories: string[][]
): string[] {
windowbucket = windowbucket.replace('"', '\\"');
afkbucket = afkbucket.replace('"', '\\"');
const params: DesktopQueryParams = {
bid_window: windowbucket,
bid_afk: afkbucket,
classes: classes,
filter_afk: filterAFK,
};
const code =
`
${canonicalEvents(params)}
events = categorize(events, ${JSON.stringify(params.classes)});
` +
(filterCategories
? `events = filter_keyvals(events, "$category", ${JSON.stringify(filterCategories)});`
: '') +
`
title_events = sort_by_duration(merge_events_by_keys(events, ["app", "title"]));
app_events = sort_by_duration(merge_events_by_keys(title_events, ["app"]));
cat_events = sort_by_duration(merge_events_by_keys(events, ["$category"]));
events = sort_by_timestamp(events);
app_events = limit_events(app_events, ${default_limit});
title_events = limit_events(title_events, ${default_limit});
duration = sum_durations(events);
RETURN = {"app_events": app_events, "title_events": title_events, "cat_events": cat_events, "duration": duration, "active_events": not_afk};`;
return querystr_to_array(code);
}

export function appQuery(
appbucket: string,
classes,
Expand Down Expand Up @@ -109,7 +73,7 @@ export function appQuery(
return querystr_to_array(code);
}

const appnames = {
const browser_appnames = {
chrome: [
'Google-chrome',
'chrome.exe',
Expand Down Expand Up @@ -140,7 +104,7 @@ const appnames = {
// Returns a list of (browserName, bucketId) pairs for found browser buckets
function browsersWithBuckets(browserbuckets: string[]): [string, string][] {
const browsername_to_bucketid: [string, string | undefined][] = _.map(
Object.keys(appnames),
Object.keys(browser_appnames),
browserName => {
const bucketId = _.find(browserbuckets, bucket_id => _.includes(bucket_id, browserName));
return [browserName, bucketId];
Expand All @@ -152,35 +116,28 @@ function browsersWithBuckets(browserbuckets: string[]): [string, string][] {

// Returns a list of active browser events (where the browser was the active window) from all browser buckets
function browserEvents(params: DesktopQueryParams): string {
// If multiple browser buckets were found
// AFK filtered later in the process
let code = `
${canonicalEvents({ ...params, filter_afk: false })}
window = events;
events = [];
browser_events = [];
`;

_.each(browsersWithBuckets(params.bid_browsers), ([browserName, bucketId]) => {
const appnames_str = JSON.stringify(appnames[browserName]);
const browser_appnames_str = JSON.stringify(browser_appnames[browserName]);
code += `events_${browserName} = flood(query_bucket("${bucketId}"));
window_${browserName} = filter_keyvals(window, "app", ${appnames_str});
${
params.filter_afk
? `window_${browserName} = filter_period_intersect(window_${browserName}, not_afk);`
: ''
}
window_${browserName} = filter_keyvals(events, "app", ${browser_appnames_str});
events_${browserName} = filter_period_intersect(events_${browserName}, window_${browserName});
events_${browserName} = split_url_events(events_${browserName});
events = sort_by_timestamp(concat(events, events_${browserName}));`;
browser_events = sort_by_timestamp(concat(browser_events, events_${browserName}));`;
});
return code;
}

export function browserSummaryQuery(
export function fullDesktopQuery(
browserbuckets: string[],
windowbucket: string,
afkbucket: string,
filterAFK = true
filterAFK = true,
classes,
filterCategories: string[][]
): string[] {
// Escape `"`
browserbuckets = _.map(browserbuckets, b => b.replace('"', '\\"'));
Expand All @@ -192,21 +149,51 @@ export function browserSummaryQuery(
bid_window: windowbucket,
bid_afk: afkbucket,
bid_browsers: browserbuckets,
classes: {},
classes: classes,
filter_afk: filterAFK,
};

return querystr_to_array(
`${browserEvents(params)}
urls = merge_events_by_keys(events, ["url"]);
urls = sort_by_duration(urls);
urls = limit_events(urls, ${default_limit});
domains = split_url_events(events);
domains = merge_events_by_keys(domains, ["$domain"]);
domains = sort_by_duration(domains);
domains = limit_events(domains, ${default_limit});
`
${canonicalEvents(params)}
events = categorize(events, ${JSON.stringify(params.classes)});
` +
(filterCategories
? `events = filter_keyvals(events, "$category", ${JSON.stringify(filterCategories)});`
: '') +
`
title_events = sort_by_duration(merge_events_by_keys(events, ["app", "title"]));
app_events = sort_by_duration(merge_events_by_keys(title_events, ["app"]));
cat_events = [];
cat_events = sort_by_duration(merge_events_by_keys(events, ["$category"]));
app_events = limit_events(app_events, ${default_limit});
title_events = limit_events(title_events, ${default_limit});
duration = sum_durations(events);
RETURN = {"domains": domains, "urls": urls, "duration": duration};`
${browserEvents(params)}
browser_events = split_url_events(browser_events);
browser_urls = merge_events_by_keys(browser_events, ["url"]);
browser_urls = sort_by_duration(browser_urls);
browser_urls = limit_events(browser_urls, ${default_limit});
browser_domains = merge_events_by_keys(browser_events, ["$domain"]);
browser_domains = sort_by_duration(browser_domains);
browser_domains = limit_events(browser_domains, ${default_limit});
browser_duration = sum_durations(browser_events);
RETURN = {
"window": {
"app_events": app_events,
"title_events": title_events,
"cat_events": cat_events,
"duration": duration
},
"browser": {
"domains": browser_domains,
"urls": browser_urls,
"duration": browser_duration
}
};`
);
}

Expand Down Expand Up @@ -234,6 +221,7 @@ export function dailyActivityQuery(afkbucket: string): string[] {
return [
'afkbucket = "' + afkbucket + '";',
'not_afk = flood(query_bucket(afkbucket));',
'not_afk = filter_keyvals(not_afk, "status", ["not-afk"]);',
'not_afk = merge_events_by_keys(not_afk, ["status"]);',
'RETURN = not_afk;',
];
Expand All @@ -245,8 +233,7 @@ export function dailyActivityQueryAndroid(androidbucket: string): string[] {
}

export default {
windowQuery,
browserSummaryQuery,
fullDesktopQuery,
appQuery,
dailyActivityQuery,
dailyActivityQueryAndroid,
Expand Down
54 changes: 15 additions & 39 deletions src/store/modules/activity.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ const _state = {
active: {
available: false,
duration: 0,
events: [],
history: {},
},

Expand Down Expand Up @@ -133,22 +132,14 @@ const actions = {
await dispatch('set_available', query_options);

if (state.window.available) {
await dispatch('query_window', query_options);
await dispatch('query_desktop_full', query_options);
} else if (state.android.available) {
await dispatch('query_android', query_options);
} else {
console.log(
'Cannot query windows as we are missing either an afk/window bucket pair or an android bucket'
);
await dispatch('query_window_empty', query_options);
}

if (state.browser.available) {
await dispatch('query_browser', query_options);
} else {
console.log(
'Cannot call query_browser as we are missing either an afk, window or browser bucket'
);
await dispatch('query_browser_empty', query_options);
}

Expand All @@ -174,22 +165,6 @@ const actions = {
}
},

async query_window({ state, commit }, { timeperiod, filterAFK, filterCategories }: QueryOptions) {
const periods = [timeperiodToStr(timeperiod)];
const start = moment();
const classes = loadClassesForQuery();
const q = queries.windowQuery(
state.buckets.window[0],
state.buckets.afk[0],
filterAFK,
classes,
filterCategories
);
const data = await this._vm.$aw.query(periods, q);
commit('query_window_completed', data[0]);
console.info(`Completed window query in ${moment().diff(start)}ms`);
},

async query_android({ state, commit }, { timeperiod, filterCategories }: QueryOptions) {
const periods = [timeperiodToStr(timeperiod)];
const classes = loadClassesForQuery();
Expand All @@ -204,21 +179,27 @@ const actions = {
title_events: [],
cat_events: [],
duration: 0,
active_events: [],
};
commit('query_window_completed', data);
},

async query_browser({ state, commit }, { timeperiod, filterAFK }: QueryOptions) {
async query_desktop_full(
{ state, commit },
{ timeperiod, filterCategories, filterAFK }: QueryOptions)
{
const periods = [timeperiodToStr(timeperiod)];
const q = queries.browserSummaryQuery(
const classes = loadClassesForQuery();
const q = queries.fullDesktopQuery(
state.buckets.browser,
state.buckets.window[0],
state.buckets.afk[0],
filterAFK
filterAFK,
classes,
filterCategories
);
const data = await this._vm.$aw.query(periods, q);
commit('query_browser_completed', data[0]);
commit('query_browser_completed', data[0].browser);
commit('query_window_completed', data[0].window);
},

async query_browser_empty({ commit }) {
Expand Down Expand Up @@ -430,9 +411,6 @@ const actions = {
app_events,
title_events,
cat_events,
active_events: [
{ timestamp: new Date().toISOString(), duration: 1.5 * 60 * 60, data: { afk: 'not-afk' } },
],
});

commit('browser_buckets', ['aw-watcher-firefox']);
Expand Down Expand Up @@ -492,7 +470,6 @@ const mutations = {
state.category.top = null;

state.active.duration = null;
state.active.events = null;

// Ensures that active history isn't being fully reloaded on every date change
// (see caching done in query_active_history and query_active_history_android)
Expand All @@ -516,13 +493,12 @@ const mutations = {
state.window.top_titles = data['title_events'];
state.category.top = data['cat_events'];
state.active.duration = data['duration'];
state.active.events = data['active_events'];
},

query_browser_completed(state, data) {
state.browser.top_domains = data['domains'];
state.browser.top_urls = data['urls'];
state.browser.duration = data['duration'];
state.browser.top_domains = data.domains;
state.browser.top_urls = data.urls;
state.browser.duration = data.duration;

// FIXME: This one might take up a lot of size in the request, move it to a seperate request
// (or remove entirely, since we have the other timeline now)
Expand Down

0 comments on commit f9bcaed

Please sign in to comment.