fix: Merge window and browser queries

Improves performance as events only needs to be fetched, flooded and
afk-filtered once

Author: johan-bjareholt

src/queries.ts

@@ -39,42 +39,6 @@ function canonicalEvents(params: DesktopQueryParams): string {
 
 const default_limit = 100; // Hardcoded limit per group
 
-export function windowQuery(
-  windowbucket,
-  afkbucket,
-  filterAFK,
-  classes,
-  filterCategories: string[][]
-): string[] {
-  windowbucket = windowbucket.replace('"', '\\"');
-  afkbucket = afkbucket.replace('"', '\\"');
-  const params: DesktopQueryParams = {
-    bid_window: windowbucket,
-    bid_afk: afkbucket,
-    classes: classes,
-    filter_afk: filterAFK,
-  };
-  const code =
-    `
-      ${canonicalEvents(params)}
-      events = categorize(events, ${JSON.stringify(params.classes)});
-    ` +
-    (filterCategories
-      ? `events = filter_keyvals(events, "$category", ${JSON.stringify(filterCategories)});`
-      : '') +
-    `
-    title_events = sort_by_duration(merge_events_by_keys(events, ["app", "title"]));
-    app_events   = sort_by_duration(merge_events_by_keys(title_events, ["app"]));
-    cat_events   = sort_by_duration(merge_events_by_keys(events, ["$category"]));
-
-    events = sort_by_timestamp(events);
-    app_events  = limit_events(app_events, ${default_limit});
-    title_events  = limit_events(title_events, ${default_limit});
-    duration = sum_durations(events);
-    RETURN  = {"app_events": app_events, "title_events": title_events, "cat_events": cat_events, "duration": duration, "active_events": not_afk};`;
-  return querystr_to_array(code);
-}
-
 export function appQuery(
   appbucket: string,
   classes,
@@ -109,7 +73,7 @@ export function appQuery(
   return querystr_to_array(code);
 }
 
-const appnames = {
+const browser_appnames = {
   chrome: [
     'Google-chrome',
     'chrome.exe',
@@ -140,7 +104,7 @@ const appnames = {
 // Returns a list of (browserName, bucketId) pairs for found browser buckets
 function browsersWithBuckets(browserbuckets: string[]): [string, string][] {
   const browsername_to_bucketid: [string, string | undefined][] = _.map(
-    Object.keys(appnames),
+    Object.keys(browser_appnames),
     browserName => {
       const bucketId = _.find(browserbuckets, bucket_id => _.includes(bucket_id, browserName));
       return [browserName, bucketId];
@@ -152,35 +116,28 @@ function browsersWithBuckets(browserbuckets: string[]): [string, string][] {
 
 // Returns a list of active browser events (where the browser was the active window) from all browser buckets
 function browserEvents(params: DesktopQueryParams): string {
-  // If multiple browser buckets were found
-  // AFK filtered later in the process
   let code = `
-    ${canonicalEvents({ ...params, filter_afk: false })}
-    window = events;
-    events = [];
+    browser_events = [];
   `;
 
   _.each(browsersWithBuckets(params.bid_browsers), ([browserName, bucketId]) => {
-    const appnames_str = JSON.stringify(appnames[browserName]);
+    const browser_appnames_str = JSON.stringify(browser_appnames[browserName]);
     code += `events_${browserName} = flood(query_bucket("${bucketId}"));
-       window_${browserName} = filter_keyvals(window, "app", ${appnames_str});
-       ${
-         params.filter_afk
-           ? `window_${browserName} = filter_period_intersect(window_${browserName}, not_afk);`
-           : ''
-       }
+       window_${browserName} = filter_keyvals(events, "app", ${browser_appnames_str});
        events_${browserName} = filter_period_intersect(events_${browserName}, window_${browserName});
        events_${browserName} = split_url_events(events_${browserName});
-       events = sort_by_timestamp(concat(events, events_${browserName}));`;
+       browser_events = sort_by_timestamp(concat(browser_events, events_${browserName}));`;
   });
   return code;
 }
 
-export function browserSummaryQuery(
+export function fullDesktopQuery(
   browserbuckets: string[],
   windowbucket: string,
   afkbucket: string,
-  filterAFK = true
+  filterAFK = true,
+  classes,
+  filterCategories: string[][]
 ): string[] {
   // Escape `"`
   browserbuckets = _.map(browserbuckets, b => b.replace('"', '\\"'));
@@ -192,21 +149,51 @@ export function browserSummaryQuery(
     bid_window: windowbucket,
     bid_afk: afkbucket,
     bid_browsers: browserbuckets,
-    classes: {},
+    classes: classes,
     filter_afk: filterAFK,
   };
 
   return querystr_to_array(
-    `${browserEvents(params)}
-    urls = merge_events_by_keys(events, ["url"]);
-    urls = sort_by_duration(urls);
-    urls = limit_events(urls, ${default_limit});
-    domains = split_url_events(events);
-    domains = merge_events_by_keys(domains, ["$domain"]);
-    domains = sort_by_duration(domains);
-    domains = limit_events(domains, ${default_limit});
+    `
+    ${canonicalEvents(params)}
+    events = categorize(events, ${JSON.stringify(params.classes)});
+    ` +
+    (filterCategories
+      ? `events = filter_keyvals(events, "$category", ${JSON.stringify(filterCategories)});`
+      : '') +
+    `
+    title_events = sort_by_duration(merge_events_by_keys(events, ["app", "title"]));
+    app_events   = sort_by_duration(merge_events_by_keys(title_events, ["app"]));
+    cat_events = [];
+    cat_events   = sort_by_duration(merge_events_by_keys(events, ["$category"]));
+
+    app_events  = limit_events(app_events, ${default_limit});
+    title_events  = limit_events(title_events, ${default_limit});
     duration = sum_durations(events);
-    RETURN = {"domains": domains, "urls": urls, "duration": duration};`
+
+    ${browserEvents(params)}
+    browser_events = split_url_events(browser_events);
+    browser_urls = merge_events_by_keys(browser_events, ["url"]);
+    browser_urls = sort_by_duration(browser_urls);
+    browser_urls = limit_events(browser_urls, ${default_limit});
+    browser_domains = merge_events_by_keys(browser_events, ["$domain"]);
+    browser_domains = sort_by_duration(browser_domains);
+    browser_domains = limit_events(browser_domains, ${default_limit});
+    browser_duration = sum_durations(browser_events);
+
+    RETURN = {
+        "window": {
+            "app_events": app_events,
+            "title_events": title_events,
+            "cat_events": cat_events,
+            "duration": duration
+        },
+        "browser": {
+            "domains": browser_domains,
+            "urls": browser_urls,
+            "duration": browser_duration
+        }
+    };`
   );
 }
 
@@ -234,6 +221,7 @@ export function dailyActivityQuery(afkbucket: string): string[] {
   return [
     'afkbucket = "' + afkbucket + '";',
     'not_afk = flood(query_bucket(afkbucket));',
+    'not_afk = filter_keyvals(not_afk, "status", ["not-afk"]);',
     'not_afk = merge_events_by_keys(not_afk, ["status"]);',
     'RETURN = not_afk;',
   ];
@@ -245,8 +233,7 @@ export function dailyActivityQueryAndroid(androidbucket: string): string[] {
 }
 
 export default {
-  windowQuery,
-  browserSummaryQuery,
+  fullDesktopQuery,
   appQuery,
   dailyActivityQuery,
   dailyActivityQueryAndroid,

src/store/modules/activity.ts

@@ -63,7 +63,6 @@ const _state = {
   active: {
     available: false,
     duration: 0,
-    events: [],
     history: {},
   },
 
@@ -133,22 +132,14 @@ const actions = {
       await dispatch('set_available', query_options);
 
       if (state.window.available) {
-        await dispatch('query_window', query_options);
+        await dispatch('query_desktop_full', query_options);
       } else if (state.android.available) {
         await dispatch('query_android', query_options);
       } else {
         console.log(
           'Cannot query windows as we are missing either an afk/window bucket pair or an android bucket'
         );
         await dispatch('query_window_empty', query_options);
-      }
-
-      if (state.browser.available) {
-        await dispatch('query_browser', query_options);
-      } else {
-        console.log(
-          'Cannot call query_browser as we are missing either an afk, window or browser bucket'
-        );
         await dispatch('query_browser_empty', query_options);
       }
 
@@ -174,22 +165,6 @@ const actions = {
     }
   },
 
-  async query_window({ state, commit }, { timeperiod, filterAFK, filterCategories }: QueryOptions) {
-    const periods = [timeperiodToStr(timeperiod)];
-    const start = moment();
-    const classes = loadClassesForQuery();
-    const q = queries.windowQuery(
-      state.buckets.window[0],
-      state.buckets.afk[0],
-      filterAFK,
-      classes,
-      filterCategories
-    );
-    const data = await this._vm.$aw.query(periods, q);
-    commit('query_window_completed', data[0]);
-    console.info(`Completed window query in ${moment().diff(start)}ms`);
-  },
-
   async query_android({ state, commit }, { timeperiod, filterCategories }: QueryOptions) {
     const periods = [timeperiodToStr(timeperiod)];
     const classes = loadClassesForQuery();
@@ -204,21 +179,27 @@ const actions = {
       title_events: [],
       cat_events: [],
       duration: 0,
-      active_events: [],
     };
     commit('query_window_completed', data);
   },
 
-  async query_browser({ state, commit }, { timeperiod, filterAFK }: QueryOptions) {
+  async query_desktop_full(
+    { state, commit },
+    { timeperiod, filterCategories, filterAFK }: QueryOptions)
+  {
     const periods = [timeperiodToStr(timeperiod)];
-    const q = queries.browserSummaryQuery(
+    const classes = loadClassesForQuery();
+    const q = queries.fullDesktopQuery(
       state.buckets.browser,
       state.buckets.window[0],
       state.buckets.afk[0],
-      filterAFK
+      filterAFK,
+      classes,
+      filterCategories
     );
     const data = await this._vm.$aw.query(periods, q);
-    commit('query_browser_completed', data[0]);
+    commit('query_browser_completed', data[0].browser);
+    commit('query_window_completed', data[0].window);
   },
 
   async query_browser_empty({ commit }) {
@@ -430,9 +411,6 @@ const actions = {
       app_events,
       title_events,
       cat_events,
-      active_events: [
-        { timestamp: new Date().toISOString(), duration: 1.5 * 60 * 60, data: { afk: 'not-afk' } },
-      ],
     });
 
     commit('browser_buckets', ['aw-watcher-firefox']);
@@ -492,7 +470,6 @@ const mutations = {
     state.category.top = null;
 
     state.active.duration = null;
-    state.active.events = null;
 
     // Ensures that active history isn't being fully reloaded on every date change
     // (see caching done in query_active_history and query_active_history_android)
@@ -516,13 +493,12 @@ const mutations = {
     state.window.top_titles = data['title_events'];
     state.category.top = data['cat_events'];
     state.active.duration = data['duration'];
-    state.active.events = data['active_events'];
   },
 
   query_browser_completed(state, data) {
-    state.browser.top_domains = data['domains'];
-    state.browser.top_urls = data['urls'];
-    state.browser.duration = data['duration'];
+    state.browser.top_domains = data.domains;
+    state.browser.top_urls = data.urls;
+    state.browser.duration = data.duration;
 
     // FIXME: This one might take up a lot of size in the request, move it to a seperate request
     // (or remove entirely, since we have the other timeline now)