Skip to content

Commit

Permalink
fix: refactor/cleanup of queries & activity store
Browse files Browse the repository at this point in the history
  • Loading branch information
@ErikBjare
ErikBjare committed Jul 6, 2022
1 parent 1dfc290 commit c8a968b
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 124 deletions.
180 changes: 98 additions & 82 deletions src/queries.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,22 @@ function escape_doublequote(s: string) {
return s.replace(/"/g, '\\"');
}

// Hostname safe for using as a variable name
function safeHostname(hostname: string): string {
return hostname.replace(/[^a-zA-Z0-9_]/g, '');
}

interface Rule {
type: string;
regex?: string;
}

type Category = [string[], Rule];

interface BaseQueryParams {
include_audible?: boolean;
classes: [string[], Rule][];
filter_classes: string[][];
categories: Category[];
filter_categories: string[][];
bid_browsers?: string[];
return_variable?: string;
}
Expand All @@ -37,6 +44,42 @@ interface AndroidQueryParams extends BaseQueryParams {
bid_android: string;
}

interface MultiQueryParams extends BaseQueryParams {
hosts: string[];
filter_afk: boolean;
// This can be used to override params on a per-host basis
host_params: { [host: string]: DesktopQueryParams | AndroidQueryParams };
}

function get_params(
params: MultiQueryParams,
host: string
): DesktopQueryParams | AndroidQueryParams {
// Return the params for a given host, based on the self params and any overrides in host_params.
// If no overrides are found, return the base params.
const new_params: DesktopQueryParams = {
...params,
bid_window: 'aw-watcher-window_' + host,
bid_afk: 'aw-watcher-afk_' + host,
bid_browsers: [],
return_variable: 'events_' + safeHostname(host),
};

const host_params = params.host_params[host];
if (host_params) {
if (!isDesktopParams(host_params)) {
console.error(`Invalid host_params for host ${host}: ${JSON.stringify(host_params)}`);
}
// Only override the params if they are defined and set to a truthy value
Object.keys(host_params).forEach(key => {
if (host_params[key] && host_params[key].length > 0) {
new_params[key] = host_params[key];
}
});
}
return new_params;
}

function isDesktopParams(object: any): object is DesktopQueryParams {
return 'bid_window' in object;
}
Expand All @@ -45,16 +88,20 @@ function isAndroidParams(object: any): object is AndroidQueryParams {
return 'bid_android' in object;
}

function isMultiParams(object: any): object is MultiQueryParams {
return 'hosts' in object;
}

// Constructs a query that returns a fully-detailed list of events from the merging of several sources (window, afk, web).
// Performs:
// - AFK filtering (if filter_afk is true)
// - Categorization (if classes specified)
// - Filters by category (if filter_classes set)
// - Categorization (if categories specified)
// - Filters by category (if filter_categories set)
// Puts it's results in `events` and `not_afk` (if not_afk available for platform).
export function canonicalEvents(params: DesktopQueryParams | AndroidQueryParams): string {
// Needs escaping for regex patterns like '\w' to work (JSON.stringify adds extra unecessary escaping)
const classes_str = JSON.stringify(params.classes).replace(/\\\\/g, '\\');
const cat_filter_str = JSON.stringify(params.filter_classes);
const categories_str = JSON.stringify(params.categories).replace(/\\\\/g, '\\');
const cat_filter_str = JSON.stringify(params.filter_categories);

// For simplicity, we assume that bid_window and bid_android are exchangeable (note however it needs special treatment)
const bid_window = isDesktopParams(params) ? params.bid_window : params.bid_android;
Expand Down Expand Up @@ -84,26 +131,48 @@ export function canonicalEvents(params: DesktopQueryParams | AndroidQueryParams)
? 'events = filter_period_intersect(events, not_afk);'
: '',
// Categorize
params.classes ? `events = categorize(events, ${classes_str});` : '',
params.categories ? `events = categorize(events, ${categories_str});` : '',
// Filter out selected categories
params.filter_classes ? `events = filter_keyvals(events, "$category", ${cat_filter_str});` : '',
params.filter_categories
? `events = filter_keyvals(events, "$category", ${cat_filter_str});`
: '',
// "Return" events by setting variable named with return_variable if set
params.return_variable ? `${params.return_variable} = events;` : '',
].join('\n');
}

export function canonicalMultideviceEvents(params: MultiQueryParams): string {
// First, query each device individually
const queries: string[] = _.map(params.hosts, hostname => {
return canonicalEvents(get_params(params, hostname));
});

// Now we need to combine the queries to get a single series of events.
// To do this, we can use the union_no_overlap function, which merges events
// but avoids overlaps by giving priority according to the order of hosts.
let query = queries.join('\n');
if (queries.length > 1) {
query += 'events = [];';
for (let i = 0; i < queries.length; i++) {
query += `events = union_no_overlap(events, events_${safeHostname(params.hosts[i])});`;
}
}

return query;
}

const default_limit = 100; // Hardcoded limit per group

export function appQuery(
appbucket: string,
classes: [string[], Rule][],
filterCategories: string[][]
categories: Category[],
filter_categories: string[][]
): string[] {
appbucket = escape_doublequote(appbucket);
const params: AndroidQueryParams = {
bid_android: appbucket,
classes: classes,
filter_classes: filterCategories,
categories,
filter_categories,
};

const code = `
Expand Down Expand Up @@ -195,34 +264,16 @@ function browserEvents(params: DesktopQueryParams): string {
return code;
}

export function fullDesktopQuery(
browserbuckets: string[],
windowbucket: string,
afkbucket: string,
filterAFK = true,
classes: [string[], Rule][] = [],
filterCategories: string[][],
include_audible: boolean
): string[] {
// Escape `"`
browserbuckets = _.map(browserbuckets, escape_doublequote);
windowbucket = escape_doublequote(windowbucket);
afkbucket = escape_doublequote(afkbucket);

// TODO: Get classes
const params: DesktopQueryParams = {
bid_window: windowbucket,
bid_afk: afkbucket,
bid_browsers: browserbuckets,
classes: classes,
filter_classes: filterCategories,
filter_afk: filterAFK,
include_audible,
};

export function fullDesktopQuery(params: DesktopQueryParams): string[] {
return querystr_to_array(
`
${canonicalEvents(params)}
${canonicalEvents({
...params,
// Escape `"`
bid_window: escape_doublequote(params.bid_window),
bid_afk: escape_doublequote(params.bid_afk),
bid_browsers: _.map(params.bid_browsers, escape_doublequote),
})}
title_events = sort_by_duration(merge_events_by_keys(events, ["app", "title"]));
app_events = sort_by_duration(merge_events_by_keys(title_events, ["app"]));
cat_events = sort_by_duration(merge_events_by_keys(events, ["$category"]));
Expand Down Expand Up @@ -261,53 +312,18 @@ export function fullDesktopQuery(
// Performs a query that combines data from multiple devices.
// A multidevice-variant of fullDesktopQuery (with limitations).
//
// 1. Performs one canonicalEvents query per device.
// 2. Combines the results into a single list of events using the transform union_no_overlap (which gives priority to events earlier in the list of devices).
// 1. Performs one canonicalEvents query per device.
// 2. Combines the results into a single list of events using the transform union_no_overlap (which gives priority to events earlier in the list of devices).
// 3. Compute the statistics of interest.
//
// NOTE: Events from devices are picked in the order of the hostnames array, such that if overlaps are detected the conflict will be resolved by choosing events from the earlier device.
// NOTE: Only supports desktop devices (for now)
// NOTE: Doesn't support browser buckets (and therefore not browser audible detection either)
// This is due to the 'unknown' hostname of browser buckets (will hopefully be fixed soon).
export function multideviceQuery(
hostnames: string[],
filterAFK = true,
classes: [string[], Rule][] = [],
filterCategories: string[][]
): string[] {
function safeHostname(hostname: string): string {
return hostname.replace(/[^a-zA-Z0-9_]/g, '');
}

// First, query each device individually
const queries: string[] = _.map(hostnames, hostname => {
const params: DesktopQueryParams = {
bid_window: 'aw-watcher-window_' + hostname,
bid_afk: 'aw-watcher-afk_' + hostname,
bid_browsers: [],
classes: classes,
filter_classes: filterCategories,
filter_afk: filterAFK,
return_variable: 'events_' + safeHostname(hostname),
};

return canonicalEvents(params);
});

// Now we need to combine the queries to get a single series of events.
// To do this, we can use the union_no_overlap function, which merges events
// but avoids overlaps by giving priority according to the order of `hostnames`.
let query = queries.join('\n');
if (queries.length > 1) {
query += 'events = [];';
for (let i = 0; i < queries.length; i++) {
query += `events = union_no_overlap(events, events_${safeHostname(hostnames[i])});`;
}
}

export function multideviceQuery(params: MultiQueryParams): string[] {
return querystr_to_array(
`
${query}
${canonicalMultideviceEvents(params)}
title_events = sort_by_duration(merge_events_by_keys(events, ["app", "title"]));
app_events = sort_by_duration(merge_events_by_keys(title_events, ["app"]));
cat_events = sort_by_duration(merge_events_by_keys(events, ["$category"]));
Expand Down Expand Up @@ -368,12 +384,12 @@ export function activityQueryAndroid(androidbucket: string): string[] {

// Returns a query that yields a dict with a key "cat_events" which is an
// array of one event per category, with the duration of each event set to the sum of the category durations.
export function categoryQuery(params: DesktopQueryParams): string[] {
export function categoryQuery(params: MultiQueryParams | DesktopQueryParams): string[] {
const q = `
${canonicalEvents(params)}
cat_events = sort_by_duration(merge_events_by_keys(events, ["$category"]));
RETURN = { "cat_events": cat_events };
`;
${isMultiParams(params) ? canonicalMultideviceEvents(params) : canonicalEvents(params)}
cat_events = sort_by_duration(merge_events_by_keys(events, ["$category"]));
RETURN = { "cat_events": cat_events };
`;
return querystr_to_array(q);
}

Expand Down
Loading

0 comments on commit c8a968b

Please sign in to comment.