Merge pull request github#2433 from github/robertbrignull/query-discovery

Hook queries panel up to real data

Author: robertbrignull

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -134,6 +134,11 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve queries`.
+ */
+export type ResolvedQueries = string[];
+
 /**
  * The expected output of `codeql resolve tests`.
  */
@@ -731,6 +736,20 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Finds all available queries in a given directory.
+   * @param queryDir Root of directory tree to search for queries.
+   * @returns The list of queries that were found.
+   */
+  public async resolveQueries(queryDir: string): Promise<ResolvedQueries> {
+    const subcommandArgs = [queryDir];
+    return await this.runJsonCodeQlCliCommand<ResolvedQueries>(
+      ["resolve", "queries"],
+      subcommandArgs,
+      "Resolving queries",
+    );
+  }
+
   /**
    * Finds all available QL tests in a given directory.
    * @param testPath Root of directory tree to search for tests.

extensions/ql-vscode/src/extension.ts

@@ -755,7 +755,7 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(databaseUI);
 
-  QueriesModule.initialize(app);
+  QueriesModule.initialize(app, cliServer);
 
   void extLogger.log("Initializing evaluator log viewer.");
   const evalLogViewer = new EvalLogViewer();

extensions/ql-vscode/src/queries-panel/queries-module.ts

@@ -1,33 +1,43 @@
+import { CodeQLCliServer } from "../codeql-cli/cli";
 import { extLogger } from "../common";
 import { App, AppMode } from "../common/app";
 import { isCanary, showQueriesPanel } from "../config";
 import { DisposableObject } from "../pure/disposable-object";
 import { QueriesPanel } from "./queries-panel";
+import { QueryDiscovery } from "./query-discovery";
 
 export class QueriesModule extends DisposableObject {
   private queriesPanel: QueriesPanel | undefined;
+  private queryDiscovery: QueryDiscovery | undefined;
 
   private constructor(readonly app: App) {
     super();
   }
 
-  private initialize(app: App): void {
+  private initialize(app: App, cliServer: CodeQLCliServer): void {
     if (app.mode === AppMode.Production || !isCanary() || !showQueriesPanel()) {
       // Currently, we only want to expose the new panel when we are in development and canary mode
       // and the developer has enabled the "Show queries panel" flag.
       return;
     }
     void extLogger.log("Initializing queries panel.");
 
-    this.queriesPanel = new QueriesPanel();
+    this.queryDiscovery = new QueryDiscovery(app, cliServer);
+    this.push(this.queryDiscovery);
+    this.queryDiscovery.refresh();
+
+    this.queriesPanel = new QueriesPanel(this.queryDiscovery);
     this.push(this.queriesPanel);
   }
 
-  public static initialize(app: App): QueriesModule {
+  public static initialize(
+    app: App,
+    cliServer: CodeQLCliServer,
+  ): QueriesModule {
     const queriesModule = new QueriesModule(app);
     app.subscriptions.push(queriesModule);
 
-    queriesModule.initialize(app);
+    queriesModule.initialize(app, cliServer);
     return queriesModule;
   }
 }

extensions/ql-vscode/src/queries-panel/queries-panel.ts

@@ -2,15 +2,16 @@ import * as vscode from "vscode";
 import { DisposableObject } from "../pure/disposable-object";
 import { QueryTreeDataProvider } from "./query-tree-data-provider";
 import { QueryTreeViewItem } from "./query-tree-view-item";
+import { QueryDiscovery } from "./query-discovery";
 
 export class QueriesPanel extends DisposableObject {
   private readonly dataProvider: QueryTreeDataProvider;
   private readonly treeView: vscode.TreeView<QueryTreeViewItem>;
 
-  public constructor() {
+  public constructor(queryDiscovery: QueryDiscovery) {
     super();
 
-    this.dataProvider = new QueryTreeDataProvider();
+    this.dataProvider = new QueryTreeDataProvider(queryDiscovery);
 
     this.treeView = vscode.window.createTreeView("codeQLQueries", {
       treeDataProvider: this.dataProvider,

extensions/ql-vscode/src/queries-panel/query-discovery.ts

@@ -0,0 +1,132 @@
+import { dirname, basename, normalize, relative } from "path";
+import { Discovery } from "../common/discovery";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import {
+  Event,
+  EventEmitter,
+  RelativePattern,
+  Uri,
+  WorkspaceFolder,
+} from "vscode";
+import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watcher";
+import { App } from "../common/app";
+import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
+import { getOnDiskWorkspaceFoldersObjects } from "../helpers";
+
+/**
+ * The results of discovering queries.
+ */
+interface QueryDiscoveryResults {
+  /**
+   * A tree of directories and query files.
+   * May have multiple roots because of multiple workspaces.
+   */
+  queries: FileTreeDirectory[];
+
+  /**
+   * File system paths to watch. If any ql file changes in these directories
+   * or any subdirectories, then this could signify a change in queries.
+   */
+  watchPaths: Uri[];
+}
+
+/**
+ * Discovers all query files contained in the QL packs in a given workspace folder.
+ */
+export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
+  private results: QueryDiscoveryResults | undefined;
+
+  private readonly onDidChangeQueriesEmitter = this.push(
+    new EventEmitter<void>(),
+  );
+  private readonly watcher: MultiFileSystemWatcher = this.push(
+    new MultiFileSystemWatcher(),
+  );
+
+  constructor(app: App, private readonly cliServer: CodeQLCliServer) {
+    super("Query Discovery");
+
+    this.push(app.onDidChangeWorkspaceFolders(this.refresh.bind(this)));
+    this.push(this.watcher.onDidChange(this.refresh.bind(this)));
+  }
+
+  public get queries(): FileTreeDirectory[] | undefined {
+    return this.results?.queries;
+  }
+
+  /**
+   * Event to be fired when the set of discovered queries may have changed.
+   */
+  public get onDidChangeQueries(): Event<void> {
+    return this.onDidChangeQueriesEmitter.event;
+  }
+
+  protected async discover(): Promise<QueryDiscoveryResults> {
+    const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+    if (workspaceFolders.length === 0) {
+      return {
+        queries: [],
+        watchPaths: [],
+      };
+    }
+
+    const queries = await this.discoverQueries(workspaceFolders);
+
+    return {
+      queries,
+      watchPaths: workspaceFolders.map((f) => f.uri),
+    };
+  }
+
+  protected update(results: QueryDiscoveryResults): void {
+    this.results = results;
+
+    this.watcher.clear();
+    for (const watchPath of results.watchPaths) {
+      // Watch for changes to any `.ql` file
+      this.watcher.addWatch(new RelativePattern(watchPath, "**/*.{ql}"));
+      // need to explicitly watch for changes to directories themselves.
+      this.watcher.addWatch(new RelativePattern(watchPath, "**/"));
+    }
+    this.onDidChangeQueriesEmitter.fire();
+  }
+
+  /**
+   * Discover all queries in the specified directory and its subdirectories.
+   * @returns A `QueryDirectory` object describing the contents of the directory, or `undefined` if
+   *   no queries were found.
+   */
+  private async discoverQueries(
+    workspaceFolders: readonly WorkspaceFolder[],
+  ): Promise<FileTreeDirectory[]> {
+    const rootDirectories = [];
+    for (const workspaceFolder of workspaceFolders) {
+      rootDirectories.push(
+        await this.discoverQueriesInWorkspace(workspaceFolder),
+      );
+    }
+    return rootDirectories;
+  }
+
+  private async discoverQueriesInWorkspace(
+    workspaceFolder: WorkspaceFolder,
+  ): Promise<FileTreeDirectory> {
+    const fullPath = workspaceFolder.uri.fsPath;
+    const name = workspaceFolder.name;
+
+    const rootDirectory = new FileTreeDirectory(fullPath, name);
+
+    const resolvedQueries = await this.cliServer.resolveQueries(fullPath);
+    for (const queryPath of resolvedQueries) {
+      const relativePath = normalize(relative(fullPath, queryPath));
+      const dirName = dirname(relativePath);
+      const parentDirectory = rootDirectory.createDirectory(dirName);
+      parentDirectory.addChild(
+        new FileTreeLeaf(queryPath, basename(queryPath)),
+      );
+    }
+
+    rootDirectory.finish();
+    return rootDirectory;
+  }
+}

extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts

@@ -1,45 +1,56 @@
-import * as vscode from "vscode";
+import { Event, EventEmitter, TreeDataProvider, TreeItem } from "vscode";
 import { QueryTreeViewItem } from "./query-tree-view-item";
 import { DisposableObject } from "../pure/disposable-object";
+import { QueryDiscovery } from "./query-discovery";
+import { FileTreeNode } from "../common/file-tree-nodes";
 
 export class QueryTreeDataProvider
   extends DisposableObject
-  implements vscode.TreeDataProvider<QueryTreeViewItem>
+  implements TreeDataProvider<QueryTreeViewItem>
 {
   private queryTreeItems: QueryTreeViewItem[];
 
-  public constructor() {
+  private readonly onDidChangeTreeDataEmitter = this.push(
+    new EventEmitter<void>(),
+  );
+
+  public constructor(private readonly queryDiscovery: QueryDiscovery) {
     super();
 
+    queryDiscovery.onDidChangeQueries(() => {
+      this.queryTreeItems = this.createTree();
+      this.onDidChangeTreeDataEmitter.fire();
+    });
+
     this.queryTreeItems = this.createTree();
   }
 
+  public get onDidChangeTreeData(): Event<void> {
+    return this.onDidChangeTreeDataEmitter.event;
+  }
+
   private createTree(): QueryTreeViewItem[] {
-    // Temporary mock data, just to populate the tree view.
-    return [
-      new QueryTreeViewItem("custom-pack", [
-        new QueryTreeViewItem("custom-pack/example.ql", []),
-      ]),
-      new QueryTreeViewItem("ql", [
-        new QueryTreeViewItem("ql/javascript", [
-          new QueryTreeViewItem("ql/javascript/example.ql", []),
-        ]),
-        new QueryTreeViewItem("ql/go", [
-          new QueryTreeViewItem("ql/go/security", [
-            new QueryTreeViewItem("ql/go/security/query1.ql", []),
-            new QueryTreeViewItem("ql/go/security/query2.ql", []),
-          ]),
-        ]),
-      ]),
-    ];
+    return (this.queryDiscovery.queries || []).map(
+      this.convertFileTreeNode.bind(this),
+    );
+  }
+
+  private convertFileTreeNode(
+    fileTreeDirectory: FileTreeNode,
+  ): QueryTreeViewItem {
+    return new QueryTreeViewItem(
+      fileTreeDirectory.name,
+      fileTreeDirectory.path,
+      fileTreeDirectory.children.map(this.convertFileTreeNode.bind(this)),
+    );
   }
 
   /**
    * Returns the UI presentation of the element that gets displayed in the view.
    * @param item The item to represent.
    * @returns The UI presentation of the item.
    */
-  public getTreeItem(item: QueryTreeViewItem): vscode.TreeItem {
+  public getTreeItem(item: QueryTreeViewItem): TreeItem {
     return item;
   }
 

extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts

@@ -1,9 +1,12 @@
 import * as vscode from "vscode";
-import { basename } from "path";
 
 export class QueryTreeViewItem extends vscode.TreeItem {
-  constructor(path: string, public readonly children: QueryTreeViewItem[]) {
-    super(basename(path));
+  constructor(
+    name: string,
+    path: string,
+    public readonly children: QueryTreeViewItem[],
+  ) {
+    super(name);
     this.tooltip = path;
     this.collapsibleState = this.children.length
       ? vscode.TreeItemCollapsibleState.Collapsed