create problem markers from task output

What's included in this PR:
- support taskDefinition, problemMatcher, & problemPattern contributed by Theia plugins.
- support customizing tasks by adding named & anonymous problem matchers and patterns in .theia/tasks.json.
- problem markers are created on parsing task output with the plugin-defined or user-defined problem matchers and patterns.

What's not included:
- prompt users to provide problem matcher and / or patterns before starting tasks

Signed-off-by: elaihau <liang.huang@ericsson.com>

packages/markers/src/browser/marker-manager.ts

@@ -55,11 +55,20 @@ export class MarkerCollection<T> {
         if (markerData.length > 0) {
             this.owner2Markers.set(owner, markerData.map(data => this.createMarker(owner, data)));
         } else {
-            this.owner2Markers.delete(owner);
+            this.removeMarkers(owner);
         }
         return before || [];
     }
 
+    addMarkers(owner: string, markerData: T[]): Marker<T>[] {
+        if (markerData.length > 0) {
+            const existing = this.owner2Markers.get(owner) || [];
+            const toAdd = markerData.map(data => this.createMarker(owner, data));
+            this.owner2Markers.set(owner, [...existing, ...toAdd]);
+        }
+        return this.owner2Markers.get(owner) || [];
+    }
+
     protected createMarker(owner: string, data: T): Readonly<Marker<T>> {
         return Object.freeze({
             uri: this.uri.toString(),
@@ -95,6 +104,9 @@ export class MarkerCollection<T> {
         }
     }
 
+    removeMarkers(owner: string): void {
+        this.owner2Markers.delete(owner);
+    }
 }
 
 export interface Uri2MarkerEntry {
@@ -160,6 +172,22 @@ export abstract class MarkerManager<D extends object> {
         return oldMarkers;
     }
 
+    /*
+     * adds markers for the given uri and owner with the given data, without touching the exsting markers associated with the same uri and owner.
+     */
+    addMarkers(uri: URI, owner: string, data: D[]): Marker<D>[] {
+        const uriString = uri.toString();
+        const collection = this.uri2MarkerCollection.get(uriString) || new MarkerCollection<D>(uri, this.getKind());
+        const newMarkers = collection.addMarkers(owner, data);
+        if (collection.empty) {
+            this.uri2MarkerCollection.delete(uri.toString());
+        } else {
+            this.uri2MarkerCollection.set(uriString, collection);
+        }
+        this.fireOnDidChangeMarkers(uri);
+        return newMarkers;
+    }
+
     /*
      * returns all markers that satisfy the given filter.
      */
@@ -197,4 +225,10 @@ export abstract class MarkerManager<D extends object> {
         }
     }
 
+    cleanMarkersByOwner(owner: string): void {
+        this.uri2MarkerCollection.forEach((collection, uri) => {
+            collection.removeMarkers(owner);
+            this.fireOnDidChangeMarkers(new URI(uri));
+        });
+    }
 }

packages/plugin-ext/src/common/plugin-protocol.ts

@@ -22,6 +22,7 @@ import { ExtPluginApi } from './plugin-ext-api-contribution';
 import { IJSONSchema, IJSONSchemaSnippet } from '@theia/core/lib/common/json-schema';
 import { RecursivePartial } from '@theia/core/lib/common/types';
 import { PreferenceSchema, PreferenceSchemaProperties } from '@theia/core/lib/common/preferences/preference-schema';
+import { TaskDefinitionContribution, ProblemMatcherContribution, ProblemPatternContribution } from '@theia/task/lib/common';
 
 export const hostedServicePath = '/services/hostedPlugin';
 
@@ -66,6 +67,9 @@ export interface PluginPackageContribution {
     keybindings?: PluginPackageKeybinding[];
     debuggers?: PluginPackageDebuggersContribution[];
     snippets: PluginPackageSnippetsContribution[];
+    taskDefinitions?: TaskDefinitionContribution[];
+    problemMatchers?: ProblemMatcherContribution[];
+    problemPatterns?: ProblemPatternContribution[];
 }
 
 export interface PluginPackageViewContainer {
@@ -359,6 +363,9 @@ export interface PluginContribution {
     keybindings?: Keybinding[];
     debuggers?: DebuggerContribution[];
     snippets?: SnippetContribution[];
+    taskDefinitions?: TaskDefinitionContribution[];
+    problemMatchers?: ProblemMatcherContribution[];
+    problemPatterns?: ProblemPatternContribution[];
 }
 
 export interface SnippetContribution {

packages/plugin-ext/src/hosted/browser/hosted-plugin.ts

@@ -197,7 +197,7 @@ export class HostedPluginSupport {
             }
 
             if (plugin.model.contributes) {
-                this.contributionHandler.handleContributions(plugin.model.contributes);
+                this.contributionHandler.handleContributions(plugin.model.contributes, plugin.model.id);
             }
         }
 

packages/plugin-ext/src/hosted/node/scanners/scanner-theia.ts

@@ -56,6 +56,7 @@ import { deepClone } from '@theia/core/lib/common/objects';
 import { FileUri } from '@theia/core/lib/node/file-uri';
 import { PreferenceSchema, PreferenceSchemaProperties } from '@theia/core/lib/common/preferences/preference-schema';
 import { RecursivePartial } from '@theia/core/lib/common/types';
+import { TaskDefinitionContribution, ProblemMatcherContribution, ProblemPatternContribution } from '@theia/task/lib/common/task-protocol';
 
 namespace nls {
     export function localize(key: string, _default: string) {
@@ -184,6 +185,18 @@ export class TheiaPluginScanner implements PluginScanner {
             contributions.debuggers = debuggers;
         }
 
+        if (rawPlugin.contributes!.taskDefinitions) {
+            contributions.taskDefinitions = rawPlugin.contributes!.taskDefinitions as TaskDefinitionContribution[];
+        }
+
+        if (rawPlugin.contributes!.problemMatchers) {
+            contributions.problemMatchers = rawPlugin.contributes!.problemMatchers as ProblemMatcherContribution[];
+        }
+
+        if (rawPlugin.contributes!.problemPatterns) {
+            contributions.problemPatterns = rawPlugin.contributes!.problemPatterns as ProblemPatternContribution[];
+        }
+
         contributions.snippets = this.readSnippets(rawPlugin);
         return contributions;
     }

packages/plugin-ext/src/main/browser/plugin-contribution-handler.ts

@@ -27,6 +27,7 @@ import { MonacoSnippetSuggestProvider } from '@theia/monaco/lib/browser/monaco-s
 import { PluginSharedStyle } from './plugin-shared-style';
 import { CommandRegistry } from '@theia/core';
 import { BuiltinThemeProvider } from '@theia/core/lib/browser/theming';
+import { TaskDefinitionRegistry, ProblemMatcherRegistry, ProblemPatternRegistry } from '@theia/task/lib/common';
 
 @injectable()
 export class PluginContributionHandler {
@@ -60,7 +61,16 @@ export class PluginContributionHandler {
     @inject(PluginSharedStyle)
     protected readonly style: PluginSharedStyle;
 
-    handleContributions(contributions: PluginContribution): void {
+    @inject(TaskDefinitionRegistry)
+    protected readonly taskDefinitionRegistry: TaskDefinitionRegistry;
+
+    @inject(ProblemMatcherRegistry)
+    protected readonly problemMatcherRegistry: ProblemMatcherRegistry;
+
+    @inject(ProblemPatternRegistry)
+    protected readonly problemPatternRegistry: ProblemPatternRegistry;
+
+    handleContributions(contributions: PluginContribution, modelId: string): void {
         if (contributions.configuration) {
             this.updateConfigurationSchema(contributions.configuration);
         }
@@ -150,6 +160,18 @@ export class PluginContributionHandler {
                 });
             }
         }
+
+        if (contributions.taskDefinitions) {
+            contributions.taskDefinitions.forEach(def => this.taskDefinitionRegistry.register(def, modelId));
+        }
+
+        if (contributions.problemPatterns) {
+            contributions.problemPatterns.forEach(pattern => this.problemPatternRegistry.register(pattern.name!, pattern));
+        }
+
+        if (contributions.problemMatchers) {
+            contributions.problemMatchers.forEach(matcher => this.problemMatcherRegistry.register(matcher));
+        }
     }
 
     protected pluginCommandIconId = 0;

packages/plugin-ext/src/plugin/types-impl.ts

@@ -1728,12 +1728,14 @@ export class Task {
         if (this.taskExecution instanceof ProcessExecution) {
             Object.assign(this.taskDefinition, {
                 type: 'process',
-                id: this.taskExecution.computeId()
+                id: this.taskExecution.computeId(),
+                taskType: this.taskDefinition!.type
             });
         } else if (this.taskExecution instanceof ShellExecution) {
             Object.assign(this.taskDefinition, {
                 type: 'shell',
-                id: this.taskExecution.computeId()
+                id: this.taskExecution.computeId(),
+                taskType: this.taskDefinition!.type
             });
         }
     }

packages/process/src/node/process.ts

@@ -108,6 +108,9 @@ export abstract class Process {
         return this.errorEmitter.event;
     }
 
+    abstract onData(listener: (buffer: string) => void): void;
+    abstract onDataClosed(listener: (exitCode: number, signal?: number) => void): void;
+
     protected emitOnStarted() {
         this.startEmitter.fire({});
     }

packages/process/src/node/raw-process.ts

@@ -141,6 +141,14 @@ export class RawProcess extends Process {
         }
     }
 
+    onData(listener: (buffer: string) => void): void {
+        this.output.on('data', listener);
+    }
+
+    onDataClosed(listener: (exitCode: number, signal?: number) => void): void {
+        this.output.on('close', listener);
+    }
+
     get pid() {
         return this.process.pid;
     }

packages/process/src/node/terminal-process.ts

@@ -123,4 +123,11 @@ export class TerminalProcess extends Process {
         this.terminal.write(data);
     }
 
+    onData(listener: (buffer: string) => void): void {
+        this.terminal.on('data', listener);
+    }
+
+    onDataClosed(listener: (exitCode: number, signal?: number) => void): void {
+        this.terminal.on('exit', listener);
+    }
 }

packages/task/package.json

@@ -10,7 +10,8 @@
     "@theia/terminal": "^0.5.0",
     "@theia/variable-resolver": "^0.5.0",
     "@theia/workspace": "^0.5.0",
-    "jsonc-parser": "^2.0.2"
+    "jsonc-parser": "^2.0.2",
+    "vscode-uri": "^1.0.1"
   },
   "publishConfig": {
     "access": "public"

packages/task/src/browser/task-configurations.ts

@@ -15,7 +15,7 @@
  ********************************************************************************/
 
 import { inject, injectable } from 'inversify';
-import { TaskConfiguration } from '../common/task-protocol';
+import { TaskConfiguration, TaskCustomization, TaskDefinitionRegistry } from '../common';
 import { Disposable, DisposableCollection, ResourceProvider } from '@theia/core/lib/common';
 import URI from '@theia/core/lib/common/uri';
 import { FileSystemWatcher, FileChangeEvent } from '@theia/filesystem/lib/browser/filesystem-watcher';
@@ -47,6 +47,8 @@ export class TaskConfigurations implements Disposable {
      * For the inner map (i.e., task config map), the key is task label and value TaskConfiguration
      */
     protected tasksMap = new Map<string, Map<string, TaskConfiguration>>();
+    protected taskCustomizations: TaskCustomization[] = [];
+
     protected watchedConfigFileUris: string[] = [];
     protected watchersMap = new Map<string, Disposable>(); // map of watchers for task config files, where the key is folder uri
 
@@ -66,6 +68,9 @@ export class TaskConfigurations implements Disposable {
     @inject(OpenerService)
     protected readonly openerService: OpenerService;
 
+    @inject(TaskDefinitionRegistry)
+    protected readonly taskDefinitionRegistry: TaskDefinitionRegistry;
+
     constructor(
         @inject(FileSystemWatcher) protected readonly watcherServer: FileSystemWatcher,
         @inject(FileSystem) protected readonly fileSystem: FileSystem
@@ -173,6 +178,10 @@ export class TaskConfigurations implements Disposable {
         this.tasksMap.delete(source);
     }
 
+    getTaskCustomizations(type: string): TaskCustomization[] {
+        return this.taskCustomizations.filter(c => c.type === type);
+    }
+
     /** returns the string uri of where the config file would be, if it existed under a given root directory */
     protected getConfigFileUri(rootDir: string): string {
         return new URI(rootDir).resolve(this.TASKFILEPATH).resolve(this.TASKFILE).toString();
@@ -198,48 +207,46 @@ export class TaskConfigurations implements Disposable {
      * If reading a config file wasn't successful then does nothing.
      */
     protected async refreshTasks(configFileUri: string) {
-        const tasksConfigsArray = await this.readTasks(configFileUri);
-        if (tasksConfigsArray) {
+        const configuredTasksArray = await this.readTasks(configFileUri);
+        if (configuredTasksArray) {
             // only clear tasks map when successful at parsing the config file
             // this way we avoid clearing and re-filling it multiple times if the
             // user is editing the file in the auto-save mode, having momentarily
             // non-parsing JSON.
             this.removeTasks(configFileUri);
 
-            if (tasksConfigsArray.length > 0) {
+            if (configuredTasksArray.length > 0) {
                 const newTaskMap = new Map<string, TaskConfiguration>();
-                for (const task of tasksConfigsArray) {
+                for (const task of configuredTasksArray) {
                     newTaskMap.set(task.label, task);
                 }
                 const source = this.getSourceFolderFromConfigUri(configFileUri);
                 this.tasksMap.set(source, newTaskMap);
             }
         }
+
+        const cutomizations = await this.readTaskCustomizations(configFileUri);
+        if (cutomizations) {
+            this.taskCustomizations.length = 0;
+            this.taskCustomizations = cutomizations;
+        }
     }
 
     /** parses a config file and extracts the tasks launch configurations */
     protected async readTasks(uri: string): Promise<TaskConfiguration[] | undefined> {
-        if (!await this.fileSystem.exists(uri)) {
-            return undefined;
-        } else {
-            try {
-                const response = await this.fileSystem.resolveContent(uri);
-
-                const strippedContent = jsoncparser.stripComments(response.content);
-                const errors: ParseError[] = [];
-                const tasks = jsoncparser.parse(strippedContent, errors);
-
-                if (errors.length) {
-                    for (const e of errors) {
-                        console.error(`Error parsing ${uri}: error: ${e.error}, length:  ${e.length}, offset:  ${e.offset}`);
-                    }
-                } else {
-                    return this.filterDuplicates(tasks['tasks']).map(t => Object.assign(t, { _source: t.source || this.getSourceFolderFromConfigUri(uri) }));
+        const taskConfigs = await this.getTaskConfigurationsFromFile(uri);
+        if (taskConfigs) {
+            const filtered = this.filterDuplicates(taskConfigs);
+            const isContributed = await Promise.all(filtered.map(t => this.isContributedTask(t)));
+            const configuredTasks: TaskConfiguration[] = [];
+            filtered.forEach((t, index) => {
+                if (!isContributed[index]) {
+                    configuredTasks.push(t);
                 }
-            } catch (err) {
-                console.error(`Error(s) reading config file: ${uri}`);
-            }
+            });
+            return configuredTasks.map(t => Object.assign(t, { _source: t.source || this.getSourceFolderFromConfigUri(uri) }));
         }
+        return undefined;
     }
 
     /** Adds given task to a config file and opens the file to provide ability to edit task configuration. */
@@ -301,4 +308,50 @@ export class TaskConfigurations implements Disposable {
     private getSourceFolderFromConfigUri(configFileUri: string): string {
         return new URI(configFileUri).parent.parent.path.toString();
     }
+
+    protected async readTaskCustomizations(uri: string): Promise<TaskCustomization[] | undefined> {
+        const taskConfigs = await this.getTaskConfigurationsFromFile(uri);
+        if (taskConfigs) {
+            const isContributed = await Promise.all(taskConfigs.map((t: TaskConfiguration) => this.isContributedTask(t)));
+            const customizations: TaskCustomization[] = [];
+            taskConfigs.forEach((t: TaskConfiguration, index: number) => {
+                if (isContributed[index]) {
+                    customizations.push(t);
+                }
+            });
+            return customizations;
+        }
+        return undefined;
+    }
+
+    private async isContributedTask(task: TaskConfiguration): Promise<boolean> {
+        const taskDefinition = await this.taskDefinitionRegistry.getDefinition(task);
+        // it is considered as a customization if the task definition registry finds a def for the task configuration
+        return !!taskDefinition;
+    }
+
+    private async getTaskConfigurationsFromFile(uri: string): Promise<TaskConfiguration[] | undefined> {
+        if (!await this.fileSystem.exists(uri)) {
+            return undefined;
+        } else {
+            try {
+                const response = await this.fileSystem.resolveContent(uri);
+
+                const strippedContent = jsoncparser.stripComments(response.content);
+                const errors: ParseError[] = [];
+                const tasks = jsoncparser.parse(strippedContent, errors)['tasks'];
+
+                if (errors.length) {
+                    for (const e of errors) {
+                        console.error(`Error parsing ${uri}: error: ${e.error}, length:  ${e.length}, offset:  ${e.offset}`);
+                    }
+                    return [];
+                } else {
+                    return tasks as TaskConfiguration[];
+                }
+            } catch (err) {
+                console.error(`Error(s) reading config file: ${uri}`);
+            }
+        }
+    }
 }