Merge branch 'microsoft:main' into main

47-studio-org · Oct 21, 2022 · 67687bc · 67687bc
2 parents 9e768f7 + a0ab7ca
commit 67687bc
Show file tree

Hide file tree

Showing 662 changed files with 2,863,001 additions and 173,959 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -12,10 +12,4 @@
 # This should make it easy to add new rules without breaking existing ones.
 
 # Global rule:
-* @cleemullins @munozemilio @joshgummersall
-
-/build/**                          @munozemilio
-/data/**                           @munozemilio
-/packages/**                       @munozemilio
-/scripts/**                        @munozemilio
-/specs/**                          @scheyal
+* @microsoft/botframework-sdk
diff --git a/.github/ISSUE_TEMPLATE/bf-cli-bug-report.md b/.github/ISSUE_TEMPLATE/bf-cli-bug-report.md
@@ -2,7 +2,7 @@
 name: BF CLI Bug Report
 about: Create a bug report for a bug you found in the Bot Framework CLI
 title: ''
-labels: ''
+labels: 'needs-triage, bug'
 assignees: ''
 
 ---

diff --git a/.github/ISSUE_TEMPLATE/bf-cli-feature-request.md b/.github/ISSUE_TEMPLATE/bf-cli-feature-request.md
@@ -2,7 +2,7 @@
 name: BF CLI Feature request
 about: Suggest a feature for the Bot Framework CLI
 title: ''
-labels: ''
+labels: 'needs-triage, feature-request'
 assignees: ''
 
 ---

diff --git a/.gitignore b/.gitignore
@@ -81,3 +81,12 @@ packages/dialog/tsconfig.tsbuildinfo
 
 # BuildInfo
 tsconfig.tsbuildinfo
+
+# download NLR model file folder
+model_dte_roberta_for_test_command
+**/*.blu
+**/*.json
+**/model/**/
+**/**assessment**.html
+**/**assessment**.txt
+
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -4,17 +4,74 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "bf CLI",
+      "name": "Orchestrator Tests",
+      "preLaunchTask": "${defaultBuildTask}",
+      "runtimeExecutable": "npm",
+      "runtimeArgs": [
+        "run",
+        "test",
+        "--",
+        "--timeout",
+        "999999",
+        "--colors",
+        "-g",
+        "Test.0600 OrchestratorHelper.parseLuFile()"
+      ],
+      "cwd": "${workspaceFolder}/packages/orchestratorlib",
+      "internalConsoleOptions": "openOnSessionStart",
+      "outputCapture": "std",
+      "outFiles": [
+        "./packages/orchestratorlib/lib/**"
+      ]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "bf Orchestrator CLI",
       "preLaunchTask": "${defaultBuildTask}",
       "program": "${workspaceFolder}/packages/cli/bin/run",
       "outputCapture": "std",
       "outFiles": [
-        "${workspaceFolder}/packages/luis/lib/**"
+        "${workspaceFolder}/packages/orchestrator/lib/**"
       ],
       "args": [
-        "luis:build",
-        "--luConfig",
-        "${env:TEMP}/sandwich.out/luconfig.json"
+        "orchestrator:create",
+        "--in",
+        "D:/src/BotBuilder-Samples/experimental/orchestrator/csharp_dotnetcore/01.dispatch-bot/CognitiveModels",
+        "--out",
+        "D:/src/BotBuilder-Samples/experimental/orchestrator/csharp_dotnetcore/01.dispatch-bot/generated",
+        "--model",
+        "D:/src/BotBuilder-Samples/experimental/orchestrator/csharp_dotnetcore/01.dispatch-bot/model",
+        "--entityModel",
+        "D:/src/BotBuilder-Samples/experimental/orchestrator/csharp_dotnetcore/01.dispatch-bot/model/entity",
+        "--hierarchical"
+      ],
+      "internalConsoleOptions": "openOnSessionStart",
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "bf Orchestrator form",
+      "preLaunchTask": "${defaultBuildTask}",
+      "program": "${workspaceFolder}/packages/orchestrator/bin/run",
+      "outputCapture": "std",
+      "outFiles": [
+        "${workspaceFolder}/packages/orchestrator/lib/**"
+      ],
+      "args": [
+        "orchestrator:create",
+        "--in",
+        "C:/Users/chrim/AppData/Local/Temp/sandwich.out/language-understanding/en-us/sandwich.en-us.lu",
+        "--out",
+        "C:/Users/chrim/AppData/Local/Temp/sandwich.out/language-understanding/en-us/sandwich.en-us.blu",
+        "--model",
+        "c:/tmp/basemodel",
+        "--entityModel",
+        "c:/tmp/basemodel/entity",
+        "--hierarchical",
+        "-d",
+        "--refresh"
       ],
       "internalConsoleOptions": "openOnSessionStart",
       "cwd": "${workspaceFolder}"
@@ -39,7 +96,7 @@
       "internalConsoleOptions": "openOnSessionStart",
       "outputCapture": "std",
       "outFiles": [
-        "./package/dialog/lib/**"
+        "./packages/dialog/lib/**"
       ]
     },
     {
@@ -62,7 +119,7 @@
       "internalConsoleOptions": "openOnSessionStart",
       "outputCapture": "std",
       "outFiles": [
-        "./package/dialog/lib/**"
+        "./packages/dialog/lib/**"
       ]
     },
     {
@@ -132,6 +189,26 @@
         "./packages/lu/lib/**"
       ]
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Dialog merge root-package schema",
+      "preLaunchTask": "${defaultBuildTask}",
+      "program": "${workspaceFolder}/packages/dialog/bin/run",
+      "outputCapture": "std",
+      "outFiles": [
+        "./packages/dialog/lib/**"
+      ],
+      "args": [
+        "dialog:merge",
+        "npm/node_modules/root-package/package.json",
+        "-o",
+        "oracles/root-package",
+        "--verbose"
+      ],
+      "internalConsoleOptions": "openOnSessionStart",
+      "cwd": "${workspaceFolder}/packages/dialog/test/commands/dialog/"
+    },
     {
       "type": "node",
       "request": "launch",

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
@@ -0,0 +1,9 @@
+# Microsoft Open Source Code of Conduct
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+
+Resources:
+
+- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
+- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
+- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns
diff --git a/README.md b/README.md
@@ -5,17 +5,17 @@
 # BF Command Line Interface
 [![Build Status](https://fuselabs.visualstudio.com/SDK_Public/_apis/build/status/microsoft.botframework-cli?branchName=main)](https://fuselabs.visualstudio.com/SDK_Public/_build/latest?definitionId=713&branchName=main)
 
-The new BF Command Line Interface (CLI) tool replaces the collection of standalone tools used to manage Bot Framework bots and related services. We have ported most tools and are in process of porting the rest. The new BF CLI aggregates the collection of cross-platform tools into one cohesive and consistent interface.
+BF Command Line Interface (CLI) tool replaces and aggregates the collection of standalone tools used to manage Bot Framework bots and related services into one cohesive and consistent interface.
 
-The old tools will be deprecated in subsequent releases. All new investments, bug fixes, and new features will be implemented in the new consolidated BF CLI alone.
+The old tools have been deprecated. All new investments, bug fixes, and new features will be implemented in the new consolidated BF CLI alone.
 
 ## Installation
 
 BF is based on the Node.js platform and the [OClif](https://github.com/oclif/oclif) framework where it inherits its command line parsing style, and plugin architecture platform.
 
 You must download the following prerequisites:
 
-* [Node.js](https://nodejs.org/) version 12
+* [Node.js](https://nodejs.org/) version 14
 
 Install the tool using the following command:
 
@@ -24,38 +24,44 @@ $ npm i -g @microsoft/botframework-cli
 $ bf
 ~~~
 
+## Troubleshooting
+
+For installation failures in some Linux platforms, try one of the following workarounds:
+- Upgrade npm to version 7.9
+- Install without the '-g' option and use [npx](https://www.npmjs.com/package/npx)
+
+~~~
+$ npm i @microsoft/botframework-cli
+$ npx bf
+~~~
+
 ## Available Commands
+
 The following commands are currently available:
 * [Chatdown][1]
 * [Config][2]
 * [Dialog][3]
 * [Luis][4]
 * [QnAMaker][5]
-
-<!--
-Preview commands (Install running [bf plugins](https://github.com/microsoft/botframework-cli/tree/main/packages/plugins#microsoftbf-cli-plugins))
-* [Dialog](https://github.com/microsoft/botframework-cli/tree/main/packages/dialog)
--->
-
-#### Future Commands
-The following commands will be ported in upcoming releases:
-* Dispatch
+* [Orchestrator][6]
 
 See [Porting Map](https://github.com/microsoft/botframework-cli/blob/main/PortingMap.md) for a mapping reference between old and new tools
 
 ## Overview
 
 The  Bot Framework Command Line Interface (BF CLI) cross-platform tool is used to manage Bot Framework bots and related services. It is part of the [Microsoft Bot Framework](https://github.com/Microsoft/botframework), a comprehensive framework for building enterprise-grade conversational AI experiences.  In particular, BF CLI provides fundamental functionality when used in conjunction with Continuous Integration, and Continuous Deployment (CI/CD) pipelines.
 
-As you build your bot, you may also need to integrate AI services like [LUIS.ai](http://luis.ai) for language understanding, [QnAMaker.ai](http://qnamaker.ai) for your bot to respond to simple questions in a Q&A format, and more. The _[bf luis](./packages/cli#bf-luis)_ command is used to convert, and translate language definition _.lu_ files or generate corresponding source (C# or JavaScript) code. Then, use the [Luis Tool](https://github.com/microsoft/botbuilder-tools/tree/master/packages/LUIS)  to deploy the local files, train, test, and publish them as Language Understanding models within the LUIS service. If used to define QnAMaker question/answer Knowledgebase, use the _[bf qnamaker](./packages/cli#bf-qnamaker)_ command to create and manage QnAMaker assets both locally, and on the QnAMaker service. Please refer to the[ _lu_ library documentation](./packages/lu)  for extended discussion on how to work with .lu file formats. _Note: You may be familiar with the Luis command if you used the legacy [LuDown](https://github.com/microsoft/botbuilder-tools/tree/master/packages/Ludown) and [LuisGen](https://github.com/microsoft/botbuilder-tools/tree/master/packages/LUISGen) tools._
+As you build your bot, you may also need to integrate AI services like [LUIS.ai](http://luis.ai) for language understanding, [QnAMaker.ai](http://qnamaker.ai) for your bot to respond to simple questions in a Q&A format, and more. The _[bf luis](./packages/cli#bf-luis)_ command is used to convert, and translate language definition _.lu_ files or generate corresponding source (C# or JavaScript) code as well as deploy the local files, train, test, and publish them as Language Understanding models within the LUIS service. If used to define QnAMaker question/answer Knowledgebase, use the _[bf qnamaker](./packages/cli#bf-qnamaker)_ command to create and manage QnAMaker assets both locally, and on the QnAMaker service. Please refer to the[ _lu_ library documentation](./packages/lu)  for extended discussion on how to work with .lu file formats. _Note: You may be familiar with the Luis command if you used the legacy [LuDown](https://github.com/microsoft/botbuilder-tools/tree/master/packages/Ludown) and [LuisGen](https://github.com/microsoft/botbuilder-tools/tree/master/packages/LUISGen) tools._
 
-As your bot grows in sophistication, use [Dispatch](https://github.com/Microsoft/botbuilder-tools/tree/master/packages/Dispatch) CLI  to create and evaluate LUIS models used to dispatch intent across multiple bot modules such as LUIS models, QnA knowledgebases, and assist in routing messages to backend bot skills.
+As your bot grows in sophistication, use [bf orchestrator][6] command to generate a dispatcher LU from the language assets in your solution similar to the legacy [dispatch](https://github.com/Microsoft/botbuilder-tools/tree/master/packages/Dispatch) CLI. Then, you may use the [Orchestrator][7] recognizer for dispatching across subsequent services and skills. Use bf orchestrator command to evaluate language model quality. Use Orchestrator to to route intents across multiple bot modules such as LUIS models, QnA knowledgebases, and assist in routing messages to backend bot skills.
 
 To test and refine your bot, you can use the new [V4 Bot Framework Emulator](https://github.com/Microsoft/BotFramework-Emulator/releases). The Bot Framework Emulator is a cross-platform [Electron](https://electronjs.org/) application that enables you to test and debug your bots on local machine or in the cloud.
 
 Also, during early designs stages you may want to create mockup of conversations between the user and the bot for the specific scenarios your bot will support. Use [bf chatdown](./packages/cli#bf-chatdown) command to author conversation mockup .chat files and convert them into rich transcripts and view the conversations in the the Emulator.
 
-Lastly, with the [Azure CLI Bot extension](./AzureCli.md) (_az bot_ command), you can create, download, publish, configure channels with the [Azure Bot Service](https://azure.microsoft.com/en-us/services/bot-service/). It is a plugin that extends the functionality of [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) to manage your Azure Bot Service assets.
+With the [Azure CLI Bot extension](./AzureCli.md) (_az bot_ command), you can create, download, publish, configure channels with the [Azure Bot Service](https://azure.microsoft.com/en-us/services/bot-service/). It is a plugin that extends the functionality of [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) to manage your Azure Bot Service assets.
+
+Lastly, although the CLI is primarily used in code-first solutions and CI/CD pipeline automation, since the underlying libraries are also used in [Bot Framework Composer][8], it can be used to provision and organize some assets for Bot Framework Composer (such as .dialog generation, Orchestrator evaluation, exporting LUIS & QnAMaker models).  
 
 
 ### See Also
@@ -64,6 +70,7 @@ Lastly, with the [Azure CLI Bot extension](./AzureCli.md) (_az bot_ command), yo
 * [Azure Bot Service](https://docs.microsoft.com/en-us/azure/bot-service/bot-service-overview-introduction?view=azure-bot-service-4.0)
 * [LUIS](https://docs.microsoft.com/en-us/azure/cognitive-services/luis/index)
 * [QnAMaker](https://docs.microsoft.com/en-us/azure/cognitive-services/qnamaker/)
+* [Orchestrator][7]
 * [Bot Design Guidelines](https://docs.microsoft.com/en-us/azure/bot-service/bot-service-design-principles)
 
 ## Issues and Feature Requests
@@ -126,5 +133,6 @@ Copyright (c) Microsoft Corporation. All rights reserved.
 [3]: ./packages/dialog/README.md
 [4]: ./packages/luis/README.md
 [5]: ./packages/qnamaker/README.md
-
-
+[6]: ./packages/orchestrator/README.md
+[7]: https://aka.ms/bf-orchestrator
+[8]: https://docs.microsoft.com/en-us/composer/introduction
diff --git a/SECURITY.md b/SECURITY.md
@@ -0,0 +1,41 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+  * Full paths of source file(s) related to the manifestation of the issue
+  * The location of the affected source code (tag/branch/commit or direct URL)
+  * Any special configuration required to reproduce the issue
+  * Step-by-step instructions to reproduce the issue
+  * Proof-of-concept or exploit code (if possible)
+  * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->
diff --git a/build/bf-cli-build-test-steps.yml b/build/bf-cli-build-test-steps.yml
@@ -30,16 +30,20 @@ steps:
   continueOnError: true
 
 - task: NodeTool@0
-  displayName: 'Use Node 12.x'
+  displayName: 'Use Node 14.x'
   inputs:
-    versionSpec: 12.x
+    versionSpec: 14.x
 
-- task: Npm@1
-  displayName: 'npm install --global @microsoft/rush'
+- script: 'node ./common/scripts/set-registry-artifacts.js'
+  displayName: 'Point /common/config/rush/.npmrc to artifacts url'
+
+- task: npmAuthenticate@0
+  displayName: 'npm Authenticate common/config/rush/.npmrc'
   inputs:
-    command: custom
-    verbose: false
-    customCommand: 'install --global @microsoft/rush@5.33.0'
+    workingFile: common/config/rush/.npmrc
+
+- script: 'npm install --global @microsoft/rush@5.33.0'
+  displayName: 'npm install --global @microsoft/rush'
 
 - script: 'rush update'
   displayName: 'rush update'
@@ -60,6 +64,13 @@ steps:
     Contents: '**/*.tgz'
     TargetFolder: '$(Build.ArtifactStagingDirectory)/drop'
     flattenFolders: true
+
+- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+  displayName: 'generate software bill of materials'
+  inputs:
+    BuildDropPath: '$(Build.ArtifactStagingDirectory)/drop'
+    PackageName: 'Microsoft Bot Framework CLI'
+    PackageVersion: $(buildVersion)
 
 - task: PublishBuildArtifacts@1
   displayName: 'Publish Artifact: drop'
@@ -71,3 +82,12 @@ steps:
   displayName: 'Dir workspace except node_modules'
   continueOnError: true
   condition: succeededOrFailed()
+
+#Run component governance and fail build if not passed
+- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+  displayName: 'Component Detection'
+  inputs:
+    sourceScanPath: './packages'
+    ignoreDirectories: './common/temp/pnpm-local/node_modules/pnpm,./common/temp/pnpm-store,./common/config/rush/pnpm-lock.yaml,./common/temp/pnpm-lock.yaml'
+    failOnAlert: true
+