Allow Custom Pre-Installs During a Manual Build (#9533)

* adjustments to allow an argument w/ additional packages in it
* adding a wrapper ci.yml that can be invoked against the entire repo
* passing around a BuildDocs param. this means that the test-all can pass 'no docs' but the default will still generate them
* deleting the nightly build ymls. with some clever parametering we don't need them anymore!

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -5,6 +5,8 @@ parameters:
   BuildTargetingString: 'azure-*'
   TestTimeoutInMinutes: 0
   ToxEnvParallel: '--tenvparallel'
+  InjectedPackages: ''
+  BuildDocs: true
   TestMatrix:
     Linux_Python27:
       OSName: 'Linux'
@@ -70,6 +72,7 @@ jobs:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
+        BuildDocs: ${{ parameters.BuildDocs }}
 
   - job: 'Analyze'
     condition: and(succeededOrFailed(), ne(variables['Skip.Analyze'], 'true'))
@@ -131,6 +134,7 @@ jobs:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         ToxTestEnv: $(toxenv)
         ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
+        InjectedPackages: ${{ parameters.InjectedPackages }}
         BeforeTestSteps: 
           - task: DownloadPipelineArtifact@0
             inputs:

eng/pipelines/templates/jobs/archetype-sdk-tests.yml

@@ -6,6 +6,8 @@ parameters:
   BuildTargetingString: 'azure-*'
   AdditionalTestArgs: ''
   TestMarkArgument: ''
+  InjectedPackages: ''
+  BuildDocs: true
   JobName: Test
   Matrix:
     Linux_Python35:
@@ -59,4 +61,6 @@ jobs:
           ToxTestEnv: "whl"
           AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
           TestMarkArgument: ${{ parameters.TestMarkArgument }}
+          InjectedPackages: ${{ parameters.InjectedPackages }}
+          BuildDocs: ${{parameters.BuildDocs}}
 

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -2,6 +2,9 @@ parameters:
   Artifacts: []
   ServiceDirectory: not-specified
   ToxEnvParallel: '--tenvparallel'
+  InjectedPackages: ''
+  BuildDocs: true
+
 
 stages:
   - stage: Build
@@ -10,6 +13,8 @@ stages:
       parameters:
         ServiceDirectory: ${{parameters.ServiceDirectory}}
         ToxEnvParallel: ${{parameters.ToxEnvParallel}}
+        BuildDocs: ${{parameters.BuildDocs}}
+        InjectedPackages: ${{parameters.InjectedPackages}}
 
   # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
   - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:

eng/pipelines/templates/stages/cosmos-sdk-client.yml

@@ -3,6 +3,8 @@ parameters:
   ServiceDirectory: not-specified
   EmulatorMsiUrl: 'https://aka.ms/cosmosdb-emulator'
   EmulatorStartParameters: ''
+  InjectedPackages: ''
+  BuildDocs: true
 
 stages:
   - stage: Build
@@ -12,7 +14,8 @@ stages:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           # Skip emulator tests in the CI run
           TestMarkArgument: not cosmosEmulator
-
+          BuildDocs: ${{parameters.BuildDocs}}
+          InjectedPackages: ${{parameters.InjectedPackages}}
       - job: Emulator
         strategy:
           matrix:
@@ -42,6 +45,8 @@ stages:
               PythonVersion: $(PythonVersion)
               OSName: $(OSName)
               ToxTestEnv: 'whl,sdist'
+              BuildDocs: ${{parameters.BuildDocs}}
+              InjectedPackages: ${{parameters.InjectedPackages}}
 
   # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
   - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:

eng/pipelines/templates/steps/analyze.yml

@@ -32,7 +32,7 @@ steps:
   - script: |
       cd eng/versioning
       pip install -r requirements.txt
-      python find_invalid_versions.py --always-succeed --service ${{parameters.ServiceDirectory}}
+      python find_invalid_versions.py --always-succeed --service=${{parameters.ServiceDirectory}}
     displayName: Find Invalid Versions
 
   - template: eng/pipelines/templates/scripts/verify-path-length.yml@azure-sdk-tools

eng/pipelines/templates/steps/build-artifacts.yml

@@ -2,6 +2,7 @@ parameters:
   BeforePublishSteps: []
   BuildTargetingString: 'azure-*'
   ServiceDirectory: ''
+  BuildDocs: true
 
 steps:
   - script: |
@@ -57,6 +58,7 @@ steps:
 
   - task: PythonScript@0
     displayName: 'Generate Docs'
+    condition: and(succeededOrFailed(), ${{parameters.BuildDocs}})
     inputs:
       scriptPath: 'scripts/devops_tasks/setup_execute_tests.py'
       arguments: >-
@@ -79,7 +81,7 @@ steps:
     artifact: packages 
 
   - task: PublishBuildArtifacts@1
-    condition: succeededOrFailed()
+    condition: and(succeededOrFailed(), ${{parameters.BuildDocs}})
     displayName: 'Publish Artifacts'
     inputs:
       artifactName: 'documentation'

eng/pipelines/templates/steps/build-test.yml

@@ -11,6 +11,7 @@ parameters:
   ToxTestEnv: ""
   RunCoverage: ne(variables['CoverageArg'], '--disablecov')
   ToxEnvParallel: ''
+  InjectedPackages: ''
 
 steps:
   - pwsh: |
@@ -42,6 +43,7 @@ steps:
         --mark_arg="${{ parameters.TestMarkArgument }}" 
         --service="${{ parameters.ServiceDirectory }}" 
         --toxenv="${{ parameters.ToxTestEnv }}"
+        --injected-packages="${{ parameters.InjectedPackages }}"
         ${{ parameters.ToxEnvParallel }}
     env: ${{ parameters.EnvVars }}