From 95cc66202ebf2d8a2af4c8e9e07448db1cc5b410 Mon Sep 17 00:00:00 2001 From: Paul Higinbotham Date: Tue, 28 Apr 2020 10:26:46 -0700 Subject: [PATCH] Move PSThreadJob module to this repo, and rename to Microsoft.PowerShell.ThreadJob --- .../Microsoft.PowerShell.ThreadJob/.ci/ci.yml | 154 ++ .../.ci/compliance.yml | 109 ++ .../.ci/release.yml | 23 + .../.ci/test.yml | 57 + .../Microsoft.PowerShell.ThreadJob/README.md | 53 + .../Microsoft.PowerShell.ThreadJob/build.ps1 | 121 ++ .../doBuild.ps1 | 73 + .../about_Microsoft.PowerShell.ThreadJob.md | 57 + .../pspackageproject.json | 8 + .../sign-module-files.xml | 9 + .../src/Microsoft.PowerShell.ThreadJob.psd1 | 64 + .../src/code/AssemblyInfo.cs | 33 + .../Microsoft.PowerShell.Threadjob.csproj | 27 + .../src/code/Resources.Designer.cs | 135 ++ .../src/code/Resources.resx | 144 ++ .../src/code/ThreadJob.cs | 1341 +++++++++++++++++ ...icrosoft.PowerShell.ThreadJob.CL.Tests.ps1 | 232 +++ .../Microsoft.PowerShell.ThreadJob.Tests.ps1 | 439 ++++++ 18 files changed, 3079 insertions(+) create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/.ci/ci.yml create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/.ci/compliance.yml create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/.ci/release.yml create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/.ci/test.yml create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/README.md create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/build.ps1 create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/doBuild.ps1 create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/help/en-US/about_Microsoft.PowerShell.ThreadJob.md create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/pspackageproject.json create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/sign-module-files.xml create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/src/Microsoft.PowerShell.ThreadJob.psd1 create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/src/code/AssemblyInfo.cs create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/src/code/Microsoft.PowerShell.Threadjob.csproj create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/src/code/Resources.Designer.cs create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/src/code/Resources.resx create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/src/code/ThreadJob.cs create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/test/Microsoft.PowerShell.ThreadJob.CL.Tests.ps1 create mode 100644 Modules/Microsoft.PowerShell.ThreadJob/test/Microsoft.PowerShell.ThreadJob.Tests.ps1 diff --git a/Modules/Microsoft.PowerShell.ThreadJob/.ci/ci.yml b/Modules/Microsoft.PowerShell.ThreadJob/.ci/ci.yml new file mode 100644 index 0000000..8423bf0 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/.ci/ci.yml @@ -0,0 +1,154 @@ +name: $(BuildDefinitionName)-$(date:yyMM).$(date:dd)$(rev:rrr) +trigger: + # Batch merge builds together while a merge build is running + batch: true + branches: + include: + - master +pr: + branches: + include: + - master + +stages: +- stage: Build + displayName: Build PowerShell Package + jobs: + - job: BuildPkg + displayName: Build Package + pool: + name: Package ES CodeHub Lab E + steps: + - powershell: | + $powerShellPath = Join-Path -Path $env:AGENT_TEMPDIRECTORY -ChildPath 'powershell' + Invoke-WebRequest -Uri https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/install-powershell.ps1 -outfile ./install-powershell.ps1 + ./install-powershell.ps1 -Destination $powerShellPath + $vstsCommandString = "vso[task.setvariable variable=PATH]$powerShellPath;$env:PATH" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Install PowerShell Core + + - task: UseDotNet@2 + displayName: 'Install .NET Core 3.1.100 sdk' + inputs: + packageType: sdk + version: 3.1.100 + + - task: PkgESSetupBuild@10 + displayName: 'Package ES - Setup Build' + inputs: + productName: PSRemotingTools + useDfs: false + + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture environment for build + condition: succeededOrFailed() + + - pwsh: | + dir $env:USERPROFILE\Documents\PowerShell\Modules\* -Directory -ErrorAction SilentlyContinue | Remove-Item -Recurse -Force -Verbose -ErrorAction SilentlyContinue + displayName: Clean PowerShell modules directory + + - pwsh: | + Get-Module -Name PowerShellGet -ListAvailable + Import-Module -Name PowerShellGet + displayName: Import PowerShellGet Module + + - pwsh: | + Install-Module -Name "platyPS","Pester" -Force + displayName: Install dependencies + - pwsh: | + Install-Module -Name "PSScriptAnalyzer" -RequiredVersion 1.18.0 -Force + displayName: Install PSScriptAnalyzer + - pwsh: | + Install-Module -Name PSPackageProject -Force + displayName: Install PSPackageProject module + - pwsh: | + $(Build.SourcesDirectory)/build.ps1 -Build -BuildConfiguration Release + displayName: Build and publish artifact + + - pwsh: | + Install-Module -Name PSPackageProject -Force + $config = Get-PSPackageProjectConfiguration + $signSrcPath = "$($config.BuildOutputPath)\$($config.ModuleName)" + $signOutPath = "$($config.BuildOutputPath)\$($config.ModuleName)\Signed" + if (! (Test-Path -Path $signOutPath)) { + $null = New-Item -Path $signOutPath -ItemType Directory + } + $signXmlPath = "$($config.SourcePath)\..\sign-module-files.xml" + # Set signing src path variable + $vstsCommandString = "vso[task.setvariable variable=signSrcPath]${signSrcPath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + # Set signing out path variable + $vstsCommandString = "vso[task.setvariable variable=signOutPath]${signOutPath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + # Set signing xml path + $vstsCommandString = "vso[task.setvariable variable=signXmlPath]${signXmlPath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Set up for code signing + + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture environment for code signing + condition: succeededOrFailed() + + - task: PkgESCodeSign@10 + displayName: Sign build files + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + signConfigXml: '$(signXmlPath)' + inPathRoot: '$(signSrcPath)' + outPathRoot: '$(signOutPath)' + binVersion: Production + binVersionOverride: '' + condition: and(and(succeeded(), eq(variables['Build.Reason'], 'Manual')), ne(variables['SkipSigning'], 'True')) + +- stage: Compliance + displayName: Compliance + dependsOn: Build + jobs: + - job: ComplianceJob + pool: + name: Package ES CodeHub Lab E + steps: + - template: compliance.yml + +- stage: Test + displayName: Test Package + dependsOn: Build + jobs: + - template: test.yml + parameters: + jobName: TestPkgWin + displayName: PowerShell Core on Windows + imageName: windows-2019 + +# - template: test.yml +# parameters: +# jobName: TestPkgWinPS +# displayName: Windows PowerShell on Windows +# imageName: windows-2019 +# powershellExecutable: powershell + + - template: test.yml + parameters: + jobName: TestPkgUbuntu16 + displayName: PowerShell Core on Ubuntu 16.04 + imageName: ubuntu-16.04 + + - template: test.yml + parameters: + jobName: TestPkgWinMacOS + displayName: PowerShell Core on macOS + imageName: macOS-10.14 + +- stage: Release + displayName: Release Package + condition: and(and(succeeded(), eq(variables['Build.Reason'], 'Manual')), eq(variables['Publish'], 'True')) + jobs: + - template: release.yml + diff --git a/Modules/Microsoft.PowerShell.ThreadJob/.ci/compliance.yml b/Modules/Microsoft.PowerShell.ThreadJob/.ci/compliance.yml new file mode 100644 index 0000000..149f417 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/.ci/compliance.yml @@ -0,0 +1,109 @@ +steps: + +- powershell: | + $powerShellPath = Join-Path -Path $env:AGENT_TEMPDIRECTORY -ChildPath 'powershell' + Invoke-WebRequest -Uri https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/install-powershell.ps1 -outfile ./install-powershell.ps1 + ./install-powershell.ps1 -Destination $powerShellPath + $vstsCommandString = "vso[task.setvariable variable=PATH]$powerShellPath;$env:PATH" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Install PowerShell Core + +- pwsh: | + Install-Module -Name "platyPS","Pester" -Force + displayName: Install platyPS + +- pwsh: | + Install-Module -Name "PSScriptAnalyzer" -RequiredVersion 1.18.0 -Force + displayName: Install PSScripAnalyzer + +- pwsh: | + Install-Module -Name PSPackageProject -Force + displayName: Install PSPackageProject module + +- task: DownloadBuildArtifacts@0 + displayName: 'Download artifacts' + inputs: + buildType: current + downloadType: specifc + itemPattern: '**/*.nupkg' + downloadPath: '$(System.ArtifactsDirectory)' + +- pwsh: | + $sourceName = 'pspackageproject-local-repo' + Register-PSRepository -Name $sourceName -SourceLocation '$(System.ArtifactsDirectory)' -ErrorAction Ignore + $config = Get-PSPackageProjectConfiguration + $buildOutputPath = $config.BuildOutputPath + $null = New-Item -ItemType Directory -Path $buildOutputPath -Verbose + $moduleName = $config.ModuleName + Save-Module -Repository $sourceName -Name $moduleName -Path $config.BuildOutputPath + $vstsCommandString = "vso[task.setvariable variable=BUILD_SOURCE]$($config.BuildOutputPath)" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Extract product artifact + +- pwsh: | + $config = Get-PSPackageProjectConfiguration + dir "$($config.BuildOutputPath)/*" -r 2>$null + displayName: 'BuildOutputPath directory' + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 + displayName: 'Run Defender Scan' + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 + displayName: 'Run CredScan' + inputs: + toolMajorVersion: V2 + debugMode: false + continueOnError: true + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-binskim.BinSkim@3 + displayName: 'Run BinSkim ' + inputs: + InputType: Basic + AnalyzeTarget: '$(BUILD_SOURCE)\Microsoft.PowerShell.SecretManagement\Microsoft.PowerShell.SecretManagement.dll' + AnalyzeSymPath: 'SRV*' + AnalyzeVerbose: true + AnalyzeHashes: true + AnalyzeStatistics: true + continueOnError: true + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 + displayName: 'Run PoliCheck' + inputs: + targetType: F + optionsFC: 0 + optionsXS: 0 + optionsPE: '1|2|3|4' + optionsHMENABLE: 0 +# optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb' +# optionsFTPATH: '$(Build.SourcesDirectory)\tools\terms\FileTypeSet.xml' + toolVersion: 5.8.2.1 + continueOnError: true + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 + displayName: 'Publish Security Analysis Logs to Build Artifacts' + continueOnError: true + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1 + displayName: 'TSA upload to Codebase: PSThreadJob_201912 Stamp: Azure' + inputs: + codeBaseName: PSThreadJob_201912 + tsaVersion: TsaV2 + uploadFortifySCA: false + uploadFxCop: false + uploadModernCop: false + uploadPREfast: false + uploadRoslyn: false + uploadTSLint: false + uploadAPIScan: false + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1 + displayName: 'Create Security Analysis Report' + inputs: + TsvFile: false + APIScan: false + BinSkim: false + CredScan: true + PoliCheck: true + PoliCheckBreakOn: Severity2Above diff --git a/Modules/Microsoft.PowerShell.ThreadJob/.ci/release.yml b/Modules/Microsoft.PowerShell.ThreadJob/.ci/release.yml new file mode 100644 index 0000000..650d624 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/.ci/release.yml @@ -0,0 +1,23 @@ +parameters: + jobName: release + imageName: windows-2019 + displayName: Release + +jobs: +- job: ${{ parameters.jobName }} + pool: + vmImage: ${{ parameters.imageName }} + displayName: ${{ parameters.displayName }} + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifacts' + inputs: + buildType: current + downloadType: single + artifactName: NuPkg + downloadPath: '$(System.ArtifactsDirectory)' + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet' + - pwsh: | + nuget push $(System.ArtifactsDirectory)\nupkg\*.nupkg -ApiKey $(NuGetApiKey) -Source https://www.powershellgallery.com/api/v2/package/ -NonInteractive + displayName: Publish Package diff --git a/Modules/Microsoft.PowerShell.ThreadJob/.ci/test.yml b/Modules/Microsoft.PowerShell.ThreadJob/.ci/test.yml new file mode 100644 index 0000000..a12e28b --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/.ci/test.yml @@ -0,0 +1,57 @@ +parameters: + jobName: TestPkgWin + imageName: windows-2019 + displayName: PowerShell Core on Windows + powershellExecutable: pwsh + +jobs: +- job: ${{ parameters.jobName }} + pool: + vmImage: ${{ parameters.imageName }} + displayName: ${{ parameters.displayName }} + steps: + - ${{ parameters.powershellExecutable }}: | + Install-Module -Name "platyPS","Pester" -Force + displayName: Install dependencies + + - ${{ parameters.powershellExecutable }}: | + Install-Module -Name "PSScriptAnalyzer" -RequiredVersion 1.18.0 -Force + displayName: Install dependencies + + - ${{ parameters.powershellExecutable }}: | + Install-Module -Name PSPackageProject -Force + displayName: Install PSPackageProject module + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifacts' + inputs: + buildType: current + downloadType: specific + itemPattern: '**/*.nupkg' + downloadPath: '$(System.ArtifactsDirectory)' + + - ${{ parameters.powershellExecutable }}: | + $sourceName = 'pspackageproject-local-repo' + Register-PSRepository -Name $sourceName -SourceLocation '$(System.ArtifactsDirectory)' -ErrorAction Ignore + $config = Get-PSPackageProjectConfiguration + $buildOutputPath = $config.BuildOutputPath + $null = New-Item -ItemType Directory -Path $buildOutputPath -Verbose + $moduleName = $config.ModuleName + Save-Module -Repository $sourceName -Name $moduleName -Path $config.BuildOutputPath + displayName: Extract product artifact + + - ${{ parameters.powershellExecutable }}: | + Invoke-PSPackageProjectTest -Type Functional + displayName: Execute functional tests + errorActionPreference: continue + + - ${{ parameters.powershellExecutable }}: | + Invoke-PSPackageProjectTest -Type StaticAnalysis + displayName: Execute static analysis tests + errorActionPreference: continue + condition: succeededOrFailed() + + - ${{ parameters.powershellExecutable }}: | + Unregister-PSRepository -Name 'pspackageproject-local-repo' -ErrorAction Ignore + displayName: Unregister temporary PSRepository + condition: always() diff --git a/Modules/Microsoft.PowerShell.ThreadJob/README.md b/Modules/Microsoft.PowerShell.ThreadJob/README.md new file mode 100644 index 0000000..1b9a3dd --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/README.md @@ -0,0 +1,53 @@ +# Microsoft.PowerShell.ThreadJob +A PowerShell module for running concurrent jobs based on threads rather than processes + +PowerShell's built-in BackgroundJob jobs (Start-Job) are run in separate processes on the local machine. They provide excellent isolation but are resource heavy. Running hundreds of BackgroundJob jobs can quickly absorb system resources by creating hundreds of processes. There is no throttling mechanism and so all jobs are started immediately and are all run currently. + +This module extends the existing PowerShell BackgroundJob to include a new thread based ThreadJob job. It is a lighter weight solution for running concurrent PowerShell scripts that works within the existing PowerShell job infrastructure. So these jobs work with existing PowerShell job cmdlets. + +ThreadJob jobs will tend to run much faster because there is lower overhead and they do not use the remoting serialization system as BackgroundJob jobs do. And they will use up fewer system resources. In addition output objects returned from the job will be 'live' since they are not re-hydrated from the serialization system. However, there is less isolation. If one ThreadJob job crashes the process then all ThreadJob jobs running in that process will be terminated. + +This module exports a single cmdlet, Start-ThreadJob, which works similarly to the existing Start-Job cmdlet. The main difference is that the jobs which are created run in separate threads within the local process. + +Also ThreadJob jobs support a ThrottleLimit parameter to limit the number of running jobs, and thus running threads, at a time. If more jobs are started then they go into a queue and wait until the current number of jobs drops below the throttle limit. + +## Examples + +```powershell +PS C:\> Start-ThreadJob -ScriptBlock { 1..100 | % { sleep 1; "Output $_" } } -ThrottleLimit 2 +PS C:\> Start-ThreadJob -ScriptBlock { 1..100 | % { sleep 1; "Output $_" } } +PS C:\> Start-ThreadJob -ScriptBlock { 1..100 | % { sleep 1; "Output $_" } } +PS C:\> get-job + +Id Name PSJobTypeName State HasMoreData Location Command +-- ---- ------------- ----- ----------- -------- ------- +1 Job1 ThreadJob Running True PowerShell 1..100 | % { sleep 1;... +2 Job2 ThreadJob Running True PowerShell 1..100 | % { sleep 1;... +3 Job3 ThreadJob NotStarted False PowerShell 1..100 | % { sleep 1;... +``` + +```powershell +PS C:\> $job = Start-ThreadJob { Get-Process -id $pid } +PS C:\> $myProc = Receive-Job $job +# !!Don't do this. $myProc is a live object!! +PS C:\> $myProc.Kill() +``` + +```powershell +# start five background jobs each running 1 second +PS C:\> Measure-Command {1..5 | % {Start-Job {Sleep 1}} | Wait-Job} | Select TotalSeconds +PS C:\> Measure-Command {1..5 | % {Start-ThreadJob {Sleep 1}} | Wait-Job} | Select TotalSeconds + +TotalSeconds +------------ + 5.7665849 # jobs creation time > 4.7 sec; results may vary + 1.5735008 # jobs creation time < 0.6 sec (8 times less!) +``` + +## Installing + +You can install this module from [PowerShell Gallery](https://www.powershellgallery.com/packages/Microsoft.PowerShell.ThreadJob/1.1.2) using this command: + +```powershell +Install-Module -Name Microsoft.PowerShell.ThreadJob -Scope CurrentUser +``` diff --git a/Modules/Microsoft.PowerShell.ThreadJob/build.ps1 b/Modules/Microsoft.PowerShell.ThreadJob/build.ps1 new file mode 100644 index 0000000..7d44f4d --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/build.ps1 @@ -0,0 +1,121 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +param ( + [Parameter(ParameterSetName="build")] + [switch] + $Clean, + + [Parameter(ParameterSetName="build")] + [switch] + $Build, + + [Parameter(ParameterSetName="build")] + [switch] + $Test, + + [Parameter(ParameterSetName="build")] + [string[]] + [ValidateSet("Functional","StaticAnalysis")] + $TestType = @("Functional"), + + [Parameter(ParameterSetName="help")] + [switch] + $UpdateHelp, + + [ValidateSet("Debug", "Release")] + [string] $BuildConfiguration = "Debug", + + [ValidateSet("net461", "netcoreapp2.1", "netstandard2.0")] + [string] $BuildFramework = "net461" +) + +if ( ! ( Get-Module -ErrorAction SilentlyContinue PSPackageProject) ) { + Install-Module PSPackageProject +} + +$config = Get-PSPackageProjectConfiguration -ConfigPath $PSScriptRoot + +$script:ModuleName = $config.ModuleName +$script:SrcPath = $config.SourcePath +$script:OutDirectory = $config.BuildOutputPath +$script:TestPath = $config.TestPath + +$script:ModuleRoot = $PSScriptRoot +$script:Culture = $config.Culture +$script:HelpPath = $config.HelpPath + +$script:BuildConfiguration = $BuildConfiguration +$script:BuildFramework = $BuildFramework + +. "$PSScriptRoot\doBuild.ps1" + +# The latest DotNet (3.1.1) is needed to perform binary build. +$dotNetCmd = Get-Command -Name dotNet -ErrorAction SilentlyContinue +$dotnetVersion = $null +if ($dotNetCmd -ne $null) { + $info = dotnet --info + foreach ($item in $info) { + $index = $item.IndexOf('Version:') + if ($index -gt -1) { + $versionStr = $item.SubString('Version:'.Length + $index) + $null = [version]::TryParse($versionStr, [ref] $dotnetVersion) + break + } + } +} +# DotNet 3.1.1 is installed in ci.yml. Just check installation and version here. +Write-Verbose -Verbose -Message "Installed DotNet found: $($dotNetCmd -ne $null), version: $versionStr" +<# +$dotNetVersionOk = ($dotnetVersion -ne $null) -and ((($dotnetVersion.Major -eq 3) -and ($dotnetVersion.Minor -ge 1)) -or ($dotnetVersion.Major -gt 3)) +if (! $dotNetVersionOk) { + + Write-Verbose -Verbose -Message "Installing dotNet..." + $installObtainUrl = "https://dotnet.microsoft.com/download/dotnet-core/scripts/v1" + + Remove-Item -ErrorAction SilentlyContinue -Recurse -Force ~\AppData\Local\Microsoft\dotnet + $installScript = "dotnet-install.ps1" + Invoke-WebRequest -Uri $installObtainUrl/$installScript -OutFile $installScript + + & ./$installScript -Channel 'release' -Version '3.1.101' + Write-Verbose -Verbose -Message "dotNet installation complete." +} +#> + +if ($Clean -and (Test-Path $OutDirectory)) +{ + Remove-Item -Path $OutDirectory -Force -Recurse -ErrorAction Stop -Verbose + + if (Test-Path "${SrcPath}/code/bin") + { + Remove-Item -Path "${SrcPath}/code/bin" -Recurse -Force -ErrorAction Stop -Verbose + } + + if (Test-Path "${SrcPath}/code/obj") + { + Remove-Item -Path "${SrcPath}/code/obj" -Recurse -Force -ErrorAction Stop -Verbose + } +} + +if (-not (Test-Path $OutDirectory)) +{ + $script:OutModule = New-Item -ItemType Directory -Path (Join-Path $OutDirectory $ModuleName) +} +else +{ + $script:OutModule = Join-Path $OutDirectory $ModuleName +} + +if ($Build.IsPresent) +{ + $sb = (Get-Item Function:DoBuild).ScriptBlock + Invoke-PSPackageProjectBuild -BuildScript $sb +} + +if ( $Test.IsPresent ) { + Invoke-PSPackageProjectTest -Type $TestType +} + +if ($UpdateHelp.IsPresent) { + Add-PSPackageProjectCmdletHelp -ProjectRoot $ModuleRoot -ModuleName $ModuleName -Culture $Culture +} diff --git a/Modules/Microsoft.PowerShell.ThreadJob/doBuild.ps1 b/Modules/Microsoft.PowerShell.ThreadJob/doBuild.ps1 new file mode 100644 index 0000000..f3ca6a1 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/doBuild.ps1 @@ -0,0 +1,73 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# +.DESCRIPTION +Implement build and packaging of the package and place the output $OutDirectory/$ModuleName +#> +function DoBuild +{ + Write-Verbose -Verbose -Message "Starting DoBuild with configuration: $BuildConfiguration, framework: $BuildFramework" + + # Module build out path + $BuildOutPath = "${OutDirectory}/${ModuleName}" + Write-Verbose -Verbose -Message "Module output file path: '$BuildOutPath'" + + # Module build source path + $BuildSrcPath = "bin/${BuildConfiguration}/${BuildFramework}/publish" + Write-Verbose -Verbose -Message "Module build source path: '$BuildSrcPath'" + + # Copy psd1 file + Write-Verbose -Verbose "Copy-Item ${SrcPath}/${ModuleName}.psd1 to ${OutDirectory}/${ModuleName}" + Copy-Item "${SrcPath}/${ModuleName}.psd1" "${OutDirectory}/${ModuleName}" + + # Copy format files here + Write-Verbose -Verbose "Copy-Item ${SrcPath}/${ModuleName}.format.ps1xml to ${OutDirectory}/${ModuleName}" + copy-item "${SrcPath}/${ModuleName}.format.ps1xml" "${OutDirectory}/${ModuleName}" + + # Copy help + Write-Verbose -Verbose -Message "Copying help files to '$BuildOutPath'" + copy-item -Recurse "${HelpPath}/${Culture}" "$BuildOutPath" + + if ( Test-Path "${SrcPath}/code" ) { + Write-Verbose -Verbose -Message "Building assembly and copying to '$BuildOutPath'" + # build code and place it in the staging location + Push-Location "${SrcPath}/code" + try { + # Build source + Write-Verbose -Verbose -Message "Building with configuration: $BuildConfiguration, framework: $BuildFramework" + Write-Verbose -Verbose -Message "Building location: PSScriptRoot: $PSScriptRoot, PWD: $pwd" + dotnet publish --configuration $BuildConfiguration --framework $BuildFramework --output $BuildSrcPath + + # Debug: Check + + # Place build results + if (! (Test-Path -Path "$BuildSrcPath/${ModuleName}.dll")) + { + throw "Expected binary was not created: $BuildSrcPath/${ModuleName}.dll" + } + + Write-Verbose -Verbose -Message "Copying $BuildSrcPath/${ModuleName}.dll to $BuildOutPath" + Copy-Item "$BuildSrcPath/${ModuleName}.dll" -Dest "$BuildOutPath" + + if (Test-Path -Path "$BuildSrcPath/${ModuleName}.pdb") + { + Write-Verbose -Verbose -Message "Copying $BuildSrcPath/${ModuleName}.pdb to $BuildOutPath" + Copy-Item -Path "$BuildSrcPath/${ModuleName}.pdb" -Dest "$BuildOutPath" + } + } + catch { + # Write-Error "dotnet build failed with error: $_" + Write-Verbose -Verbose -Message "dotnet build failed with error: $_" + } + finally { + Pop-Location + } + } + else { + Write-Verbose -Verbose -Message "No code to build in '${SrcPath}/code'" + } + + ## Add build and packaging here + Write-Verbose -Verbose -Message "Ending DoBuild" +} diff --git a/Modules/Microsoft.PowerShell.ThreadJob/help/en-US/about_Microsoft.PowerShell.ThreadJob.md b/Modules/Microsoft.PowerShell.ThreadJob/help/en-US/about_Microsoft.PowerShell.ThreadJob.md new file mode 100644 index 0000000..03a0d20 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/help/en-US/about_Microsoft.PowerShell.ThreadJob.md @@ -0,0 +1,57 @@ +# Microsoft.PowerShell.ThreadJob +## about_Microsoft.PowerShell.ThreadJob + +``` +ABOUT TOPIC NOTE: +The first header of the about topic should be the topic name. +The second header contains the lookup name used by the help system. + +IE: +# Some Help Topic Name +## SomeHelpTopicFileName + +This will be transformed into the text file +as `about_SomeHelpTopicFileName`. +Do not include file extensions. +The second header should have no spaces. +``` + +# SHORT DESCRIPTION +{{ Short Description Placeholder }} + +``` +ABOUT TOPIC NOTE: +About topics can be no longer than 80 characters wide when rendered to text. +Any topics greater than 80 characters will be automatically wrapped. +The generated about topic will be encoded UTF-8. +``` + +# LONG DESCRIPTION +{{ Long Description Placeholder }} + +## Optional Subtopics +{{ Optional Subtopic Placeholder }} + +# EXAMPLES +{{ Code or descriptive examples of how to leverage the functions described. }} + +# NOTE +{{ Note Placeholder - Additional information that a user needs to know.}} + +# TROUBLESHOOTING NOTE +{{ Troubleshooting Placeholder - Warns users of bugs}} + +{{ Explains behavior that is likely to change with fixes }} + +# SEE ALSO +{{ See also placeholder }} + +{{ You can also list related articles, blogs, and video URLs. }} + +# KEYWORDS +{{List alternate names or titles for this topic that readers might use.}} + +- {{ Keyword Placeholder }} +- {{ Keyword Placeholder }} +- {{ Keyword Placeholder }} +- {{ Keyword Placeholder }} diff --git a/Modules/Microsoft.PowerShell.ThreadJob/pspackageproject.json b/Modules/Microsoft.PowerShell.ThreadJob/pspackageproject.json new file mode 100644 index 0000000..1a81614 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/pspackageproject.json @@ -0,0 +1,8 @@ +{ + "ModuleName": "Microsoft.PowerShell.ThreadJob", + "Culture": "en-US", + "BuildOutputPath": "out", + "HelpPath": "help", + "TestPath": "test", + "SourcePath": "src" +} diff --git a/Modules/Microsoft.PowerShell.ThreadJob/sign-module-files.xml b/Modules/Microsoft.PowerShell.ThreadJob/sign-module-files.xml new file mode 100644 index 0000000..038c148 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/sign-module-files.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/Modules/Microsoft.PowerShell.ThreadJob/src/Microsoft.PowerShell.ThreadJob.psd1 b/Modules/Microsoft.PowerShell.ThreadJob/src/Microsoft.PowerShell.ThreadJob.psd1 new file mode 100644 index 0000000..d2fcd3a --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/src/Microsoft.PowerShell.ThreadJob.psd1 @@ -0,0 +1,64 @@ +# +# Module manifest for module 'Microsoft.PowerShell.ThreadJob' +# + +@{ + +# Script module or binary module file associated with this manifest. +RootModule = '.\Microsoft.PowerShell.ThreadJob.dll' + +# Version number of this module. +ModuleVersion = '2.1.0' + +# ID used to uniquely identify this module +GUID = '0e7b895d-2fec-43f7-8cae-11e8d16f6e40' + +Author = 'Microsoft Corporation' +CompanyName = 'Microsoft Corporation' +Copyright = '(c) Microsoft Corporation. All rights reserved.' + +# Description of the functionality provided by this module +Description = " +PowerShell's built-in BackgroundJob jobs (Start-Job) are run in separate processes on the local machine. +They provide excellent isolation but are resource heavy. Running hundreds of BackgroundJob jobs can quickly +absorb system resources. + +This module extends the existing PowerShell BackgroundJob to include a new thread based ThreadJob job. This is a +lighter weight solution for running concurrent PowerShell scripts that works within the existing PowerShell job +infrastructure. + +ThreadJob jobs will tend to run quicker because there is lower overhead and they do not use the remoting serialization +system. And they will use up fewer system resources. In addition output objects returned from the job will be +'live' since they are not re-hydrated from the serialization system. However, there is less isolation. If one +ThreadJob job crashes the process then all ThreadJob jobs running in that process will be terminated. + +This module exports a single cmdlet, Start-ThreadJob, which works similarly to the existing Start-Job cmdlet. +The main difference is that the jobs which are created run in separate threads within the local process. + +One difference is that ThreadJob jobs support a ThrottleLimit parameter to limit the number of running jobs, +and thus active threads, at a time. If more jobs are started then they go into a queue and wait until the current +number of jobs drops below the throttle limit. + +Source for this module is at GitHub. Please submit any issues there. +https://github.com/PowerShell/Modules/Modules/Microsoft.PowerShell.ThreadJob + +Added Runspace cleanup. +Added Using variable expression support. +Added StreamingHost parameter to stream host data writes to a provided host object. +Added Information stream handling. +Bumped version to 2.0.0, and now only support PowerShell version 5.1 and higher. +Fixed using keyword bug with PowerShell preview version, and removed unneeded version check. +Added setting current working directory to running jobs, when available. +Added help URI to module. +Change module name to Microsoft.PowerShell.ThreadJob, add literalpath for current directory. +" + +# Minimum version of the Windows PowerShell engine required by this module +PowerShellVersion = '5.1' + +# Cmdlets to export from this module +CmdletsToExport = 'Start-ThreadJob' + +HelpInfoURI = 'https://go.microsoft.com/fwlink/?linkid=2113345' + +} diff --git a/Modules/Microsoft.PowerShell.ThreadJob/src/code/AssemblyInfo.cs b/Modules/Microsoft.PowerShell.ThreadJob/src/code/AssemblyInfo.cs new file mode 100644 index 0000000..10dba83 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/src/code/AssemblyInfo.cs @@ -0,0 +1,33 @@ +/********************************************************************++ +Copyright (c) Microsoft Corporation. All rights reserved. +--********************************************************************/ + +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyDescription("Implements PowerShell Start-ThreadJob")] +[assembly: AssemblyCopyright("")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("aba48637-8365-4c8f-90b5-dc424f5f5281")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] diff --git a/Modules/Microsoft.PowerShell.ThreadJob/src/code/Microsoft.PowerShell.Threadjob.csproj b/Modules/Microsoft.PowerShell.ThreadJob/src/code/Microsoft.PowerShell.Threadjob.csproj new file mode 100644 index 0000000..26e282e --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/src/code/Microsoft.PowerShell.Threadjob.csproj @@ -0,0 +1,27 @@ + + + + Library + Microsoft.PowerShell.ThreadJob + Microsoft.PowerShell.ThreadJob + 2.1.0.0 + 2.1.0 + 2.1.0 + net461;netcoreapp2.1 + + + + False + C:\Windows\Microsoft.NET\assembly\GAC_MSIL\System.Management.Automation\v4.0_3.0.0.0__31bf3856ad364e35\System.Management.Automation.dll + + + + + + + + + + + + diff --git a/Modules/Microsoft.PowerShell.ThreadJob/src/code/Resources.Designer.cs b/Modules/Microsoft.PowerShell.ThreadJob/src/code/Resources.Designer.cs new file mode 100644 index 0000000..c41ea3a --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/src/code/Resources.Designer.cs @@ -0,0 +1,135 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.PowerShell.ThreadJob.Properties { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Microsoft.PowerShell.ThreadJob.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to Unable to parse script file.. + /// + internal static string CannotParseScriptFile { + get { + return ResourceManager.GetString("CannotParseScriptFile", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot start job because it is not in NotStarted state.. + /// + internal static string CannotStartJob { + get { + return ResourceManager.GetString("CannotStartJob", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Invalid file path extension. Extension should be .ps1.. + /// + internal static string FilePathExt { + get { + return ResourceManager.GetString("FilePathExt", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to FilePath cannot contain wildcards.. + /// + internal static string FilePathWildcards { + get { + return ResourceManager.GetString("FilePathWildcards", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to No script block or script file was provided for the job to run.. + /// + internal static string NoScriptToRun { + get { + return ResourceManager.GetString("NoScriptToRun", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot get the value of the Using expression {0}. Start-ThreadJob only supports using variable expressions.. + /// + internal static string UsingNotVariableExpression { + get { + return ResourceManager.GetString("UsingNotVariableExpression", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Unable to find Using variable {0}.. + /// + internal static string UsingVariableNotFound { + get { + return ResourceManager.GetString("UsingVariableNotFound", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot run trusted script file {0} in FullLanguage mode because an initialization script block is included in the job, and the script block is not trusted.. + /// + internal static string CannotRunTrustedFileInFL{ + get { + return ResourceManager.GetString("CannotRunTrustedFileInFL", resourceCulture); + } + } + } +} diff --git a/Modules/Microsoft.PowerShell.ThreadJob/src/code/Resources.resx b/Modules/Microsoft.PowerShell.ThreadJob/src/code/Resources.resx new file mode 100644 index 0000000..2d079b6 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/src/code/Resources.resx @@ -0,0 +1,144 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Unable to parse script file. + + + Cannot start job because it is not in NotStarted state. + + + Invalid file path extension. Extension should be .ps1. + + + FilePath cannot contain wildcards. + + + No script block or script file was provided for the job to run. + + + Cannot get the value of the Using expression {0}. Start-ThreadJob only supports using variable expressions. + + + Unable to find Using variable {0}. + + + Cannot run trusted script file {0} in FullLanguage mode because an initialization script block is included in the job, and the script block is not trusted. + + \ No newline at end of file diff --git a/Modules/Microsoft.PowerShell.ThreadJob/src/code/ThreadJob.cs b/Modules/Microsoft.PowerShell.ThreadJob/src/code/ThreadJob.cs new file mode 100644 index 0000000..dcc7467 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/src/code/ThreadJob.cs @@ -0,0 +1,1341 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Linq; +using System.Management.Automation; +using System.Management.Automation.Host; +using System.Management.Automation.Language; +using System.Management.Automation.Runspaces; +using System.Management.Automation.Security; +using System.Text; +using System.Threading; + +namespace Microsoft.PowerShell.ThreadJob +{ + [Cmdlet(VerbsLifecycle.Start, "ThreadJob")] + [OutputType(typeof(ThreadJob))] + public sealed class StartThreadJobCommand : PSCmdlet + { + #region Private members + + private bool _processFirstRecord; + private string _command; + private string _currentLocationPath; + private ThreadJob _threadJob; + + #endregion + + #region Parameters + + private const string ScriptBlockParameterSet = "ScriptBlock"; + private const string FilePathParameterSet = "FilePath"; + + [Parameter(ParameterSetName = ScriptBlockParameterSet, Mandatory=true, Position=0)] + [ValidateNotNullAttribute] + public ScriptBlock ScriptBlock { get; set; } + + [Parameter(ParameterSetName = FilePathParameterSet, Mandatory=true, Position=0)] + [ValidateNotNullOrEmpty] + public string FilePath { get; set; } + + [Parameter(ParameterSetName = ScriptBlockParameterSet)] + [Parameter(ParameterSetName = FilePathParameterSet)] + [ValidateNotNullOrEmpty] + public string Name { get; set; } + + [Parameter(ParameterSetName = ScriptBlockParameterSet)] + [Parameter(ParameterSetName = FilePathParameterSet)] + [ValidateNotNull] + public ScriptBlock InitializationScript { get; set; } + + [Parameter(ParameterSetName = ScriptBlockParameterSet, ValueFromPipeline=true)] + [Parameter(ParameterSetName = FilePathParameterSet, ValueFromPipeline=true)] + [ValidateNotNull] + public PSObject InputObject { get; set; } + + [Parameter(ParameterSetName = ScriptBlockParameterSet)] + [Parameter(ParameterSetName = FilePathParameterSet)] + public Object[] ArgumentList { get; set; } + + [Parameter(ParameterSetName = ScriptBlockParameterSet)] + [Parameter(ParameterSetName = FilePathParameterSet)] + [ValidateRange(1, 1000000)] + public int ThrottleLimit { get; set; } + + [Parameter(ParameterSetName = ScriptBlockParameterSet)] + [Parameter(ParameterSetName = FilePathParameterSet)] + public PSHost StreamingHost { get; set; } + + #endregion + + #region Overrides + + protected override void BeginProcessing() + { + base.BeginProcessing(); + + if (ParameterSetName.Equals(ScriptBlockParameterSet)) + { + _command = ScriptBlock.ToString(); + } + else + { + _command = FilePath; + } + + try + { + _currentLocationPath = SessionState.Path.CurrentLocation.Path; + } + catch (PSInvalidOperationException) + { + } + } + + protected override void ProcessRecord() + { + base.ProcessRecord(); + + if (!_processFirstRecord) + { + if (StreamingHost != null) + { + _threadJob = new ThreadJob(Name, _command, ScriptBlock, FilePath, InitializationScript, ArgumentList, + InputObject, this, _currentLocationPath, StreamingHost); + } + else + { + _threadJob = new ThreadJob(Name, _command, ScriptBlock, FilePath, InitializationScript, ArgumentList, + InputObject, this, _currentLocationPath); + } + + ThreadJob.StartJob(_threadJob, ThrottleLimit); + WriteObject(_threadJob); + + _processFirstRecord = true; + } + else + { + // Inject input. + if (InputObject != null) + { + _threadJob.InjectInput(InputObject); + } + } + } + + protected override void EndProcessing() + { + base.EndProcessing(); + + _threadJob.CloseInputStream(); + } + + #endregion + } + + public sealed class ThreadJobSourceAdapter : JobSourceAdapter + { + #region Members + + private ConcurrentDictionary _repository; + + #endregion + + #region Constructor + + /// + /// Constructor + /// + public ThreadJobSourceAdapter() + { + Name = "ThreadJobSourceAdapter"; + _repository = new ConcurrentDictionary(); + } + + #endregion + + #region JobSourceAdapter Implementation + + /// + /// NewJob + /// + public override Job2 NewJob(JobInvocationInfo specification) + { + var job = specification.Parameters[0][0].Value as ThreadJob; + if (job != null) + { + _repository.TryAdd(job.InstanceId, job); + } + return job; + } + + /// + /// GetJobs + /// + public override IList GetJobs() + { + return _repository.Values.ToArray(); + } + + /// + /// GetJobsByName + /// + public override IList GetJobsByName(string name, bool recurse) + { + List rtnList = new List(); + foreach (var job in _repository.Values) + { + if (job.Name.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + rtnList.Add(job); + } + } + return rtnList; + } + + /// + /// GetJobsByCommand + /// + public override IList GetJobsByCommand(string command, bool recurse) + { + List rtnList = new List(); + foreach (var job in _repository.Values) + { + if (job.Command.Equals(command, StringComparison.OrdinalIgnoreCase)) + { + rtnList.Add(job); + } + } + return rtnList; + } + + /// + /// GetJobByInstanceId + /// + public override Job2 GetJobByInstanceId(Guid instanceId, bool recurse) + { + Job2 job; + if (_repository.TryGetValue(instanceId, out job)) + { + return job; + } + return null; + } + + /// + /// GetJobBySessionId + /// + public override Job2 GetJobBySessionId(int id, bool recurse) + { + foreach (var job in _repository.Values) + { + if (job.Id == id) + { + return job; + } + } + return null; + } + + /// + /// GetJobsByState + /// + public override IList GetJobsByState(JobState state, bool recurse) + { + List rtnList = new List(); + foreach (var job in _repository.Values) + { + if (job.JobStateInfo.State == state) + { + rtnList.Add(job); + } + } + return rtnList; + } + + /// + /// GetJobsByFilter + /// + public override IList GetJobsByFilter(Dictionary filter, bool recurse) + { + throw new PSNotSupportedException(); + } + + /// + /// RemoveJob + /// + public override void RemoveJob(Job2 job) + { + Job2 removeJob; + if (_repository.TryGetValue(job.InstanceId, out removeJob)) + { + removeJob.StopJob(); + _repository.TryRemove(job.InstanceId, out removeJob); + } + } + + #endregion + } + + internal sealed class ThreadJobDebugger : Debugger + { + #region Members + + private Debugger _wrappedDebugger; + private string _jobName; + + #endregion + + #region Constructor + + private ThreadJobDebugger() { } + + public ThreadJobDebugger( + Debugger debugger, + string jobName) + { + if (debugger == null) + { + throw new PSArgumentNullException("debugger"); + } + + _wrappedDebugger = debugger; + _jobName = jobName ?? string.Empty; + + // Create handlers for wrapped debugger events. + _wrappedDebugger.BreakpointUpdated += HandleBreakpointUpdated; + _wrappedDebugger.DebuggerStop += HandleDebuggerStop; + } + + #endregion + + #region Debugger overrides + + /// + /// Evaluates provided command either as a debugger specific command + /// or a PowerShell command. + /// + /// PowerShell command. + /// Output. + /// DebuggerCommandResults. + public override DebuggerCommandResults ProcessCommand(PSCommand command, PSDataCollection output) + { + // Special handling for the prompt command. + if (command.Commands[0].CommandText.Trim().Equals("prompt", StringComparison.OrdinalIgnoreCase)) + { + return HandlePromptCommand(output); + } + + return _wrappedDebugger.ProcessCommand(command, output); + } + + /// + /// Adds the provided set of breakpoints to the debugger. + /// + /// Breakpoints. + public override void SetBreakpoints(IEnumerable breakpoints) + { + _wrappedDebugger.SetBreakpoints(breakpoints); + } + + /// + /// Sets the debugger resume action. + /// + /// DebuggerResumeAction. + public override void SetDebuggerAction(DebuggerResumeAction resumeAction) + { + _wrappedDebugger.SetDebuggerAction(resumeAction); + } + + /// + /// Stops a running command. + /// + public override void StopProcessCommand() + { + _wrappedDebugger.StopProcessCommand(); + } + + /// + /// Returns current debugger stop event arguments if debugger is in + /// debug stop state. Otherwise returns null. + /// + /// DebuggerStopEventArgs. + public override DebuggerStopEventArgs GetDebuggerStopArgs() + { + return _wrappedDebugger.GetDebuggerStopArgs(); + } + + /// + /// Sets the parent debugger, breakpoints, and other debugging context information. + /// + /// Parent debugger. + /// List of breakpoints. + /// Debugger mode. + /// PowerShell host. + /// Current path. + public override void SetParent( + Debugger parent, + IEnumerable breakPoints, + DebuggerResumeAction? startAction, + PSHost host, + PathInfo path) + { + // For now always enable step mode debugging. + SetDebuggerStepMode(true); + } + + /// + /// Sets the debugger mode. + /// + public override void SetDebugMode(DebugModes mode) + { + _wrappedDebugger.SetDebugMode(mode); + + base.SetDebugMode(mode); + } + + /// + /// Returns IEnumerable of CallStackFrame objects. + /// + /// + public override IEnumerable GetCallStack() + { + return _wrappedDebugger.GetCallStack(); + } + + /// + /// Sets debugger stepping mode. + /// + /// True if stepping is to be enabled. + public override void SetDebuggerStepMode(bool enabled) + { + _wrappedDebugger.SetDebuggerStepMode(enabled); + } + + /// + /// True when debugger is stopped at a breakpoint. + /// + public override bool InBreakpoint + { + get { return _wrappedDebugger.InBreakpoint; } + } + + #endregion + + #region Private methods + + private void HandleDebuggerStop(object sender, DebuggerStopEventArgs e) + { + this.RaiseDebuggerStopEvent(e); + } + + private void HandleBreakpointUpdated(object sender, BreakpointUpdatedEventArgs e) + { + this.RaiseBreakpointUpdatedEvent(e); + } + + private DebuggerCommandResults HandlePromptCommand(PSDataCollection output) + { + // Nested debugged runspace prompt should look like: + // [DBG]: [JobName]: PS C:\>> + string promptScript = "'[DBG]: '" + " + " + "'[" + CodeGeneration.EscapeSingleQuotedStringContent(_jobName) + "]: '" + " + " + @"""PS $($executionContext.SessionState.Path.CurrentLocation)>> """; + PSCommand promptCommand = new PSCommand(); + promptCommand.AddScript(promptScript); + _wrappedDebugger.ProcessCommand(promptCommand, output); + + return new DebuggerCommandResults(null, true); + } + + #endregion + } + + /// + /// ThreadJob + /// + public sealed class ThreadJob : Job2, IJobDebugger + { + #region Private members + + private ScriptBlock _sb; + private string _filePath; + private ScriptBlock _initSb; + private object[] _argumentList; + private Dictionary _usingValuesMap; + private PSDataCollection _input; + private Runspace _rs; + private System.Management.Automation.PowerShell _ps; + private PSDataCollection _output; + private bool _runningInitScript; + private PSHost _streamingHost; + private Debugger _jobDebugger; + private string _currentLocationPath; + + private const string VERBATIM_ARGUMENT = "--%"; + + private static ThreadJobQueue s_JobQueue; + + #endregion + + #region Properties + + /// + /// Specifies the job definition for the JobManager + /// + public JobDefinition ThreadJobDefinition + { + get; + private set; + } + + #endregion + + #region Constructors + + // Constructors + static ThreadJob() + { + s_JobQueue = new ThreadJobQueue(5); + } + + private ThreadJob() + { } + + /// + /// Constructor. + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public ThreadJob( + string name, + string command, + ScriptBlock sb, + string filePath, + ScriptBlock initSb, + object[] argumentList, + PSObject inputObject, + PSCmdlet psCmdlet, + string currentLocationPath) + : this(name, command, sb, filePath, initSb, argumentList, inputObject, psCmdlet, currentLocationPath, null) + { + } + + /// + /// Constructor. + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public ThreadJob( + string name, + string command, + ScriptBlock sb, + string filePath, + ScriptBlock initSb, + object[] argumentList, + PSObject inputObject, + PSCmdlet psCmdlet, + string currentLocationPath, + PSHost streamingHost) + : base(command, name) + { + _sb = sb; + _filePath = filePath; + _initSb = initSb; + _argumentList = argumentList; + _input = new PSDataCollection(); + if (inputObject != null) + { + _input.Add(inputObject); + } + _output = new PSDataCollection(); + _streamingHost = streamingHost; + _currentLocationPath = currentLocationPath; + + this.PSJobTypeName = "ThreadJob"; + + // Get script block to run. + if (!string.IsNullOrEmpty(_filePath)) + { + _sb = GetScriptBlockFromFile(_filePath, psCmdlet); + if (_sb == null) + { + throw new InvalidOperationException(Properties.Resources.CannotParseScriptFile); + } + } + else if (_sb == null) + { + throw new PSArgumentNullException(Properties.Resources.NoScriptToRun); + } + + // Create Runspace/PowerShell object and state callback. + // The job script/command will run in a separate thread associated with the Runspace. + var iss = InitialSessionState.CreateDefault2(); + + // Determine session language mode for Windows platforms + WarningRecord lockdownWarning = null; + if (Environment.OSVersion.Platform.ToString().Equals("Win32NT", StringComparison.OrdinalIgnoreCase)) + { + bool enforceLockdown = (SystemPolicy.GetSystemLockdownPolicy() == SystemEnforcementMode.Enforce); + if (enforceLockdown && !string.IsNullOrEmpty(_filePath)) + { + // If script source is a file, check to see if it is trusted by the lock down policy + enforceLockdown = (SystemPolicy.GetLockdownPolicy(_filePath, null) == SystemEnforcementMode.Enforce); + + if (!enforceLockdown && (_initSb != null)) + { + // Even if the script file is trusted, an initialization script cannot be trusted, so we have to enforce + // lock down. Otherwise untrusted script could be run in FullLanguage mode along with the trusted file script. + enforceLockdown = true; + lockdownWarning = new WarningRecord( + string.Format( + CultureInfo.InvariantCulture, + Properties.Resources.CannotRunTrustedFileInFL, + _filePath)); + } + } + + iss.LanguageMode = enforceLockdown ? PSLanguageMode.ConstrainedLanguage : PSLanguageMode.FullLanguage; + } + + if (_streamingHost != null) + { + _rs = RunspaceFactory.CreateRunspace(_streamingHost, iss); + } + else + { + _rs = RunspaceFactory.CreateRunspace(iss); + } + _ps = System.Management.Automation.PowerShell.Create(); + _ps.Runspace = _rs; + _ps.InvocationStateChanged += (sender, psStateChanged) => + { + var newStateInfo = psStateChanged.InvocationStateInfo; + + // Update Job state. + switch (newStateInfo.State) + { + case PSInvocationState.Running: + SetJobState(JobState.Running); + break; + + case PSInvocationState.Stopped: + SetJobState(JobState.Stopped, newStateInfo.Reason, disposeRunspace:true); + break; + + case PSInvocationState.Failed: + SetJobState(JobState.Failed, newStateInfo.Reason, disposeRunspace:true); + break; + + case PSInvocationState.Completed: + if (_runningInitScript) + { + // Begin running main script. + _runningInitScript = false; + RunScript(); + } + else + { + SetJobState(JobState.Completed, newStateInfo.Reason, disposeRunspace:true); + } + break; + } + }; + + // Get any using variables. + var usingAsts = _sb.Ast.FindAll(ast => ast is UsingExpressionAst, searchNestedScriptBlocks: true).Cast(); + if (usingAsts != null && + usingAsts.FirstOrDefault() != null) + { + // Get using variables as dictionary, since we now only support PowerShell version 5.1 and greater + _usingValuesMap = GetUsingValuesAsDictionary(usingAsts, psCmdlet); + } + + // Hook up data streams. + this.Output = _output; + this.Output.EnumeratorNeverBlocks = true; + + this.Error = _ps.Streams.Error; + this.Error.EnumeratorNeverBlocks = true; + + this.Progress = _ps.Streams.Progress; + this.Progress.EnumeratorNeverBlocks = true; + + this.Verbose = _ps.Streams.Verbose; + this.Verbose.EnumeratorNeverBlocks = true; + + this.Warning = _ps.Streams.Warning; + this.Warning.EnumeratorNeverBlocks = true; + if (lockdownWarning != null) + { + this.Warning.Add(lockdownWarning); + } + + this.Debug = _ps.Streams.Debug; + this.Debug.EnumeratorNeverBlocks = true; + + this.Information = _ps.Streams.Information; + this.Information.EnumeratorNeverBlocks = true; + + // Create the JobManager job definition and job specification, and add to the JobManager. + ThreadJobDefinition = new JobDefinition(typeof(ThreadJobSourceAdapter), "", Name); + Dictionary parameterCollection = new Dictionary(); + parameterCollection.Add("NewJob", this); + var jobSpecification = new JobInvocationInfo(ThreadJobDefinition, parameterCollection); + var newJob = psCmdlet.JobManager.NewJob(jobSpecification); + System.Diagnostics.Debug.Assert(newJob == this, "JobManager must return this job"); + } + + #endregion + + #region Public methods + + /// + /// StartJob + /// + public override void StartJob() + { + if (this.JobStateInfo.State != JobState.NotStarted) + { + throw new Exception(Properties.Resources.CannotStartJob); + } + + // Initialize Runspace state + _rs.Open(); + + // Set current location path on the runspace, if available. + if (_currentLocationPath != null) + { + using (var ps = System.Management.Automation.PowerShell.Create()) + { + ps.Runspace = _rs; + ps.AddCommand("Set-Location").AddParameter("LiteralPath", _currentLocationPath).Invoke(); + } + } + + // If initial script block provided then execute. + if (_initSb != null) + { + // Run initial script and then the main script. + _ps.Commands.Clear(); + _ps.AddScript(_initSb.ToString()); + _runningInitScript = true; + _ps.BeginInvoke(_input, _output); + } + else + { + // Run main script. + RunScript(); + } + } + + /// + /// InjectInput + /// + /// + public void InjectInput(PSObject psObject) + { + if (psObject != null) + { + _input.Add(psObject); + } + } + + /// + /// CloseInputStream + /// + public void CloseInputStream() + { + _input.Complete(); + } + + /// + /// StartJob + /// + /// + /// + public static void StartJob(ThreadJob job, int throttleLimit) + { + s_JobQueue.EnqueueJob(job, throttleLimit); + } + + /// + /// Dispose + /// + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + if (_ps.InvocationStateInfo.State == PSInvocationState.Running) + { + _ps.Stop(); + } + _ps.Dispose(); + + _input.Complete(); + _output.Complete(); + } + + base.Dispose(disposing); + } + + /// + /// StatusMessage + /// + public override string StatusMessage + { + get { return string.Empty; } + } + + /// + /// HasMoreData + /// + public override bool HasMoreData + { + get + { + return (this.Output.Count > 0 || + this.Error.Count > 0 || + this.Progress.Count > 0 || + this.Verbose.Count > 0 || + this.Debug.Count > 0 || + this.Warning.Count > 0); + } + } + + /// + /// Location + /// + public override string Location + { + get { return "PowerShell"; } + } + + /// + /// StopJob + /// + public override void StopJob() + { + _ps.Stop(); + } + + /// + /// ReportError + /// + /// + public void ReportError(Exception e) + { + try + { + SetJobState(JobState.Failed); + + this.Error.Add( + new ErrorRecord(e, "ThreadJobError", ErrorCategory.InvalidOperation, this)); + } + catch (ObjectDisposedException) + { + // Ignore. Thrown if Job is disposed (race condition.). + } + catch (PSInvalidOperationException) + { + // Ignore. Thrown if Error collection is closed (race condition.). + } + } + + #endregion + + #region Base class overrides + + /// + /// OnStartJobCompleted + /// + /// + protected override void OnStartJobCompleted(AsyncCompletedEventArgs eventArgs) + { + base.OnStartJobCompleted(eventArgs); + } + + /// + /// StartJobAsync + /// + public override void StartJobAsync() + { + this.StartJob(); + this.OnStartJobCompleted( + new AsyncCompletedEventArgs(null, false, this)); + } + + /// + /// StopJob + /// + /// + /// + public override void StopJob(bool force, string reason) + { + _ps.Stop(); + } + + /// + /// OnStopJobCompleted + /// + /// + protected override void OnStopJobCompleted(AsyncCompletedEventArgs eventArgs) + { + base.OnStopJobCompleted(eventArgs); + } + + /// + /// StopJobAsync + /// + public override void StopJobAsync() + { + _ps.BeginStop((iasync) => { OnStopJobCompleted(new AsyncCompletedEventArgs(null, false, this)); }, null); + } + + /// + /// StopJobAsync + /// + /// + /// + public override void StopJobAsync(bool force, string reason) + { + _ps.BeginStop((iasync) => { OnStopJobCompleted(new AsyncCompletedEventArgs(null, false, this)); }, null); + } + + #region Not implemented + + /// + /// SuspendJob + /// + public override void SuspendJob() + { + throw new NotImplementedException(); + } + + /// + /// SuspendJob + /// + /// + /// + public override void SuspendJob(bool force, string reason) + { + throw new NotImplementedException(); + } + + /// + /// ResumeJobAsync + /// + public override void ResumeJobAsync() + { + throw new NotImplementedException(); + } + + /// + /// ResumeJob + /// + public override void ResumeJob() + { + throw new NotImplementedException(); + } + + /// + /// SuspendJobAsync + /// + public override void SuspendJobAsync() + { + throw new NotImplementedException(); + } + + /// + /// SuspendJobAsync + /// + /// + /// + public override void SuspendJobAsync(bool force, string reason) + { + throw new NotImplementedException(); + } + + /// + /// UnblockJobAsync + /// + public override void UnblockJobAsync() + { + throw new NotImplementedException(); + } + + /// + /// UnblockJob + /// + public override void UnblockJob() + { + throw new NotImplementedException(); + } + + #endregion + + #endregion + + #region IJobDebugger + + /// + /// Job Debugger + /// + public Debugger Debugger + { + get + { + if (_jobDebugger == null && _rs.Debugger != null) + { + _jobDebugger = new ThreadJobDebugger(_rs.Debugger, this.Name); + } + + return _jobDebugger; + } + } + + /// + /// IsAsync + /// + public bool IsAsync + { + get; + set; + } + + #endregion + + #region Private methods + + // Private methods + private void RunScript() + { + _ps.Commands.Clear(); + _ps.AddScript(_sb.ToString()); + + if (_argumentList != null) + { + foreach (var arg in _argumentList) + { + _ps.AddArgument(arg); + } + } + + // Using variables + if (_usingValuesMap != null && _usingValuesMap.Count > 0) + { + _ps.AddParameter(VERBATIM_ARGUMENT, _usingValuesMap); + } + + _ps.BeginInvoke(_input, _output); + } + + private ScriptBlock GetScriptBlockFromFile(string filePath, PSCmdlet psCmdlet) + { + if (WildcardPattern.ContainsWildcardCharacters(filePath)) + { + throw new ArgumentException(Properties.Resources.FilePathWildcards); + } + + if (!filePath.EndsWith(".ps1", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException(Properties.Resources.FilePathExt); + } + + ProviderInfo provider = null; + string resolvedPath = psCmdlet.GetResolvedProviderPathFromPSPath(filePath, out provider).FirstOrDefault(); + if (!string.IsNullOrEmpty(resolvedPath)) + { + Token[] tokens; + ParseError[] errors; + ScriptBlockAst scriptBlockAst = Parser.ParseFile(resolvedPath, out tokens, out errors); + if (scriptBlockAst != null && errors.Length == 0) + { + return scriptBlockAst.GetScriptBlock(); + } + + foreach (var error in errors) + { + this.Error.Add( + new ErrorRecord( + new ParseException(error.Message), "ThreadJobError", ErrorCategory.InvalidData, this)); + } + } + + return null; + } + + private void SetJobState(JobState jobState, Exception reason, bool disposeRunspace = false) + { + base.SetJobState(jobState, reason); + if (disposeRunspace) + { + _rs.Dispose(); + } + } + + private static Dictionary GetUsingValuesAsDictionary(IEnumerable usingAsts, PSCmdlet psCmdlet) + { + Dictionary usingValues = new Dictionary(); + + foreach (var usingAst in usingAsts) + { + var varAst = usingAst.SubExpression as VariableExpressionAst; + if (varAst == null) + { + var msg = string.Format(CultureInfo.InvariantCulture, + Properties.Resources.UsingNotVariableExpression, + new object[] { usingAst.Extent.Text }); + throw new PSInvalidOperationException(msg); + } + + try + { + var usingValue = psCmdlet.GetVariableValue(varAst.VariablePath.UserPath); + var usingKey = GetUsingExpressionKey(usingAst); + if (!usingValues.ContainsKey(usingKey)) + { + usingValues.Add(usingKey, usingValue); + } + } + catch (Exception ex) + { + var msg = string.Format(CultureInfo.InvariantCulture, + Properties.Resources.UsingVariableNotFound, + new object[] { usingAst.Extent.Text }); + throw new PSInvalidOperationException(msg, ex); + } + } + + return usingValues; + } + + /// + /// This method creates a dictionary key for a Using expression value that is bound to + /// a thread job script block parameter. PowerShell version 5.0+ recognizes this and performs + /// the correct Using parameter argument binding. + /// + /// A using expression + /// Base64 encoded string as the key of the UsingExpressionAst + private static string GetUsingExpressionKey(UsingExpressionAst usingAst) + { + string usingAstText = usingAst.ToString(); + if (usingAst.SubExpression is VariableExpressionAst) + { + usingAstText = usingAstText.ToLowerInvariant(); + } + + return Convert.ToBase64String(Encoding.Unicode.GetBytes(usingAstText.ToCharArray())); + } + + #endregion + } + + /// + /// ThreadJobQueue + /// + internal sealed class ThreadJobQueue + { + #region Private members + + // Private members + ConcurrentQueue _jobQueue = new ConcurrentQueue(); + object _syncObject = new object(); + int _throttleLimit = 5; + int _currentJobs; + bool _haveRunningJobs; + private ManualResetEvent _processJobsHandle = new ManualResetEvent(true); + + #endregion + + #region Constructors + + /// + /// Constructor + /// + public ThreadJobQueue() + { } + + /// + /// Constructor + /// + /// + public ThreadJobQueue(int throttleLimit) + { + _throttleLimit = throttleLimit; + } + + #endregion + + #region Public properties + + /// + /// ThrottleLimit + /// + public int ThrottleLimit + { + get { return _throttleLimit; } + set + { + if (value > 0) + { + lock (_syncObject) + { + _throttleLimit = value; + if (_currentJobs < _throttleLimit) + { + _processJobsHandle.Set(); + } + } + } + } + } + + /// + /// CurrentJobs + /// + public int CurrentJobs + { + get { return _currentJobs; } + } + + /// + /// Count + /// + public int Count + { + get { return _jobQueue.Count; } + } + + #endregion + + #region Public methods + + /// + /// EnqueueJob + /// + /// + /// + public void EnqueueJob(ThreadJob job, int throttleLimit) + { + if (job == null) + { + throw new ArgumentNullException("job"); + } + + ThrottleLimit = throttleLimit; + job.StateChanged += new EventHandler(HandleJobStateChanged); + + lock (_syncObject) + { + _jobQueue.Enqueue(job); + + if (_haveRunningJobs) + { + return; + } + + if (_jobQueue.Count > 0) + { + _haveRunningJobs = true; + System.Threading.ThreadPool.QueueUserWorkItem(new WaitCallback(ServiceJobs)); + } + } + } + + #endregion + + #region Private methods + + private void HandleJobStateChanged(object sender, JobStateEventArgs e) + { + ThreadJob job = sender as ThreadJob; + JobState state = e.JobStateInfo.State; + if (state == JobState.Completed || + state == JobState.Stopped || + state == JobState.Failed) + { + job.StateChanged -= new EventHandler(HandleJobStateChanged); + DecrementCurrentJobs(); + } + } + + private void IncrementCurrentJobs() + { + lock (_syncObject) + { + if (++_currentJobs >= _throttleLimit) + { + _processJobsHandle.Reset(); + } + } + } + + private void DecrementCurrentJobs() + { + lock (_syncObject) + { + if ((_currentJobs > 0) && + (--_currentJobs < _throttleLimit)) + { + _processJobsHandle.Set(); + } + } + } + + private void ServiceJobs(object toProcess) + { + while (true) + { + lock (_syncObject) + { + if (_jobQueue.Count == 0) + { + _haveRunningJobs = false; + return; + } + } + + _processJobsHandle.WaitOne(); + + ThreadJob job; + if (_jobQueue.TryDequeue(out job)) + { + try + { + // Start job running on its own thread/runspace. + IncrementCurrentJobs(); + job.StartJob(); + } + catch (Exception e) + { + DecrementCurrentJobs(); + job.ReportError(e); + } + } + } + } + + #endregion + } +} diff --git a/Modules/Microsoft.PowerShell.ThreadJob/test/Microsoft.PowerShell.ThreadJob.CL.Tests.ps1 b/Modules/Microsoft.PowerShell.ThreadJob/test/Microsoft.PowerShell.ThreadJob.CL.Tests.ps1 new file mode 100644 index 0000000..cdc2e77 --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/test/Microsoft.PowerShell.ThreadJob.CL.Tests.ps1 @@ -0,0 +1,232 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +## +## ---------- +## Test Note: +## ---------- +## Since these tests change session and system state (constrained language and system lockdown) +## they will all use try/finally blocks instead of Pester AfterEach/AfterAll to ensure session +## and system state is restored. +## Pester AfterEach, AfterAll is not reliable when the session is constrained language or locked down. +## + +function Get-RandomFileName +{ + [System.IO.Path]::GetFileNameWithoutExtension([IO.Path]::GetRandomFileName()) +} + +if ($IsWindows) +{ + $code = @' + + #region Using directives + + using System; + using System.Management.Automation; + + #endregion + + /// Adds a new type to the Application Domain + [Cmdlet("Invoke", "LanguageModeTestingSupportCmdlet")] + public sealed class InvokeLanguageModeTestingSupportCmdlet : PSCmdlet + { + [Parameter()] + public SwitchParameter EnableFullLanguageMode { get; set; } + + [Parameter()] + public SwitchParameter SetLockdownMode { get; set; } + + [Parameter()] + public SwitchParameter RevertLockdownMode { get; set; } + + protected override void BeginProcessing() + { + if (EnableFullLanguageMode) + { + SessionState.LanguageMode = PSLanguageMode.FullLanguage; + } + + if (SetLockdownMode) + { + Environment.SetEnvironmentVariable("__PSLockdownPolicy", "0x80000007", EnvironmentVariableTarget.Machine); + } + + if (RevertLockdownMode) + { + Environment.SetEnvironmentVariable("__PSLockdownPolicy", null, EnvironmentVariableTarget.Machine); + } + } + } +'@ + + if (-not (Get-Command Invoke-LanguageModeTestingSupportCmdlet -ErrorAction Ignore)) + { + $moduleName = Get-RandomFileName + $moduleDirectory = join-path $TestDrive\Modules $moduleName + if (-not (Test-Path $moduleDirectory)) + { + $null = New-Item -ItemType Directory $moduleDirectory -Force + } + + try + { + Add-Type -TypeDefinition $code -OutputAssembly $moduleDirectory\TestCmdletForConstrainedLanguage.dll -ErrorAction Ignore + } catch {} + + Import-Module -Name $moduleDirectory\TestCmdletForConstrainedLanguage.dll + } +} + +try +{ + $defaultParamValues = $PSDefaultParameterValues.Clone() + $PSDefaultParameterValues["it:Skip"] = !$IsWindows + + Describe "ThreadJob Constrained Language Tests" -Tags 'Feature','RequireAdminOnWindows' { + + BeforeAll { + + $sb = { $ExecutionContext.SessionState.LanguageMode } + + $scriptTrustedFilePath = Join-Path $TestDrive "ThJobTrusted_System32.ps1" + @' + Write-Output $ExecutionContext.SessionState.LanguageMode +'@ | Out-File -FilePath $scriptTrustedFilePath + + $scriptUntrustedFilePath = Join-Path $TestDrive "ThJobUntrusted.ps1" + @' + Write-Output $ExecutionContext.SessionState.LanguageMode +'@ | Out-File -FilePath $scriptUntrustedFilePath + } + + AfterAll { + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + } + + It "ThreadJob script must run in ConstrainedLanguage mode with system lock down" { + + try + { + $ExecutionContext.SessionState.LanguageMode = "ConstrainedLanguage" + Invoke-LanguageModeTestingSupportCmdlet -SetLockdownMode + + $results = Start-ThreadJob -ScriptBlock { $ExecutionContext.SessionState.LanguageMode } | Wait-Job | Receive-Job + } + finally + { + Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode + } + + $results | Should -BeExactly "ConstrainedLanguage" + } + + It "ThreadJob script block using variable must run in ConstrainedLanguage mode with system lock down" { + + try + { + $ExecutionContext.SessionState.LanguageMode = "ConstrainedLanguage" + Invoke-LanguageModeTestingSupportCmdlet -SetLockdownMode + + $results = Start-ThreadJob -ScriptBlock { & $using:sb } | Wait-Job | Receive-Job + } + finally + { + Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode + } + + $results | Should -BeExactly "ConstrainedLanguage" + } + + It "ThreadJob script block argument variable must run in ConstrainedLanguage mode with system lock down" { + + try + { + $ExecutionContext.SessionState.LanguageMode = "ConstrainedLanguage" + Invoke-LanguageModeTestingSupportCmdlet -SetLockdownMode + + $results = Start-ThreadJob -ScriptBlock { param ($sb) & $sb } -ArgumentList $sb | Wait-Job | Receive-Job + } + finally + { + Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode + } + + $results | Should -BeExactly "ConstrainedLanguage" + } + + It "ThreadJob script block piped variable must run in ConstrainedLanguage mode with system lock down" { + + try + { + $ExecutionContext.SessionState.LanguageMode = "ConstrainedLanguage" + Invoke-LanguageModeTestingSupportCmdlet -SetLockdownMode + + $results = $sb | Start-ThreadJob -ScriptBlock { $input | ForEach-Object { & $_ } } | Wait-Job | Receive-Job + } + finally + { + Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode + } + + $results | Should -BeExactly "ConstrainedLanguage" + } + + It "ThreadJob untrusted script file must run in ConstrainedLanguage mode with system lock down" { + try + { + $ExecutionContext.SessionState.LanguageMode = "ConstrainedLanguage" + Invoke-LanguageModeTestingSupportCmdlet -SetLockdownMode + + $results = Start-ThreadJob -File $scriptUntrustedFilePath | Wait-Job | Receive-Job + } + finally + { + Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode + } + + $results | Should -BeExactly "ConstrainedLanguage" + } + + It "ThreadJob trusted script file must run in FullLanguage mode with system lock down" { + try + { + $ExecutionContext.SessionState.LanguageMode = "ConstrainedLanguage" + Invoke-LanguageModeTestingSupportCmdlet -SetLockdownMode + + $results = Start-ThreadJob -File $scriptTrustedFilePath | Wait-Job | Receive-Job + } + finally + { + Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode + } + + $results | Should -BeExactly "FullLanguage" + } + + It "ThreadJob trusted script file *with* untrusted initialization script must run in ConstrainedLanguage mode with system lock down" { + try + { + $ExecutionContext.SessionState.LanguageMode = "ConstrainedLanguage" + Invoke-LanguageModeTestingSupportCmdlet -SetLockdownMode + + $results = Start-ThreadJob -File $scriptTrustedFilePath -InitializationScript { "Hello" } | Wait-Job | Receive-Job 3>&1 + } + finally + { + Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode + } + + $results.Count | Should -BeExactly 3 -Because "Includes init script, file script, warning output" + $results[0] | Should -BeExactly "Hello" -Because "This is the expected initialization script output" + $results[1] | Should -BeExactly "ConstrainedLanguage" -Because "This is the expected script file language mode" + } + } +} +finally +{ + if ($defaultParamValues -ne $null) + { + $Global:PSDefaultParameterValues = $defaultParamValues + } +} diff --git a/Modules/Microsoft.PowerShell.ThreadJob/test/Microsoft.PowerShell.ThreadJob.Tests.ps1 b/Modules/Microsoft.PowerShell.ThreadJob/test/Microsoft.PowerShell.ThreadJob.Tests.ps1 new file mode 100644 index 0000000..edfd37a --- /dev/null +++ b/Modules/Microsoft.PowerShell.ThreadJob/test/Microsoft.PowerShell.ThreadJob.Tests.ps1 @@ -0,0 +1,439 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Helper function to wait for job to reach a running or completed state +# Job state can go to "Running" before the underlying runspace thread is running +# so we always first wait 100 mSec before checking state. +function Wait-ForJobRunning +{ + param ( + $job + ) + + $iteration = 10 + Do + { + Start-Sleep -Milliseconds 100 + } + Until (($job.State -match "Running|Completed|Failed") -or (--$iteration -eq 0)) + + if ($job.State -notmatch "Running|Completed|Failed") + { + throw ("Cannot start job '{0}'. Job state is '{1}'" -f $job,$job.State) + } +} + +Describe 'Basic ThreadJob Tests' -Tags 'CI' { + + BeforeAll { + + $scriptFilePath1 = Join-Path $testdrive "TestThreadJobFile1.ps1" + @' + for ($i=0; $i -lt 10; $i++) + { + Write-Output "Hello $i" + } +'@ > $scriptFilePath1 + + $scriptFilePath2 = Join-Path $testdrive "TestThreadJobFile2.ps1" + @' + param ($arg1, $arg2) + Write-Output $arg1 + Write-Output $arg2 +'@ > $scriptFilePath2 + + $scriptFilePath3 = Join-Path $testdrive "TestThreadJobFile3.ps1" + @' + $input | foreach { + Write-Output $_ + } +'@ > $scriptFilePath3 + + $scriptFilePath4 = Join-Path $testdrive "TestThreadJobFile4.ps1" + @' + Write-Output $using:Var1 + Write-Output $($using:Array1)[2] + Write-Output @(,$using:Array1) +'@ > $scriptFilePath4 + + $scriptFilePath5 = Join-Path $testdrive "TestThreadJobFile5.ps1" + @' + param ([string]$param1) + Write-Output "$param1 $using:Var1 $using:Var2" +'@ > $scriptFilePath5 + + $WaitForCountFnScript = @' + function Wait-ForExpectedRSCount + { + param ( + $expectedRSCount + ) + + $iteration = 20 + while ((@(Get-Runspace).Count -ne $expectedRSCount) -and ($iteration-- -gt 0)) + { + Start-Sleep -Milliseconds 100 + } + } +'@ + } + + AfterEach { + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + } + + It 'ThreadJob with ScriptBlock' { + + $job = Start-ThreadJob -ScriptBlock { "Hello" } + $results = $job | Receive-Job -Wait + $results | Should -Be "Hello" + } + + It 'ThreadJob with ScriptBlock and Initialization script' { + + $job = Start-ThreadJob -ScriptBlock { "Goodbye" } -InitializationScript { "Hello" } + $results = $job | Receive-Job -Wait + $results[0] | Should -Be "Hello" + $results[1] | Should -Be "Goodbye" + } + + It 'ThreadJob with ScriptBlock and Argument list' { + + $job = Start-ThreadJob -ScriptBlock { param ($arg1, $arg2) $arg1; $arg2 } -ArgumentList @("Hello","Goodbye") + $results = $job | Receive-Job -Wait + $results[0] | Should -Be "Hello" + $results[1] | Should -Be "Goodbye" + } + + It 'ThreadJob with ScriptBlock and piped input' { + + $job = "Hello","Goodbye" | Start-ThreadJob -ScriptBlock { $input | ForEach-Object { $_ } } + $results = $job | Receive-Job -Wait + $results[0] | Should -Be "Hello" + $results[1] | Should -Be "Goodbye" + } + + It 'ThreadJob with ScriptBlock and Using variables' { + + $Var1 = "Hello" + $Var2 = "Goodbye" + $Var3 = 102 + $Var4 = 1..5 + $global:GVar1 = "GlobalVar" + $job = Start-ThreadJob -ScriptBlock { + Write-Output $using:Var1 + Write-Output $using:Var2 + Write-Output $using:Var3 + Write-Output ($using:Var4)[1] + Write-Output @(,$using:Var4) + Write-Output $using:GVar1 + } + + $results = $job | Receive-Job -Wait + $results[0] | Should -Be $Var1 + $results[1] | Should -Be $Var2 + $results[2] | Should -Be $Var3 + $results[3] | Should -Be 2 + $results[4] | Should -Be $Var4 + $results[5] | Should -Be $global:GVar1 + } + + It 'ThreadJob with ScriptBlock and Using variables and argument list' { + + $Var1 = "Hello" + $Var2 = 52 + $job = Start-ThreadJob -ScriptBlock { + param ([string] $param1) + + "$using:Var1 $param1 $using:Var2" + } -ArgumentList "There" + + $results = $job | Receive-Job -Wait + $results | Should -Be "Hello There 52" + } + + It 'ThreadJob with ScriptFile' { + + $job = Start-ThreadJob -FilePath $scriptFilePath1 + $results = $job | Receive-Job -Wait + $results | Should -HaveCount 10 + $results[9] | Should -Be "Hello 9" + } + + It 'ThreadJob with ScriptFile and Initialization script' { + + $job = Start-ThreadJob -FilePath $scriptFilePath1 -Initialization { "Goodbye" } + $results = $job | Receive-Job -Wait + $results | Should -HaveCount 11 + $results[0] | Should -Be "Goodbye" + } + + It 'ThreadJob with ScriptFile and Argument list' { + + $job = Start-ThreadJob -FilePath $scriptFilePath2 -ArgumentList @("Hello","Goodbye") + $results = $job | Receive-Job -Wait + $results[0] | Should -Be "Hello" + $results[1] | Should -Be "Goodbye" + } + + It 'ThreadJob with ScriptFile and piped input' { + + $job = "Hello","Goodbye" | Start-ThreadJob -FilePath $scriptFilePath3 + $results = $job | Receive-Job -Wait + $results[0] | Should -Be "Hello" + $results[1] | Should -Be "Goodbye" + } + + It 'ThreadJob with ScriptFile and Using variables' { + + $Var1 = "Hello!" + $Array1 = 1..10 + + $job = Start-ThreadJob -FilePath $scriptFilePath4 + $results = $job | Receive-Job -Wait + $results[0] | Should -Be $Var1 + $results[1] | Should -Be 3 + $results[2] | Should -Be $Array1 + } + + It 'ThreadJob with ScriptFile and Using variables with argument list' { + + $Var1 = "There" + $Var2 = 60 + $job = Start-ThreadJob -FilePath $scriptFilePath5 -ArgumentList "Hello" + $results = $job | Receive-Job -Wait + $results | Should -Be "Hello There 60" + } + + It 'ThreadJob with terminating error' { + + $job = Start-ThreadJob -ScriptBlock { throw "MyError!" } + $job | Wait-Job + $job.JobStateInfo.Reason.Message | Should -Be "MyError!" + } + + It 'ThreadJob and Error stream output' { + + $job = Start-ThreadJob -ScriptBlock { Write-Error "ErrorOut" } | Wait-Job + $job.Error | Should -Be "ErrorOut" + } + + It 'ThreadJob and Warning stream output' { + + $job = Start-ThreadJob -ScriptBlock { Write-Warning "WarningOut" } | Wait-Job + $job.Warning | Should -Be "WarningOut" + } + + It 'ThreadJob and Verbose stream output' { + + $job = Start-ThreadJob -ScriptBlock { $VerbosePreference = 'Continue'; Write-Verbose "VerboseOut" } | Wait-Job + $job.Verbose | Should Match "VerboseOut" + } + + It 'ThreadJob and Verbose stream output' { + + $job = Start-ThreadJob -ScriptBlock { $DebugPreference = 'Continue'; Write-Debug "DebugOut" } | Wait-Job + $job.Debug | Should -Be "DebugOut" + } + + It 'ThreadJob and Information stream output' { + + $job = Start-ThreadJob -ScriptBlock { Write-Information "InformationOutput" -InformationAction Continue } | Wait-Job + $job.Information | Should -Be "InformationOutput" + } + + It 'ThreadJob and Host stream output' { + + # Host stream data is automatically added to the Information stream + $job = Start-ThreadJob -ScriptBlock { Write-Host "HostOutput" } | Wait-Job + $job.Information | Should -Be "HostOutput" + } + + It 'ThreadJob ThrottleLimit and Queue' { + + try + { + # Start four thread jobs with ThrottleLimit set to two + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + $job1 = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } -ThrottleLimit 2 + $job2 = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } + $job3 = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } + $job4 = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } + + # Allow jobs to start + Wait-ForJobRunning $job2 + + Get-Job | Where-Object { ($_.PSJobTypeName -eq "ThreadJob") -and ($_.State -eq "Running") } | Should -HaveCount 2 + Get-Job | Where-Object { ($_.PSJobTypeName -eq "ThreadJob") -and ($_.State -eq "NotStarted") } | Should -HaveCount 2 + } + finally + { + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + } + + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Should -HaveCount 0 + } + + It 'ThreadJob Runspaces should be cleaned up at completion' { + + $script = $WaitForCountFnScript + @' + + try + { + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + $rsStartCount = @(Get-Runspace).Count + + # Start four thread jobs with ThrottleLimit set to two + $Job1 = Start-ThreadJob -ScriptBlock { "Hello 1!" } -ThrottleLimit 5 + $job2 = Start-ThreadJob -ScriptBlock { "Hello 2!" } + $job3 = Start-ThreadJob -ScriptBlock { "Hello 3!" } + $job4 = Start-ThreadJob -ScriptBlock { "Hello 4!" } + + $null = $job1,$job2,$job3,$job4 | Wait-Job + + # Allow for runspace clean up to happen + Wait-ForExpectedRSCount $rsStartCount + + Write-Output (@(Get-Runspace).Count -eq $rsStartCount) + } + finally + { + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + } +'@ + + $result = & "$PSHOME/pwsh" -c $script + $result | Should -BeExactly "True" + } + + It 'ThreadJob Runspaces should be cleaned up after job removal' { + + $script = $WaitForCountFnScript + @' + + try { + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + $rsStartCount = @(Get-Runspace).Count + + # Start four thread jobs with ThrottleLimit set to two + $Job1 = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } -ThrottleLimit 2 + $job2 = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } + $job3 = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } + $job4 = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } + + Wait-ForExpectedRSCount ($rsStartCount + 4) + Write-Output (@(Get-Runspace).Count -eq ($rsStartCount + 4)) + + # Stop two jobs + $job1 | Remove-Job -Force + $job3 | Remove-Job -Force + + Wait-ForExpectedRSCount ($rsStartCount + 2) + Write-Output (@(Get-Runspace).Count -eq ($rsStartCount + 2)) + } + finally + { + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + } + + Wait-ForExpectedRSCount $rsStartCount + Write-Output (@(Get-Runspace).Count -eq $rsStartCount) +'@ + + $result = & "$PSHOME/pwsh" -c $script + $result | Should -BeExactly "True","True","True" + } + + It 'ThreadJob jobs should work with Receive-Job -AutoRemoveJob' { + + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + + $job1 = Start-ThreadJob -ScriptBlock { 1..2 | ForEach-Object { Start-Sleep -Milliseconds 100; "Output $_" } } -ThrottleLimit 5 + $job2 = Start-ThreadJob -ScriptBlock { 1..2 | ForEach-Object { Start-Sleep -Milliseconds 100; "Output $_" } } + $job3 = Start-ThreadJob -ScriptBlock { 1..2 | ForEach-Object { Start-Sleep -Milliseconds 100; "Output $_" } } + $job4 = Start-ThreadJob -ScriptBlock { 1..2 | ForEach-Object { Start-Sleep -Milliseconds 100; "Output $_" } } + + $null = $job1,$job2,$job3,$job4 | Receive-Job -Wait -AutoRemoveJob + + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Should -HaveCount 0 + } + + It 'ThreadJob jobs should run in FullLanguage mode by default' { + + $result = Start-ThreadJob -ScriptBlock { $ExecutionContext.SessionState.LanguageMode } | Wait-Job | Receive-Job + $result | Should -Be "FullLanguage" + } + + It 'ThreadJob jobs should run in the current working directory' { + + $threadJobCurrentLocation = Start-ThreadJob -ScriptBlock { $pwd } | Wait-Job | Receive-Job + $threadJobCurrentLocation.Path | Should -BeExactly $pwd.Path + } +} + +Describe 'Job2 class API tests' -Tags 'CI' { + + AfterEach { + Get-Job | Where-Object PSJobTypeName -eq "ThreadJob" | Remove-Job -Force + } + + It 'Verifies StopJob API' { + + $job = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } -ThrottleLimit 5 + Wait-ForJobRunning $job + $job.StopJob($true, "No Reason") + $job.JobStateInfo.State | Should -Be "Stopped" + } + + It 'Verifies StopJobAsync API' { + + $job = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } -ThrottleLimit 5 + Wait-ForJobRunning $job + $job.StopJobAsync($true, "No Reason") + Wait-Job $job + $job.JobStateInfo.State | Should -Be "Stopped" + } + + It 'Verifies StartJobAsync API' { + + $jobRunning = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } -ThrottleLimit 1 + $jobNotRunning = Start-ThreadJob -ScriptBlock { Start-Sleep -Seconds 60 } + + $jobNotRunning.JobStateInfo.State | Should -Be "NotStarted" + + # StartJobAsync starts jobs synchronously for ThreadJob jobs + $jobNotRunning.StartJobAsync() + Wait-ForJobRunning $jobNotRunning + $jobNotRunning.JobStateInfo.State | Should -Be "Running" + } + + It 'Verifies JobSourceAdapter Get-Jobs' { + + $job = Start-ThreadJob -ScriptBlock { "Hello" } | Wait-Job + + $getJob = Get-Job -InstanceId $job.InstanceId 2> $null + $getJob | Should -Be $job + + $getJob = Get-Job -Name $job.Name 2> $null + $getJob | Should -Be $job + + $getJob = Get-Job -Command ' "hello" ' 2> $null + $getJob | Should -Be $job + + $getJob = Get-Job -State $job.JobStateInfo.State 2> $null + $getJob | Should -Be $job + + $getJob = Get-Job -Id $job.Id 2> $null + $getJob | Should -Be $job + + # Get-Job -Filter is not supported + $result = Get-Job -Filter @{Id = ($job.Id)} 3> $null + $result | Should -BeNullOrEmpty + } + + It 'Verifies terminating job error' { + + $job = Start-ThreadJob -ScriptBlock { throw "My Job Error!" } | Wait-Job + $results = $job | Receive-Job 2>&1 + $results.ToString() | Should -Be "My Job Error!" + } +}