diff --git a/.azure-pipelines/weekly-examples-update.yml b/.azure-pipelines/weekly-examples-update.yml index 23e05c5a41f..4f38c431e56 100644 --- a/.azure-pipelines/weekly-examples-update.yml +++ b/.azure-pipelines/weekly-examples-update.yml @@ -2,10 +2,10 @@ trigger: none # disable triggers based on commits. pr: none # disable as a PR gate. -name: 'PowerShellExamplesUpdate Check' +name: 'PowerShellExamplesUpdateV1 Check' schedules: - cron: "0 3 * * FRI" # every Friday at 3AM UTC (off hours for Redmond, Nairobi and Montréal) - displayName: 'PowerShellExamplesUpdate Check' + displayName: 'PowerShellExamplesUpdateV1' branches: include: - dev @@ -28,12 +28,11 @@ resources: ref: dev jobs: -- job: PowerShellExamplesUpdate +- job: PowerShellExamplesUpdateV1 pool: name: ${{ parameters.BuildAgent }} timeoutInMinutes: ${{ parameters.PipelineTimeout }} steps: - - template: ./common-templates/update-sdkversion.yml - task: PowerShell@2 name: "ComputeBranch" @@ -56,13 +55,21 @@ jobs: git status - task: PowerShell@2 - displayName: 'Update Examples From Graph Reference' + displayName: 'Update Examples From API reference - V1' continueOnError: false inputs: targetType: 'filePath' pwsh: true filePath: tools\ExamplesGenerator.ps1 + - task: PublishBuildArtifacts@1 + displayName: 'Publish Examples to be reviewed as artifact' + inputs: + PathtoPublish: 'examplesreport' + ArtifactName: 'ExamplesToBeReviewed' + publishLocation: 'Container' + # StoreAsTar: true + - task: PowerShell@2 displayName: Pushing to github env: @@ -71,10 +78,12 @@ jobs: targetType: inline pwsh: true script: | + git config --global user.email "GraphTooling@service.microsoft.com" + git config --global user.name "Microsoft Graph DevX Tooling" git status git add . - git commit -m "Updating examples" - git push --set-upstream origin $(ComputeBranch.WeeklyExamplesBranch) + git commit -m "Updating v1 examples" + git push --set-upstream https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git $(ComputeBranch.WeeklyExamplesBranch) git status - template: ./common-templates/create-pr.yml @@ -82,4 +91,4 @@ jobs: BaseBranch: "dev" TargetBranch: $(ComputeBranch.WeeklyExamplesBranch) Title: "[v1] Examples Update" - Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." + Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file diff --git a/.gitignore b/.gitignore index 24670b3a0b8..d379648302a 100644 --- a/.gitignore +++ b/.gitignore @@ -358,4 +358,7 @@ MigrationBackup/ .vscode/ #Custom Environment Files -localenv.json \ No newline at end of file +localenv.json + +#Wrong Examples report Folder +examplesreport/ \ No newline at end of file diff --git a/tools/ExamplesGenerator.ps1 b/tools/ExamplesGenerator.ps1 index f3148c718e9..ceafae2f28c 100644 --- a/tools/ExamplesGenerator.ps1 +++ b/tools/ExamplesGenerator.ps1 @@ -2,7 +2,15 @@ # Licensed under the MIT License. Param( $ModulesToGenerate = @(), - [string] $ModuleMappingConfigPath = (Join-Path $PSScriptRoot "..\config\ModulesMapping.jsonc") + $Available = @(), + [hashtable]$V1CommandGetVariantList= @{}, + [hashtable]$BetaCommandGetVariantList= @{}, + [hashtable]$V1CommandListVariantList= @{}, + [hashtable]$BetaCommandListVariantList= @{}, + [string] $ModuleMappingConfigPath = (Join-Path $PSScriptRoot "..\config\ModulesMapping.jsonc"), + [string] $FolderForExamplesToBeReviewed = (Join-Path $PSScriptRoot "..\examplesreport"), + [string] $ExamplesToBeReviewed = "ExamplesToBeReviewed.csv", + $MetaDataJsonFile = (Join-Path $PSScriptRoot "../src/Authentication/Authentication/custom/common/MgCommandMetadata.json") ) function Start-Generator { Param( @@ -32,10 +40,9 @@ function Start-Generator { $ProfilePathMapping = "examples\v1.0-beta" } $ModulePath = Join-Path $PSScriptRoot "..\src\$GraphModule\$GraphModule\$ProfilePathMapping" - Get-ExternalDocsUrl -ManualExternalDocsUrl $ManualExternalDocsUrl -GenerationMode $GenerationMode -GraphProfilePath $ModulePath -Command $GraphCommand -GraphProfile $ProfilePath -Module -$Module + Get-ExternalDocsUrl -ManualExternalDocsUrl $ManualExternalDocsUrl -GenerationMode $GenerationMode -GraphProfilePath $ModulePath -Command $GraphCommand -GraphProfile $ProfilePath -Module $GraphModule } - } function Get-FilesByProfile { Param( @@ -71,9 +78,7 @@ function Get-Files { [string] $Module = "Users", [Hashtable] $OpenApiContent ) - $ModuleManifestFile = (Join-Path $PSScriptRoot "..\src\$Module\$Module\Microsoft.Graph.$Module.psd1") - $ModuleManifestFileContent = Get-Content -Path $ModuleManifestFile - + try { if (Test-Path $GraphProfilePath) { @@ -81,28 +86,18 @@ function Get-Files { #Extract command over here $Command = [System.IO.Path]::GetFileNameWithoutExtension($File) - #Check for cmdlet existence from the module manifest file - if ($ModuleManifestFileContent | Select-String -pattern $Command) { - #Extract URI path - $Uripaths = Find-MgGraphCommand -Command $Command $UriPath = $null - if (-not([string]::IsNullOrEmpty($Uripaths))) { - if ($Uripaths.APIVersion.Contains($GraphProfile)) { - if ($Uripaths.Length -gt 1) { - $UriPath = $UriPaths.URI[0].ToString() - } - else { - $UriPath = $UriPaths.URI.ToString() - } - } - - if ($UriPath) { - $Method = $UriPaths.Method - Get-ExternalDocsUrl -GraphProfile $GraphProfile -Url -UriPath $UriPath -Command $Command -OpenApiContent $OpenApiContent -GraphProfilePath $GraphProfilePath -Method $Method -Module $Module - } + if($GraphProfile -eq "beta"){ + $UriPath = $BetaCommandGetVariantList[$Command] + }else{ + $UriPath = $V1CommandGetVariantList[$Command] + } + + if ($UriPath) { + $Method = $UriPaths.Method + Get-ExternalDocsUrl -GraphProfile $GraphProfile -Url -UriPath $UriPath -Command $Command -OpenApiContent $OpenApiContent -GraphProfilePath $GraphProfilePath -Method $Method -Module $Module } - } } } @@ -135,18 +130,19 @@ function Get-ExternalDocsUrl { if (-not([string]::IsNullOrEmpty($ManualExternalDocsUrl))) { - Start-WebScrapping -GraphProfile $GraphProfile -ExternalDocUrl $ManualExternalDocsUrl -Command $Command -GraphProfilePath $GraphProfilePath + Start-WebScrapping -GraphProfile $GraphProfile -ExternalDocUrl $ManualExternalDocsUrl -Command $Command -GraphProfilePath $GraphProfilePath -UriPath $UriPath -Module $Module } } else { if ($UriPath) { - if ($openApiContent.openapi && $openApiContent.info.version) { - foreach ($path in $openApiContent.paths) { + + if ($OpenApiContent.openapi && $OpenApiContent.info.version) { + foreach ($Path in $OpenApiContent.paths) { $MethodName = $Method | Out-String - - $externalDocUrl = $path[$UriPath].get.externalDocs.url - if ([string]::IsNullOrEmpty($externalDocUrl)) { + $ExternalDocUrl = $Path[$UriPath].values.externalDocs.url + + if ([string]::IsNullOrEmpty($ExternalDocUrl)) { $PathSplit = $UriPath.Split("/") $PathToAppend = $PathSplit[$PathSplit.Count - 1] if ($PathToAppend.StartsWith("{") -or $PathToAppend.StartsWith("$")) { @@ -158,29 +154,85 @@ function Get-ExternalDocsUrl { $PathRebuild += $PathSplit[$i] + "/" } $RebuiltPath = $PathRebuild + "microsoft.graph." + $PathToAppend - $externalDocUrl = $path[$RebuiltPath].get.externalDocs.url + $ExternalDocUrl = $path[$RebuiltPath].get.externalDocs.url + if ([string]::IsNullOrEmpty($ExternalDocUrl)) { + $GetPath = $null + if($GraphProfile -eq "beta"){ + $GetPath = $BetaCommandListVariantList[$Command] + }else{ + $GetPath = $V1CommandListVariantList[$Command] + } + if(-not([string]::IsNullOrEmpty($GetPath))){ + $ExternalDocUrl = $Path[$GetPath].get.externalDocs.url + } + + } } } if ($MethodName -eq "POST") { - $externalDocUrl = $path[$UriPath].post.externalDocs.url + $ExternalDocUrl = $Path[$UriPath].post.externalDocs.url + if ([string]::IsNullOrEmpty($ExternalDocUrl)) { + $PostPath = $null + if($GraphProfile -eq "beta"){ + $PostPath = $BetaCommandListVariantList[$Command] + }else{ + $PostPath = $V1CommandListVariantList[$Command] + } + if(-not([string]::IsNullOrEmpty($PostPath))){ + $ExternalDocUrl = $Path[$PostPath].post.externalDocs.url + } + } } if ($MethodName -eq "PATCH") { - $externalDocUrl = $path[$UriPath].patch.externalDocs.url + $ExternalDocUrl = $Path[$UriPath].patch.externalDocs.url + if ([string]::IsNullOrEmpty($ExternalDocUrl)) { + $PatchPath = $null + if($GraphProfile -eq "beta"){ + $PatchPath = $BetaCommandListVariantList[$Command] + }else{ + $PatchPath = $V1CommandListVariantList[$Command] + } + if(-not([string]::IsNullOrEmpty($PatchPath))){ + $ExternalDocUrl = $Path[$PatchPath].post.externalDocs.url + } + } } if ($MethodName -eq "DELETE") { - $externalDocUrl = $path[$UriPath].delete.externalDocs.url + $ExternalDocUrl = $Path[$UriPath].delete.externalDocs.url + if ([string]::IsNullOrEmpty($ExternalDocUrl)) { + $DeletePath = $null + if($GraphProfile -eq "beta"){ + $DeletePath = $BetaCommandListVariantList[$Command] + }else{ + $DeletePath = $V1CommandListVariantList[$Command] + } + if(-not([string]::IsNullOrEmpty($DeletePath))){ + $ExternalDocUrl = $Path[$DeletePath].post.externalDocs.url + } + } } - + if ($MethodName -eq "PUT") { - $externalDocUrl = $path[$UriPath].put.externalDocs.url + $ExternalDocUrl = $Path[$UriPath].put.externalDocs.url + if ([string]::IsNullOrEmpty($ExternalDocUrl)) { + $PutPath = $null + if($GraphProfile -eq "beta"){ + $PutPath = $BetaCommandListVariantList[$Command] + }else{ + $PutPath = $V1CommandListVariantList[$Command] + } + if(-not([string]::IsNullOrEmpty($PutPath))){ + $ExternalDocUrl = $Path[$PutPath].post.externalDocs.url + } + } } - if (-not([string]::IsNullOrEmpty($externalDocUrl))) { - Start-WebScrapping -GraphProfile $GraphProfile -ExternalDocUrl $externalDocUrl -Command $Command -GraphProfilePath $GraphProfilePath + if (-not([string]::IsNullOrEmpty($ExternalDocUrl))) { + Start-WebScrapping -GraphProfile $GraphProfile -ExternalDocUrl $ExternalDocUrl -Command $Command -GraphProfilePath $GraphProfilePath -UriPath $UriPath -Module $Module } - } + } } } @@ -194,8 +246,10 @@ function Start-WebScrapping { [string] $ExternalDocUrl = "https://learn.microsoft.com/en-us/graph/api/user-get?view=graph-rest-1.0&tabs=powershell", [ValidateNotNullOrEmpty()] [string] $Command = "Get-MgUser", + [string] $UriPath, + [string] $Module = "Users", [string] $GraphProfilePath = (Join-Path $PSScriptRoot "..\src\Users\Users\examples\v1.0") - ) + ) $ExampleFile = "$GraphProfilePath/$Command.md" $url = $ExternalDocUrl @@ -212,7 +266,7 @@ function Start-WebScrapping { if ($checkPowershell.Contains('lang-powershell')) { $result = $node.InnerHtml $result = $result.Replace('"', '"') - $ExampleList.Add($result) + $EL = $ExampleList.Add($result) } } foreach ($header in $headers) { @@ -220,12 +274,12 @@ function Start-WebScrapping { if ($checkPowershell.Contains('Example')) { $result = $header.InnerHtml - $HeaderList.Add($result) + $HL = $HeaderList.Add($result) } } - Update-ExampleFile -GraphProfile $GraphProfile -HeaderList $HeaderList -ExampleList $ExampleList -ExampleFile $ExampleFile -Description $Description + Update-ExampleFile -GraphProfile $GraphProfile -HeaderList $HeaderList -ExampleList $ExampleList -ExampleFile $ExampleFile -Description $Description -Command $Command -ExternalDocUrl $ExternalDocUrl -UriPath $UriPath -Module $Module } function Update-ExampleFile { @@ -235,59 +289,129 @@ function Update-ExampleFile { [System.Collections.ArrayList] $HeaderList, [System.Collections.ArrayList] $ExampleList, [string] $ExampleFile, - [string] $Description + [string] $UriPath, + [string] $Description, + [string] $Module = "Users", + [string] $Command = "Get-MgUser", + [ValidateNotNullOrEmpty()] + [string] $ExternalDocUrl = "https://learn.microsoft.com/en-us/graph/api/user-get?view=graph-rest-1.0&tabs=powershell" ) $Content = Get-Content -Path $ExampleFile - $SearchText = "{{ Add description here }}" + $SearchText = "Example" + $SearchTextForNewImports = "{{ Add description here }}" $ReplaceEverything = $False if ($HeaderList.Count -eq 0) { for ($d = 0; $d -lt $ExampleList.Count; $d++) { $sum = $d + 1 - $HeaderList.Add("Example " + $sum + ": Code snippet".Trim()) + $HL = $HeaderList.Add("Example " + $sum + ": Code snippet".Trim()) } } if ($HeaderList.Count -ne $ExampleList.Count) { $HeaderList.Clear() for ($d = 0; $d -lt $ExampleList.Count; $d++) { $sum = $d + 1 - $HeaderList.Add("Example " + $sum + ": Code snippet".Trim()) + $H = $HeaderList.Add("Example " + $sum + ": Code snippet".Trim()) } } - if ($Content | Select-String -pattern $SearchText) { + if (($Content | Select-String -pattern $SearchTextForNewImports)) { $ReplaceEverything = $True - #Start-Sleep 5 - } - $headCount = $HeaderList.Count - $exampleCount = $ExampleList.Count - if ($ReplaceEverything -and $exampleCount -gt 0 -and $headCount -eq $exampleCount) { + + $HeadCount = $HeaderList.Count + $ExampleCount = $ExampleList.Count + $WrongExamplesCount = 0; + #===========================Importing new examples into files ============================================# + if ($ReplaceEverything -and $ExampleCount -gt 0 -and $HeadCount -eq $ExampleCount) { Clear-Content $ExampleFile -Force - for ($d = 0; $d -lt $headerList.Count; $d++) { - $codeValue = $exampleList[$d].Trim() - $titleValue = "### " + $headerList[$d].Trim() - $code = "``````powershell`r$codeValue`r`n``````" + for ($d = 0; $d -lt $HeaderList.Count; $d++) { + $CodeValue = $ExampleList[$d].Trim() + if($CodeValue.Contains($Command)){ + $TitleValue = "### " + $HeaderList[$d].Trim() + $Code = "``````powershell`r$CodeValue`r`n``````" - $totalText = "$titleValue`r`n`n$code`r`n$description`r`n" - Add-Content -Path $ExampleFile -Value $totalText + $TotalText = "$TitleValue`r`n`n$Code`r`n$Description`r`n" + Add-Content -Path $ExampleFile -Value $TotalText + }else{ + $WrongExamplesCount++ + + } } - git config --global user.email "timwamalwa@gmail.com" - git config --global user.name "Timothy Wamalwa" - git add $ExampleFile - git commit -m "Examples update on $ExampleFile-$GraphProfile" } - else { - if ($headCount -ne $exampleCount) { - Write-Host "Mismatch in Title and Snippet count i.e Title ="$headerList.Count " Snippet = "$exampleList.Count + $PatternToSearch = "Import-Module Microsoft.Graph.$Module" + if(($Content | Select-String -pattern $SearchText) -and ($Content | Select-String -pattern "This example shows")){ + $ContainsPatternToSearch = $False + foreach($List in $ExampleList){ + if($List.Contains($PatternToSearch)){ + $ContainsPatternToSearch = $True + } } - else { - Write-Host "No examples found from the reference docs" + if($ContainsPatternToSearch){ + Clear-Content $ExampleFile -Force + #Replace everything + for ($d = 0; $d -lt $HeaderList.Count; $d++) { + $CodeValue = $ExampleList[$d].Trim() + $TitleValue = "### " + $HeaderList[$d].Trim() + $Code = "``````powershell`r$CodeValue`r`n``````" + + $TotalText = "$TitleValue`r`n`n$Code`r`n$Description`r`n" + Add-Content -Path $ExampleFile -Value $TotalText + } + + }else{ + Clear-Content $ExampleFile -Force + #Replace everything with boiler plate code + $DefaultBoilerPlate = "### Example 1: {{ Add title here }}`r`n``````powershell`r`n PS C:\> {{ Add code here }}`r`n`n{{ Add output here }}`r`n```````n`n{{ Add description here }}`r`n`n### Example 2: {{ Add title here }}`r`n``````powershell`r`n PS C:\> {{ Add code here }}`r`n`n{{ Add output here }}`r`n```````n`n{{ Add description here }}`r`n`n" + Add-Content -Path $ExampleFile -Value $DefaultBoilerPlate.Trim() } - } + if($WrongExamplesCount -gt 0){ + $DefaultBoilerPlate = "### Example 1: {{ Add title here }}`r`n``````powershell`r`n PS C:\> {{ Add code here }}`r`n`n{{ Add output here }}`r`n```````n`n{{ Add description here }}`r`n`n### Example 2: {{ Add title here }}`r`n``````powershell`r`n PS C:\> {{ Add code here }}`r`n`n{{ Add output here }}`r`n```````n`n{{ Add description here }}`r`n`n" + Add-Content -Path $ExampleFile -Value $DefaultBoilerPlate.Trim() + #Log api path api version and equivalent external doc url giving wron examples + #Create folder and file if it doesn't exist + #The artifact below will be ignored on git. + #You can download the artificat from the generator pipeline + + if(-not(Test-Path -PathType Container $FolderForExamplesToBeReviewed)){ + New-Item -ItemType Directory -Force -Path $FolderForExamplesToBeReviewed + } + if (-not (Test-Path "$FolderForExamplesToBeReviewed\$ExamplesToBeReviewed")) { + "Command, ExternalDocsUrl, ApiVersion" | Out-File -FilePath "$FolderForExamplesToBeReviewed\$ExamplesToBeReviewed" -Encoding ASCII + } + + $File = Get-Content "$FolderForExamplesToBeReviewed\$ExamplesToBeReviewed" + $containsWord = $File | % { $_ -match "$Command, $ExternalDocUrl, $GraphProfile, $UriPath" } + if (-not($containsWord -contains $true)) { + "$Command, $ExternalDocUrl, $GraphProfile, $UriPath" | Out-File -FilePath "$FolderForExamplesToBeReviewed\$ExamplesToBeReviewed" -Append -Encoding ASCII + } + } + +} +$JsonContent = Get-Content -Path $MetaDataJsonFile +$DeserializedContent = $JsonContent | ConvertFrom-Json +foreach($Data in $DeserializedContent) +{ + if($Data.ApiVersion -eq "beta") + { + if((-not($Data.Variants[0].Contains("List")))){ + $Beta = $BetaCommandGetVariantList.Add($Data.Command, $Data.Uri) + }else{ + $Beta1 = $BetaCommandListVariantList.Add($Data.Command, $Data.Uri) + } + } + + if($Data.ApiVersion -eq "v1.0") + { + if((-not($Data.Variants[0].Contains("List")))){ + $V1 = $V1CommandGetVariantList.Add($Data.Command, $Data.Uri) + }else{ + $V11 = $V1CommandListVariantList.Add($Data.Command, $Data.Uri) + } + } } if (!(Get-Module "powershell-yaml" -ListAvailable -ErrorAction SilentlyContinue)) { Install-Module "powershell-yaml" -AcceptLicense -Scope CurrentUser -Force @@ -305,12 +429,21 @@ if ($ModulesToGenerate.Count -eq 0) { [HashTable] $ModuleMapping = Get-Content $ModuleMappingConfigPath | ConvertFrom-Json -AsHashTable $ModulesToGenerate = $ModuleMapping.Keys } - Start-Generator -ModulesToGenerate $ModulesToGenerate -GenerationMode "auto" #Comment the above and uncomment the below start command, if you manually want to manually pass ExternalDocs url. #This is for scenarios where the correponding external docs url to the uri path gotten from Find-MgGraph command, is missing on the openapi.yml file for a particular module. -#Ensure that you pass all correct parameters as oer the already existing example - +#Ensure that you pass all correct parameters as per the already existing example #Start-Generator -GenerationMode "manual" -ManualExternalDocsUrl "https://docs.microsoft.com/graph/api/serviceprincipal-post-approleassignedto?view=graph-rest-1.0&tabs=http" -GraphCommand "New-MgServicePrincipalAppRoleAssignedTo" -GraphModule "Applications" -Profile "v1.0" -Write-Host -ForegroundColor Green "-------------Done-------------" + +#The below tests are ran manually. Comment the above Start-Generator with Generation mode set to auto and uncomment the below test +#---------------------------------------------------------------------------------------------------------------------------------# +#1. Test for making corrections and updating auto imported examples. I.e Examples that were not handwritten +#Start-Generator -GenerationMode "manual" -ManualExternalDocsUrl "https://docs.microsoft.com/graph/api/directoryobject-getmembergroups?view=graph-rest-beta" -GraphCommand "Get-MgBetaApplicationMemberGroup" -GraphModule "Applications" -Profile "beta" + +#2. Test for ensuring that a handwritten example is not tampered with +#Start-Generator -GenerationMode "manual" -ManualExternalDocsUrl "https://docs.microsoft.com/graph/api/user-get?view=graph-rest-1.0" -GraphCommand "Get-MgUser" -GraphModule "Users" -Profile "v1.0" + +#3. Test for updates from api reference +#Start-Generator -GenerationMode "manual" -ManualExternalDocsUrl "https://docs.microsoft.com/graph/api/serviceprincipal-post-approleassignedto?view=graph-rest-1.0" -GraphCommand "New-MgServicePrincipalAppRoleAssignedTo" -GraphModule "Applications" -Profile "v1.0" +Write-Host -ForegroundColor Green "-------------Done-------------" \ No newline at end of file