Skip to content

Commit

Permalink
Add infrastructure to enable 2-machine network testing on 1ES Azure p…
Browse files Browse the repository at this point in the history 
…ools. (#4364)

* secnetperf script for 1es azure pools

* more debug logging

* fix infinite loop issue

* dumb syntax issue

* remove C:\ remove

* wip

* use config pwsh 7

* skip preparing machine for now

* add support for starting stopping server passively

* fix syntax issue

* create the file

* more logging

* remove ps1 extension

* fix ps1 issue

* get rid of 30s timeout

* NULL in case of Az linux

* wip

* GOAL: Get a successful secnetperf run for linux

* lets get some data

* add stop server logic

* p2p module remove

* ensure we stop using remote pwsh if it's not supported

* get some answers

* print more logs

* && does not exist lmao

* collect me some data

* add run id and stuff

* wrap in a try catch

* wip

* wip

* get some stdout

* forget about core dumps in 1es

* revert 1es change
  • Loading branch information
@ProjectsByJackHe
ProjectsByJackHe committed Jun 21, 2024
1 parent f5bec53 commit f61ad9b
Show file tree
Hide file tree
Showing 2 changed files with 279 additions and 59 deletions.
190 changes: 182 additions & 8 deletions scripts/secnetperf-helpers.psm1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,71 @@ function Start-RemoteServer {
throw "Server failed to start!"
}

# Passively starts the server on the remote machine by queuing up a new script to execute.
function Start-RemoteServerPassive {
param ($Session, $Command, $RemoteStateDir, $RunId, $SyncerSecret)
if ($Session -eq "NOT_SUPPORTED") {
Write-Host "Command to start server: $Command"
$headers = @{
"secret" = "$SyncerSecret"
}
$url = "https://netperfapiwebapp.azurewebsites.net"
try {
$Response = Invoke-WebRequest -Uri "$url/getkeyvalue?key=$RunId" -Headers $headers
} catch {
Write-Host "Unable to fetch state. Creating a new one now."
$state = [pscustomobject]@{
value=[pscustomobject]@{
"SeqNum" = 0
"Commands" = @($Command)
}}
$StateJson = $state | ConvertTo-Json
$Response = Invoke-WebRequest -Uri "$url/setkeyvalue?key=$RunId" -Headers $headers -Method Post -Body $StateJson -ContentType "application/json"
if ($Response.StatusCode -ne 200) {
Write-GHError "[Start-Remote-Passive] Failed to set the key value!"
throw "Failed to set the key value!"
}
return
}
$CurrState = $Response.Content | ConvertFrom-Json
$CurrState.Commands += $Command
$CurrState = [pscustomobject]@{
value=$CurrState
}
$StateJson = $CurrState | ConvertTo-Json
$Response = Invoke-WebRequest -Uri "$url/setkeyvalue?key=$RunId" -Headers $headers -Method Post -Body $StateJson -ContentType "application/json"
if ($Response.StatusCode -ne 200) {
Write-GHError "[Start-Remote-Passive] Failed to set the key value!"
throw "Failed to set the key value!"
}
return
}

Invoke-Command -Session $Session -ScriptBlock {
if (!(Test-Path $Using:RemoteStateDir)) {
New-Item -ItemType Directory $Using:RemoteStateDir | Out-Null
}
# Fetch all files names inside the directory
$files = Get-ChildItem $Using:RemoteStateDir
# Find the highest lexicographically sorted file name
$max = 0
foreach ($file in $files) {
# Remove .ps1 extension from file.Name
$filename = $file.Name.split(".")[0]
$num = [int]($filename -replace "[^0-9]", "")
if ($num -gt $max) {
$max = $num
}
}
$newmax = $max + 1
# Create a new file with the next number
Write-Host "Creating new file execute_$newmax.ps1"
New-Item -ItemType File (Join-Path $Using:RemoteStateDir ("execute_$newmax.ps1")) | Out-Null
$newFile = Join-Path $Using:RemoteStateDir ("execute_$newmax.ps1")
Set-Content -Path $newFile -Value $Using:Command
}
}

# Sends a special UDP packet to tell the remote secnetperf to shutdown, and then
# waits for the job to complete. Finally, it returns the console output of the
# job.
Expand All @@ -306,6 +371,83 @@ function Stop-RemoteServer {
return $RemoteResult -join "`n"
}

function Stop-RemoteServerAsyncAwait {
param ($Session, $RemoteStateDir, $RemoteAddress, $RunId, $SyncerSecret)
# Ping side-channel socket on 9999 to tell the app to die
$Socket = New-Object System.Net.Sockets.UDPClient
$BytesToSend = @(
0x57, 0xe6, 0x15, 0xff, 0x26, 0x4f, 0x0e, 0x57,
0x88, 0xab, 0x07, 0x96, 0xb2, 0x58, 0xd1, 0x1c
)
if ($Session -eq "NOT_SUPPORTED") {
for ($i = 0; $i -lt 30; $i++) {
$Socket.Send($BytesToSend, $BytesToSend.Length, $RemoteAddress, 9999) | Out-Null
Start-Sleep -Seconds 8 | Out-Null
$headers = @{
"secret" = "$SyncerSecret"
}
$url = "https://netperfapiwebapp.azurewebsites.net"
$Response = Invoke-WebRequest -Uri "$url/getkeyvalue?key=$RunId" -Headers $headers
if (!($Response.StatusCode -eq 200)) {
Write-GHError "[Stop-Remote-Passive] Failed to get the key value!"
throw "Failed to get the key value!"
}
$CurrState = $Response.Content | ConvertFrom-Json
if ($CurrState.SeqNum -eq $CurrState.Commands.Count) {
return
}
}
Write-GHError "[Stop-Remote-Passive] SeqNum less than Commands Count!"
throw "Unable to stop the remote server in time!"
}
$done = $false
$MaxSeqNum = 0
for ($i = 0; $i -lt 30; $i++) {
$Socket.Send($BytesToSend, $BytesToSend.Length, $RemoteAddress, 9999) | Out-Null
Start-Sleep -Seconds 5 | Out-Null
$files = Invoke-Command -Session $Session -ScriptBlock { Get-ChildItem $Using:RemoteStateDir }
foreach ($file in $files) {
$filename = $file.Name.split(".")[0]
$num = [int]($filename -replace "[^0-9]", "")
if ($num -gt $MaxSeqNum) {
$MaxSeqNum = $num
}
}
# Check if completed_<MaxSeqNum>.txt exists
foreach ($file in $files) {
if ($file.Name -eq "completed_$MaxSeqNum.txt") {
$done = $true
break
}
}
if ($done) {
break
}
}
if (!$done) {
Write-GHError "[Stop-Remote-Passive] Unable to find a corresponding 'completed_$MaxSeqNum' file!"
throw "Unable to stop the remote server in time!"
}
}

function Wait-StartRemoteServerPassive {
param ($FullPath, $RemoteName, $OutputDir)

for ($i = 0; $i -lt 30; $i++) {
Start-Sleep -Seconds 5 | Out-Null
Write-Host "Attempt $i to start the remote server, command: $FullPath -target:$RemoteName"
$Process = Start-LocalTest $FullPath "-target:$RemoteName" $OutputDir
$ConsoleOutput = Wait-LocalTest $Process $OutputDir $false 30000 $true
Write-Host "Wait-StartRemoteServerPassive: $ConsoleOutput"
$DidMatch = $ConsoleOutput -match "Completed" # Look for the special string to indicate success.
if ($DidMatch) {
return
}
}

throw "Unable to start the remote server in time!"
}

# Creates a new local process to asynchronously run the test.
function Start-LocalTest {
param ($FullPath, $FullArgs, $OutputDir, $UseSudo)
Expand Down Expand Up @@ -337,7 +479,7 @@ function Start-LocalTest {

# Waits for a local test process to complete, and then returns the console output.
function Wait-LocalTest {
param ($Process, $OutputDir, $testKernel, $TimeoutMs)
param ($Process, $OutputDir, $testKernel, $TimeoutMs, $Silent = $false)
$StdOut = $Process.StandardOutput.ReadToEndAsync()
$StdError = $Process.StandardError.ReadToEndAsync()
# Wait for the process to exit.
Expand All @@ -351,6 +493,10 @@ function Wait-LocalTest {
$Out = $StdOut.Result.Trim()
if ($Out.Length -ne 0) { Write-Host $Out }
} catch {}
if ($Silent) {
Write-Host "Silently ignoring Client timeout!"
return ""
}
throw "secnetperf: Client timed out!"
}
# Verify the process cleanly exitted.
Expand All @@ -360,15 +506,27 @@ function Wait-LocalTest {
$Out = $StdOut.Result.Trim()
if ($Out.Length -ne 0) { Write-Host $Out }
} catch {}
if ($Silent) {
Write-Host "Silently ignoring Client exit code: $($Process.ExitCode)"
return ""
}
throw "secnetperf: Nonzero exit code: $($Process.ExitCode)"
}
# Wait for the output streams to flush.
[System.Threading.Tasks.Task]::WaitAll(@($StdOut, $StdError))
$consoleTxt = $StdOut.Result.Trim()
if ($consoleTxt.Length -eq 0) {
if ($Silent) {
Write-Host "Silently ignoring Client no console output!"
return ""
}
throw "secnetperf: No console output (possibly crashed)!"
}
if ($consoleTxt.Contains("Error")) {
if ($Silent) {
Write-Host "Silently ignoring Client error: $($consoleTxt)"
return ""
}
throw "secnetperf: $($consoleTxt.Substring(7))" # Skip over the "Error: " prefix
}
return $consoleTxt
Expand Down Expand Up @@ -460,7 +618,7 @@ function Get-LatencyOutput {

# Invokes secnetperf with the given arguments for both TCP and QUIC.
function Invoke-Secnetperf {
param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter)
param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter, $Environment, $RunId, $SyncerSecret)

$values = @(@(), @())
$latency = $null
Expand All @@ -485,7 +643,7 @@ function Invoke-Secnetperf {
$execMode = $ExeArgs.Substring(0, $ExeArgs.IndexOf(" ")) # First arg is the exec mode
$clientPath = Repo-Path $SecNetPerfPath
$serverArgs = "$execMode -io:$io"
$clientArgs = "-target:netperf-peer $ExeArgs -tcp:$tcp -trimout -watchdog:25000"
$clientArgs = "-target:$RemoteName $ExeArgs -tcp:$tcp -trimout -watchdog:25000"
if ($io -eq "xdp" -or $io -eq "qtip") {
$serverArgs += " -pollidle:10000"
$clientArgs += " -pollidle:10000"
Expand Down Expand Up @@ -524,15 +682,17 @@ function Invoke-Secnetperf {
$artifactDir = Repo-Path "artifacts/logs/$artifactName"
$remoteArtifactDir = "$RemoteDir/artifacts/logs/$artifactName"
New-Item -ItemType Directory $artifactDir -ErrorAction Ignore | Out-Null
Invoke-Command -Session $Session -ScriptBlock {
New-Item -ItemType Directory $Using:remoteArtifactDir -ErrorAction Ignore | Out-Null
if (!($Session -eq "NOT_SUPPORTED")) {
Invoke-Command -Session $Session -ScriptBlock {
New-Item -ItemType Directory $Using:remoteArtifactDir -ErrorAction Ignore | Out-Null
}
}

$clientOut = (Join-Path $artifactDir "client.console.log")
$serverOut = (Join-Path $artifactDir "server.console.log")

# Start logging on both sides, if configured.
if ($LogProfile -ne "" -and $LogProfile -ne "NULL") {
if ($LogProfile -ne "" -and $LogProfile -ne "NULL" -and !($Session -eq "NOT_SUPPORTED")) {
Invoke-Command -Session $Session -ScriptBlock {
try { & "$Using:RemoteDir/scripts/log.ps1" -Cancel } catch {} # Cancel any previous logging
& "$Using:RemoteDir/scripts/log.ps1" -Start -Profile $Using:LogProfile -ProfileInScriptDirectory
Expand All @@ -547,7 +707,17 @@ function Invoke-Secnetperf {

# Start the server running.
"> secnetperf $serverArgs" | Add-Content $serverOut
$job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo

$StateDir = "C:/_state"
if (!$IsWindows) {
$StateDir = "/etc/_state"
}
if ($Environment -eq "azure") {
Start-RemoteServerPassive $Session "$sudo$RemoteDir/$SecNetPerfPath $serverArgs" $StateDir $RunId $SyncerSecret
Wait-StartRemoteServerPassive "$sudo$clientPath" $RemoteName $artifactDir
} else {
$job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo
}

# Run the test multiple times, failing (for now) only if all tries fail.
# TODO: Once all failures have been fixed, consider all errors fatal.
Expand Down Expand Up @@ -586,7 +756,11 @@ function Invoke-Secnetperf {
$testFailures = $true
} finally {
# Stop the server.
try { Stop-RemoteServer $job $RemoteName | Add-Content $serverOut } catch { }
if ($Environment -eq "azure") {
Stop-RemoteServerAsyncAwait $Session $StateDir $RemoteName $RunId $SyncerSecret
} else {
try { Stop-RemoteServer $job $RemoteName | Add-Content $serverOut } catch { }
}

# Stop any logging and copy the logs to the artifacts folder.
if ($LogProfile -ne "" -and $LogProfile -ne "NULL") {
Expand Down
Loading

0 comments on commit f61ad9b

Please sign in to comment.