diff --git a/.buildkite/nightly.definition.yaml b/.buildkite/nightly.definition.yaml new file mode 100644 index 00000000..f696ee5d --- /dev/null +++ b/.buildkite/nightly.definition.yaml @@ -0,0 +1,11 @@ +agent_queue_id: trigger-pipelines +description: Nightly build and deployment of Unreal Example project +github: + branch_configuration: [] + default_branch: master + pull_request_branch_filter_configuration: [] +teams: +- name: Everyone + permission: BUILD_AND_READ +- name: gbu/develop/unreal + permission: MANAGE_BUILD_AND_READ diff --git a/.buildkite/nightly.steps.yaml b/.buildkite/nightly.steps.yaml new file mode 100644 index 00000000..5b543d74 --- /dev/null +++ b/.buildkite/nightly.steps.yaml @@ -0,0 +1,25 @@ +common: &common + agents: + - "agent_count=1" + - "capable_of_building=gdk-for-unreal" + - "environment=production" + - "machine_type=quad" + - "permission_set=builder" + - "platform=windows" + - "scaler_version=2" + - "queue=${CI_WINDOWS_BUILDER_QUEUE:-v3-1569490589-f9345a43b21a6cec-------z}" + timeout_in_minutes: 60 # TODO(ENG-548): reduce timeout once agent-cold-start is optimised. + retry: + automatic: + # This is designed to trap and retry failures because agent lost connection. Agent exits with -1 in this case. + - exit_status: -1 + limit: 3 + plugins: + - ca-johnson/taskkill#v4.1: ~ + +steps: + - label: "build-and-deploy-:windows:" + command: powershell -NoProfile -NonInteractive -InputFormat Text -Command ./ci/setup-and-build.ps1 + <<: *common # This folds the YAML named anchor into this step. Overrides, if any, should follow, not precede. + artifact_paths: + - "UnrealEngine/Engine/Programs/AutomationTool/Saved/Logs/*" diff --git a/Game/Source/GDKShooter.Target.cs b/Game/Source/GDKShooter.Target.cs index b8677abd..e3d35404 100644 --- a/Game/Source/GDKShooter.Target.cs +++ b/Game/Source/GDKShooter.Target.cs @@ -9,7 +9,5 @@ public GDKShooterTarget(TargetInfo Target) : base(Target) { Type = TargetType.Game; ExtraModuleNames.Add("GDKShooter"); - // TODO: UNR-1791 for long-term fix - GlobalDefinitions.Add("UE_ALLOW_MAP_OVERRIDE_IN_SHIPPING=1"); } } diff --git a/ci/common.ps1 b/ci/common.ps1 new file mode 100644 index 00000000..3d409533 --- /dev/null +++ b/ci/common.ps1 @@ -0,0 +1,58 @@ +function Write-Log() { + param( + [string] $msg, + [Parameter(Mandatory=$false)] [bool] $expand = $false + ) + if ($expand) { + Write-Output "+++ $($msg)" + } else { + Write-Output "--- $($msg)" + } +} + +function Start-Event() { + param( + [string] $event_name, + [string] $event_parent + ) + + # Start this tracing span. + Start-Process -NoNewWindow "imp-ci" -ArgumentList @(` + "events", "new", ` + "--name", "$($event_name)", ` + "--child-of", "$($event_parent)" + ) | Out-Null + + Write-Log "--- $($event_name)" +} + +function Finish-Event() { + param( + [string] $event_name, + [string] $event_parent + ) + + # Emit the end marker for this tracing span. + Start-Process -NoNewWindow "imp-ci" -ArgumentList @(` + "events", "new", ` + "--name", "$($event_name)", ` + "--child-of", "$($event_parent)" + ) | Out-Null +} + +## Checks whether the specified environment variable has been set. If it has, return its value. Else return the default value. +function Get-Env-Variable-Value-Or-Default() { + param( + [string] $environment_variable_name, + [string] $default_value + ) + + If (Test-Path env:$environment_variable_name) { + $environment_variable_value = Get-Content -Path env:$environment_variable_name + return $environment_variable_value + } Else { + return $default_value + } +} + +$ErrorActionPreference = 'Stop' \ No newline at end of file diff --git a/ci/deploy.ps1 b/ci/deploy.ps1 new file mode 100644 index 00000000..9e08c077 --- /dev/null +++ b/ci/deploy.ps1 @@ -0,0 +1,132 @@ +param( + [string] $launch_deployment = "false", + [string] $gdk_branch_name = "master" +) + +. "$PSScriptRoot\common.ps1" + +Start-Event "deploy-game" "build-unreal-gdk-example-project-:windows:" + # Use the shortened commit hash gathered during GDK plugin clone and the current date and time to distinguish the deployment + $date_and_time = Get-Date -Format "MMdd_HHmm" + $deployment_name = "exampleproject_${date_and_time}_$($gdk_commit_hash)" + $assembly_name = "$($deployment_name)_asm" + +pushd "spatial" + + Start-Event "build-worker-configurations" "deploy-unreal-gdk-example-project-:windows:" + $build_configs_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath "spatial" -ArgumentList @(` + "build", ` + "build-config" + ) + + if ($build_configs_process.ExitCode -ne 0) { + Write-Log "Failed to build worker configurations for the project. Error: $($build_configs_process.ExitCode)" + Throw "Failed to build worker configurations" + } + Finish-Event "build-worker-configurations" "deploy-unreal-gdk-example-project-:windows:" + + Start-Event "prepare-for-run" "deploy-unreal-gdk-example-project-:windows:" + $prepare_for_run_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath "spatial" -ArgumentList @(` + "prepare-for-run", ` + "--log_level=debug" + ) + + if ($prepare_for_run_process.ExitCode -ne 0) { + Write-Log "Failed to prepare for a Spatial cloud launch. Error: $($prepare_for_run_process.ExitCode)" + Throw "Spatial prepare for run failed" + } + Finish-Event "prepare-for-run" "deploy-unreal-gdk-example-project-:windows:" + + Start-Event "upload-assemblies" "deploy-unreal-gdk-example-project-:windows:" + $upload_assemblies_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath "spatial" -ArgumentList @(` + "cloud", ` + "upload", ` + "$assembly_name", ` + "--project_name=$project_name", ` + "--log_level=debug", ` + "--force" + ) + + if ($upload_assemblies_process.ExitCode -ne 0) { + Write-Log "Failed to upload assemblies to cloud. Error: $($upload_assemblies_process.ExitCode)" + Throw "Failed to upload assemblies" + } + Finish-Event "upload-assemblies" "deploy-unreal-gdk-example-project-:windows:" + + Start-Event "launch-deployment" "deploy-unreal-gdk-example-project-:windows:" + # Determine whether deployment should be launched (it will by default) + if ($launch_deployment -eq "true") { + $launch_deployment_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath "spatial" -ArgumentList @(` + "cloud", ` + "launch", ` + "$assembly_name", ` + "$deployment_launch_configuration", ` + "$deployment_name", ` + "--project_name=$project_name", ` + "--snapshot=$deployment_snapshot_path", ` + "--cluster_region=$deployment_cluster_region", ` + "--log_level=debug", ` + "--tags=ttl_1_hours" + ) + + if ($launch_deployment_process.ExitCode -ne 0) { + Write-Log "Failed to launch a Spatial cloud deployment. Error: $($launch_deployment_process.ExitCode)" + Throw "Deployment launch failed" + } + + # Send a Slack notification with a link to the new deployment and to the build. + # Read Slack webhook secret from the vault and extract the Slack webhook URL from it. + $slack_webhook_secret = "$(imp-ci secrets read --environment=production --buildkite-org=improbable --secret-type=slack-webhook --secret-name=unreal-gdk-slack-web-hook)" + $slack_webhook_url = $slack_webhook_secret | ConvertFrom-Json | %{$_.url} + + $deployment_url = "https://console.improbable.io/projects/${project_name}/deployments/${deployment_name}/overview" + $build_url = "$env:BUILDKITE_BUILD_URL" + + $json_message = [ordered]@{ + text = "Example Project build succeeded and a deployment has been launched." + attachments= @( + @{ + fallback = "Find deployment here: $deployment_url and build here: $build_url" + fields= @( + @{ + title = "GDK branch" + value = "$gdk_branch_name" + short = "true" + } + @{ + title = "Example Project branch" + value = "$env:BUILDKITE_BRANCH" + short = "true" + } + ) + actions= @( + @{ + type = "button" + text = "Take me to the deployment" + url = "$deployment_url" + style = "primary" + } + @{ + type = "button" + text = "Take me to the build" + url = "$build_url" + style = "primary" + } + ) + } + ) + } + + $json_request = $json_message | ConvertTo-Json -Depth 10 + + Invoke-WebRequest -UseBasicParsing "$slack_webhook_url" -ContentType "application/json" -Method POST -Body "$json_request" + + } + else { + Write-Log "Deployment will not be launched as you have passed in an argument specifying that it should not be (START_DEPLOYMENT=${launch_deployment}). Remove it to have your build launch a deployment." + } + + Finish-Event "launch-deployment" "deploy-unreal-gdk-example-project-:windows:" + +popd +Finish-Event "deploy-game" "build-unreal-gdk-example-project-:windows:" diff --git a/ci/setup-and-build.ps1 b/ci/setup-and-build.ps1 new file mode 100644 index 00000000..97414de2 --- /dev/null +++ b/ci/setup-and-build.ps1 @@ -0,0 +1,151 @@ +param( + [string] $exampleproject_home = (get-item "$($PSScriptRoot)").parent.FullName, ## The root of the repo + [string] $gdk_repo = "git@github.com:spatialos/UnrealGDK.git", + [string] $gcs_publish_bucket = "io-internal-infra-unreal-artifacts-production/UnrealEngine", + [string] $deployment_launch_configuration = "one_worker_test.json", + [string] $deployment_snapshot_path = "snapshots/FPS-Start_Small.snapshot", + [string] $deployment_cluster_region = "eu", + [string] $project_name = "unreal_gdk" +) + +. "$PSScriptRoot\common.ps1" + +# When a build is launched custom environment variables can be specified. +# Parse them here to use the set value or the default. +$gdk_branch_name = Get-Env-Variable-Value-Or-Default -environment_variable_name "GDK_BRANCH" -default_value "master" +$launch_deployment = Get-Env-Variable-Value-Or-Default -environment_variable_name "START_DEPLOYMENT" -default_value "true" + +$gdk_home = "${exampleproject_home}\Game\Plugins\UnrealGDK" + +pushd "$exampleproject_home" + Start-Event "clone-gdk-plugin" "build-unreal-gdk-example-project-:windows:" + pushd "Game" + New-Item -Name "Plugins" -ItemType Directory -Force + pushd "Plugins" + Start-Process -Wait -PassThru -NoNewWindow -FilePath "git" -ArgumentList @(` + "clone", ` + "$gdk_repo", ` + "--depth 1", ` + "-b $gdk_branch_name" + ) + popd + popd + Finish-Event "clone-gdk-plugin" "build-unreal-gdk-example-project-:windows:" + + Start-Event "get-gdk-head-commit" "build-unreal-gdk-example-project-:windows:" + pushd $gdk_home + # Get the short commit hash of this gdk build for later use in assembly name + $gdk_commit_hash = (git rev-parse HEAD).Substring(0,7) + Write-Log "GDK at commit: $gdk_commit_hash on branch $gdk_branch_name" + popd + Finish-Event "get-gdk-head-commit" "build-unreal-gdk-example-project-:windows:" + + Start-Event "set-up-gdk-plugin" "build-unreal-gdk-example-project-:windows:" + pushd $gdk_home + # Invoke the GDK's setup script + &"$($gdk_home)\ci\setup-gdk.ps1" + popd + Finish-Event "set-up-gdk-plugin" "build-unreal-gdk-example-project-:windows:" + + # Use the cached engine version or set it up if it has not been cached yet. + Start-Event "set-up-engine" "build-unreal-gdk-example-project-:windows:" + + $engine_directory = "${exampleproject_home}\UnrealEngine" + &"$($gdk_home)\ci\get-engine.ps1" -unreal_path "$engine_directory" + + Finish-Event "set-up-engine" "build-unreal-gdk-example-project-:windows:" + + + Start-Event "associate-uproject-with-engine" "build-unreal-gdk-example-project-:windows:" + pushd $engine_directory + $unreal_version_selector_path = "Engine\Binaries\Win64\UnrealVersionSelector.exe" + + $find_engine_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath $unreal_version_selector_path -ArgumentList @(` + "-switchversionsilent", ` + "${exampleproject_home}\Game\GDKShooter.uproject", ` + "$engine_directory" + ) + + if ($find_engine_process.ExitCode -ne 0) { + Write-Log "Failed to set Unreal Engine association for the project. Error: $($find_engine_process.ExitCode)" + Throw "Failed to set Engine association" + } + popd + Finish-Event "associate-uproject-with-engine" "build-unreal-gdk-example-project-:windows:" + + + $build_script_path = "$($gdk_home)\SpatialGDK\Build\Scripts\BuildWorker.bat" + + Start-Event "build-editor" "build-unreal-gdk-example-project-:windows:" + # Build the project editor to allow the snapshot and schema commandlet to run + $build_editor_proc = Start-Process -PassThru -NoNewWindow -FilePath $build_script_path -ArgumentList @(` + "GDKShooterEditor", ` + "Win64", ` + "Development", ` + "GDKShooter.uproject" + ) + + # Explicitly hold on to the process handle. + # This works around an issue whereby Wait-Process would fail to find build_editor_proc + $build_editor_handle = $build_editor_proc.Handle + + Wait-Process -Id (Get-Process -InputObject $build_editor_proc).id + if ($build_editor_proc.ExitCode -ne 0) { + Write-Log "Failed to build Win64 Development Editor. Error: $($build_editor_proc.ExitCode)" + Throw "Failed to build Win64 Development Editor" + } + Finish-Event "build-editor" "build-unreal-gdk-example-project-:windows:" + + # Invoke the GDK commandlet to generate schema and snapshot. Note: this needs to be run prior to cooking + Start-Event "generate-schema" "build-unreal-gdk-example-project-:windows:" + pushd "UnrealEngine/Engine/Binaries/Win64" + Start-Process -Wait -PassThru -NoNewWindow -FilePath ((Convert-Path .) + "\UE4Editor.exe") -ArgumentList @(` + "$($exampleproject_home)/Game/GDKShooter.uproject", ` + "-run=GenerateSchemaAndSnapshots", ` + "-MapPaths=`"/Maps/FPS-Start_Small`"" + ) + + # TODO (GV-29) this is also being done as part of setup.bat. We should look into calling setup.bat instead of this, but need to make sure it doesn't brake if called after setup-gdk.ps1 + $core_gdk_schema_path = "$($gdk_home)\SpatialGDK\Extras\schema\*" + $schema_std_lib_path = "$($gdk_home)\SpatialGDK\Binaries\ThirdParty\Improbable\Programs\schema\*" + New-Item -Path "$($exampleproject_home)\spatial\schema\unreal" -Name "gdk" -ItemType Directory -Force + New-Item -Path "$($exampleproject_home)\spatial" -Name "\build\dependencies\schema\standard_library" -ItemType Directory -Force + Copy-Item "$($core_gdk_schema_path)" -Destination "$($exampleproject_home)\spatial\schema\unreal\gdk" -Force -Recurse + Copy-Item "$($schema_std_lib_path)" -Destination "$($exampleproject_home)\spatial\build\dependencies\schema\standard_library" -Force -Recurse + popd + Finish-Event "generate-schema" "build-unreal-gdk-example-project-:windows:" + + Start-Event "build-win64-client" "build-unreal-gdk-example-project-:windows:" + $build_client_proc = Start-Process -PassThru -NoNewWindow -FilePath $build_script_path -ArgumentList @(` + "GDKShooter", ` + "Win64", ` + "Development", ` + "GDKShooter.uproject" + ) + $build_client_handle = $build_client_proc.Handle + Wait-Process -Id (Get-Process -InputObject $build_client_proc).id + if ($build_client_proc.ExitCode -ne 0) { + Write-Log "Failed to build Win64 Development Client. Error: $($build_client_proc.ExitCode)" + Throw "Failed to build Win64 Development Client" + } + Finish-Event "build-win64-client" "build-unreal-gdk-example-project-:windows:" + + Start-Event "build-linux-worker" "build-unreal-gdk-example-project-:windows:" + $build_server_proc = Start-Process -PassThru -NoNewWindow -FilePath $build_script_path -ArgumentList @(` + "GDKShooterServer", ` + "Linux", ` + "Development", ` + "GDKShooter.uproject" + ) + $build_server_handle = $build_server_proc.Handle + Wait-Process -Id (Get-Process -InputObject $build_server_proc).id + + if ($build_server_proc.ExitCode -ne 0) { + Write-Log "Failed to build Linux Development Server. Error: $($build_server_proc.ExitCode)" + Throw "Failed to build Linux Development Server" + } + Finish-Event "build-linux-worker" "build-unreal-gdk-example-project-:windows:" + + # Deploy the project to SpatialOS + &$PSScriptRoot"\deploy.ps1" -launch_deployment "$launch_deployment" -gdk_branch_name "$gdk_branch_name" +popd \ No newline at end of file