UNR-2019: Example Project CI

.buildkite/nightly.definition.yaml

@@ -0,0 +1,11 @@
+agent_queue_id: trigger-pipelines
+description: Nightly build and deployment of Unreal Example project
+github:
+  branch_configuration: []
+  default_branch: master
+  pull_request_branch_filter_configuration: []
+teams:
+- name: Everyone
+  permission: BUILD_AND_READ
+- name: gbu/develop/unreal
+  permission: MANAGE_BUILD_AND_READ

.buildkite/nightly.steps.yaml

@@ -0,0 +1,25 @@
+common: &common
+  agents:
+    - "agent_count=1"
+    - "capable_of_building=gdk-for-unreal"
+    - "environment=production"
+    - "machine_type=quad"
+    - "permission_set=builder"
+    - "platform=windows"
+    - "scaler_version=2"
+    - "queue=${CI_WINDOWS_BUILDER_QUEUE:-v3-1569490589-f9345a43b21a6cec-------z}"
+  timeout_in_minutes: 60 # TODO(ENG-548): reduce timeout once agent-cold-start is optimised.
+  retry:
+    automatic:
+        # This is designed to trap and retry failures because agent lost connection. Agent exits with -1 in this case.
+      - exit_status: -1
+        limit: 3
+  plugins:
+    - ca-johnson/taskkill#v4.1: ~
+
+steps:
+  - label: "build-and-deploy-:windows:"
+    command: powershell -NoProfile -NonInteractive -InputFormat Text -Command ./ci/setup-and-build.ps1
+    <<: *common # This folds the YAML named anchor into this step. Overrides, if any, should follow, not precede.
+    artifact_paths:
+      - "UnrealEngine/Engine/Programs/AutomationTool/Saved/Logs/*"

Game/Source/GDKShooter.Target.cs

@@ -9,7 +9,5 @@ public GDKShooterTarget(TargetInfo Target) : base(Target)
 	{
 		Type = TargetType.Game;
 		ExtraModuleNames.Add("GDKShooter");
-		// TODO: UNR-1791 for long-term fix
-		GlobalDefinitions.Add("UE_ALLOW_MAP_OVERRIDE_IN_SHIPPING=1");
 	}
 }

ci/common.ps1

@@ -0,0 +1,58 @@
+function Write-Log() {
+  param(
+    [string] $msg,
+    [Parameter(Mandatory=$false)] [bool] $expand = $false
+  )
+  if ($expand) {
+      Write-Output "+++ $($msg)"
+  } else {
+      Write-Output "--- $($msg)"
+  }
+}
+
+function Start-Event() {
+    param(
+        [string] $event_name,
+        [string] $event_parent
+    )
+
+    # Start this tracing span.
+    Start-Process -NoNewWindow "imp-ci" -ArgumentList @(`
+        "events", "new", `
+        "--name", "$($event_name)", `
+        "--child-of", "$($event_parent)"
+    ) | Out-Null
+
+    Write-Log "--- $($event_name)"
+}
+
+function Finish-Event() {
+    param(
+        [string] $event_name,
+        [string] $event_parent
+    )
+
+    # Emit the end marker for this tracing span.
+    Start-Process -NoNewWindow "imp-ci"  -ArgumentList @(`
+        "events", "new", `
+        "--name", "$($event_name)", `
+        "--child-of", "$($event_parent)"
+    ) | Out-Null
+}
+
+## Checks whether the specified environment variable has been set. If it has, return its value. Else return the default value.
+function Get-Env-Variable-Value-Or-Default() {
+    param(
+        [string] $environment_variable_name,
+        [string] $default_value
+    )
+
+    If (Test-Path env:$environment_variable_name) {
+        $environment_variable_value = Get-Content -Path env:$environment_variable_name
+        return $environment_variable_value
+    } Else {
+        return $default_value
+    }
+}
+
+$ErrorActionPreference = 'Stop'

ci/deploy.ps1

@@ -0,0 +1,132 @@
+param(
+  [string] $launch_deployment = "false",
+  [string] $gdk_branch_name = "master"
+)
+
+. "$PSScriptRoot\common.ps1"
+
+Start-Event "deploy-game" "build-unreal-gdk-example-project-:windows:"
+    # Use the shortened commit hash gathered during GDK plugin clone and the current date and time to distinguish the deployment
+    $date_and_time = Get-Date -Format "MMdd_HHmm"
+    $deployment_name = "exampleproject_${date_and_time}_$($gdk_commit_hash)"
+    $assembly_name = "$($deployment_name)_asm"
+
+pushd "spatial"
+
+    Start-Event "build-worker-configurations" "deploy-unreal-gdk-example-project-:windows:"
+        $build_configs_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath "spatial" -ArgumentList @(`
+            "build", `
+            "build-config"
+        )
+
+        if ($build_configs_process.ExitCode -ne 0) {
+            Write-Log "Failed to build worker configurations for the project. Error: $($build_configs_process.ExitCode)"
+            Throw "Failed to build worker configurations"
+        }
+    Finish-Event "build-worker-configurations" "deploy-unreal-gdk-example-project-:windows:"
+
+    Start-Event "prepare-for-run" "deploy-unreal-gdk-example-project-:windows:"
+        $prepare_for_run_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath "spatial" -ArgumentList @(`
+            "prepare-for-run", `
+            "--log_level=debug"
+        )
+
+        if ($prepare_for_run_process.ExitCode -ne 0) {
+            Write-Log "Failed to prepare for a Spatial cloud launch. Error: $($prepare_for_run_process.ExitCode)"
+            Throw "Spatial prepare for run failed"
+        }
+    Finish-Event "prepare-for-run" "deploy-unreal-gdk-example-project-:windows:"
+
+    Start-Event "upload-assemblies" "deploy-unreal-gdk-example-project-:windows:"
+        $upload_assemblies_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath "spatial" -ArgumentList @(`
+            "cloud", `
+            "upload", `
+            "$assembly_name", `
+            "--project_name=$project_name", `
+            "--log_level=debug", `
+            "--force"
+        )
+
+        if ($upload_assemblies_process.ExitCode -ne 0) {
+            Write-Log "Failed to upload assemblies to cloud. Error: $($upload_assemblies_process.ExitCode)"
+            Throw "Failed to upload assemblies"
+        }
+    Finish-Event "upload-assemblies" "deploy-unreal-gdk-example-project-:windows:"
+
+    Start-Event "launch-deployment" "deploy-unreal-gdk-example-project-:windows:"
+        # Determine whether deployment should be launched (it will by default)
+        if ($launch_deployment -eq "true") {
+            $launch_deployment_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath "spatial" -ArgumentList @(`
+                "cloud", `
+                "launch", `
+                "$assembly_name", `
+                "$deployment_launch_configuration", `
+                "$deployment_name", `
+                "--project_name=$project_name", `
+                "--snapshot=$deployment_snapshot_path", `
+                "--cluster_region=$deployment_cluster_region", `
+                "--log_level=debug", `
+                "--tags=ttl_1_hours"
+            )
+
+            if ($launch_deployment_process.ExitCode -ne 0) {
+                Write-Log "Failed to launch a Spatial cloud deployment. Error: $($launch_deployment_process.ExitCode)"
+                Throw "Deployment launch failed"
+            }
+
+            # Send a Slack notification with a link to the new deployment and to the build.
+            # Read Slack webhook secret from the vault and extract the Slack webhook URL from it.
+            $slack_webhook_secret = "$(imp-ci secrets read --environment=production --buildkite-org=improbable --secret-type=slack-webhook --secret-name=unreal-gdk-slack-web-hook)"
+            $slack_webhook_url = $slack_webhook_secret | ConvertFrom-Json | %{$_.url}
+
+            $deployment_url = "https://console.improbable.io/projects/${project_name}/deployments/${deployment_name}/overview"
+            $build_url = "$env:BUILDKITE_BUILD_URL"
+
+            $json_message = [ordered]@{
+                text = "Example Project build succeeded and a deployment has been launched."
+                attachments= @(
+                        @{
+                            fallback = "Find deployment here: $deployment_url and build here: $build_url"
+                            fields= @(
+                                    @{
+                                        title = "GDK branch"
+                                        value = "$gdk_branch_name"
+                                        short = "true"
+                                    }
+                                    @{
+                                        title = "Example Project branch"
+                                        value = "$env:BUILDKITE_BRANCH"
+                                        short = "true"
+                                    }
+                                )
+                            actions= @(
+                                    @{
+                                        type = "button"
+                                        text = "Take me to the deployment"
+                                        url = "$deployment_url"
+                                        style = "primary"
+                                    }
+                                    @{
+                                        type = "button"
+                                        text = "Take me to the build"
+                                        url = "$build_url"
+                                        style = "primary"
+                                    }
+                                )
+                        }
+                    )
+                }
+
+            $json_request = $json_message | ConvertTo-Json -Depth 10
+
+            Invoke-WebRequest -UseBasicParsing "$slack_webhook_url" -ContentType "application/json" -Method POST -Body "$json_request"
+
+        }
+        else {
+            Write-Log "Deployment will not be launched as you have passed in an argument specifying that it should not be (START_DEPLOYMENT=${launch_deployment}). Remove it to have your build launch a deployment."
+        }
+
+    Finish-Event "launch-deployment" "deploy-unreal-gdk-example-project-:windows:"
+
+popd
+Finish-Event "deploy-game" "build-unreal-gdk-example-project-:windows:"

ci/setup-and-build.ps1

@@ -0,0 +1,151 @@
+param(
+  [string] $exampleproject_home = (get-item "$($PSScriptRoot)").parent.FullName, ## The root of the repo
+  [string] $gdk_repo = "git@github.com:spatialos/UnrealGDK.git",
+  [string] $gcs_publish_bucket = "io-internal-infra-unreal-artifacts-production/UnrealEngine",
+  [string] $deployment_launch_configuration = "one_worker_test.json",
+  [string] $deployment_snapshot_path = "snapshots/FPS-Start_Small.snapshot",
+  [string] $deployment_cluster_region = "eu",
+  [string] $project_name = "unreal_gdk"
+)
+
+. "$PSScriptRoot\common.ps1"
+
+# When a build is launched custom environment variables can be specified.
+# Parse them here to use the set value or the default.
+$gdk_branch_name = Get-Env-Variable-Value-Or-Default -environment_variable_name "GDK_BRANCH" -default_value "master"
+$launch_deployment = Get-Env-Variable-Value-Or-Default -environment_variable_name "START_DEPLOYMENT" -default_value "true"
+
+$gdk_home = "${exampleproject_home}\Game\Plugins\UnrealGDK"
+
+pushd "$exampleproject_home"
+    Start-Event "clone-gdk-plugin" "build-unreal-gdk-example-project-:windows:"
+        pushd "Game"
+            New-Item -Name "Plugins" -ItemType Directory -Force
+            pushd "Plugins"
+            Start-Process -Wait -PassThru -NoNewWindow -FilePath "git" -ArgumentList @(`
+                "clone", `
+                "$gdk_repo", `
+                "--depth 1", `
+                "-b $gdk_branch_name"
+            )
+            popd
+        popd
+    Finish-Event "clone-gdk-plugin" "build-unreal-gdk-example-project-:windows:"
+
+    Start-Event "get-gdk-head-commit" "build-unreal-gdk-example-project-:windows:"
+        pushd $gdk_home
+            # Get the short commit hash of this gdk build for later use in assembly name
+            $gdk_commit_hash = (git rev-parse HEAD).Substring(0,7)
+            Write-Log "GDK at commit: $gdk_commit_hash on branch $gdk_branch_name"
+        popd
+    Finish-Event "get-gdk-head-commit" "build-unreal-gdk-example-project-:windows:"
+
+    Start-Event "set-up-gdk-plugin" "build-unreal-gdk-example-project-:windows:"
+        pushd $gdk_home
+            # Invoke the GDK's setup script
+            &"$($gdk_home)\ci\setup-gdk.ps1"
+        popd
+    Finish-Event "set-up-gdk-plugin" "build-unreal-gdk-example-project-:windows:"
+
+    # Use the cached engine version or set it up if it has not been cached yet.
+    Start-Event "set-up-engine" "build-unreal-gdk-example-project-:windows:"
+
+        $engine_directory = "${exampleproject_home}\UnrealEngine"
+        &"$($gdk_home)\ci\get-engine.ps1" -unreal_path "$engine_directory"
+
+    Finish-Event "set-up-engine" "build-unreal-gdk-example-project-:windows:"
+
+
+    Start-Event "associate-uproject-with-engine" "build-unreal-gdk-example-project-:windows:"
+        pushd $engine_directory
+            $unreal_version_selector_path = "Engine\Binaries\Win64\UnrealVersionSelector.exe"
+
+            $find_engine_process = Start-Process -Wait -PassThru -NoNewWindow -FilePath $unreal_version_selector_path -ArgumentList @(`
+                "-switchversionsilent", `
+                "${exampleproject_home}\Game\GDKShooter.uproject", `
+                "$engine_directory"
+            )
+
+            if ($find_engine_process.ExitCode -ne 0) {
+                Write-Log "Failed to set Unreal Engine association for the project. Error: $($find_engine_process.ExitCode)"
+                Throw "Failed to set Engine association"
+            }
+        popd
+    Finish-Event "associate-uproject-with-engine" "build-unreal-gdk-example-project-:windows:"
+
+
+    $build_script_path = "$($gdk_home)\SpatialGDK\Build\Scripts\BuildWorker.bat"
+
+    Start-Event "build-editor" "build-unreal-gdk-example-project-:windows:"
+        # Build the project editor to allow the snapshot and schema commandlet to run
+        $build_editor_proc = Start-Process -PassThru -NoNewWindow -FilePath $build_script_path -ArgumentList @(`
+            "GDKShooterEditor", `
+            "Win64", `
+            "Development", `
+            "GDKShooter.uproject"
+        )
+
+        # Explicitly hold on to the process handle. 
+        # This works around an issue whereby Wait-Process would fail to find build_editor_proc 
+        $build_editor_handle = $build_editor_proc.Handle
+
+        Wait-Process -Id (Get-Process -InputObject $build_editor_proc).id
+        if ($build_editor_proc.ExitCode -ne 0) {
+            Write-Log "Failed to build Win64 Development Editor. Error: $($build_editor_proc.ExitCode)"
+            Throw "Failed to build Win64 Development Editor"
+        }
+    Finish-Event "build-editor" "build-unreal-gdk-example-project-:windows:"
+
+    # Invoke the GDK commandlet to generate schema and snapshot. Note: this needs to be run prior to cooking 
+    Start-Event "generate-schema" "build-unreal-gdk-example-project-:windows:"
+        pushd "UnrealEngine/Engine/Binaries/Win64"
+            Start-Process -Wait -PassThru -NoNewWindow -FilePath ((Convert-Path .) + "\UE4Editor.exe") -ArgumentList @(`
+                "$($exampleproject_home)/Game/GDKShooter.uproject", `
+                "-run=GenerateSchemaAndSnapshots", `
+                "-MapPaths=`"/Maps/FPS-Start_Small`""
+            )
+
+            # TODO (GV-29) this is also being done as part of setup.bat. We should look into calling setup.bat instead of this, but need to make sure it doesn't brake if called after setup-gdk.ps1
+            $core_gdk_schema_path = "$($gdk_home)\SpatialGDK\Extras\schema\*"
+            $schema_std_lib_path = "$($gdk_home)\SpatialGDK\Binaries\ThirdParty\Improbable\Programs\schema\*"
+            New-Item -Path "$($exampleproject_home)\spatial\schema\unreal" -Name "gdk" -ItemType Directory -Force
+            New-Item -Path "$($exampleproject_home)\spatial" -Name "\build\dependencies\schema\standard_library" -ItemType Directory -Force
+            Copy-Item "$($core_gdk_schema_path)" -Destination "$($exampleproject_home)\spatial\schema\unreal\gdk" -Force -Recurse
+            Copy-Item "$($schema_std_lib_path)" -Destination "$($exampleproject_home)\spatial\build\dependencies\schema\standard_library" -Force -Recurse
+        popd
+    Finish-Event "generate-schema" "build-unreal-gdk-example-project-:windows:"
+
+    Start-Event "build-win64-client" "build-unreal-gdk-example-project-:windows:"
+        $build_client_proc = Start-Process -PassThru -NoNewWindow -FilePath $build_script_path -ArgumentList @(`
+            "GDKShooter", `
+            "Win64", `
+            "Development", `
+            "GDKShooter.uproject"
+        )       
+        $build_client_handle = $build_client_proc.Handle
+        Wait-Process -Id (Get-Process -InputObject $build_client_proc).id
+        if ($build_client_proc.ExitCode -ne 0) {
+            Write-Log "Failed to build Win64 Development Client. Error: $($build_client_proc.ExitCode)"
+            Throw "Failed to build Win64 Development Client"
+        }
+    Finish-Event "build-win64-client" "build-unreal-gdk-example-project-:windows:"
+
+    Start-Event "build-linux-worker" "build-unreal-gdk-example-project-:windows:"
+        $build_server_proc = Start-Process -PassThru -NoNewWindow -FilePath $build_script_path -ArgumentList @(`
+            "GDKShooterServer", `
+            "Linux", `
+            "Development", `
+            "GDKShooter.uproject"
+        )       
+        $build_server_handle = $build_server_proc.Handle
+        Wait-Process -Id (Get-Process -InputObject $build_server_proc).id
+
+        if ($build_server_proc.ExitCode -ne 0) {
+            Write-Log "Failed to build Linux Development Server. Error: $($build_server_proc.ExitCode)"
+            Throw "Failed to build Linux Development Server"
+        }
+    Finish-Event "build-linux-worker" "build-unreal-gdk-example-project-:windows:"
+
+    # Deploy the project to SpatialOS
+    &$PSScriptRoot"\deploy.ps1" -launch_deployment "$launch_deployment" -gdk_branch_name "$gdk_branch_name"
+popd