From a8564747f55af4254c6ccddb080ba704db30c941 Mon Sep 17 00:00:00 2001 From: Sophia Date: Wed, 10 Jun 2020 14:16:08 +0200 Subject: [PATCH 1/6] set up auto version for master --- .yamato/_preview_publish.yml | 4 ++++ .yamato/ruamel/jobs/preview_publish/pb_auto_version.py | 1 + 2 files changed, 5 insertions(+) diff --git a/.yamato/_preview_publish.yml b/.yamato/_preview_publish.yml index f38721af89f..c520b877e62 100644 --- a/.yamato/_preview_publish.yml +++ b/.yamato/_preview_publish.yml @@ -16,6 +16,10 @@ auto-version: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - upm-ci utils auto-version bump --package-path com.unity.render-pipelines.core --package-path com.unity.render-pipelines.universal --package-path com.unity.render-pipelines.high-definition --package-path com.unity.shadergraph --package-path com.unity.visualeffectgraph --package-path com.unity.render-pipelines.high-definition-config --package-path com.unity.render-pipelines.lightweight - upm-ci utils auto-version commit --push + triggers: + branches: + only: + - master artifacts: packages: paths: diff --git a/.yamato/ruamel/jobs/preview_publish/pb_auto_version.py b/.yamato/ruamel/jobs/preview_publish/pb_auto_version.py index b3c5faa4f9b..42c528e3617 100644 --- a/.yamato/ruamel/jobs/preview_publish/pb_auto_version.py +++ b/.yamato/ruamel/jobs/preview_publish/pb_auto_version.py @@ -23,6 +23,7 @@ def get_job_definition(self, agent, packages, target_branch, auto_version): f'npm install upm-ci-utils@stable -g --registry {NPM_UPMCI_INSTALL_URL}', f'upm-ci utils auto-version bump {bump_packages_args}', f'upm-ci utils auto-version commit --push']) + job.add_trigger_integration_branch(target_branch) job.add_artifacts_packages() # if auto_version is True: # job.add_trigger_integration_branch(target_branch) From a0e44081b4180eb57dc3658d46f6d2dd8221069d Mon Sep 17 00:00:00 2001 From: Sophia Date: Wed, 10 Jun 2020 14:48:10 +0200 Subject: [PATCH 2/6] create "promote all - nightly" job and edit dependencies --- .yamato/_preview_publish.yml | 31 +++++++++++++++++++ .yamato/ruamel/build.py | 6 +++- .../ruamel/jobs/preview_publish/pb_promote.py | 19 ++++++++---- .../preview_publish/pb_promote_all_preview.py | 22 +++++++++++++ .yamato/ruamel/jobs/shared/namer.py | 5 ++- 5 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 .yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py diff --git a/.yamato/_preview_publish.yml b/.yamato/_preview_publish.yml index c520b877e62..de792ecaba7 100644 --- a/.yamato/_preview_publish.yml +++ b/.yamato/_preview_publish.yml @@ -34,6 +34,16 @@ publish_all_preview: - .yamato/_preview_publish.yml#publish_vfx - .yamato/_preview_publish.yml#publish_config - .yamato/_preview_publish.yml#publish_lwrp +promote_all_preview: + name: Promote all preview packages - nightly + dependencies: + - .yamato/_preview_publish.yml#promote_core + - .yamato/_preview_publish.yml#promote_universal + - .yamato/_preview_publish.yml#promote_hdrp + - .yamato/_preview_publish.yml#promote_shadergraph + - .yamato/_preview_publish.yml#promote_vfx + - .yamato/_preview_publish.yml#promote_config + - .yamato/_preview_publish.yml#promote_lwrp wait_for_nightly: name: Wait for nightly dependencies: @@ -86,7 +96,10 @@ promote_core: variables: UPMCI_PROMOTION: 1 dependencies: + - .yamato/_preview_publish.yml#publish_core - .yamato/_packages.yml#pack_core + - .yamato/_packages.yml#test_core_OSX_trunk + - .yamato/_packages.yml#test_core_Win_trunk commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - upm-ci package promote --package-path com.unity.render-pipelines.core @@ -121,7 +134,10 @@ promote_universal: variables: UPMCI_PROMOTION: 1 dependencies: + - .yamato/_preview_publish.yml#publish_universal - .yamato/_packages.yml#pack_universal + - .yamato/_packages.yml#test_universal_OSX_trunk + - .yamato/_packages.yml#test_universal_Win_trunk commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - upm-ci package promote --package-path com.unity.render-pipelines.universal @@ -156,7 +172,10 @@ promote_hdrp: variables: UPMCI_PROMOTION: 1 dependencies: + - .yamato/_preview_publish.yml#publish_hdrp - .yamato/_packages.yml#pack_hdrp + - .yamato/_packages.yml#test_hdrp_OSX_trunk + - .yamato/_packages.yml#test_hdrp_Win_trunk commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - upm-ci package promote --package-path com.unity.render-pipelines.high-definition @@ -191,7 +210,10 @@ promote_shadergraph: variables: UPMCI_PROMOTION: 1 dependencies: + - .yamato/_preview_publish.yml#publish_shadergraph - .yamato/_packages.yml#pack_shadergraph + - .yamato/_packages.yml#test_shadergraph_OSX_trunk + - .yamato/_packages.yml#test_shadergraph_Win_trunk commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - upm-ci package promote --package-path com.unity.shadergraph @@ -226,7 +248,10 @@ promote_vfx: variables: UPMCI_PROMOTION: 1 dependencies: + - .yamato/_preview_publish.yml#publish_vfx - .yamato/_packages.yml#pack_vfx + - .yamato/_packages.yml#test_vfx_OSX_trunk + - .yamato/_packages.yml#test_vfx_Win_trunk commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - upm-ci package promote --package-path com.unity.visualeffectgraph @@ -261,7 +286,10 @@ promote_config: variables: UPMCI_PROMOTION: 1 dependencies: + - .yamato/_preview_publish.yml#publish_config - .yamato/_packages.yml#pack_config + - .yamato/_packages.yml#test_config_OSX_trunk + - .yamato/_packages.yml#test_config_Win_trunk commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - upm-ci package promote --package-path com.unity.render-pipelines.high-definition-config @@ -296,7 +324,10 @@ promote_lwrp: variables: UPMCI_PROMOTION: 1 dependencies: + - .yamato/_preview_publish.yml#publish_lwrp - .yamato/_packages.yml#pack_lwrp + - .yamato/_packages.yml#test_lwrp_OSX_trunk + - .yamato/_packages.yml#test_lwrp_Win_trunk commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - upm-ci package promote --package-path com.unity.render-pipelines.lightweight diff --git a/.yamato/ruamel/build.py b/.yamato/ruamel/build.py index 1ce1122015b..dbbd82c6e99 100644 --- a/.yamato/ruamel/build.py +++ b/.yamato/ruamel/build.py @@ -21,6 +21,7 @@ from jobs.preview_publish.pb_promote import PreviewPublish_PromoteJob from jobs.preview_publish.pb_auto_version import PreviewPublish_AutoVersionJob from jobs.preview_publish.pb_publish_all_preview import PreviewPublish_PublishAllPreviewJob +from jobs.preview_publish.pb_promote_all_preview import PreviewPublish_PromoteAllPreviewJob from jobs.preview_publish.pb_wait_for_nightly import PreviewPublish_WaitForNightlyJob from jobs.templates.template_pack import Template_PackJob from jobs.templates.template_test import Template_TestJob @@ -224,6 +225,9 @@ def create_preview_publish_jobs(metafile_name): job = PreviewPublish_PublishAllPreviewJob(metafile["packages"], target_branch, metafile["publishing"]["auto_publish"]) yml[job.job_id] = job.yml + job = PreviewPublish_PromoteAllPreviewJob(metafile["packages"], target_branch, metafile["publishing"]["auto_publish"]) + yml[job.job_id] = job.yml + job = PreviewPublish_WaitForNightlyJob(metafile["packages"], metafile["platforms"], target_editor) yml[job.job_id] = job.yml @@ -233,7 +237,7 @@ def create_preview_publish_jobs(metafile_name): job = PreviewPublish_PublishJob(get_agent(metafile["agent_publish"]), package, metafile["platforms"], target_editor) yml[job.job_id] = job.yml - job = PreviewPublish_PromoteJob(get_agent(metafile["agent_promote"]), package) + job = PreviewPublish_PromoteJob(get_agent(metafile["agent_promote"]), package, metafile["platforms"], target_editor) yml[job.job_id] = job.yml dump_yml(pb_filepath(), yml) diff --git a/.yamato/ruamel/jobs/preview_publish/pb_promote.py b/.yamato/ruamel/jobs/preview_publish/pb_promote.py index f32f432b256..df565114e7b 100644 --- a/.yamato/ruamel/jobs/preview_publish/pb_promote.py +++ b/.yamato/ruamel/jobs/preview_publish/pb_promote.py @@ -5,22 +5,29 @@ class PreviewPublish_PromoteJob(): - def __init__(self, agent, package): + def __init__(self, agent, package, platforms, target_editor): self.job_id = pb_job_id_promote(package["name"]) - self.yml = self.get_job_definition(agent, package).get_yml() + self.yml = self.get_job_definition(agent, package, platforms, target_editor).get_yml() - def get_job_definition(self, agent, package): - + def get_job_definition(self, agent, package, platforms, target_editor): if package["publish_source"] != True: - raise Exception('Tried to promote package for which "publish_source" set to false.') + raise Exception('Tried to promote package for which "publish_source" set to false.') + + # define dependencies + dependencies = [ + f'{pb_filepath()}#{pb_job_id_publish(package["name"])}', + f'{packages_filepath()}#{package_job_id_pack(package["name"])}'] + + for platform in platforms: + dependencies.append(f'{packages_filepath()}#{package_job_id_test(package["name"], platform["os"], target_editor)}') # construct job job = YMLJob() job.set_name(f'[{package["name"]}] Production Promote') job.set_agent(agent) job.add_var_custom('UPMCI_PROMOTION', 1) - job.add_dependencies([f'{packages_filepath()}#{package_job_id_pack(package["name"])}']) + job.add_dependencies(dependencies) job.add_commands([ f'npm install upm-ci-utils@stable -g --registry {NPM_UPMCI_INSTALL_URL}', f'upm-ci {package["type"]} promote --{package["type"]}-path {package["path"]}']) diff --git a/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py b/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py new file mode 100644 index 00000000000..289b01236eb --- /dev/null +++ b/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py @@ -0,0 +1,22 @@ +from ruamel.yaml.scalarstring import DoubleQuotedScalarString as dss +from ..shared.namer import * +from ..shared.yml_job import YMLJob + +class PreviewPublish_PromoteAllPreviewJob(): + + def __init__(self, packages, target_branch, auto_publish): + self.job_id = pb_job_id_promote_all_preview() + self.yml = self.get_job_definition(packages, target_branch, auto_publish).get_yml() + + + def get_job_definition(self, packages, target_branch, auto_publish): + + # construct job + job = YMLJob() + job.set_name(f'Promote all preview packages - nightly') + job.add_dependencies([f'{pb_filepath()}#{pb_job_id_promote(package["name"])}' for package in packages]) + # if auto_publish is True: + # job.add_trigger_recurrent(target_branch, 'daily') + return job + + \ No newline at end of file diff --git a/.yamato/ruamel/jobs/shared/namer.py b/.yamato/ruamel/jobs/shared/namer.py index de9ae748ca8..5339f248871 100644 --- a/.yamato/ruamel/jobs/shared/namer.py +++ b/.yamato/ruamel/jobs/shared/namer.py @@ -103,4 +103,7 @@ def pb_job_id_wait_for_nightly(): return f'wait_for_nightly' def pb_job_id_publish_all_preview(): - return f'publish_all_preview' \ No newline at end of file + return f'publish_all_preview' + +def pb_job_id_promote_all_preview(): + return f'promote_all_preview' \ No newline at end of file From 86a3a0b177c05a081a2a892b4b9b2988e7fb7330 Mon Sep 17 00:00:00 2001 From: Sophia Date: Wed, 10 Jun 2020 14:58:23 +0200 Subject: [PATCH 3/6] fix dependencies for promote_all --- .yamato/_preview_publish.yml | 1 + .../ruamel/jobs/preview_publish/pb_promote_all_preview.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.yamato/_preview_publish.yml b/.yamato/_preview_publish.yml index de792ecaba7..039d24f6093 100644 --- a/.yamato/_preview_publish.yml +++ b/.yamato/_preview_publish.yml @@ -37,6 +37,7 @@ publish_all_preview: promote_all_preview: name: Promote all preview packages - nightly dependencies: + - .yamato/_preview_publish.yml#publish_all_preview - .yamato/_preview_publish.yml#promote_core - .yamato/_preview_publish.yml#promote_universal - .yamato/_preview_publish.yml#promote_hdrp diff --git a/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py b/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py index 289b01236eb..e4fb17a351a 100644 --- a/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py +++ b/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py @@ -11,10 +11,15 @@ def __init__(self, packages, target_branch, auto_publish): def get_job_definition(self, packages, target_branch, auto_publish): + dependencies = [f'{pb_filepath()}#{pb_job_id_publish_all_preview()}'] + + for package in packages: + dependencies.append(f'{pb_filepath()}#{pb_job_id_promote(package["name"])}') + # construct job job = YMLJob() job.set_name(f'Promote all preview packages - nightly') - job.add_dependencies([f'{pb_filepath()}#{pb_job_id_promote(package["name"])}' for package in packages]) + job.add_dependencies(dependencies) # if auto_publish is True: # job.add_trigger_recurrent(target_branch, 'daily') return job From d8b7cfa1fb9f75195fb703693389f80badb9ebfb Mon Sep 17 00:00:00 2001 From: Sophia Date: Wed, 10 Jun 2020 15:01:41 +0200 Subject: [PATCH 4/6] Revert "fix dependencies for promote_all" This reverts commit 86a3a0b177c05a081a2a892b4b9b2988e7fb7330. --- .yamato/_preview_publish.yml | 1 - .../ruamel/jobs/preview_publish/pb_promote_all_preview.py | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.yamato/_preview_publish.yml b/.yamato/_preview_publish.yml index 039d24f6093..de792ecaba7 100644 --- a/.yamato/_preview_publish.yml +++ b/.yamato/_preview_publish.yml @@ -37,7 +37,6 @@ publish_all_preview: promote_all_preview: name: Promote all preview packages - nightly dependencies: - - .yamato/_preview_publish.yml#publish_all_preview - .yamato/_preview_publish.yml#promote_core - .yamato/_preview_publish.yml#promote_universal - .yamato/_preview_publish.yml#promote_hdrp diff --git a/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py b/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py index e4fb17a351a..289b01236eb 100644 --- a/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py +++ b/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py @@ -11,15 +11,10 @@ def __init__(self, packages, target_branch, auto_publish): def get_job_definition(self, packages, target_branch, auto_publish): - dependencies = [f'{pb_filepath()}#{pb_job_id_publish_all_preview()}'] - - for package in packages: - dependencies.append(f'{pb_filepath()}#{pb_job_id_promote(package["name"])}') - # construct job job = YMLJob() job.set_name(f'Promote all preview packages - nightly') - job.add_dependencies(dependencies) + job.add_dependencies([f'{pb_filepath()}#{pb_job_id_promote(package["name"])}' for package in packages]) # if auto_publish is True: # job.add_trigger_recurrent(target_branch, 'daily') return job From 354335bf928b4d7863393caf45248d703654815a Mon Sep 17 00:00:00 2001 From: Sophia Date: Wed, 10 Jun 2020 15:06:33 +0200 Subject: [PATCH 5/6] add nightly trigger to preview promote job --- .yamato/_preview_publish.yml | 4 ++++ .yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.yamato/_preview_publish.yml b/.yamato/_preview_publish.yml index de792ecaba7..ba3f8471292 100644 --- a/.yamato/_preview_publish.yml +++ b/.yamato/_preview_publish.yml @@ -44,6 +44,10 @@ promote_all_preview: - .yamato/_preview_publish.yml#promote_vfx - .yamato/_preview_publish.yml#promote_config - .yamato/_preview_publish.yml#promote_lwrp + triggers: + recurring: + - branch: master + frequency: daily wait_for_nightly: name: Wait for nightly dependencies: diff --git a/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py b/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py index 289b01236eb..acd4c481a2a 100644 --- a/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py +++ b/.yamato/ruamel/jobs/preview_publish/pb_promote_all_preview.py @@ -15,8 +15,8 @@ def get_job_definition(self, packages, target_branch, auto_publish): job = YMLJob() job.set_name(f'Promote all preview packages - nightly') job.add_dependencies([f'{pb_filepath()}#{pb_job_id_promote(package["name"])}' for package in packages]) - # if auto_publish is True: - # job.add_trigger_recurrent(target_branch, 'daily') + if auto_publish is True: + job.add_trigger_recurrent(target_branch, 'daily') return job \ No newline at end of file From 08060f10254656c32eabae4d0d0b6ecd235ddb45 Mon Sep 17 00:00:00 2001 From: Sophia Date: Wed, 10 Jun 2020 15:18:50 +0200 Subject: [PATCH 6/6] Update pb_promote.py --- .yamato/ruamel/jobs/preview_publish/pb_promote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/ruamel/jobs/preview_publish/pb_promote.py b/.yamato/ruamel/jobs/preview_publish/pb_promote.py index df565114e7b..9fcb125505c 100644 --- a/.yamato/ruamel/jobs/preview_publish/pb_promote.py +++ b/.yamato/ruamel/jobs/preview_publish/pb_promote.py @@ -12,7 +12,7 @@ def __init__(self, agent, package, platforms, target_editor): def get_job_definition(self, agent, package, platforms, target_editor): if package["publish_source"] != True: - raise Exception('Tried to promote package for which "publish_source" set to false.') + raise Exception('Tried to promote package for which "publish_source" set to false.') # define dependencies dependencies = [