Organize manifests in folders.

.github/workflows/manifests.yml

@@ -4,7 +4,7 @@ on:
   push:
   pull_request:
     paths:
-      - manifests/*.yml
+      - manifests/**/*.yml
   schedule:
     - cron: "0 0 * * *"
 
@@ -17,7 +17,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - id: set-matrix
-        run: echo "::set-output name=matrix::$(ls manifests/opensearch*.yml | jq -R -s -c 'split("\n")[:-1]')"
+        run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | jq -R -s -c 'split("\n")[:-1]')"
 
   check:
     needs: list-manifests

bundle-workflow/Jenkinsfile

@@ -15,7 +15,7 @@ pipeline {
                             parameters([
                                     string(
                                             defaultValue: '',
-                                            name: 'BUILD_MANIFEST',
+                                            name: 'INPUT_MANIFEST',
                                             trim: true
                                     )
                             ])
@@ -43,7 +43,7 @@ pipeline {
                         script {
                             sh "rm -rf ~/.m2/"
                             git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main'
-                            sh "./bundle-workflow/build.sh manifests/$BUILD_MANIFEST --snapshot"
+                            sh "./bundle-workflow/build.sh manifests/$INPUT_MANIFEST --snapshot"
                             withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) {
                                 sh('$WORKSPACE/publish/publish-snapshot.sh $WORKSPACE/artifacts/$ARTIFACT_PATH/maven')
                             }
@@ -107,7 +107,7 @@ pipeline {
 
 void build() {
     git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main'
-    sh "./bundle-workflow/build.sh manifests/$BUILD_MANIFEST"
+    sh "./bundle-workflow/build.sh manifests/$INPUT_MANIFEST"
     sh './bundle-workflow/assemble.sh artifacts/manifest.yml'
 
     script { manifest = readYaml(file: 'artifacts/manifest.yml') }
@@ -120,6 +120,6 @@ void build() {
 
 void publishNotification(icon, msg) {
     withCredentials([string(credentialsId: 'BUILD_NOTICE_WEBHOOK', variable: 'TOKEN')]) {
-        sh """curl -XPOST --header "Content-Type: application/json" --data '{"result_text": "$icon ${env.JOB_NAME} [${env.BUILD_NUMBER}] $msg ${env.BUILD_URL}\nManifest: ${BUILD_MANIFEST}"}' $TOKEN"""
+        sh """curl -XPOST --header "Content-Type: application/json" --data '{"result_text": "$icon ${env.JOB_NAME} [${env.BUILD_NUMBER}] $msg ${env.BUILD_URL}\nManifest: ${INPUT_MANIFEST}"}' $TOKEN"""
     }
 }

bundle-workflow/README.md

@@ -19,16 +19,18 @@ The bundle workflow builds a complete OpenSearch distribution from source. You c
 
 Each build requires a manifest to be passed as input. We currently have the following input manifests.
 
-| name                                                                  | description                                                   |
-|-----------------------------------------------------------------------|---------------------------------------------------------------|
-| [opensearch-1.0.0.yml](/manifests/opensearch-1.0.0.yml)               | Manifest to reproduce 1.0.0 build.                            |
-| [opensearch-1.0.0-maven.yml](/manifests/opensearch-1.0.0-maven.yml)   | One-time manifest to build maven artifacts for 1.0 from tags. |
-| [opensearch-1.1.0.yml](/manifests/opensearch-1.1.0.yml)               | Manifest to build upcoming 1.x release.                       |
+| name                                                                        | description                                                   |
+|-----------------------------------------------------------------------------|---------------------------------------------------------------|
+| [opensearch-1.0.0.yml](/manifests/1.0.0/opensearch-1.0.0.yml)               | Manifest to reproduce 1.0.0 build.                            |
+| [opensearch-1.0.0-maven.yml](/manifests/1.0.0/opensearch-1.0.0-maven.yml)   | One-time manifest to build maven artifacts for 1.0 from tags. |
+| [opensearch-1.1.0.yml](/manifests/1.1.0/opensearch-1.1.0.yml)               | Manifest for 1.1.0, the next version.                         |
+| [opensearch-1.2.0.yml](/manifests/1.2.0/opensearch-1.2.0.yml)               | Manifest for 1.2.0, the following version.                    |
+| [opensearch-2.0.0.yml](/manifests/2.0.0/opensearch-2.0.0.yml)               | Manifest for 2.0.0, the next major version of OpenSearch.     |
 
 The following example builds a shapshot version of OpenSearch 1.1.0.
 
 ```bash
-./bundle-workflow/build.sh manifests/opensearch-1.1.0.yml --snapshot
+./bundle-workflow/build.sh manifests/1.1.0/opensearch-1.1.0.yml --snapshot
 ```
 
 The [OpenSearch repo](https://github.com/opensearch-project/OpenSearch) is built first, followed by [common-utils](https://github.com/opensearch-project/common-utils), and all declared plugin repositories. These dependencies are published to maven local under `~/.m2`, and subsequent project builds pick those up. All final output is placed into an `artifacts` folder along with a build output `manifest.yml` that contains output details.
@@ -181,7 +183,7 @@ The following checks are available.
 The following example sanity-checks components in the the OpenSearch 1.1.0 manifest.
 
 ```bash
-./bundle-workflow/ci.sh manifests/opensearch-1.1.0.yml --snapshot
+./bundle-workflow/ci.sh manifests/1.1.0/opensearch-1.1.0.yml --snapshot
 ```
 
 The following options are available.

bundle-workflow/src/manifests/input_manifests.py

@@ -14,7 +14,7 @@
 
 class InputManifests(Manifests):
     def __init__(self):
-        files = glob.glob(os.path.join(self.manifests_path, "opensearch-*.yml"))
+        files = glob.glob(os.path.join(self.manifests_path, "**/opensearch-*.yml"))
         # there's an opensearch-1.0.0-maven.yml that we want to skip
         files = [f for f in files if re.search(r"/opensearch-([0-9.]*)\.yml$", f)]
         super().__init__(InputManifest, files)

bundle-workflow/src/manifests_workflow/input_manifests.py

@@ -31,7 +31,7 @@ def manifests_path(self):
     def files(self):
         results = []
         for filename in glob.glob(
-            os.path.join(self.manifests_path(), "opensearch-*.yml")
+            os.path.join(self.manifests_path(), "**/opensearch-*.yml")
         ):
             # avoids the -maven manifest
             match = re.search(r"^opensearch-([0-9.]*).yml$", os.path.basename(filename))
@@ -116,8 +116,8 @@ def update(self, keep=False):
                     )
 
                 manifest = InputManifest(data)
-                manifest_path = os.path.join(
-                    self.manifests_path(), f"opensearch-{release_version}.yml"
-                )
+                manifest_dir = os.path.join(self.manifests_path(), release_version)
+                os.makedirs(manifest_dir, exist_ok=True)
+                manifest_path = os.path.join(manifest_dir, f"opensearch-{release_version}.yml")
                 manifest.to_file(manifest_path)
                 logging.info(f"Wrote {manifest_path}")

bundle-workflow/tests/tests_build_workflow/test_build_args.py

@@ -24,7 +24,7 @@ class TestBuildArgs(unittest.TestCase):
 
     OPENSEARCH_MANIFEST = os.path.realpath(
         os.path.join(
-            os.path.dirname(__file__), "../../../manifests/opensearch-1.1.0.yml"
+            os.path.dirname(__file__), "../../../manifests/1.1.0/opensearch-1.1.0.yml"
         )
     )
 

bundle-workflow/tests/tests_ci_workflow/test_ci_args.py

@@ -19,7 +19,7 @@ class TestCiArgs(unittest.TestCase):
 
     OPENSEARCH_MANIFEST = os.path.realpath(
         os.path.join(
-            os.path.dirname(__file__), "../../../manifests/opensearch-1.1.0.yml"
+            os.path.dirname(__file__), "../../../manifests/1.1.0/opensearch-1.1.0.yml"
         )
     )
 

bundle-workflow/tests/tests_manifests/test_input_manifest.py

@@ -20,7 +20,7 @@ def setUp(self):
         )
 
     def test_1_0(self):
-        path = os.path.join(self.manifests_path, "opensearch-1.0.0.yml")
+        path = os.path.join(self.manifests_path, "1.0.0/opensearch-1.0.0.yml")
         manifest = InputManifest.from_path(path)
         self.assertEqual(manifest.version, "1.0")
         self.assertEqual(manifest.build.name, "OpenSearch")
@@ -37,7 +37,7 @@ def test_1_0(self):
             self.assertIsInstance(component.ref, str)
 
     def test_1_1(self):
-        path = os.path.join(self.manifests_path, "opensearch-1.1.0.yml")
+        path = os.path.join(self.manifests_path, "1.1.0/opensearch-1.1.0.yml")
         manifest = InputManifest.from_path(path)
         self.assertEqual(manifest.version, "1.0")
         self.assertEqual(manifest.build.name, "OpenSearch")
@@ -66,7 +66,7 @@ def test_1_1(self):
         self.assertEqual(alerting_component.checks[1].args, "alerting")
 
     def test_to_dict(self):
-        path = os.path.join(self.manifests_path, "opensearch-1.1.0.yml")
+        path = os.path.join(self.manifests_path, "1.1.0/opensearch-1.1.0.yml")
         manifest = InputManifest.from_path(path)
         data = manifest.to_dict()
         with open(path) as f:

bundle-workflow/tests/tests_manifests_workflow/test_input_manifests.py

@@ -18,7 +18,7 @@ def test_files(self):
         self.assertTrue(len(files) >= 2)
         manifest = os.path.realpath(
             os.path.join(
-                os.path.dirname(__file__), "../../../manifests/opensearch-1.1.0.yml"
+                os.path.dirname(__file__), "../../../manifests/1.1.0/opensearch-1.1.0.yml"
             )
         )
         self.assertTrue(manifest in files)
@@ -29,6 +29,7 @@ def test_manifests_path(self):
         )
         self.assertEqual(path, InputManifests.manifests_path())
 
+    @patch("os.makedirs")
     @patch("os.chdir")
     @patch("manifests_workflow.input_manifests.InputManifest.from_path")
     @patch("manifests_workflow.input_manifests.ComponentOpenSearchMin")
@@ -62,8 +63,8 @@ def test_update(
         self.assertEqual(mock_input_manifest().to_file.call_count, 2)
         calls = [
             call(
-                os.path.join(InputManifests.manifests_path(), "opensearch-0.10.0.yml")
+                os.path.join(InputManifests.manifests_path(), "0.10.0/opensearch-0.10.0.yml")
             ),
-            call(os.path.join(InputManifests.manifests_path(), "opensearch-0.9.0.yml")),
+            call(os.path.join(InputManifests.manifests_path(), "0.9.0/opensearch-0.9.0.yml")),
         ]
         mock_input_manifest().to_file.assert_has_calls(calls)

manifests/opensearch-1.2.0.yml → manifests/1.2.0/opensearch-1.2.0.yml

@@ -5,4 +5,7 @@ components:
 - name: OpenSearch
   ref: 1.x
   repository: https://github.com/opensearch-project/OpenSearch.git
+  checks:
+    - gradle:publish
+    - gradle:properties:version
 schema-version: '1.0'

manifests/opensearch-2.0.0.yml → manifests/2.0.0/opensearch-2.0.0.yml

@@ -5,4 +5,7 @@ components:
 - name: OpenSearch
   ref: main
   repository: https://github.com/opensearch-project/OpenSearch.git
+  checks:
+    - gradle:publish
+    - gradle:properties:version
 schema-version: '1.0'