Add Ubuntu Build in Pipeline

.dockerignore

@@ -0,0 +1,11 @@
+# Local excludes in root directory
+/GNUmakefile
+/config.log
+/config.status
+/VERSION
+/CMakeLists.txt
+
+# Reduce docker context
+/gpAux/gpdemo/datadirs
+**/*.so
+**/*.o

concourse/pipelines/5X_STABLE-generated.yml

@@ -12,7 +12,7 @@
 ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -t prod).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2018-10-25 11:33:03.993936
+## Generated by gen_pipeline.py at: 2018-11-07 16:18:53.533099
 ## Template file: gpdb-tpl.yml
 ## OS Types: ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16']
 ## Test Sections: ['ICW', 'CS', 'MPP', 'MM', 'DPM', 'UD', 'FileRep', 'AA']
@@ -35,6 +35,7 @@ groups:
   - compile_gpdb_centos7
   - compile_gpdb_sles11
   - compile_gpdb_ubuntu16
+  - compile_gpdb_open_source_ubuntu16
   - compile_gpdb_open_source_centos6
   - compile_gpdb_binary_swap_centos6
   - compile_gpdb_windows_cl
@@ -152,6 +153,7 @@ groups:
   - compile_gpdb_centos7
   - compile_gpdb_sles11
   - compile_gpdb_ubuntu16
+  - compile_gpdb_open_source_ubuntu16
   - compile_gpdb_open_source_centos6
   - compile_gpdb_binary_swap_centos6
   - compile_gpdb_windows_cl
@@ -396,6 +398,12 @@ resources:
     private_key: {{gpaddon-git-key}}
     uri: {{gpaddon-git-remote}}
 
+- name: docker-in-concourse
+  type: git
+  source:
+    uri: git@github.com:pivotal/docker-in-concourse
+    private_key: {{docker-in-concourse-git-key}}
+
 - name: debian_release
   type: git
   source:
@@ -432,6 +440,14 @@ resources:
     repository: pivotaldata/ubuntu-gpdb-debian-dev
     tag: '16.04'
 
+- name: ubuntu-gpcloud-dev-16
+  type: docker-image
+  source:
+    repository: pivotaldata/ubuntu-gpcloud-dev
+    tag: '16.04'
+    username: {{docker_username}}
+    password: {{docker_password}}
+
 - name: centos-mingw
   type: docker-image
   source:
@@ -536,6 +552,15 @@ resources:
     secret_access_key: {{bucket-secret-access-key}}
     versioned_file: {{bin_gpdb_sles11_versioned_file}}
 
+- name: bin_gpdb_ubuntu16
+  type: s3
+  source:
+    access_key_id: {{bucket-access-key-id}}
+    bucket: {{bucket-name}}
+    region_name: {{aws-region}}
+    secret_access_key: {{bucket-secret-access-key}}
+    versioned_file: {{bin_gpdb_ubuntu16_versioned_file}}
+
 - name: bin_gpdb_windows_clients
   type: s3
   source:
@@ -636,13 +661,22 @@ resources:
     regexp: deliverables/greenplum-loaders-5.(.*)-sles11-x86_64.zip
 
 - name: deb_package_ubuntu16
+  type: s3
+  source:
+    access_key_id: ((bucket-access-key-id))
+    bucket: ((bucket-name))
+    region_name: ((aws-region))
+    secret_access_key: ((bucket-secret-access-key))
+    regexp: deb_package_ubuntu16/greenplum-db_(.*)_amd64.deb
+
+- name: deb_package_open_source_ubuntu16
   type: s3
   source:
     access_key_id: {{bucket-access-key-id}}
     bucket: {{bucket-name}}
     region_name: {{aws-region}}
     secret_access_key: {{bucket-secret-access-key}}
-    versioned_file: {{deb_package_ubuntu16_versioned_file}}
+    versioned_file: {{deb_package_open_source_ubuntu16_versioned_file}}
 
 - name: nightly-trigger
   type: time
@@ -984,6 +1018,7 @@ jobs:
     - get: centos-mingw
     - get: ubuntu-gpdb-dev-16
     - get: ubuntu-gpdb-debian-dev-16
+    - get: ubuntu-gpcloud-dev-16
 
 - name: compile_gpdb_centos6
   plan:
@@ -1113,6 +1148,31 @@ jobs:
       file: gpdb_artifacts/greenplum-loaders-*-sles11-x86_64.zip
 
 - name: compile_gpdb_ubuntu16
+  plan:
+  - aggregate:
+    - get: reduced-frequency-trigger
+      trigger: ((reduced-frequency-trigger-flag))
+    - get: gpdb_src
+      passed: [gate_compile_start]
+      trigger: ((gpdb_src-trigger-flag))
+    - get: gpaddon_src
+      passed: [gate_compile_start]
+    - get: docker-in-concourse
+    - get: ubuntu-gpcloud-dev-16
+      passed: [gate_compile_start]
+  - task: compile_gpdb
+    file: gpdb_src/concourse/tasks/compile_gpdb_ubuntu16.yml
+    image: ubuntu-gpcloud-dev-16
+    privileged: true
+  - aggregate:
+    - put: bin_gpdb_ubuntu16
+      params:
+        file: gpdb_artifacts/bin_gpdb.tar.gz
+    - put: deb_package_ubuntu16
+      params:
+        file: gpdb_artifacts/greenplum-db*.deb
+
+- name: compile_gpdb_open_source_ubuntu16
   plan:
   - aggregate:
     - get: reduced-frequency-trigger
@@ -1142,9 +1202,9 @@ jobs:
     - put: compiled_bits_ubuntu16
       params:
         file: compiled_bits_ubuntu16/compiled_bits_ubuntu16.tar.gz
-    - put: deb_package_ubuntu16
+    - put: deb_package_open_source_ubuntu16
       params:
-        file: {{deb_package_ubuntu16_versioned_file}}
+        file: {{deb_package_open_source_ubuntu16_versioned_file}}
 
 - name: compile_gpdb_open_source_centos6
   public: true
@@ -1336,6 +1396,7 @@ jobs:
       - compile_gpdb_centos7
       - compile_gpdb_sles11
       - compile_gpdb_ubuntu16
+      - compile_gpdb_open_source_ubuntu16
       trigger: true
     - get: bin_gpdb_sles11
       passed:
@@ -1351,10 +1412,10 @@ jobs:
       - compile_gpdb_centos6
     - get: compiled_bits_ubuntu16
       passed:
-      - compile_gpdb_ubuntu16
-    - get: deb_package_ubuntu16
+      - compile_gpdb_open_source_ubuntu16
+    - get: deb_package_open_source_ubuntu16
       passed:
-      - compile_gpdb_ubuntu16
+      - compile_gpdb_open_source_ubuntu16
 
 ## ======================================================================
 ##  ___ ______        __
@@ -1386,7 +1447,7 @@ jobs:
       - get: compiled_bits_ubuntu16
         passed:
         - gate_compile_end
-      - get: deb_package_ubuntu16
+      - get: deb_package_open_source_ubuntu16
         passed:
         - gate_compile_end
 
@@ -1584,15 +1645,15 @@ jobs:
       passed:
       - gate_icw_start
     - get: ubuntu-gpdb-debian-dev-16
-    - get: deb_package_ubuntu16
+    - get: deb_package_open_source_ubuntu16
       passed:
       - gate_icw_start
       trigger: true
   - task: test_gpdb_deb_package
     file: gpdb_src/concourse/tasks/deb_test_package.yml
     image: ubuntu-gpdb-debian-dev-16
     params:
-      DEBIAN_PACKAGE: {{deb_package_ubuntu16_versioned_file}}
+      DEBIAN_PACKAGE: {{deb_package_open_source_ubuntu16_versioned_file}}
 
 - name: icw_extensions_gpcloud
   plan:
@@ -4154,6 +4215,7 @@ jobs:
       - compile_gpdb_centos7
       - compile_gpdb_sles11
       - compile_gpdb_ubuntu16
+      - compile_gpdb_open_source_ubuntu16
       - compile_gpdb_windows_cl
       - compile_gpdb_aix7_remote
       - icw_planner_centos6

concourse/pipelines/README.md

@@ -83,8 +83,8 @@ fly -t gpdb-prod \
     set-pipeline \
     -p 5X_STABLE \
     -c 5X_STABLE-generated.yml \
-    -l ~/workspace/continuous-integration/secrets/gpdb_common-ci-secrets.yml \
-    -l ~/workspace/continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml
+    -l ~/workspace/gp-continuous-integration/secrets/gpdb_common-ci-secrets.yml \
+    -l ~/workspace/gp-continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml
 
 ======================================================================
 Validate Pipeline Release Jobs
@@ -127,9 +127,9 @@ fly -t gpdb-dev \
     set-pipeline \
     -p gpdb-dpm-curry \
     -c gpdb-dpm-curry.yml \
-    -l ~/workspace/continuous-integration/secrets/gpdb_common-ci-secrets.yml \
-    -l ~/workspace/continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml \
-    -l ~/workspace/continuous-integration/secrets/ccp_ci_secrets_gpdb-dev.yml \
+    -l ~/workspace/gp-continuous-integration/secrets/gpdb_common-ci-secrets.yml \
+    -l ~/workspace/gp-continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml \
+    -l ~/workspace/gp-continuous-integration/secrets/ccp_ci_secrets_gpdb-dev.yml \
     -v bucket-name=gpdb5-concourse-builds-dev \
     -v gpdb-git-remote=<https://github.com/<github-user>/gpdb> \
     -v gpdb-git-branch=<branch-name>
@@ -156,9 +156,9 @@ fly -t gpdb-dev \
     set-pipeline \
     -p gpdb-cs-durant \
     -c gpdb-cs-durant.yml \
-    -l ~/workspace/continuous-integration/secrets/gpdb_common-ci-secrets.yml \
-    -l ~/workspace/continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml \
-    -l ~/workspace/continuous-integration/secrets/ccp_ci_secrets_gpdb-dev.yml \
+    -l ~/workspace/gp-continuous-integration/secrets/gpdb_common-ci-secrets.yml \
+    -l ~/workspace/gp-continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml \
+    -l ~/workspace/gp-continuous-integration/secrets/ccp_ci_secrets_gpdb-dev.yml \
     -v bucket-name=gpdb5-concourse-builds-dev \
     -v gpdb-git-remote=<https://github.com/<github-user>/gpdb> \
     -v gpdb-git-branch=<branch-name>

concourse/pipelines/gen_pipeline.py

@@ -170,17 +170,17 @@ def how_to_use_generated_pipeline_message():
         msg += '    set-pipeline \\\n'
         msg += '    -p 5X_STABLE \\\n'
         msg += '    -c %s \\\n' % ARGS.output_filepath
-        msg += '    -l ~/workspace/continuous-integration/secrets/gpdb_common-ci-secrets.yml \\\n'
-        msg += '    -l ~/workspace/continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml\n'
+        msg += '    -l ~/workspace/gp-continuous-integration/secrets/gpdb_common-ci-secrets.yml \\\n'
+        msg += '    -l ~/workspace/gp-continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml\n'
     else:
         msg += 'NOTE: You can set the developer pipeline with the following:\n\n'
         msg += 'fly -t gpdb-dev \\\n'
         msg += '    set-pipeline \\\n'
         msg += '    -p %s \\\n' % os.path.basename(ARGS.output_filepath).rsplit('.', 1)[0]
         msg += '    -c %s \\\n' % ARGS.output_filepath
-        msg += '    -l ~/workspace/continuous-integration/secrets/gpdb_common-ci-secrets.yml \\\n'
-        msg += '    -l ~/workspace/continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml \\\n'
-        msg += '    -l ~/workspace/continuous-integration/secrets/ccp_ci_secrets_gpdb-dev.yml \\\n'
+        msg += '    -l ~/workspace/gp-continuous-integration/secrets/gpdb_common-ci-secrets.yml \\\n'
+        msg += '    -l ~/workspace/gp-continuous-integration/secrets/gpdb_5X_STABLE-ci-secrets.yml \\\n'
+        msg += '    -l ~/workspace/gp-continuous-integration/secrets/ccp_ci_secrets_gpdb-dev.yml \\\n'
         msg += '    -v bucket-name=gpdb5-concourse-builds-dev \\\n'
         msg += '    -v gpdb-git-remote=%s \\\n' % suggested_git_remote()
         msg += '    -v gpdb-git-branch=%s \n' % suggested_git_branch()