Continuous Integration: independent release process

This commit introduces several new pipelines to aid a PXF release process independent of any underlying SQL database. This commit thus includes scripts for making and managing a PXF RPM, which includes the C extension, whose code now lives under external-table/. These are the new pipelines: * pivnet_artifacts: a pipeline that fetches releases of GPDB from PivNet, aka TanzuNet * pxf-build: pipeline that builds and releases new PXF artifacts * dev-pxf-build: dev version of pxf-build, much abbreviated * pxf-certification: pipeline that tests existing PXF artifacts against new versions of Greenplum * pxf_pr_pipeline: new PR pipeline which uses the new build process Version is now managed at the top level, in a file `version`. By changing this file to a non-SNAPSHOT version, we can create a release using the pxf-build pipeline. This causes a new tag/release to be triggered on GitHub (Releases API), as well as a new SNAPSHOT version to be committed. Further, by uploading an OSL file to the appropriate cloud location, we trigger publication of our release to the Release Engineering team. Along with an independent release, we are now also installing PXF not under GPHOME but separately in PXF_HOME. Thus this commit also has changes to the PXF CLI to reflect the new install location. This includes a new command called `pxf [cluster] register` to re-install the PXF C extension under GPHOME. Other noteworthy changes include: * The new pipelines make use of smaller images that don't include PXF dependencies, but instead pull in those dependencies via tarballs that are pre-built. * The new pattern for setting pipelines is to have targets in concourse/Makefile. * Cloud tests have been hardened to reduce flakiness * In CI scripts we source ~gpadmin/.pxfrc which is cached on each image * Slack integrations have been added for pxf-certification, pxf-build, and dev-pxf-build pipelines Co-authored-by: Oliver Albertini <oalbertini@vmware.com> Co-authored-by: Alexander Denissov <adenissov@pivotal.io> Co-authored-by: Francisco Guerrero <aguerrero@pivotal.io>
greenplum-db · Jun 3, 2020 · 2674c5a · 2674c5a
1 parent 5938ad7
commit 2674c5a
Show file tree

Hide file tree

Showing 102 changed files with 7,114 additions and 592 deletions.
diff --git a/.github/workflows/create-release-on-tag.yml b/.github/workflows/create-release-on-tag.yml
@@ -0,0 +1,27 @@
+on:
+  push:
+    # Sequence of patterns matched against refs/tags
+    tags:
+      - '*' # match on any tag
+
+name: Create Release
+
+jobs:
+  build:
+    name: Create Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: PXF Version ${{ github.ref }}
+          body: |
+            ## PLACE CHANGELOG diff in here
+          draft: false
+          prerelease: false
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,10 @@
 *.pyc
 *.log
 *.out
+*.rpm
+*.deb
+*.o
+*.so
+build/
 server/*/out
 server/tmp
-server/pxf-cli/go/pkg
diff --git a/Makefile b/Makefile
@@ -1,18 +1,19 @@
-ifeq "$(PXF_HOME)" ""
-    ifneq "$(GPHOME)" ""
-        PXF_HOME = "$(GPHOME)/pxf"
-    endif
-endif
+include common.mk
 
-SHELL := /bin/bash
+PXF_VERSION ?= $(shell cat version)
+export PXF_VERSION
 
-export PXF_HOME
+LICENSE ?= ASL 2.0
+VENDOR ?= Open Source
 
 default: all
 
-.PHONY: all cli server install tar clean test help
+.PHONY: all external-table cli server install stage tar rpm rpm-tar clean test it help
 
-all: cli server
+all: external-table cli server
+
+external-table:
+	make -C external-table
 
 cli:
 	make -C cli/go/src/pxf-cli
@@ -21,6 +22,8 @@ server:
 	make -C server
 
 clean:
+	rm -rf build
+	make -C external-table clean-all
 	make -C cli/go/src/pxf-cli clean
 	make -C server clean
 
@@ -32,20 +35,78 @@ it:
 	make -C automation TEST=$(TEST)
 
 install:
+	make -C external-table install
 	make -C cli/go/src/pxf-cli install
 	make -C server install
 
-tar:
-	make -C cli/go/src/pxf-cli tar
-	make -C server tar
+stage:
+	rm -rf build/stage
+	make -C external-table stage
+	make -C cli/go/src/pxf-cli stage
+	make -C server stage
+	set -e ;\
+	GP_MAJOR_VERSION=$$(cat external-table/build/metadata/gp_major_version) ;\
+	GP_BUILD_ARCH=$$(cat external-table/build/metadata/build_arch) ;\
+	PXF_PACKAGE_NAME=pxf-gpdb$${GP_MAJOR_VERSION}-$${PXF_VERSION}-$${GP_BUILD_ARCH} ;\
+	mkdir -p build/stage/$${PXF_PACKAGE_NAME} ;\
+	cp -a external-table/build/stage/* build/stage/$${PXF_PACKAGE_NAME} ;\
+	cp -a cli/build/stage/* build/stage/$${PXF_PACKAGE_NAME} ;\
+	cp -a server/build/stage/* build/stage/$${PXF_PACKAGE_NAME} ;\
+	echo $$(git rev-parse --verify HEAD) > build/stage/$${PXF_PACKAGE_NAME}/pxf/commit.sha ;\
+	cp package/install_binary build/stage/$${PXF_PACKAGE_NAME}/install_component
+
+tar: stage
+	rm -rf build/dist
+	mkdir -p build/dist
+	tar -czf build/dist/$(shell ls build/stage).tar.gz -C build/stage $(shell ls build/stage)
+
+rpm:
+	make -C external-table stage
+	make -C cli/go/src/pxf-cli stage
+	make -C server stage
+	set -e ;\
+	GP_MAJOR_VERSION=$$(cat external-table/build/metadata/gp_major_version) ;\
+	PXF_MAIN_VERSION=$${PXF_VERSION//-SNAPSHOT/} ;\
+	if [[ $${PXF_VERSION} == *"-SNAPSHOT" ]]; then PXF_RELEASE=SNAPSHOT; else PXF_RELEASE=1; fi ;\
+	rm -rf build/rpmbuild ;\
+	mkdir -p build/rpmbuild/{BUILD,RPMS,SOURCES,SPECS} ;\
+	mkdir -p build/rpmbuild/SOURCES/gpextable ;\
+	cp -a external-table/build/stage/* build/rpmbuild/SOURCES/gpextable ;\
+	cp -a cli/build/stage/pxf/* build/rpmbuild/SOURCES ;\
+	cp -a server/build/stage/pxf/* build/rpmbuild/SOURCES ;\
+	echo $$(git rev-parse --verify HEAD) > build/rpmbuild/SOURCES/commit.sha ;\
+	cp package/*.spec build/rpmbuild/SPECS/ ;\
+	rpmbuild \
+	--define "_topdir $${PWD}/build/rpmbuild" \
+	--define "pxf_version $${PXF_MAIN_VERSION}" \
+	--define "pxf_release $${PXF_RELEASE}" \
+	--define "license ${LICENSE}" \
+	--define "vendor ${VENDOR}" \
+	-bb $${PWD}/build/rpmbuild/SPECS/pxf-gp$${GP_MAJOR_VERSION}.spec
+
+rpm-tar: rpm
+	rm -rf build/{stagerpm,distrpm}
+	mkdir -p build/{stagerpm,distrpm}
+	set -e ;\
+	GP_MAJOR_VERSION=$$(cat external-table/build/metadata/gp_major_version) ;\
+	PXF_RPM_FILE=$$(find build/rpmbuild/RPMS -name pxf-gp$${GP_MAJOR_VERSION}-*.rpm) ;\
+	PXF_RPM_BASE_NAME=$$(basename $${PXF_RPM_FILE%*.rpm}) ;\
+	PXF_PACKAGE_NAME=$${PXF_RPM_BASE_NAME%.*} ;\
+	mkdir -p build/stagerpm/$${PXF_PACKAGE_NAME} ;\
+	cp $${PXF_RPM_FILE} build/stagerpm/$${PXF_PACKAGE_NAME} ;\
+	cp package/install_rpm build/stagerpm/$${PXF_PACKAGE_NAME}/install_component ;\
+	tar -czf build/distrpm/$${PXF_PACKAGE_NAME}.tar.gz -C build/stagerpm $${PXF_PACKAGE_NAME}
 
 help:
 	@echo
 	@echo 'Possible targets'
-	@echo	'  - all (cli, server)'
+	@echo	'  - all (external-table, cli, server)'
+	@echo	'  - external-table - build Greenplum external table extension'
 	@echo	'  - cli - install Go CLI dependencies and build Go CLI'
 	@echo	'  - server - install PXF server dependencies and build PXF server'
-	@echo	'  - clean - clean up CLI and server binaries'
+	@echo	'  - clean - clean up external-table, CLI and server binaries'
 	@echo	'  - test - runs tests for PXF Go CLI and server'
-	@echo	'  - install - install PXF CLI and server'
-	@echo	'  - tar - bundle PXF CLI along with tomcat into a single tarball'
+	@echo	'  - install - install PXF external table extension, CLI and server'
+	@echo	'  - tar - bundle PXF external table extension, CLI, server and tomcat into a single tarball'
+	@echo	'  - rpm - create PXF RPM / DEB package'
+	@echo	'  - rpm-tar - bundle PXF RPM / DEB package along with helper scripts into a single tarball'
diff --git a/README.md b/README.md
@@ -7,14 +7,17 @@
 Introduction
 ============
 
-PXF is an extensible framework that allows a distributed database like GPDB to query external data files, whose metadata is not managed by the database.
-PXF includes built-in connectors for accessing data that exists inside HDFS files, Hive tables, HBase tables and more.
-Users can also create their own connectors to other data storages or processing engines.
-To create these connectors using JAVA plugins, see the PXF API and Reference Guide onGPDB.
+PXF is an extensible framework that allows a distributed database like Greenplum to query external data files, whose metadata is not managed by the database.
+PXF includes built-in connectors for accessing data that exists inside HDFS files, Hive tables, HBase tables, JDBC-accessible databases and more.
+Users can also create their own connectors to other data storage or processing engines.
 
-
-Package Contents
+Repository Contents
 ================
+## external-table/
+Contains the Greenplum extension implementing an External Table protocol handler
+
+## fdw/
+Contains the Greenplum extension implementing a Foreign Data Wrapper (FDW) for PXF
 
 ## server/
 Contains the server side code of PXF along with the PXF Service and all the Plugins
@@ -34,14 +37,14 @@ Resources for PXF's Continuous Integration pipelines
 ## regression/
 Contains the end-to-end (integration) tests for PXF against the various datasources, utilizing the PostgreSQL testing framework `pg_regress`
 
-## fdw/
-Contains the Greenplum extension implementing a Foreign Data Wrapper (FDW) for PXF.
+## downloads/
+An empty directory that serves as a staging location for Greenplum RPMs for the development Docker image
 
 PXF Development
 =================
-Below are the steps to build and install PXF along with its dependencies including GPDB and Hadoop.
+Below are the steps to build and install PXF along with its dependencies including Greenplum and Hadoop.
 
-To start, ensure you have a `~/workspace` directory and have cloned the `pxf` and its prerequisites(shown below) under it.
+To start, ensure you have a `~/workspace` directory and have cloned the `pxf` and its prerequisites (shown below) under it.
 (The name `workspace` is not strictly required but will be used throughout this guide.)
 ```bash
 mkdir -p ~/workspace
@@ -58,50 +61,53 @@ ln -s ~/<git_repos_root> ~/workspace
 
 To build PXF, you must have:
 
-- JDK 1.8 to compile (PXF runs on Java 8 and Java 11)
-- Go (1.9 or later)
-- unzip
-
-Export your `JAVA_HOME`.
-
-```
-export JAVA_HOME=<PATH_TO_YOUR_JAVA_HOME>
-```
-
-To install Go on CentOS, `sudo yum install go`.
-
-For other platforms, see the [Go downloads page](https://golang.org/dl/).
-
-Make sure to export your `GOPATH` and add go to your `PATH`. For example:
-
-```
-export GOPATH=$HOME/go
-export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin
-```
-
-Once you have installed Go, you will need the `dep` and `ginkgo` tools, which install Go dependencies and run Go tests,
-respectively. Assuming `go` is on your `PATH`, you can run:
-
-```
-go get github.com/golang/dep/cmd/dep
-go get github.com/onsi/ginkgo/ginkgo
-```
-
-to install them.
-
-## How to Build
-PXF uses gradle for build and has a wrapper makefile for abstraction
+1. GCC compiler, `make` system, `unzip` package, `maven` for running integration tests
+2. Installed Greenplum DB
+
+    Either download and install Greenplum RPM or build Greenplum from the source by following instructions in the [GPDB README](https://github.com/greenplum-db/gpdb).
+
+    Assuming you have installed Greenplum into `/usr/local/greenplum-db` directory, run its environment script:
+    ```
+    source /usr/local/greenplum-db/greenplum_path.sh
+    ```
+
+3. JDK 1.8 to compile (PXF runs on Java 8 and Java 11)
+
+    Export your `JAVA_HOME`:
+    ```
+    export JAVA_HOME=<PATH_TO_YOUR_JAVA_HOME>
+    ```
+
+4. Go (1.9 or later)
+
+    To install Go on CentOS, `sudo yum install go`. For other platforms, see the [Go downloads page](https://golang.org/dl/).
+
+    Make sure to export your `GOPATH` and add go to your `PATH`. For example:
+    ```
+    export GOPATH=$HOME/go
+    export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin
+    ```
+
+    Once you have installed Go, you will need the `dep` and `ginkgo` tools, which install Go dependencies and run Go tests,
+    respectively. Assuming `go` is on your `PATH`, you can run:
+    ```
+    go get github.com/golang/dep/cmd/dep
+    go get github.com/onsi/ginkgo/ginkgo
+    ```
+
+## How to Build PXF
+PXF uses Makefiles to build its components. PXF server component uses Gradle that is wrapped into the Makefile for convenience.
 ```bash
 cd ~/workspace/pxf
 
 # Compile & Test PXF
 make
 
-# Simply Run unittest
+# Only run unit tests
 make test
 ```
 
-## Install
+## How to Install PXF
 
 To install PXF, specify the `PXF_HOME` location, for example `/usr/local/gpdb/pxf`:
 
@@ -111,11 +117,8 @@ cd ~/workspace/pxf
 PXF_HOME=/usr/local/gpdb/pxf make install
 ```
 
-## Demonstrating Hadoop Integration
-In order to demonstrate end to end functionality you will need GPDB and Hadoop installed.
-
-### Hadoop
-We have all the related hadoop components (hdfs, hive, hbase, zookeeper, etc) mapped into simple artifact named singlecluster.
+## How to demonstrate Hadoop Integration
+In order to demonstrate end to end functionality you will need Hadoop installed. We have all the related hadoop components (hdfs, hive, hbase, zookeeper, etc) mapped into simple artifact named singlecluster.
 You can [download from here](http://storage.googleapis.com/pxf-public/singlecluster-HDP.tar.gz) and untar the `singlecluster-HDP.tar.gz` file, which contains everything needed to run Hadoop.
 
 ```bash
@@ -124,30 +127,34 @@ cd ~/workspace
 tar xzf singlecluster-HDP.tar.gz
 ```
 
-### GPDB
-```
-git clone https://github.com/greenplum-db/gpdb.git
-```
 
-You'll end up with a directory structure like this:
+# Development With Docker
+NOTE: Since the docker container will house all Single cluster Hadoop, Greenplum and PXF, we recommend that you have at least 4 cpus and 6GB memory allocated to Docker. These settings are available under docker preferences.
 
-```
-~
-└── workspace
-    ├── pxf
-    ├── singlecluster-HDP
-    └── gpdb
-```
+The following commands run the docker container and set up and switch to user gpadmin.
 
-If you already have GPDB installed and running using the instructions shown in the [GPDB README](https://github.com/greenplum-db/gpdb), 
-you can ignore the `Setup GPDB` section below and simply follow the steps in  `Setup Hadoop` and `Setup PXF`
+```bash
+# Get the latest image
+export GCR_PROJECT=<YOUR-GOOGLE_CONTAINER_REGISTRY-PROJECT-NAME>
+docker pull gcr.io/$GCR_PROJECT/gpdb-pxf-dev/gpdb6-centos7-test-pxf:latest
 
-If you don't wish to use docker, make sure you manually install JDK.
+docker run --rm -it \
+  -p 5432:5432 \
+  -p 5888:5888 \
+  -p 8000:8000 \
+  -p 5005:5005 \
+  -p 8020:8020 \
+  -p 9000:9000 \
+  -p 9090:9090 \
+  -p 50070:50070 \
+  -w /home/gpadmin/workspace \
+  -v ~/workspace/pxf:/home/gpadmin/workspace/pxf \
+  -v ~/workspace/singlecluster-HDP:/home/gpadmin/workspace/singlecluster \
+  gcr.io/$GCR_PROJECT/gpdb-pxf-dev/gpdb6-centos7-test-pxf:latest /bin/bash -c \
+  "/home/gpadmin/workspace/pxf/dev/indocker_setup.bash && /sbin/service sshd start && su - gpadmin"
 
-## Development With Docker
-NOTE: Since the docker container will house all Single cluster Hadoop, Greenplum and PXF, we recommend that you have at least 4 cpus and 6GB memory allocated to Docker. These settings are available under docker preferences.
+```
 
-The following commands run the docker container and set up and switch to user gpadmin.
 
 ```bash
 # Get the latest image