From 06e7fb0dcac2694eada6dd81788baa84b441b847 Mon Sep 17 00:00:00 2001 From: Rahul Iyer Date: Mon, 23 Jan 2017 16:58:29 -0800 Subject: [PATCH 1/3] Build: Use only major version for GPDB 5, HAWQ 2 GPDB, starting 5.0, and HAWQ, starting 2.0, are using semantic versioning for releases. This implies a binary compatibility between same major versions. Hence, we only need to compile a single MADlib binary for the same major version, with the folder name being just the major version instead of 'major.minor'. --- src/madpack/madpack.py | 40 ++++++++++++------- src/ports/greenplum/{5.0 => 5}/CMakeLists.txt | 0 ...enplum_5_0.cmake => FindGreenplum_5.cmake} | 0 src/ports/hawq/{2.0 => 2}/CMakeLists.txt | 1 - src/ports/hawq/CMakeLists.txt | 2 +- .../{FindHAWQ_2_0.cmake => FindHAWQ_2.cmake} | 0 .../postgres/cmake/PostgreSQLUtils.cmake | 36 ++++++++++++----- 7 files changed, 52 insertions(+), 27 deletions(-) rename src/ports/greenplum/{5.0 => 5}/CMakeLists.txt (100%) rename src/ports/greenplum/cmake/{FindGreenplum_5_0.cmake => FindGreenplum_5.cmake} (100%) rename src/ports/hawq/{2.0 => 2}/CMakeLists.txt (99%) rename src/ports/hawq/cmake/{FindHAWQ_2_0.cmake => FindHAWQ_2.cmake} (100%) diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py index e15bb4a7c..05077b346 100755 --- a/src/madpack/madpack.py +++ b/src/madpack/madpack.py @@ -318,12 +318,6 @@ def _get_dbver(): match = re.search("PostgreSQL[a-zA-Z\s]*(\d+\.\d+)", versionStr) elif portid == 'greenplum': match = re.search("Greenplum[a-zA-Z\s]*(\d+\.\d+)", versionStr) - # Due to the ABI incompatibility between 4.3.4 and 4.3.5, - # MADlib treat 4.3.5+ as DB version 4.3V2 that is different from 4.3 - if match and match.group(1) == '4.3': - match_details = re.search("Greenplum[a-zA-Z\s]*(\d+\.\d+.\d+)", versionStr) - if _is_rev_gte(_get_rev_num(match_details.group(1)), _get_rev_num('4.3.5')): - return '4.3ORCA' elif portid == 'hawq': match = re.search("HAWQ[a-zA-Z\s]*(\d+\.\d+)", versionStr) return None if match is None else match.group(1) @@ -1098,11 +1092,6 @@ def main(argv): # Get DB version dbver = _get_dbver() - portdir = os.path.join(maddir, "ports", portid) - if portid == "hawq" and _is_rev_gte(_get_rev_num(dbver), _get_rev_num('2.0')): - is_hawq2 = True - else: - is_hawq2 = False # Get MADlib version in DB dbrev = _get_madlib_dbrev(schema) @@ -1111,12 +1100,14 @@ def main(argv): if portid == 'hawq' and not is_hawq2 and schema.lower() != 'madlib': _error("*** Installation is currently restricted only to 'madlib' schema ***", True) + portdir = os.path.join(maddir, "ports", portid) supportedVersions = [dirItem for dirItem in os.listdir(portdir) if os.path.isdir(os.path.join(portdir, dirItem)) and - re.match("^\d+\.\d+", dirItem)] + re.match("^\d+", dirItem)] if dbver is None: - dbver = ".".join(map(str, max([map(int, versionStr.split('.')) - for versionStr in supportedVersions]))) + dbver = ".".join( + map(str, max([versionStr.split('.') + for versionStr in supportedVersions]))) _info("Could not parse version string reported by {DBMS}. Will " "default to newest supported version of {DBMS} " "({version}).".format(DBMS=ports[portid]['name'], @@ -1124,6 +1115,27 @@ def main(argv): else: _info("Detected %s version %s." % (ports[portid]['name'], dbver), True) + + if portid == "hawq": + # HAWQ (starting 2.0) and GPDB (starting 5.0) uses semantic versioning, + # which implies all HAWQ 2.x or GPDB 5.x versions will have binary + # compatibility. Hence, we can keep single folder for all 2.X / 5.X. + if (_is_rev_gte(_get_rev_num(dbver), _get_rev_num('2.0')) and + not _is_rev_gte(_get_rev_num(dbver), _get_rev_num('3.0'))): + is_hawq2 = True + dbver = '2' + elif portid == 'greenplum': + # similar to HAWQ above, collapse all 5.X versions + if (_is_rev_gte(_get_rev_num(dbver), _get_rev_num('5.0')) and + not _is_rev_gte(_get_rev_num(dbver), _get_rev_num('6.0'))): + dbver = '5' + # Due to the ABI incompatibility between 4.3.4 and 4.3.5, + # MADlib treats 4.3.5+ as DB version 4.3ORCA which is different + # from 4.3. The name is suffixed with ORCA since optimizer (ORCA) is + # 'on' by default in 4.3.5 + elif _is_rev_gte(_get_rev_num(dbver), _get_rev_num('4.3.4')): + dbver = '4.3ORCA' + if not os.path.isdir(os.path.join(portdir, dbver)): _error("This version is not among the %s versions for which " "MADlib support files have been installed (%s)." % diff --git a/src/ports/greenplum/5.0/CMakeLists.txt b/src/ports/greenplum/5/CMakeLists.txt similarity index 100% rename from src/ports/greenplum/5.0/CMakeLists.txt rename to src/ports/greenplum/5/CMakeLists.txt diff --git a/src/ports/greenplum/cmake/FindGreenplum_5_0.cmake b/src/ports/greenplum/cmake/FindGreenplum_5.cmake similarity index 100% rename from src/ports/greenplum/cmake/FindGreenplum_5_0.cmake rename to src/ports/greenplum/cmake/FindGreenplum_5.cmake diff --git a/src/ports/hawq/2.0/CMakeLists.txt b/src/ports/hawq/2/CMakeLists.txt similarity index 99% rename from src/ports/hawq/2.0/CMakeLists.txt rename to src/ports/hawq/2/CMakeLists.txt index 0a2eb24f5..a024a53f2 100644 --- a/src/ports/hawq/2.0/CMakeLists.txt +++ b/src/ports/hawq/2/CMakeLists.txt @@ -16,5 +16,4 @@ # specific language governing permissions and limitations # under the License. # ------------------------------------------------------------------------------ - add_current_hawq_version() diff --git a/src/ports/hawq/CMakeLists.txt b/src/ports/hawq/CMakeLists.txt index 8610238a3..2ba532d17 100644 --- a/src/ports/hawq/CMakeLists.txt +++ b/src/ports/hawq/CMakeLists.txt @@ -85,7 +85,7 @@ add_sql_files( "../postgres/modules" "${CMAKE_CURRENT_BINARY_DIR}/modules" ) -# Add Greenplum-specific modules. Files will be appended to SQL_TARGET_FILES. +# Add HAWQ-specific modules. Files will be appended to SQL_TARGET_FILES. add_sql_files( SQL_TARGET_FILES "modules" diff --git a/src/ports/hawq/cmake/FindHAWQ_2_0.cmake b/src/ports/hawq/cmake/FindHAWQ_2.cmake similarity index 100% rename from src/ports/hawq/cmake/FindHAWQ_2_0.cmake rename to src/ports/hawq/cmake/FindHAWQ_2.cmake diff --git a/src/ports/postgres/cmake/PostgreSQLUtils.cmake b/src/ports/postgres/cmake/PostgreSQLUtils.cmake index e5e4228a7..e17a84562 100644 --- a/src/ports/postgres/cmake/PostgreSQLUtils.cmake +++ b/src/ports/postgres/cmake/PostgreSQLUtils.cmake @@ -4,7 +4,7 @@ function(define_postgresql_features IN_VERSION OUT_FEATURES) if(NOT ${IN_VERSION} VERSION_LESS "9.0") list(APPEND ${OUT_FEATURES} __HAS_ORDERED_AGGREGATES__) endif() - + # Pass values to caller set(${OUT_FEATURES} "${${OUT_FEATURES}}" PARENT_SCOPE) endfunction(define_postgresql_features) @@ -54,7 +54,7 @@ endfunction(cpack_add_version_component) # function(determine_target_versions OUT_VERSIONS) get_subdirectories("${CMAKE_CURRENT_SOURCE_DIR}" SUPPORTED_VERSIONS) - get_filtered_list(SUPPORTED_VERSIONS "^[0-9]+.[0-9]+.*$" ${SUPPORTED_VERSIONS}) + get_filtered_list(SUPPORTED_VERSIONS "^[0-9]+.*$" ${SUPPORTED_VERSIONS}) foreach(VERSION ${SUPPORTED_VERSIONS}) string(REPLACE "." "_" VERSION_UNDERSCORE "${VERSION}") @@ -66,16 +66,30 @@ function(determine_target_versions OUT_VERSIONS) find_package(${PORT}) if(${PORT_UC}_FOUND) - # Due to the ABI incompatibility between 4.3.4 and 4.3.5, - # MADlib treat 4.3.5+ as DB version that is different from 4.3 - if(${PORT_UC} STREQUAL "GREENPLUM" AND - ${${PORT_UC}_VERSION_MAJOR} EQUAL 4 AND - ${${PORT_UC}_VERSION_MINOR} EQUAL 3 AND - ${${PORT_UC}_VERSION_PATCH} GREATER 4) - set(VERSION "4.3ORCA") - else() + if(${PORT_UC} STREQUAL "GREENPLUM") + # Starting GPDB 5.0, semantic versioning will be followed, + # implying we only need 1 folder for same major versions + if(${${PORT_UC}_VERSION_MAJOR} EQUAL 5) + set(VERSION "5") + + # Due to the ABI incompatibility between 4.3.4 and 4.3.5, + # MADlib treat 4.3.5+ as DB version that is different from 4.3 + elseif(${${PORT_UC}_VERSION_MAJOR} EQUAL 4 AND + ${${PORT_UC}_VERSION_MINOR} EQUAL 3 AND + ${${PORT_UC}_VERSION_PATCH} GREATER 4) + set(VERSION "4.3ORCA") + endif() + elseif(${PORT_UC} STREQUAL "HAWQ" AND + ${${PORT_UC}_VERSION_MAJOR} EQUAL 2) + # Starting HAWQ 2.0, semantic versioning will be followed, + # implying we only need 1 folder for same major versions + set(VERSION "2") + endif() + + if(NOT DEFINED VERSION) set(VERSION "${${PORT_UC}_VERSION_MAJOR}.${${PORT_UC}_VERSION_MINOR}") endif() + list(FIND SUPPORTED_VERSIONS "${VERSION}" _POS) if(_POS EQUAL -1) string(REPLACE ";" ", " _SUPPORTED_VERSIONS_STR "${SUPPORTED_VERSIONS}") @@ -96,7 +110,7 @@ function(determine_target_versions OUT_VERSIONS) endif(_POS EQUAL -1) endif(${PORT_UC}_FOUND) endif(NOT DEFINED ${OUT_VERSIONS}) - + # Pass values to caller set(${OUT_VERSIONS} "${${OUT_VERSIONS}}" PARENT_SCOPE) # ${PORT_UC}_${_VERSION_UNDERSCORE}_PG_CONFIG might have been set earlier! From 57cdd41c49b755e1e55d48dd8bb448ceeff65c96 Mon Sep 17 00:00:00 2001 From: Rahul Iyer Date: Wed, 25 Jan 2017 13:48:48 -0800 Subject: [PATCH 2/3] Keep default value of VERSION for simplification --- src/ports/postgres/cmake/PostgreSQLUtils.cmake | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/ports/postgres/cmake/PostgreSQLUtils.cmake b/src/ports/postgres/cmake/PostgreSQLUtils.cmake index e17a84562..7acdf9ad9 100644 --- a/src/ports/postgres/cmake/PostgreSQLUtils.cmake +++ b/src/ports/postgres/cmake/PostgreSQLUtils.cmake @@ -64,8 +64,8 @@ function(determine_target_versions OUT_VERSIONS) endforeach(VERSION) if(NOT DEFINED ${OUT_VERSIONS}) find_package(${PORT}) - if(${PORT_UC}_FOUND) + set(VERSION "${${PORT_UC}_VERSION_MAJOR}.${${PORT_UC}_VERSION_MINOR}") if(${PORT_UC} STREQUAL "GREENPLUM") # Starting GPDB 5.0, semantic versioning will be followed, # implying we only need 1 folder for same major versions @@ -86,10 +86,6 @@ function(determine_target_versions OUT_VERSIONS) set(VERSION "2") endif() - if(NOT DEFINED VERSION) - set(VERSION "${${PORT_UC}_VERSION_MAJOR}.${${PORT_UC}_VERSION_MINOR}") - endif() - list(FIND SUPPORTED_VERSIONS "${VERSION}" _POS) if(_POS EQUAL -1) string(REPLACE ";" ", " _SUPPORTED_VERSIONS_STR "${SUPPORTED_VERSIONS}") From ea52f6eafd4ed92ae32cef43bd98bceededd8926 Mon Sep 17 00:00:00 2001 From: Rahul Iyer Date: Wed, 25 Jan 2017 14:45:05 -0800 Subject: [PATCH 3/3] Get 3rd digit from Greenplum --- src/madpack/madpack.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py index 05077b346..8b0e64db5 100755 --- a/src/madpack/madpack.py +++ b/src/madpack/madpack.py @@ -317,7 +317,9 @@ def _get_dbver(): if portid == 'postgres': match = re.search("PostgreSQL[a-zA-Z\s]*(\d+\.\d+)", versionStr) elif portid == 'greenplum': - match = re.search("Greenplum[a-zA-Z\s]*(\d+\.\d+)", versionStr) + # for Greenplum the 3rd digit is necessary to differentiate + # 4.3.5+ from versions < 4.3.5 + match = re.search("Greenplum[a-zA-Z\s]*(\d+\.\d+\.\d+)", versionStr) elif portid == 'hawq': match = re.search("HAWQ[a-zA-Z\s]*(\d+\.\d+)", versionStr) return None if match is None else match.group(1)