From 2bf47049a12bee3f4e7018e9b0c24b37bdae6a35 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 13 Sep 2023 17:26:28 +0000 Subject: [PATCH 1/7] feat: add OS detection to hermetic build scripts --- .github/workflows/verify_library_generation.yaml | 3 +-- library_generation/generate_library.sh | 2 +- library_generation/utilities.sh | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index dabf01a458..ae32b9eced 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -28,8 +28,7 @@ jobs: library_generation/test/generate_library_integration_test.sh \ -p google/bigtable/v2 \ -d google-cloud-bigtable-v2-java \ - --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git \ - --os_type ${{ matrix.os }} + --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git unit_tests: strategy: matrix: diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index e20d9f1856..4932d86857 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -78,7 +78,7 @@ if [ -z "${include_samples}" ]; then fi if [ -z "${os_architecture}" ]; then - os_architecture="linux-x86_64" + os_architecture=$(detect_OS) fi mkdir -p "${destination_path}" diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 70dccc6c6c..d5865262c2 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -273,3 +273,19 @@ get_version_from_versions_txt() { version=$(grep "$key:" "${versions}" | cut -d: -f3) # 3rd field is snapshot echo "${version}" } + +detect_OS() { + if [[ "${OSTYPE}" == "linux-gnu"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then + os_architecture="linux-x86_64" + elif [[ "${OSTYPE}" == "darwin"* ]]; then + os_architecture="osx-x86_64" + elif [[ "${OSTYPE}" == "cygwin" ]] || [[ "${OSTYPE}" == "msys" ]]; then + os_architecture="win32" + else + >&2 echo 'Could not detect OS. Please specify it with --os_architecture' + >&2 echo 'Also, see https://github.com/protocolbuffers/protobuf/releases for a list of available OS (e.g. linux-aarch_64)' + exit 1 + fi + >&2 echo "Detected OS architecture: ${os_architecture}" >2 + echo "${os_architecture}" +} From 0b97a2cf7bbb0771bce80714f6ca7662dcc9a8e7 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 13 Sep 2023 17:35:52 +0000 Subject: [PATCH 2/7] skip os arch in integration test --- .../test/generate_library_integration_test.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index 1c437d328a..f0f54c2eac 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -78,11 +78,6 @@ include_samples=$(get_config_from_BUILD \ "false" ) echo "GAPIC options are transport=${transport}, rest_numeric_enums=${rest_numeric_enums}, include_samples=${include_samples}." -os_architecture="linux-x86_64" -if [[ "$os_type" == *"macos"* ]]; then - os_architecture="osx-x86_64" -fi -echo "OS Architecture is ${os_architecture}." # generate GAPIC client library echo "Generating library from ${proto_path}, to ${destination_path}..." "${library_generation_dir}"/generate_library.sh \ @@ -93,8 +88,7 @@ echo "Generating library from ${proto_path}, to ${destination_path}..." --grpc_version "${grpc_version}" \ --transport "${transport}" \ --rest_numeric_enums "${rest_numeric_enums}" \ ---include_samples "${include_samples}" \ ---os_architecture "${os_architecture}" +--include_samples "${include_samples}" echo "Generate library finished." echo "Checking out googleapis-gen repository..." From 05f2a9960368f36b1418f26a42bd189b69a5a053 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 13 Sep 2023 17:49:38 +0000 Subject: [PATCH 3/7] remove unused IT flag --- library_generation/test/generate_library_integration_test.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index f0f54c2eac..fa1585346e 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -30,10 +30,6 @@ case $key in googleapis_gen_url="$2" shift ;; - --os_type) - os_type="$2" - shift - ;; *) echo "Invalid option: [$1]" exit 1 From 686da2a20dc3ce582c421cc3056c5afa3e831537 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 13 Sep 2023 18:10:43 +0000 Subject: [PATCH 4/7] dynamic architecture resolution --- library_generation/utilities.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index d5865262c2..b9e887009f 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -276,9 +276,9 @@ get_version_from_versions_txt() { detect_OS() { if [[ "${OSTYPE}" == "linux-gnu"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then - os_architecture="linux-x86_64" + os_architecture="linux-$(uname -m)" elif [[ "${OSTYPE}" == "darwin"* ]]; then - os_architecture="osx-x86_64" + os_architecture="osx-$(uname -m)" elif [[ "${OSTYPE}" == "cygwin" ]] || [[ "${OSTYPE}" == "msys" ]]; then os_architecture="win32" else From 2dae9f1b357b6ba8c81702472b7974ef938efc10 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 13 Sep 2023 18:12:26 +0000 Subject: [PATCH 5/7] change function name --- library_generation/generate_library.sh | 2 +- library_generation/utilities.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index 4932d86857..8355b4a6a5 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -78,7 +78,7 @@ if [ -z "${include_samples}" ]; then fi if [ -z "${os_architecture}" ]; then - os_architecture=$(detect_OS) + os_architecture=$(detect_os_architecture) fi mkdir -p "${destination_path}" diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index b9e887009f..512e0074b5 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -274,7 +274,7 @@ get_version_from_versions_txt() { echo "${version}" } -detect_OS() { +detect_os_architecture() { if [[ "${OSTYPE}" == "linux-gnu"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then os_architecture="linux-$(uname -m)" elif [[ "${OSTYPE}" == "darwin"* ]]; then From 287bfa8cffc30a225d21826abd616fcd7b4a82f1 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 13 Sep 2023 18:19:23 +0000 Subject: [PATCH 6/7] use `case` in os detection func --- library_generation/utilities.sh | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 512e0074b5..7ac6c99abc 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -275,17 +275,22 @@ get_version_from_versions_txt() { } detect_os_architecture() { - if [[ "${OSTYPE}" == "linux-gnu"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then - os_architecture="linux-$(uname -m)" - elif [[ "${OSTYPE}" == "darwin"* ]]; then - os_architecture="osx-$(uname -m)" - elif [[ "${OSTYPE}" == "cygwin" ]] || [[ "${OSTYPE}" == "msys" ]]; then - os_architecture="win32" - else - >&2 echo 'Could not detect OS. Please specify it with --os_architecture' - >&2 echo 'Also, see https://github.com/protocolbuffers/protobuf/releases for a list of available OS (e.g. linux-aarch_64)' - exit 1 - fi + case "${OSTYPE}" in + "linux-gnu"*|"freebsd"*) + os_architecture="linux-$(uname -m)" + ;; + "darwin"*) + os_architecture="osx-$(uname -m)" + ;; + "cygwin"|"msys") + os_architecture="win32" + ;; + *) + >&2 echo 'Could not detect OS. Please specify it with --os_architecture' + >&2 echo 'Also, see https://github.com/protocolbuffers/protobuf/releases for a list of available OS (e.g. linux-aarch_64)' + exit 1 + ;; + esac >&2 echo "Detected OS architecture: ${os_architecture}" >2 echo "${os_architecture}" } From 6308f9b17fc7a60cd53172198f0359c37a00b863 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 13 Sep 2023 19:50:48 +0000 Subject: [PATCH 7/7] correct os detection for aarch64 uses function by Joe Wang --- library_generation/utilities.sh | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 7ac6c99abc..d7647f217a 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -275,22 +275,19 @@ get_version_from_versions_txt() { } detect_os_architecture() { - case "${OSTYPE}" in - "linux-gnu"*|"freebsd"*) - os_architecture="linux-$(uname -m)" + local os_type + local os_architecture + os_type=$(uname -sm) + case "${os_type}" in + *"Linux x86_64"*) + os_architecture="linux-x86_64" ;; - "darwin"*) - os_architecture="osx-$(uname -m)" - ;; - "cygwin"|"msys") - os_architecture="win32" + *"Darwin x86_64"*) + os_architecture="osx-x86_64" ;; *) - >&2 echo 'Could not detect OS. Please specify it with --os_architecture' - >&2 echo 'Also, see https://github.com/protocolbuffers/protobuf/releases for a list of available OS (e.g. linux-aarch_64)' - exit 1 + os_architecture="osx-aarch_64" ;; esac - >&2 echo "Detected OS architecture: ${os_architecture}" >2 echo "${os_architecture}" }