From fb7a5411e23ad0293f284e7ae9588c4020a19a9e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 31 Jul 2023 06:11:32 +0900 Subject: [PATCH] GH-36685: [R][C++] Fix illegal opcode failure with Homebrew (#36705) ### Rationale for this change Summary of this problem: https://github.com/apache/arrow/issues/31132#issuecomment-1378082076 Why this problem is happen again? Because I removed `ENV["HOMEBREW_OPTIMIZATION_LEVEL"] = "O2"` in #36583. The solution we chose by #14342 was forcing to use `-O2` for SIMD related code. It works for `-DCMAKE_BUILD_TYPE=MinSizeRel` but it doesn't work for Homebrew. Because Homebrew's CC https://github.com/Homebrew/brew/blob/master/Library/Homebrew/shims/super/cc forces to use the same `-O` flag. The default is `-Os`. If we specify `-O2`, Homebrew's CC replaces it to `-Os`. If we use `ENV["HOMEBREW_OPTIMIZATION_LEVEL"] = "O2"`, Homebrew's CC always use `-O2`. So the solution we chose by #14342 isn't used for Homebrew. But Homebrew thinks that `ENV["HOMEBREW_OPTIMIZATION_LEVEL"] = "O2"` is a workaround. So we need another solution for Homebrew. Here are candidate solutions: 1. `-DARROW_RUNTIME_SIMD_LEVEL=NONE` 2. Remove `ENV.runtime_cpu_detection if Hardware::CPU.intel?` "1. `-DARROW_RUNTIME_SIMD_LEVEL=NONE`" works because we don't use the runtime SIMD dispatch feature (the problematic feature) entirely. "2. Remove `ENV.runtime_cpu_detection if Hardware::CPU.intel?`" works but I don't know why... If `ENV.runtime_cpu_detection` is called, Homebrew's CC stops replacing `-march=*`. If we call `ENV.runtime_cpu_detection`, `-march=haswell` is used for AVX2 related code and `-march=skylake-avx512` is used for AVX512 including BMI2 related code. If we don't call `ENV.runtime_cpu_detection`, `-march=nehalem` is always used. (Note that SIMD related flags such as `-mbmi2` aren't removed by Homebrew's CC. So I think that SIMD is enabled.) I don't know why but "the one-definition-rule violation" (see the summary for details: https://github.com/apache/arrow/issues/31132#issuecomment-1378082076 ) isn't happen. FYI: CPU info for GitHub Actions macOS hosted-runner: ```console $ sysctl hw.optional machdep.cpu hw.optional.adx: 0 hw.optional.aes: 1 hw.optional.avx1_0: 1 hw.optional.avx2_0: 0 hw.optional.avx512bw: 0 hw.optional.avx512cd: 0 hw.optional.avx512dq: 0 hw.optional.avx512f: 0 hw.optional.avx512ifma: 0 hw.optional.avx512vbmi: 0 hw.optional.avx512vl: 0 hw.optional.bmi1: 0 hw.optional.bmi2: 0 hw.optional.enfstrg: 0 hw.optional.f16c: 1 hw.optional.floatingpoint: 1 hw.optional.fma: 0 hw.optional.hle: 0 hw.optional.mmx: 1 hw.optional.mpx: 0 hw.optional.rdrand: 1 hw.optional.rtm: 0 hw.optional.sgx: 0 hw.optional.sse: 1 hw.optional.sse2: 1 hw.optional.sse3: 1 hw.optional.sse4_1: 1 hw.optional.sse4_2: 1 hw.optional.supplementalsse3: 1 hw.optional.x86_64: 1 machdep.cpu.address_bits.physical: 43 machdep.cpu.address_bits.virtual: 48 machdep.cpu.arch_perf.events: 127 machdep.cpu.arch_perf.events_number: 7 machdep.cpu.arch_perf.fixed_number: 0 machdep.cpu.arch_perf.fixed_width: 0 machdep.cpu.arch_perf.number: 4 machdep.cpu.arch_perf.version: 1 machdep.cpu.arch_perf.width: 48 machdep.cpu.cache.L2_associativity: 8 machdep.cpu.cache.linesize: 64 machdep.cpu.cache.size: 256 machdep.cpu.mwait.extensions: 3 machdep.cpu.mwait.linesize_max: 4096 machdep.cpu.mwait.linesize_min: 64 machdep.cpu.mwait.sub_Cstates: 16 machdep.cpu.thermal.ACNT_MCNT: 0 machdep.cpu.thermal.core_power_limits: 0 machdep.cpu.thermal.dynamic_acceleration: 0 machdep.cpu.thermal.energy_policy: 0 machdep.cpu.thermal.fine_grain_clock_mod: 0 machdep.cpu.thermal.hardware_feedback: 0 machdep.cpu.thermal.invariant_APIC_timer: 1 machdep.cpu.thermal.package_thermal_intr: 0 machdep.cpu.thermal.sensor: 0 machdep.cpu.thermal.thresholds: 0 machdep.cpu.tlb.data.small: 64 machdep.cpu.tlb.inst.large: 8 machdep.cpu.tlb.inst.small: 64 machdep.cpu.tlb.shared: 512 machdep.cpu.tsc_ccc.denominator: 0 machdep.cpu.tsc_ccc.numerator: 0 machdep.cpu.xsave.extended_state: 7 832 832 0 machdep.cpu.xsave.extended_state1: 0 0 0 0 machdep.cpu.brand: 0 machdep.cpu.brand_string: Intel(R) Xeon(R) CPU E5-1650 v2 @ 3.50GHz machdep.cpu.core_count: 3 machdep.cpu.cores_per_package: 4 machdep.cpu.extfamily: 0 machdep.cpu.extfeature_bits: 4967106816 machdep.cpu.extfeatures: SYSCALL XD EM64T LAHF RDTSCP TSCI machdep.cpu.extmodel: 3 machdep.cpu.family: 6 machdep.cpu.feature_bits: 18427078393948011519 machdep.cpu.features: FPU VME DE PSE TSC MSR PAE MCE CX8 APIC SEP MTRR PGE MCA CMOV PAT PSE36 CLFSH MMX FXSR SSE SSE2 SS HTT SSE3 PCLMULQDQ MON VMX SSSE3 CX16 SSE4.1 SSE4.2 x2APIC POPCNT AES VMM PCID XSAVE OSXSAVE TSCTMR AVX1.0 RDRAND F16C machdep.cpu.leaf7_feature_bits: 643 0 machdep.cpu.leaf7_feature_bits_edx: 3154117632 machdep.cpu.leaf7_features: RDWRFSGS TSC_THREAD_OFFSET SMEP ERMS MDCLEAR IBRS STIBP L1DF ACAPMSR SSBD machdep.cpu.logical_per_package: 4 machdep.cpu.max_basic: 13 machdep.cpu.max_ext: 2147483656 machdep.cpu.microcode_version: 1070 machdep.cpu.model: 58 machdep.cpu.processor_flag: 0 machdep.cpu.signature: 198313 machdep.cpu.stepping: 9 machdep.cpu.thread_count: 3 machdep.cpu.vendor: GenuineIntel ``` ### What changes are included in this PR? "1. `-DARROW_RUNTIME_SIMD_LEVEL=NONE`" because it's straightforward and "2. Remove `ENV.runtime_cpu_detection if Hardware::CPU.intel?`" may also disable runtime SIMD dispatch implicitly. This also adds the following debug information for easy to debug in future: * CPU information for GitHub Actions runner * Homebrew's build logs ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * Closes: #36685 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- dev/tasks/homebrew-formulae/apache-arrow.rb | 8 +++++++- dev/tasks/macros.jinja | 4 +++- dev/tasks/r/github.macos.brew.yml | 20 ++++++++++++++++---- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index 9086fdaae4ab4..f5d7ff36ea041 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -57,7 +57,8 @@ class ApacheArrow < Formula fails_with gcc: "5" def install - # https://github.com/Homebrew/homebrew-core/issues/76537 + # This isn't for https://github.com/Homebrew/homebrew-core/issues/76537 . + # This may improve performance. ENV.runtime_cpu_detection if Hardware::CPU.intel? # link against system libc++ instead of llvm provided libc++ @@ -90,6 +91,11 @@ def install -DARROW_WITH_ZSTD=ON -DPARQUET_BUILD_EXECUTABLES=ON ] + # Disable runtime SIMD dispatch. It may cause "illegal opcode" + # error on Intel Mac because of one-definition-rule violation. + # + # https://github.com/apache/arrow/issues/36685 + args << "-DARROW_RUNTIME_SIMD_LEVEL=NONE" if OS.mac? and Hardware::CPU.intel? system "cmake", "-S", "cpp", "-B", "build", *args, *std_cmake_args system "cmake", "--build", "build" diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 475494af18ce6..1f2f9a72975eb 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -214,6 +214,7 @@ on: # see https://github.com/actions/runner-images/issues/6868 brew install --overwrite python@3.11 python@3.10 + set -x ARROW_GLIB_FORMULA=$(echo ${ARROW_FORMULA} | sed -e 's/\.rb/-glib.rb/') echo "ARROW_GLIB_FORMULA=${ARROW_GLIB_FORMULA}" >> ${GITHUB_ENV} for formula in ${ARROW_FORMULA} ${ARROW_GLIB_FORMULA}; do @@ -223,11 +224,12 @@ on: # Pin the current commit in the formula to test so that # we're not always pulling from the tip of the default branch sed -i '' -E \ - -e 's@https://github.com/apache/arrow.git"$@{{ arrow.remote }}.git", revision: "{{ arrow.head }}"@' \ + -e 's@https://github.com/apache/arrow.git", branch: "main"$@{{ arrow.remote }}.git", revision: "{{ arrow.head }}"@' \ ${formula} # Sometimes crossbow gives a remote URL with .git and sometimes not. # Make sure there's only one sed -i '' -E -e 's@.git.git@.git@' ${formula} + cat ${formula} cp ${formula} $(brew --repository homebrew/core)/Formula/ done {% endmacro %} diff --git a/dev/tasks/r/github.macos.brew.yml b/dev/tasks/r/github.macos.brew.yml index dea7564c5f80f..8a0e6f6bb96e3 100644 --- a/dev/tasks/r/github.macos.brew.yml +++ b/dev/tasks/r/github.macos.brew.yml @@ -24,21 +24,33 @@ jobs: name: "Homebrew + R package" runs-on: macOS-11 steps: + - name: Show system information + run: | + sysctl hw.optional machdep.cpu + {{ macros.github_checkout_arrow()|indent }} {{ macros.configure_homebrew_arrow(formula)|indent }} - name: Install apache-arrow env: - {{ macros.github_set_sccache_envvars()|indent(8)}} + {{ macros.github_set_sccache_envvars()|indent(8)}} run: | - brew install sccache # for testing brew install minio - + # TODO(ARROW-16907): apache/arrow@main seems to be installed already # so this does nothing on a branch/PR - brew install -v --HEAD apache-arrow + brew install -v --HEAD {{ '$(brew --repository homebrew/core)/Formula/apache-arrow.rb' }} + + mkdir -p homebrew-logs + cp -a ~/Library/Logs/Homebrew/apache-arrow homebrew-logs/ + - name: Save logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: homebrew-logs + path: homebrew-logs - uses: r-lib/actions/setup-r@v2 - name: Install dependencies