Merge branch 'master' into profiletidy

.github/workflows/benchmarkadhoc.yml

@@ -35,7 +35,7 @@ jobs:
         java: [ 8, 21 ]
         experimental: [false]
       fail-fast: false
-        
+
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:

.github/workflows/coverage.yml

@@ -15,7 +15,7 @@
 
 name: Coverage
 
-on: 
+on:
   # allow direct trigger
   workflow_dispatch:
   push:

.github/workflows/maven.yml

@@ -15,6 +15,8 @@
 
 name: Java CI
 
+# cross-build tests are done in a separate action as they currrently take a long time
+
 on:
   # allow direct trigger
   workflow_dispatch:
@@ -41,32 +43,28 @@ jobs:
         # These names are used in conditional statements below.
         java: [ 8, 11, 17, 21 ]
         experimental: [false]
-        # macos-11 / java21 keeps failing: java is loading libcrypto in an unsafe way
-        # so change it to experimental to stop failing the entire workflow
-        exclude:
-        - os: macos-11
-          java: 21
-        include:
-        - os: macos-11 
-          java: 21
-          experimental: true        
         # macos-13-arm64 does not appear to be available
         # include:
         #   - java: 21
         #     os: macos-13-arm64
-        #     experimental: true        
+        #     experimental: true
 #        include:
 #          - java: 22-ea
 #            os: ubuntu-20.04
-#            experimental: true        
+#            experimental: true
 #          - java: 22-ea
 #            os: windows-latest
-#            experimental: true        
+#            experimental: true
 #          - java: 22-ea
 #            os: macos-latest
-#            experimental: true        
+#            experimental: true
+        # We don't need to build all Java versions every time
+        # Try excluding 11, 17 on branch pushes
+        exclude:
+          - java: ${{ (github.ref != 'refs/heads/master' && github.event_name == 'push') && 11 || 99 }} 
+          - java: ${{ (github.ref != 'refs/heads/master' && github.event_name == 'push') && 17 || 99 }} 
       fail-fast: false
-        
+
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
@@ -84,33 +82,54 @@ jobs:
         java-version: ${{ matrix.java }}
     - name: OpenSSL version
       run: openssl version -a
-    - name: Build with Maven
-      # OPENSSL_HOME is needed for Windows build; not used by other builds so can set unconditionally
+    - name: OpenSSL engine (macos)
+      # need to override the libarary on macOS
+      if: ${{ matrix.os == 'macos-11' }}
+      run: |
+        echo $(openssl version -e | sed -n -e 's/engines-.*//' -e 's/: "/=/p') >> "$GITHUB_ENV"
+    - name: OpenSSL engine (windows)
+      # need to override the libarary on windows
+      if: ${{ matrix.os == 'windows-latest' }}
+      # e.g. NAME: "libcrypto-1_1-x64.dll"
+      # Not sure how to derive this automatically
+      run: |
+        openssl version -a
+        echo "NAME=libcrypto-1_1-x64.dll" >> $env:GITHUB_ENV
+    # N.B. '-V -B -ntp' is shorthand for '--show-version --batch-mode --no-transfer-progress'
+    # 
+    # The bash shell under Windows changes the openssl default library, so is not used for running tests
+    # Unfortunately that means separate steps for Windows, as it uses a different syntax for referrring to
+    # environment variables: $env:VARNAME instead of $VARNAME
+    # Also, note that Windows stores all the DLLs in the same directory.
+    # Instead of defining jni.library.path and jna.library.path we need to define
+    # jni.library.name and commons.crypto.OpenSslNativeJna to override the file names
+    - name: Build with Maven (Windows)
+      if: ${{ matrix.os == 'windows-latest' }}
+      # OPENSSL_HOME is needed for Windows build to find some header files
       # It's not clear how one is supposed to find the correct setting;
       # The value below was found by searching for openssl files under C (warning: slow)
       # Other possible values are:
       # "C:\\Miniconda\\pkgs\\openssl-1.1.1n-h2bbff1b_0\\Library"
       # "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw64\\opt"
+      # N.B. This must *not* be run under the bash shell, as that changes the default openssl library under Windows
       env:
         OPENSSL_HOME: "C:\\Miniconda\\Library"
-      run: mvn --show-version --batch-mode --no-transfer-progress -DtrimStackTrace=false
-    - name: Run Crypto (JNI)
       run: |
-        mvn --show-version --batch-mode --no-transfer-progress -q exec:java -D"exec.mainClass=org.apache.commons.crypto.Crypto" -D"commons.crypto.debug=true"
-    - name: Run OpenSslJna (JNA - don't override lib)
-      run: |
-        mvn --show-version --batch-mode --no-transfer-progress -q exec:java -D"exec.mainClass=org.apache.commons.crypto.jna.OpenSslJna" -D"commons.crypto.debug=true"
-    - name: Run OpenSslJna (JNA - macOS from openssl version -a ENGINESDIR)
-      if: ${{ matrix.os == 'macos-11' }}
+        mvn -V -B -ntp -DtrimStackTrace=false -D"jni.library.name=$env:NAME" -D"commons.crypto.OpenSslNativeJna=$env:NAME"
+    - name: Build with Maven (not Windows)
+      if: ${{ matrix.os != 'windows-latest' }}
       run: |
-        mvn --show-version --batch-mode --no-transfer-progress -q exec:java -D"exec.mainClass=org.apache.commons.crypto.jna.OpenSslJna" -D"jna.library.path=/usr/local/Cellar/openssl@1.1/1.1.1v/lib" -D"commons.crypto.debug=true"
-    - name: Run OpenSslJna (JNA - Windows from openssl version -a ENGINESDIR)
-      if: ${{ matrix.os == 'windows-latest' }}
-      run: |
-        mvn --show-version --batch-mode --no-transfer-progress -q exec:java -D"exec.mainClass=org.apache.commons.crypto.jna.OpenSslJna" -D"jna.library.path=C:\Program Files\OpenSSL\lib" -D"commons.crypto.debug=true"
+        mvn -V -B -ntp -DtrimStackTrace=false -D"jni.library.path=$ENGINESDIR" -D"jna.library.path=$ENGINESDIR"
     - name: Check benchmark code compiles
       if: ${{ matrix.java == '8' }}
       env:
         OPENSSL_HOME: "C:\\Miniconda\\Library"
       run: |
-        mvn --show-version --batch-mode --no-transfer-progress clean test-compile -Pbenchmark
+        mvn -V -B -ntp clean test-compile -Pbenchmark
+    - name: Check JNI and JNA tests are independent
+      # N.B. the default library fails with 'java is loading libcrypto in an unsafe way'
+      # so we need to define the appropriate library for each test
+      if: ${{ matrix.java == '8' && matrix.os != 'windows-latest' }}
+      run: |
+        mvn -V -B -ntp test -Ptestjni -D"jni.library.path=$ENGINESDIR" -Dcommons.crypto.OpenSslNativeJna=___
+        mvn -V -B -ntp test -Ptestjna -D"jna.library.path=$ENGINESDIR" -Djni.library.name=___

.github/workflows/maven_adhoc.yml

@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Maven adhoc
+
+on:
+  # allow direct trigger
+  workflow_dispatch:
+  # self-trigger
+  push:
+    paths:
+      - '**/maven_adhoc.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # macos-latest and ubuntu-latest uses OpenSSL 3 which breaks tests
+        # os: [macos-11, ubuntu-20.04, windows-latest]
+        os: [ macos-latest, ubuntu-latest, windows-latest ]
+        # These names are used in conditional statements below.
+        # java: [ 8, 11, 17, 21 ]
+        java: [ 21 ]
+        experimental: [false]
+
+    steps:
+    - name: OpenSSL version
+      run: openssl version -a
+    - name: Checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        persist-credentials: false
+    - name: Set up JDK ${{ matrix.java }}
+      uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java }}
+    - name: Build only
+      env:
+        OPENSSL_HOME: "C:\\Miniconda\\Library"
+      run: |
+        mvn clean test -B -V -ntp -DskipTests
+    - name: JNI test default
+      run: |
+        mvn -q exec:java -D"exec.mainClass=org.apache.commons.crypto.Crypto" -D"commons.crypto.debug=true"
+    - name: JNA test default
+      if: always()
+      run: |
+        mvn -q exec:java -D"jna.debug_load=true" -D"exec.mainClass=org.apache.commons.crypto.jna.OpenSslJna" -D"commons.crypto.debug=true"
+    - name: Maven test default
+      if: always()
+      run: |
+        mvn surefire:test -B -V -ntp -D"jna.debug_load=true" -DtrimStackTrace=false -D"commons.crypto.debug=true"

.github/workflows/maven_crossbuild.yml

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Java Cross Build
+
+# This is done as a separate action for now, as it takes a long time
+
+on:
+  # allow direct trigger
+  workflow_dispatch:
+  push:
+    paths:
+      - '**/native/**'
+      - '**/maven_crossbuild.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  build-cross:
+    strategy:
+      matrix:
+        include:
+          - platform: aarch64
+          - platform: riscv64
+            jna_override: "-Djna.version=5.12.0" # See https://github.com/java-native-access/jna/issues/1557
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          persist-credentials: false
+      - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+        with:
+            platforms: ${{ matrix.platform }}
+      - run: |
+          # Build package
+          docker compose -f src/docker/docker-compose.yaml run crypto src/docker/build.sh
+          # Run on platform
+          docker compose -f src/docker/docker-compose.yaml run crypto-${{ matrix.platform }} \
+            mvn -V -B -ntp surefire:test ${{ matrix.jna_override }}

Makefile

@@ -83,6 +83,8 @@ $(NATIVE_DLL): $(COMMONS_CRYPTO_OUT)/$(LIBNAME)
 	@mkdir -p $(@D)
 	cp $< $@
 
+# These targets should correspond with the entries in the list 'known_os_archs' defined in Makefile.common
+# e.g. linux32 corresponds with Linux-x86
 win32:
 	$(MAKE) native CROSS_PREFIX=i686-w64-mingw32- OS_NAME=Windows OS_ARCH=x86
 
@@ -99,6 +101,9 @@ mac64:
 macArm64:
 	$(MAKE) native OS_NAME=Mac OS_ARCH=arm64
 
+mac-aarch64:
+	$(MAKE) native OS_NAME=Mac OS_ARCH=aarch64
+
 linux32:
 	$(MAKE) native OS_NAME=Linux OS_ARCH=x86
 
@@ -120,8 +125,25 @@ linux-armhf:
 linux-aarch64:
 	$(MAKE) native CROSS_PREFIX=aarch64-linux-gnu- OS_NAME=Linux OS_ARCH=aarch64
 
-clean-native-linux32:
-	$(MAKE) clean-native OS_NAME=Linux OS_ARCH=x86
+# for cross-compilation on Ubuntu, install the g++-riscv64-linux-gnu
+linux-riscv64:
+	$(MAKE) native CROSS_PREFIX=riscv64-linux-gnu- OS_NAME=Linux OS_ARCH=riscv64
+
+linux-ppc: # TODO: Untested; may need additional CROSS_PREFIX define
+	$(MAKE) native OS_NAME=Linux OS_ARCH=ppc
+
+linux-ppc64: # TODO: Untested; may need additional CROSS_PREFIX define
+	$(MAKE) native OS_NAME=Linux OS_ARCH=ppc64
+
+sunos32: # TODO: Untested; may need additional CROSS_PREFIX define
+	$(MAKE) native OS_NAME=SunOS OS_ARCH=x86
+
+sunos64: # TODO: Untested; may need additional CROSS_PREFIX define
+	$(MAKE) native OS_NAME=SunOS OS_ARCH=x86_64
+
+sunos-sparc: # TODO: Untested; may need additional CROSS_PREFIX define
+	$(MAKE) native OS_NAME=SunOS OS_ARCH=sparc
+
+aix-ppc64: # TODO: Untested; may need additional CROSS_PREFIX define
+	$(MAKE) native OS_NAME=AIX OS_ARCH=ppc64
 
-clean-native-win32:
-	$(MAKE) clean-native OS_NAME=Windows OS_ARCH=x86

Makefile.common

@@ -52,8 +52,15 @@ endif
 
 
 # os=Default is meant to be generic Unix/Linux
-# The following list must include all OS entries beloe (apart from Default)
-known_os_archs := Linux-x86 Linux-x86_64 Linux-aarch64 Linux-arm Linux-armhf Linux-ppc Linux-ppc64 Mac-x86 Mac-x86_64 Mac-arm64 Mac-aarch64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc64
+# The following list must include all OS entries below (apart from Default)
+# Also there should be a target in the makefile for each of the combinations
+# For example, 'Linux-x86' is invoked by the target 'linux32'
+known_os_archs := Linux-x86 Linux-x86_64 Linux-aarch64 Linux-riscv64 Linux-arm Linux-armhf Linux-ppc Linux-ppc64 \
+                  Mac-x86 Mac-x86_64 Mac-arm64 Mac-aarch64 \
+                  FreeBSD-x86_64 \
+                  Windows-x86 Windows-x86_64 \
+                  SunOS-x86 SunOS-sparc SunOS-x86_64 \
+                  AIX-ppc64
 os_arch := $(OS_NAME)-$(OS_ARCH)
 
 ifeq (,$(findstring $(strip $(os_arch)),$(known_os_archs)))
@@ -190,6 +197,16 @@ Linux-aarch64_LIBNAME   := libcommons-crypto.so
 Linux-aarch64_LIBNAME_OSSL3 := libcommons-crypto-ossl3.so
 Linux-aarch64_COMMONS_CRYPTO_FLAGS  :=
 
+Linux-riscv64_CC        := $(CROSS_PREFIX)gcc
+Linux-riscv64_CXX       := $(CROSS_PREFIX)g++
+Linux-riscv64_STRIP     := $(CROSS_PREFIX)strip
+Linux-riscv64_CXXFLAGS  := -Ilib/inc_linux -I"$(JAVA_HOME)/include" -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -Wall -Werror
+Linux-riscv64_CFLAGS    := -Ilib/inc_linux -I"$(JAVA_HOME)/include" -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -Wall -Werror
+Linux-riscv64_LINKFLAGS := -shared -static-libgcc
+Linux-riscv64_LIBNAME   := libcommons-crypto.so
+Linux-riscv64_LIBNAME_OSSL3 := libcommons-crypto-ossl3.so
+Linux-riscv64_COMMONS_CRYPTO_FLAGS  :=
+
 ifndef Mac_INC_OPENSSL
 Mac_INC_OPENSSL   := /usr/local/opt/openssl/include
 endif