diff --git a/.circleci/build-wheel.sh b/.circleci/build-wheel.sh index d69e8c9213fe..1cd80487992b 100755 --- a/.circleci/build-wheel.sh +++ b/.circleci/build-wheel.sh @@ -23,8 +23,8 @@ if [[ "${PYBIN}" =~ $REGEX ]]; then PY_LIMITED_API="--py-limited-api=cp3${BASH_REMATCH[1]}" fi -LDFLAGS="-L/opt/pyca/cryptography/openssl/lib" \ - CFLAGS="-I/opt/pyca/cryptography/openssl/include -Wl,--exclude-libs,ALL" \ +OPENSSL_DIR="/opt/pyca/cryptography/openssl" \ + RUSTFLAGS="-Clink-arg=-Wl,--exclude-libs,ALL" \ ../../.venv/bin/python setup.py bdist_wheel "$PY_LIMITED_API" auditwheel repair --plat "${PLATFORM}" -w wheelhouse/ dist/cryptography*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2685778ed2ee..51afd7e135b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,8 +110,9 @@ jobs: if: matrix.PYTHON.OPENSSL && steps.ossl-cache.outputs.cache-hit != 'true' - name: Set CFLAGS/LDFLAGS run: | - echo "CFLAGS=${CFLAGS} -Werror=implicit-function-declaration -I${OSSL_PATH}/include" >> $GITHUB_ENV - echo "LDFLAGS=${LDFLAGS} -L${OSSL_PATH}/lib -L${OSSL_PATH}/lib64 -Wl,-rpath=${OSSL_PATH}/lib -Wl,-rpath=${OSSL_PATH}/lib64" >> $GITHUB_ENV + echo "OPENSSL_DIR=${OSSL_PATH}" >> $GITHUB_ENV + echo "CFLAGS=${CFLAGS} -Werror=implicit-function-declaration" >> $GITHUB_ENV + echo "RUSTFLAGS=-Clink-arg=-Wl,-rpath=${OSSL_PATH}/lib -Clink-arg=-Wl,-rpath=${OSSL_PATH}/lib64" >> $GITHUB_ENV if: matrix.PYTHON.OPENSSL - name: Tests run: | @@ -188,8 +189,7 @@ jobs: - {VERSION: "3.9", TOXENV: "py39"} RUST: # Cover MSRV (and likely next MSRV) and in-dev versions - - 1.41.0 - - 1.45.0 + - 1.48.0 - beta name: "${{ matrix.PYTHON.TOXENV }} with Rust ${{ matrix.RUST }}" timeout-minutes: 15 @@ -284,7 +284,7 @@ jobs: repository: "google/wycheproof" path: "wycheproof" ref: "master" - - run: python -m pip install tox coverage + - run: python -m pip install tox coverage cffi - name: Tests run: | tox -r -- --color=yes --wycheproof-root=wycheproof @@ -381,8 +381,9 @@ jobs: - name: Tests run: | CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS=1 \ - LDFLAGS="${HOME}/openssl-macos-x86-64/lib/libcrypto.a ${HOME}/openssl-macos-x86-64/lib/libssl.a" \ - CFLAGS="-I${HOME}/openssl-macos-x86-64/include -Werror -Wno-error=deprecated-declarations -Wno-error=incompatible-pointer-types-discards-qualifiers -Wno-error=unused-function -Wno-error=unused-command-line-argument -mmacosx-version-min=10.10 -march=core2 $EXTRA_CFLAGS" \ + OPENSSL_DIR="${HOME}/openssl-macos-x86-64" \ + OPENSSL_STATIC=1 \ + CFLAGS="-Werror -Wno-error=deprecated-declarations -Wno-error=incompatible-pointer-types-discards-qualifiers -Wno-error=unused-function -Wno-error=unused-command-line-argument -mmacosx-version-min=10.10 -march=core2 $EXTRA_CFLAGS" \ tox -r -- --color=yes --wycheproof-root=wycheproof env: TOXENV: ${{ matrix.PYTHON.TOXENV }} @@ -442,8 +443,7 @@ jobs: - name: Download OpenSSL run: | python .github/workflows/download_openssl.py windows openssl-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.MSVC_VERSION }} - echo "INCLUDE=C:/openssl-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.MSVC_VERSION }}/include;$INCLUDE" >> $GITHUB_ENV - echo "LIB=C:/openssl-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.MSVC_VERSION }}/lib;$LIB" >> $GITHUB_ENV + echo "OPENSSL_DIR=C:/openssl-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.MSVC_VERSION }}" >> $GITHUB_ENV echo "CL=${{ matrix.PYTHON.CL_FLAGS }}" >> $GITHUB_ENV env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.zuul.playbooks/playbooks/tox/pre.yaml b/.zuul.playbooks/playbooks/tox/pre.yaml index 33d3487a22bd..cc6e231ece53 100644 --- a/.zuul.playbooks/playbooks/tox/pre.yaml +++ b/.zuul.playbooks/playbooks/tox/pre.yaml @@ -17,6 +17,7 @@ - libssl-dev - libffi-dev - python3-dev + - pkg-config become: yes when: ansible_distribution in ['Debian', 'Ubuntu'] diff --git a/.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/files/build-wheels.sh b/.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/files/build-wheels.sh index 216a839338e8..efda176fdd8d 100644 --- a/.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/files/build-wheels.sh +++ b/.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/files/build-wheels.sh @@ -20,8 +20,8 @@ for P in ${PYTHONS}; do PY_LIMITED_API="--py-limited-api=cp3${BASH_REMATCH[1]}" fi - LDFLAGS="-L/opt/pyca/cryptography/openssl/lib" \ - CFLAGS="-I/opt/pyca/cryptography/openssl/include -Wl,--exclude-libs,ALL" \ + OPENSSL_DIR="/opt/pyca/cryptography/openssl" \ + RUSTFLAGS="-Clink-arg=-Wl,--exclude-libs,ALL" \ .venv/bin/python setup.py bdist_wheel $PY_LIMITED_API auditwheel repair --plat ${PLAT} -w wheelhouse/ dist/cryptography*.whl diff --git a/pyproject.toml b/pyproject.toml index ad29d7af36d3..0dca0e772b0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ warn_unused_ignores = true [[tool.mypy.overrides]] module = [ - "cryptography.hazmat.bindings._openssl", + "cryptography.hazmat.bindings._rust._openssl", "pretend" ] ignore_missing_imports = true diff --git a/setup.py b/setup.py index 24b9f102bbf0..8f781a6ac0d9 100644 --- a/setup.py +++ b/setup.py @@ -37,9 +37,6 @@ try: # See setup.cfg for most of the config metadata. setup( - cffi_modules=[ - "src/_cffi_src/build_openssl.py:ffi", - ], rust_extensions=[ RustExtension( "_rust", diff --git a/src/_cffi_src/build_openssl.py b/src/_cffi_src/build_openssl.py index c5ab3cb3c68f..c2c3858adeb4 100644 --- a/src/_cffi_src/build_openssl.py +++ b/src/_cffi_src/build_openssl.py @@ -112,3 +112,15 @@ def _extra_compile_args(platform): libraries=_get_openssl_libraries(sys.platform), extra_compile_args=_extra_compile_args(sys.platform), ) + +if __name__ == "__main__": + out_dir = os.getenv("OUT_DIR") + module_name, source, source_extension, kwds = ffi._assigned_source + c_file = os.path.join(out_dir, module_name + source_extension) + ffi.embedding_api( + """ + extern "Python" void _this_is_not_used(void); + """ + ) + ffi.embedding_init_code("") + ffi.emit_c_code(c_file) diff --git a/src/_cffi_src/openssl/cryptography.py b/src/_cffi_src/openssl/cryptography.py index 7cbcb591dd68..a0e22932c261 100644 --- a/src/_cffi_src/openssl/cryptography.py +++ b/src/_cffi_src/openssl/cryptography.py @@ -32,7 +32,9 @@ #include #endif #if defined(_WIN32) +#if !defined(WIN32_LEAN_AND_MEAN) #define WIN32_LEAN_AND_MEAN +#endif #include #include #include diff --git a/src/_cffi_src/utils.py b/src/_cffi_src/utils.py index bb742a4a383c..80d5c99c4fdf 100644 --- a/src/_cffi_src/utils.py +++ b/src/_cffi_src/utils.py @@ -73,6 +73,12 @@ def build_ffi( verify_source += '\n#define CRYPTOGRAPHY_PACKAGE_VERSION "{}"'.format( about["__version__"] ) + verify_source += r""" + +int make_cryptography_openssl_module(void) { + return cffi_start_python(); +} +""" ffi.cdef(cdef_source) ffi.set_source( module_name, diff --git a/src/cryptography/hazmat/bindings/_rust/_openssl.pyi b/src/cryptography/hazmat/bindings/_rust/_openssl.pyi new file mode 100644 index 000000000000..4a687d5cf0c4 --- /dev/null +++ b/src/cryptography/hazmat/bindings/_rust/_openssl.pyi @@ -0,0 +1 @@ +# An empty file to make mypy recognize this module diff --git a/src/cryptography/hazmat/bindings/openssl/binding.py b/src/cryptography/hazmat/bindings/openssl/binding.py index c8e6cc4dc981..b9d4912eeaee 100644 --- a/src/cryptography/hazmat/bindings/openssl/binding.py +++ b/src/cryptography/hazmat/bindings/openssl/binding.py @@ -11,9 +11,11 @@ import cryptography from cryptography import utils from cryptography.exceptions import InternalError -from cryptography.hazmat.bindings._openssl import ffi, lib +from cryptography.hazmat.bindings._rust import _openssl from cryptography.hazmat.bindings.openssl._conditional import CONDITIONAL_NAMES +ffi = _openssl.ffi # type: ignore +lib = _openssl.lib # type: ignore _OpenSSLErrorWithText = typing.NamedTuple( "_OpenSSLErrorWithText", [("code", int), ("lib", int), ("reason", int), ("reason_text", bytes)], diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index e4bd1d8397a9..df0563bbebd2 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -53,6 +53,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "cc" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" + [[package]] name = "cfg-if" version = "1.0.0" @@ -76,13 +82,31 @@ name = "cryptography-rust" version = "0.1.0" dependencies = [ "asn1", + "cc", "chrono", "lazy_static", + "openssl", + "openssl-sys", "ouroboros", "pem", "pyo3", ] +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "indoc" version = "0.3.6" @@ -161,6 +185,42 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" +[[package]] +name = "openssl" +version = "0.10.38" +source = "git+https://github.com/sfackler/rust-openssl.git#16f65124674afd935b52ba6e6b10f963fb440968" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "git+https://github.com/sfackler/rust-openssl.git#16f65124674afd935b52ba6e6b10f963fb440968" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-sys" +version = "0.9.72" +source = "git+https://github.com/sfackler/rust-openssl.git#16f65124674afd935b52ba6e6b10f963fb440968" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ouroboros" version = "0.13.0" @@ -240,6 +300,12 @@ dependencies = [ "regex", ] +[[package]] +name = "pkg-config" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12295df4f294471248581bc09bef3c38a5e46f1e36d6a37353621a0c6c357e1f" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -401,6 +467,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.3" diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index 617167d04429..2aed2e05a9ed 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -12,6 +12,11 @@ asn1 = { version = "0.8.7", default-features = false, features = ["derive"] } pem = "1.0" chrono = { version = "0.4", default-features = false, features = ["alloc", "clock"] } ouroboros = "0.13" +openssl = "0.10.38" +openssl-sys = "0.9.72" + +[build-dependencies] +cc = "1.0.72" [features] extension-module = ["pyo3/extension-module"] @@ -24,3 +29,7 @@ crate-type = ["cdylib"] [profile.release] lto = "thin" overflow-checks = true + +[patch.crates-io] +openssl-sys = { git = "https://github.com/sfackler/rust-openssl.git" } +openssl = { git = "https://github.com/sfackler/rust-openssl.git" } diff --git a/src/rust/build.rs b/src/rust/build.rs new file mode 100644 index 000000000000..5ed7bb0facd6 --- /dev/null +++ b/src/rust/build.rs @@ -0,0 +1,143 @@ +use std::env; +use std::io::Write; +use std::path::{Path, MAIN_SEPARATOR}; +use std::process::{Command, Stdio}; + +fn main() { + let target = env::var("TARGET").unwrap(); + let openssl_static = env::var("OPENSSL_STATIC") + .map(|x| x == "1") + .unwrap_or(false); + if target.contains("apple") && openssl_static { + // On (older) OSX we need to link against the clang runtime, + // which is hidden in some non-default path. + // + // More details at https://github.com/alexcrichton/curl-rust/issues/279. + if let Some(path) = macos_link_search_path() { + println!("cargo:rustc-link-lib=clang_rt.osx"); + println!("cargo:rustc-link-search={}", path); + } + } + + let out_dir = env::var("OUT_DIR").unwrap(); + // FIXME: maybe pyo3-build-config should provide a way to do this? + let python = env::var("PYO3_PYTHON").unwrap_or_else(|_| "python3".to_string()); + println!("cargo:rerun-if-changed=../_cffi_src/"); + let python_path = match env::var("PYTHONPATH") { + Ok(mut val) => { + if cfg!(target_os = "windows") { + val.push(';'); + } else { + val.push(':'); + } + val.push_str(&format!("..{}", MAIN_SEPARATOR)); + val + } + Err(_) => format!("..{}", MAIN_SEPARATOR), + }; + let output = Command::new(&python) + .env("PYTHONPATH", python_path) + .env("OUT_DIR", &out_dir) + .arg("../_cffi_src/build_openssl.py") + .output() + .expect("failed to execute build_openssl.py"); + if !output.status.success() { + panic!( + "failed to run build_openssl.py, stdout: \n{}\nstderr: \n{}\n", + String::from_utf8(output.stdout).unwrap(), + String::from_utf8(output.stderr).unwrap() + ); + } + + let python_impl = run_python_script( + &python, + "import platform; print(platform.python_implementation(), end='')", + ) + .unwrap(); + let python_include = run_python_script( + &python, + "import sysconfig; print(sysconfig.get_path('include'), end='')", + ) + .unwrap(); + let openssl_include = + std::env::var_os("DEP_OPENSSL_INCLUDE").expect("unable to find openssl include path"); + let openssl_c = Path::new(&out_dir).join("_openssl.c"); + + let mut build = cc::Build::new(); + build + .file(openssl_c) + .include(python_include) + .include(openssl_include) + .flag_if_supported("-Wconversion") + .flag_if_supported("-Wno-error=sign-conversion"); + + // Enable abi3 mode if we're not using PyPy. + if python_impl != "PyPy" { + // cp36 + // build.define("Py_LIMITED_API", "0x030600f0"); + } + + if cfg!(windows) { + build.define("WIN32_LEAN_AND_MEAN", None); + } + + build.compile("_openssl.a"); +} + +/// Run a python script using the specified interpreter binary. +fn run_python_script(interpreter: impl AsRef, script: &str) -> Result { + let interpreter = interpreter.as_ref(); + let out = Command::new(interpreter) + .env("PYTHONIOENCODING", "utf-8") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .and_then(|mut child| { + child + .stdin + .as_mut() + .expect("piped stdin") + .write_all(script.as_bytes())?; + child.wait_with_output() + }); + + match out { + Err(err) => Err(format!( + "failed to run the Python interpreter at {}: {}", + interpreter.display(), + err + )), + Ok(ok) if !ok.status.success() => Err(format!( + "Python script failed: {}", + String::from_utf8(ok.stderr).expect("failed to parse Python script output as utf-8") + )), + Ok(ok) => Ok( + String::from_utf8(ok.stdout).expect("failed to parse Python script output as utf-8") + ), + } +} + +fn macos_link_search_path() -> Option { + let output = Command::new("clang") + .arg("--print-search-dirs") + .output() + .ok()?; + if !output.status.success() { + println!( + "failed to run 'clang --print-search-dirs', continuing without a link search path" + ); + return None; + } + + let stdout = String::from_utf8_lossy(&output.stdout); + for line in stdout.lines() { + if line.contains("libraries: =") { + let path = line.split('=').nth(1)?; + return Some(format!("{}/lib/darwin", path)); + } + } + + println!("failed to determine link search path, continuing without it"); + None +} diff --git a/src/rust/src/lib.rs b/src/rust/src/lib.rs index 4d1bdd2e9447..776eb6eaa170 100644 --- a/src/rust/src/lib.rs +++ b/src/rust/src/lib.rs @@ -4,11 +4,19 @@ #![deny(rust_2018_idioms)] +// Force linking of OpenSSL since we're not using it in Rust side yet +#[allow(unused_extern_crates)] +extern crate openssl_sys; + mod asn1; mod x509; use std::convert::TryInto; +extern "C" { + fn make_cryptography_openssl_module() -> std::os::raw::c_int; +} + /// Returns the value of the input with the most-significant-bit copied to all /// of the bits. fn duplicate_msb_to_all(a: u8) -> u8 { @@ -91,6 +99,11 @@ fn _rust(py: pyo3::Python<'_>, m: &pyo3::types::PyModule) -> pyo3::PyResult<()> crate::x509::ocsp_resp::add_to_module(ocsp_mod)?; m.add_submodule(ocsp_mod)?; + let openssl_mod = unsafe { + make_cryptography_openssl_module(); + pyo3::types::PyModule::import(py, "_openssl")? + }; + m.add_submodule(openssl_mod)?; Ok(()) } diff --git a/tests/hazmat/backends/test_openssl_memleak.py b/tests/hazmat/backends/test_openssl_memleak.py index d83387c96ff1..d1d672a4f507 100644 --- a/tests/hazmat/backends/test_openssl_memleak.py +++ b/tests/hazmat/backends/test_openssl_memleak.py @@ -24,7 +24,10 @@ def main(argv): import cffi - from cryptography.hazmat.bindings._openssl import ffi, lib + from cryptography.hazmat.bindings._rust import _openssl + + ffi = _openssl.ffi + lib = _openssl.lib heap = {} diff --git a/tox.ini b/tox.ini index 2591c618c271..a732ec6a9e17 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ deps = -e ./vectors pytest-shard>=0.1.2 randomorder: pytest-randomly -passenv = ARCHFLAGS LDFLAGS CFLAGS INCLUDE LIB LD_LIBRARY_PATH RUSTFLAGS CARGO_TARGET_DIR LLVM_PROFILE_FILE OPENSSL_FORCE_FIPS_MODE +passenv = ARCHFLAGS LDFLAGS CFLAGS INCLUDE LIB LD_LIBRARY_PATH RUSTFLAGS CARGO_TARGET_DIR LLVM_PROFILE_FILE OPENSSL_FORCE_FIPS_MODE OPENSSL_DIR OPENSSL_STATIC commands = pip list pytest -n auto --cov=cryptography --cov=tests --durations=10 {posargs} tests/