diff --git a/Cargo.toml b/Cargo.toml index 6976cabd4a4..79582d8e644 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -209,12 +209,11 @@ debug-assertions = false overflow-checks = false rpath = false -# The release profile, used for `cargo build --release` [profile.release] opt-level = 3 -debug = true -codegen-units = 1 # TODO: remove this once rust-lang/rust#50199 and rust-lang/rust#53833 are resolved. -lto = true +debug = false +codegen-units = 1 +lto = "thin" incremental = false panic = 'abort' debug-assertions = false diff --git a/Makefile b/Makefile index 5df43a80924..6e08ea21d48 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,36 @@ +# The TiKV Makefile +# +# These are mostly light rules around cargo and in general developers +# can do daily development using cargo alone. It does do a few key +# things that developers need to understand: +# +# It turns on features that are not on by default in the cargo +# manifest but that are part of the release build (see the +# ENABLE_FEATURES variable). +# +# The `clippy` rule runs clippy with custom configuration, so +# is the correct way to run clippy against TiKV. +# +# The `test` rule runs tests in configurations that are not covered by `cargo +# test` by default. +# +# Important make rules: +# +# - `build` - create a development profile, unoptimized build +# +# - `run` - run a development build +# +# - `test` - run the test suite in a variety of configurations +# +# - `format` - reformat the code with cargo format +# +# - `clippy` - run clippy with tikv-specific configuration +# +# - `dev` - the rule that needs to pass before submitting a PR. It runs +# tests and static analysis including clippy and rustfmt +# +# - `release` - create a release profile, optimized build + SHELL := /bin/bash ENABLE_FEATURES ?= @@ -37,6 +70,7 @@ BIN_PATH = $(CURDIR)/bin GOROOT ?= $(DEPS_PATH)/go CARGO_TARGET_DIR ?= $(CURDIR)/target +# Build-time environment, captured for reporting by the application binary BUILD_INFO_GIT_FALLBACK := "Unknown (no git or not git repo)" BUILD_INFO_RUSTC_FALLBACK := "Unknown" export TIKV_BUILD_TIME := $(shell date -u '+%Y-%m-%d %I:%M:%S') @@ -54,20 +88,14 @@ default: release .PHONY: all -all: format build test +clean: + cargo clean -pre-clippy: unset-override - @rustup component add clippy -clippy: pre-clippy - @cargo clippy --all --all-targets -- \ - -A clippy::module_inception -A clippy::needless_pass_by_value -A clippy::cognitive_complexity \ - -A clippy::unreadable_literal -A clippy::should_implement_trait -A clippy::verbose_bit_mask \ - -A clippy::implicit_hasher -A clippy::large_enum_variant -A clippy::new_without_default \ - -A clippy::neg_cmp_op_on_partial_ord -A clippy::too_many_arguments \ - -A clippy::excessive_precision -A clippy::collapsible_if -A clippy::blacklisted_name \ - -A clippy::needless_range_loop -D rust-2018-idioms -A clippy::redundant_closure \ - -A clippy::match_wild_err_arm -A clippy::blacklisted_name +## Development builds +## ------------------ + +all: format build test dev: format clippy @env FAIL_POINT=1 make test @@ -75,65 +103,70 @@ dev: format clippy build: cargo build --no-default-features --features "${ENABLE_FEATURES}" -ctl: - cargo build --release --no-default-features --features "${ENABLE_FEATURES}" --bin tikv-ctl - @mkdir -p ${BIN_PATH} - @cp -f ${CARGO_TARGET_DIR}/release/tikv-ctl ${BIN_PATH}/ - run: cargo run --no-default-features --features "${ENABLE_FEATURES}" --bin tikv-server + +## Release builds (optimized dev builds) +## ---------------------------- + +# These builds are heavily optimized, but only use thinLTO, not full +# LTO, and they don't include debuginfo by default. + # An optimized build suitable for development and benchmarking, by default built # with RocksDB compiled with the "portable" option, for -march=x86-64 (an # sse2-level instruction set), but with sse4.2 and the PCLMUL instruction # enabled (the "sse" option) release: - make dist_release + cargo build --release --no-default-features --features "${ENABLE_FEATURES}" # An optimized build that builds an "unportable" RocksDB, which means it is # built with -march native. It again includes the "sse" option by default. unportable_release: - make dist_unportable_release + ROCKSDB_SYS_PORTABLE=0 make release # An optimized build with jemalloc memory profiling enabled. prof_release: - make dist_prof_release + ENABLE_FEATURES=mem-profiling make release # An optimized build instrumented with failpoints. # This is used for schrodinger chaos testing. fail_release: - make dist_fail_release + FAIL_POINT=1 make release + + +## Distribution builds (true release builds) +## ------------------- + +# These builds are fully optimized, with LTO, and they contain +# debuginfo. They take a very long time to build, so it is recommended +# not to use them. # The target used by CI/CD to build the distributable release artifacts. # Individual developers should only need to use the `dist_` rules when working # on the CI/CD system. dist_release: - make build_release + make build_dist_release @mkdir -p ${BIN_PATH} @cp -f ${CARGO_TARGET_DIR}/release/tikv-ctl ${CARGO_TARGET_DIR}/release/tikv-server ${CARGO_TARGET_DIR}/release/tikv-importer ${BIN_PATH}/ bash scripts/check-sse4_2.sh +# Build with release flag as if it were for distribution, but without +# additional sanity checks and file movement. +build_dist_release: + make x-build-dist + # Distributable bins with SSE4.2 optimizations dist_unportable_release: - ROCKSDB_SYS_PORTABLE=0 make release - -# Distributable bins with jemalloc memory profiling -dist_prof_release: - ENABLE_FEATURES=mem-profiling make release - -# Distributable bins instrumented with failpoints. -# This is used for schrodinger chaos testing. -dist_fail_release: - FAIL_POINT=1 make release + ROCKSDB_SYS_PORTABLE=0 make dist_release -# Build with release flag -build_release: - cargo build --no-default-features --release --features "${ENABLE_FEATURES}" -# unlike test, this target will trace tests and output logs when fail test is detected. -trace_test: - env CI=true SKIP_FORMAT_CHECK=true FAIL_POINT=1 ${PROJECT_DIR}/ci-build/test.sh +## Testing +## ------- +# Run tests under a variety of conditions. This should pass before +# submitting pull requests. Note though that the CI system tests TiKV +# through its own scripts and does not use this rule. test: # When SIP is enabled, DYLD_LIBRARY_PATH will not work in subshell, so we have to set it # again here. LOCAL_DIR is defined in .travis.yml. @@ -153,6 +186,10 @@ test: fi bash scripts/check-bins-for-jemalloc.sh + +## Static analysis +## --------------- + unset-override: @# unset first in case of any previous overrides @if rustup override list | grep `pwd` > /dev/null; then rustup override unset; fi @@ -164,6 +201,19 @@ format: pre-format @cargo fmt --all -- --check >/dev/null || \ cargo fmt --all +pre-clippy: unset-override + @rustup component add clippy + +clippy: pre-clippy + @cargo clippy --all --all-targets -- \ + -A clippy::module_inception -A clippy::needless_pass_by_value -A clippy::cognitive_complexity \ + -A clippy::unreadable_literal -A clippy::should_implement_trait -A clippy::verbose_bit_mask \ + -A clippy::implicit_hasher -A clippy::large_enum_variant -A clippy::new_without_default \ + -A clippy::neg_cmp_op_on_partial_ord -A clippy::too_many_arguments \ + -A clippy::excessive_precision -A clippy::collapsible_if -A clippy::blacklisted_name \ + -A clippy::needless_range_loop -D rust-2018-idioms -A clippy::redundant_closure \ + -A clippy::match_wild_err_arm -A clippy::blacklisted_name + pre-audit: $(eval LATEST_AUDIT_VERSION := $(strip $(shell cargo search cargo-audit | head -n 1 | awk '{ gsub(/"/, "", $$3); print $$3 }'))) $(eval CURRENT_AUDIT_VERSION = $(strip $(shell (cargo audit --version 2> /dev/null || echo "noop 0") | awk '{ print $$2 }'))) @@ -171,77 +221,41 @@ pre-audit: cargo install cargo-audit --force; \ fi +# Check for security vulnerabilities audit: pre-audit cargo audit -clean: - cargo clean +## Special targets +## --------------- + +# A special target for building just the tikv-ctl binary and release mode and copying it +# into BIN_PATH. It's not clear who uses this for what. If you know please document it. +ctl: + cargo build --release --no-default-features --features "${ENABLE_FEATURES}" --bin tikv-ctl + @mkdir -p ${BIN_PATH} + @cp -f ${CARGO_TARGET_DIR}/release/tikv-ctl ${BIN_PATH}/ + +# A special target for testing only "coprocessor::dag::expr" +# per https://github.com/tikv/tikv/pull/3280 expression: format clippy LOG_LEVEL=ERROR RUST_BACKTRACE=1 cargo test --features "${ENABLE_FEATURES}" "coprocessor::dag::expr" --no-default-features -- --nocapture +## The driver for script/run-cargo.sh +## ---------------------------------- -# The below x- targets are temporary, for experimenting with new profiles, -# specifically in pursuit of compile time speedups. -# -# re https://github.com/tikv/tikv/issues/4189 -# -# The idea here is that there are more "profiles" than just "dev" and "release". -# In particular, there is an optimized dev profile, here "dev-opt". The below -# profiles are intentionally named differently from the stock cargo 'dev' -# and 'release' profiles to avoid confusion, but eventually we might expect -# e.g. dev-opt to become the 'release' and 'bench' profiles, and 'dev-opt' -# to become the 'dev' and perhaps 'test' profiles; with the _real_ release -# profile being created with a config file. -# -# They can be invoked as: -# -# $ make x-build-dev-nopt-quick # An unoptimized build -# # (fastest build / slow run) -# # (no debug assertions or overflow checks) -# $ make x-build-dev-nopt # An unoptimized build -# # (fast build / slow run) -# $ make x-build-dev-opt # A mostly-optimized dev profile -# # (slower build / faster run) -# $ make x-build-prod # A release build -# # (slowest build / fastest run) -# $ make x-bench # Run benches mostly-optimized -# # (slower build / faster run) -# $ make x-test # Run tests unoptimized -# # (fast build / slow run) -# -# Use cases: -# -# testing with fastest turnaround - dev-nopt-quick -# testing - dev-nopt-quick -# casual benchmarking - dev-opt -# benchmarking with full release config - prod -# building the release for publish - prod -# -# The below rules all rely on using a .cargo/config file to override various -# profiles. Within those config files we'll experiment with compile-time -# optimizations which can't be done with Cargo.toml alone. -# -# Eventually, we'll merge as much of the configs into Cargo.toml as possible, -# and merge the below commands into the rest of the makefile. -# -# None of the build profiles has debuginfo on by default because it increases -# the build time by ~20%. The easiest way to build with debuginfo is by setting -# the DEBUG makefile variable, -# -# $ make x-build DEBUG=1 -# -# To pass extra arguments to cargo you can set CARGO_ARGS, +# Cargo only has two non-test profiles, dev and release, and we have +# more than two use cases for which a cargo profile is required. This +# is a hack to manage more cargo profiles, written in +# `etc/cargo.config.*`. These make use of the unstable +# `-Zconfig-profile` cargo option to specify profiles in +# `.cargo/config`, which `scripts/run-cargo.sh copies into place. # -# $ make x-build CARGO_ARGS="--all" +# Presently the only thing this is used for is the `dist_release` +# rules, which are used for producing release builds. -DEV_OPT_CONFIG=etc/cargo.config.dev-opt -DEV_NOPT_CONFIG=etc/cargo.config.dev-nopt -DEV_NOPT_QUICK_CONFIG=etc/cargo.config.dev-nopt-quick -PROD_CONFIG=etc/cargo.config.prod -TEST_CONFIG=etc/cargo.config.test -BENCH_CONFIG=etc/cargo.config.bench +DIST_CONFIG=etc/cargo.config.dist ifneq ($(DEBUG),) export X_DEBUG=${DEBUG} @@ -249,94 +263,9 @@ endif export X_CARGO_ARGS:=${CARGO_ARGS} -x-build-dev-nopt-quick: export X_CARGO_CMD=build -x-build-dev-nopt-quick: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-build-dev-nopt-quick: export X_CARGO_RELEASE=0 -x-build-dev-nopt-quick: export X_CARGO_CONFIG_FILE=${DEV_NOPT_QUICK_CONFIG} -x-build-dev-nopt-quick: +x-build-dist: export X_CARGO_CMD=build +x-build-dist: export X_CARGO_FEATURES=${ENABLE_FEATURES} +x-build-dist: export X_CARGO_RELEASE=1 +x-build-dist: export X_CARGO_CONFIG_FILE=${DIST_CONFIG} +x-build-dist: bash scripts/run-cargo.sh - -x-build-dev-nopt: export X_CARGO_CMD=build -x-build-dev-nopt: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-build-dev-nopt: export X_CARGO_RELEASE=0 -x-build-dev-nopt: export X_CARGO_CONFIG_FILE=${DEV_NOPT_CONFIG} -x-build-dev-nopt: - bash scripts/run-cargo.sh - -x-build-dev-opt: export X_CARGO_CMD=build -x-build-dev-opt: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-build-dev-opt: export X_CARGO_RELEASE=1 -x-build-dev-opt: export X_CARGO_CONFIG_FILE=${DEV_OPT_CONFIG} -x-build-dev-opt: - bash scripts/run-cargo.sh - -x-build-prod: export X_CARGO_CMD=build -x-build-prod: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-build-prod: export X_CARGO_RELEASE=1 -x-build-prod: export X_CARGO_CONFIG_FILE=${PROD_CONFIG} -x-build-prod: - bash scripts/run-cargo.sh - -# "run" commands for the above -# -# these need to be run with CARGO_ARGS="--bin tikv-server" etc - -x-run-dev-nopt-quick: export X_CARGO_CMD=run -x-run-dev-nopt-quick: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-run-dev-nopt-quick: export X_CARGO_RELEASE=0 -x-run-dev-nopt-quick: export X_CARGO_CONFIG_FILE=${DEV_NOPT_QUICK_CONFIG} -x-run-dev-nopt-quick: - bash scripts/run-cargo.sh - -x-run-dev-nopt: export X_CARGO_CMD=run -x-run-dev-nopt: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-run-dev-nopt: export X_CARGO_RELEASE=0 -x-run-dev-nopt: export X_CARGO_CONFIG_FILE=${DEV_NOPT_CONFIG} -x-run-dev-nopt: - bash scripts/run-cargo.sh - -x-run-dev-opt: export X_CARGO_CMD=run -x-run-dev-opt: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-run-dev-opt: export X_CARGO_RELEASE=1 -x-run-dev-opt: export X_CARGO_CONFIG_FILE=${DEV_OPT_CONFIG} -x-run-dev-opt: - bash scripts/run-cargo.sh - -x-run-prod: export X_CARGO_CMD=run -x-run-prod: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-run-prod: export X_CARGO_RELEASE=1 -x-run-prod: export X_CARGO_CONFIG_FILE=${PROD_CONFIG} -x-run-prod: - bash scripts/run-cargo.sh - -# bench and test targets - -x-test: export X_CARGO_CMD=test -x-test: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-test: export X_CARGO_RELEASE=0 -x-test: export X_CARGO_CONFIG_FILE=${TEST_CONFIG} -x-test: - bash scripts/run-cargo.sh - -x-bench: export X_CARGO_CMD=bench -x-bench: export X_CARGO_FEATURES=${ENABLE_FEATURES} -x-bench: export X_CARGO_RELEASE=0 -x-bench: export X_CARGO_CONFIG_FILE=${BENCH_CONFIG} -x-bench: - bash scripts/run-cargo.sh - -# Devs might want to use the config files but not the makefiles. -# These are rules to put each config file in place. - -x-dev-opt-config: - mkdir -p .cargo && cp -b "${DEV_OPT_CONFIG}" .cargo/config - -x-dev-nopt-config: - mkdir -p .cargo && cp -b "${DEV_NOPT_CONFIG}" .cargo/config - -x-prod-config: - mkdir -p .cargo && cp -b "${PROD_CONFIG}" .cargo/config - -x-clean: - -rm -r .cargo - cargo clean diff --git a/etc/cargo.config.bench b/etc/cargo.config.bench deleted file mode 100644 index dd5854e05df..00000000000 --- a/etc/cargo.config.bench +++ /dev/null @@ -1,15 +0,0 @@ -# Configuration for `cargo bench`, similar to dev-opt. -# For some reason we override the bench profile by -# overriding the release profile. -# See https://github.com/rust-lang/rust/issues/48683#issuecomment-473731715 - -[profile.release] -opt-level = 3 -debug = false -rpath = false -lto = "thin" -debug-assertions = false -codegen-units = 1 -# panic = "unwind" // can't set panic for bench profile -incremental = true -overflow-checks = false diff --git a/etc/cargo.config.dev-nopt b/etc/cargo.config.dev-nopt deleted file mode 100644 index d1296c658fc..00000000000 --- a/etc/cargo.config.dev-nopt +++ /dev/null @@ -1,12 +0,0 @@ -# Configuration for `cargo build` without `--release`. - -[profile.dev] -opt-level = 0 -debug = false -rpath = false -lto = false -debug-assertions = true -codegen-units = 4 -panic = "abort" -incremental = true -overflow-checks = true diff --git a/etc/cargo.config.dev-nopt-quick b/etc/cargo.config.dev-nopt-quick deleted file mode 100644 index 5203102c020..00000000000 --- a/etc/cargo.config.dev-nopt-quick +++ /dev/null @@ -1,12 +0,0 @@ -# Configuration for `cargo build` without `--release`. - -[profile.dev] -opt-level = 0 -debug = false -rpath = false -lto = false -debug-assertions = false -codegen-units = 4 -panic = "abort" -incremental = true -overflow-checks = false diff --git a/etc/cargo.config.dev-opt b/etc/cargo.config.dev-opt deleted file mode 100644 index ecc57c3554d..00000000000 --- a/etc/cargo.config.dev-opt +++ /dev/null @@ -1,12 +0,0 @@ -# Configuration for casual benchmarking. Fast build + fast runtime. - -[profile.release] -opt-level = 3 -debug = false -rpath = false -lto = "thin" -debug-assertions = false -codegen-units = 1 -panic = "abort" -incremental = false -overflow-checks = false diff --git a/etc/cargo.config.prod b/etc/cargo.config.dist similarity index 100% rename from etc/cargo.config.prod rename to etc/cargo.config.dist diff --git a/etc/cargo.config.test b/etc/cargo.config.test deleted file mode 100644 index a09c56d82ff..00000000000 --- a/etc/cargo.config.test +++ /dev/null @@ -1,15 +0,0 @@ -# Configuration for `cargo test`, similar to dev-nopt. -# For some reason we override the test profile by -# overriding the debug profile. -# See https://github.com/rust-lang/rust/issues/48683#issuecomment-473731715 - -[profile.dev] -opt-level = 0 -debug = 1 # enable line numbers by default for easy test debugging -rpath = false -lto = false -debug-assertions = true -codegen-units = 16 -# panic = "unwind" // can't set panic for test profile -incremental = true -overflow-checks = true