From ce1bab47060f78669d5382484957a57296cbfbff Mon Sep 17 00:00:00 2001 From: eksperimental Date: Fri, 25 Jan 2019 11:39:55 +0700 Subject: [PATCH 01/19] Add Reproducible Build feature to CI See https://github.com/elixir-lang/elixir/issues/8689 for more information --- .travis.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4bf615f3dbe..673a7854e3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,29 @@ install: - PATH=$(pwd)/otp/bin:$PATH script: + - export SOURCE_DATE_EPOCH=$(date +%s) - ELIXIRC_OPTS="--warnings-as-errors" ERLC_OPTS="+warning_as_errors" make compile + - mkdir -p lib/elixir/ebin_reproducible/ + lib/eex/ebin_reproducible/ + lib/iex/ebin_reproducible/ + lib/logger/ebin_reproducible/ + lib/mix/ebin_reproducible/ + - mv lib/elixir/ebin/* lib/elixir/ebin_reproducible/ + - mv lib/eex/ebin/* lib/eex/ebin_reproducible/ + - mv lib/iex/ebin/* lib/iex/ebin_reproducible/ + - mv lib/logger/ebin/* lib/logger/ebin_reproducible/ + - mv lib/mix/ebin/* lib/mix/ebin_reproducible/ + - ELIXIRC_OPTS="--warnings-as-errors" ERLC_OPTS="+warning_as_errors" make compile + - diff -r lib/elixir/ebin/ lib/elixir/ebin_reproducible/ + - diff -r lib/eex/ebin/ lib/eex/ebin_reproducible/ + - diff -r lib/iex/ebin/ lib/iex/ebin_reproducible/ + - diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ + - diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ + - rm -rf lib/elixir/ebin_reproducible/ + lib/eex/ebin_reproducible/ + lib/iex/ebin_reproducible/ + lib/logger/ebin_reproducible/ + lib/mix/ebin_reproducible/ - rm -rf .git - make test - dialyzer -pa lib/elixir/ebin --build_plt --output_plt elixir.plt --apps lib/elixir/ebin/elixir.beam lib/elixir/ebin/Elixir.Kernel.beam From 0083ab6b8e85735a9cd48b55e66b44fdba459431 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Fri, 25 Jan 2019 22:12:56 +0700 Subject: [PATCH 02/19] Move commands after tests --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 673a7854e3a..099d091e28c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,9 @@ install: script: - export SOURCE_DATE_EPOCH=$(date +%s) - ELIXIRC_OPTS="--warnings-as-errors" ERLC_OPTS="+warning_as_errors" make compile + - rm -rf .git + - make test + - dialyzer -pa lib/elixir/ebin --build_plt --output_plt elixir.plt --apps lib/elixir/ebin/elixir.beam lib/elixir/ebin/Elixir.Kernel.beam - mkdir -p lib/elixir/ebin_reproducible/ lib/eex/ebin_reproducible/ lib/iex/ebin_reproducible/ @@ -52,9 +55,6 @@ script: lib/iex/ebin_reproducible/ lib/logger/ebin_reproducible/ lib/mix/ebin_reproducible/ - - rm -rf .git - - make test - - dialyzer -pa lib/elixir/ebin --build_plt --output_plt elixir.plt --apps lib/elixir/ebin/elixir.beam lib/elixir/ebin/Elixir.Kernel.beam notifications: recipients: From 67c4d426b58e49ee3b9c8c049724b1ec3a04b0cd Mon Sep 17 00:00:00 2001 From: eksperimental Date: Fri, 25 Jan 2019 22:15:17 +0700 Subject: [PATCH 03/19] Run "make compile" after without .git dir --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 099d091e28c..1d9cf84cfed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ script: - export SOURCE_DATE_EPOCH=$(date +%s) - ELIXIRC_OPTS="--warnings-as-errors" ERLC_OPTS="+warning_as_errors" make compile - rm -rf .git + - make compile - make test - dialyzer -pa lib/elixir/ebin --build_plt --output_plt elixir.plt --apps lib/elixir/ebin/elixir.beam lib/elixir/ebin/Elixir.Kernel.beam - mkdir -p lib/elixir/ebin_reproducible/ From 3674ce78a1306dcb4d4a7bb138dec1b310cd7b63 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 00:00:13 +0700 Subject: [PATCH 04/19] do not delete files in ebin_reproducible --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1d9cf84cfed..3be163bff43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,11 +51,6 @@ script: - diff -r lib/iex/ebin/ lib/iex/ebin_reproducible/ - diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ - diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ - - rm -rf lib/elixir/ebin_reproducible/ - lib/eex/ebin_reproducible/ - lib/iex/ebin_reproducible/ - lib/logger/ebin_reproducible/ - lib/mix/ebin_reproducible/ notifications: recipients: From f3fb8a3fc0451748ac441284a5bd6f44b5197268 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 00:34:10 +0700 Subject: [PATCH 05/19] Only execute reproducible build in lastest OTP release --- .travis.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3be163bff43..76cba604a4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ sudo: false env: global: - ELIXIR_ASSERT_TIMEOUT=2000 + - LAST_OTP_RELEASE=OTP-21.2 matrix: - OTP_RELEASE=OTP-20.0 - OTP_RELEASE=OTP-20.1 @@ -52,6 +53,24 @@ script: - diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ - diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ + # Check for reproducible build only in last OTP release + - | + if [ "${OTP_RELEASE}" == "${LAST_OTP_RELEASE}" ]; then + echo "Checking for reproducible builds" + mv lib/elixir/ebin/* lib/elixir/ebin_reproducible/ + mv lib/eex/ebin/* lib/eex/ebin_reproducible/ + mv lib/iex/ebin/* lib/iex/ebin_reproducible/ + mv lib/logger/ebin/* lib/logger/ebin_reproducible/ + mv lib/mix/ebin/* lib/mix/ebin_reproducible/ + make compile + diff -r lib/elixir/ebin/ lib/elixir/ebin_reproducible/ + diff -r lib/eex/ebin/ lib/eex/ebin_reproducible/ + diff -r lib/iex/ebin/ lib/iex/ebin_reproducible/ + diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ + diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ + fi; +>>>>>>> Only execute reproducible build in lastest OTP release + notifications: recipients: - jose.valim@gmail.com From 7cc162068aab19c16339ac33398199db64803fd2 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 02:53:11 +0700 Subject: [PATCH 06/19] run tests in the future --- .travis.yml | 26 ++++++++++++++++++++++---- lib/elixir/test/elixir/file_test.exs | 1 + 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 76cba604a4b..0dfc60422c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,14 @@ language: bash sudo: false +before_install: + - sudo apt-get install -y faketime + env: global: - ELIXIR_ASSERT_TIMEOUT=2000 - - LAST_OTP_RELEASE=OTP-21.2 + - LATEST_OTP_RELEASE=false + - FUTURE=false matrix: - OTP_RELEASE=OTP-20.0 - OTP_RELEASE=OTP-20.1 @@ -12,7 +16,7 @@ env: - OTP_RELEASE=OTP-20.3 - OTP_RELEASE=OTP-21.0 - OTP_RELEASE=OTP-21.1 - - OTP_RELEASE=OTP-21.2 + - OTP_RELEASE=OTP-21.2 LATEST_OTP_RELEASE=true FUTURE=true - OTP_RELEASE=maint - OTP_RELEASE=master @@ -21,6 +25,7 @@ matrix: allow_failures: - env: OTP_RELEASE=maint - env: OTP_RELEASE=master + - env: OTP_RELEASE=OTP-21.2 FUTURE=true install: - wget -O otp.tar.gz https://repo.hex.pm/builds/otp/ubuntu-14.04/${OTP_RELEASE}.tar.gz @@ -30,6 +35,7 @@ install: - PATH=$(pwd)/otp/bin:$PATH script: + # Check for test in the future, only in the latest OTP release - export SOURCE_DATE_EPOCH=$(date +%s) - ELIXIRC_OPTS="--warnings-as-errors" ERLC_OPTS="+warning_as_errors" make compile - rm -rf .git @@ -53,10 +59,22 @@ script: - diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ - diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ - # Check for reproducible build only in last OTP release + # Run tests in the future + - | + if [ "${FUTURE}" == true ]; then + echo "Emulate \"make test\" in 12 years" + faketime -f '+12y' make test + fi; + + # Check for reproducible build and test in the future, only in the latest OTP release - | - if [ "${OTP_RELEASE}" == "${LAST_OTP_RELEASE}" ]; then + if [ "${LATEST_OTP_RELEASE}" == true ]; then echo "Checking for reproducible builds" + mkdir -p lib/elixir/ebin_reproducible/ + lib/eex/ebin_reproducible/ + lib/iex/ebin_reproducible/ + lib/logger/ebin_reproducible/ + lib/mix/ebin_reproducible/ mv lib/elixir/ebin/* lib/elixir/ebin_reproducible/ mv lib/eex/ebin/* lib/eex/ebin_reproducible/ mv lib/iex/ebin/* lib/iex/ebin_reproducible/ diff --git a/lib/elixir/test/elixir/file_test.exs b/lib/elixir/test/elixir/file_test.exs index 4f53577877b..b6e66c5146d 100644 --- a/lib/elixir/test/elixir/file_test.exs +++ b/lib/elixir/test/elixir/file_test.exs @@ -1901,6 +1901,7 @@ defmodule FileTest do end) == :cd_result end + @tag :skip test "touch with no file" do fixture = tmp_path("tmp_test.txt") time = {{2010, 4, 17}, {14, 0, 0}} From 27697559e72c9e1d8ecd3117aa3fd9207b5a16c2 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 03:31:02 +0700 Subject: [PATCH 07/19] Revert changes related to Run tests in the Future --- .travis.yml | 17 ++--------------- lib/elixir/test/elixir/file_test.exs | 1 - 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0dfc60422c8..4822b161974 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,10 @@ language: bash sudo: false -before_install: - - sudo apt-get install -y faketime - env: global: - ELIXIR_ASSERT_TIMEOUT=2000 - LATEST_OTP_RELEASE=false - - FUTURE=false matrix: - OTP_RELEASE=OTP-20.0 - OTP_RELEASE=OTP-20.1 @@ -16,7 +12,7 @@ env: - OTP_RELEASE=OTP-20.3 - OTP_RELEASE=OTP-21.0 - OTP_RELEASE=OTP-21.1 - - OTP_RELEASE=OTP-21.2 LATEST_OTP_RELEASE=true FUTURE=true + - OTP_RELEASE=OTP-21.2 LATEST_OTP_RELEASE=true - OTP_RELEASE=maint - OTP_RELEASE=master @@ -25,7 +21,6 @@ matrix: allow_failures: - env: OTP_RELEASE=maint - env: OTP_RELEASE=master - - env: OTP_RELEASE=OTP-21.2 FUTURE=true install: - wget -O otp.tar.gz https://repo.hex.pm/builds/otp/ubuntu-14.04/${OTP_RELEASE}.tar.gz @@ -35,7 +30,6 @@ install: - PATH=$(pwd)/otp/bin:$PATH script: - # Check for test in the future, only in the latest OTP release - export SOURCE_DATE_EPOCH=$(date +%s) - ELIXIRC_OPTS="--warnings-as-errors" ERLC_OPTS="+warning_as_errors" make compile - rm -rf .git @@ -59,14 +53,7 @@ script: - diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ - diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ - # Run tests in the future - - | - if [ "${FUTURE}" == true ]; then - echo "Emulate \"make test\" in 12 years" - faketime -f '+12y' make test - fi; - - # Check for reproducible build and test in the future, only in the latest OTP release + # Check for reproducible builds only in the latest OTP release - | if [ "${LATEST_OTP_RELEASE}" == true ]; then echo "Checking for reproducible builds" diff --git a/lib/elixir/test/elixir/file_test.exs b/lib/elixir/test/elixir/file_test.exs index b6e66c5146d..4f53577877b 100644 --- a/lib/elixir/test/elixir/file_test.exs +++ b/lib/elixir/test/elixir/file_test.exs @@ -1901,7 +1901,6 @@ defmodule FileTest do end) == :cd_result end - @tag :skip test "touch with no file" do fixture = tmp_path("tmp_test.txt") time = {{2010, 4, 17}, {14, 0, 0}} From acd1d706ec2d498532704a3e99f47b0e1a2e2fca Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 04:17:01 +0700 Subject: [PATCH 08/19] Run "make clean" before second "make compile" --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4822b161974..4c7f58157c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ script: mv lib/iex/ebin/* lib/iex/ebin_reproducible/ mv lib/logger/ebin/* lib/logger/ebin_reproducible/ mv lib/mix/ebin/* lib/mix/ebin_reproducible/ + make clean make compile diff -r lib/elixir/ebin/ lib/elixir/ebin_reproducible/ diff -r lib/eex/ebin/ lib/eex/ebin_reproducible/ From db810db24a3323ccfbafa27b36662edb634c073c Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 13:27:14 +0700 Subject: [PATCH 09/19] Move to Makefile --- .gitignore | 1 + .travis.yml | 40 ++-------------------------------------- Makefile | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 0760a308146..d5445818359 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /doc/ /lib/*/ebin/ +/lib/*/ebin_reproducible/ /lib/*/_build/ /lib/*/tmp/ /lib/elixir/src/*_parser.erl diff --git a/.travis.yml b/.travis.yml index 4c7f58157c5..69e076a2102 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,52 +30,16 @@ install: - PATH=$(pwd)/otp/bin:$PATH script: - - export SOURCE_DATE_EPOCH=$(date +%s) - - ELIXIRC_OPTS="--warnings-as-errors" ERLC_OPTS="+warning_as_errors" make compile - rm -rf .git - - make compile + - ELIXIRC_OPTS="--warnings-as-errors" ERLC_OPTS="+warning_as_errors" make compile - make test - dialyzer -pa lib/elixir/ebin --build_plt --output_plt elixir.plt --apps lib/elixir/ebin/elixir.beam lib/elixir/ebin/Elixir.Kernel.beam - - mkdir -p lib/elixir/ebin_reproducible/ - lib/eex/ebin_reproducible/ - lib/iex/ebin_reproducible/ - lib/logger/ebin_reproducible/ - lib/mix/ebin_reproducible/ - - mv lib/elixir/ebin/* lib/elixir/ebin_reproducible/ - - mv lib/eex/ebin/* lib/eex/ebin_reproducible/ - - mv lib/iex/ebin/* lib/iex/ebin_reproducible/ - - mv lib/logger/ebin/* lib/logger/ebin_reproducible/ - - mv lib/mix/ebin/* lib/mix/ebin_reproducible/ - - ELIXIRC_OPTS="--warnings-as-errors" ERLC_OPTS="+warning_as_errors" make compile - - diff -r lib/elixir/ebin/ lib/elixir/ebin_reproducible/ - - diff -r lib/eex/ebin/ lib/eex/ebin_reproducible/ - - diff -r lib/iex/ebin/ lib/iex/ebin_reproducible/ - - diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ - - diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ # Check for reproducible builds only in the latest OTP release - | if [ "${LATEST_OTP_RELEASE}" == true ]; then - echo "Checking for reproducible builds" - mkdir -p lib/elixir/ebin_reproducible/ - lib/eex/ebin_reproducible/ - lib/iex/ebin_reproducible/ - lib/logger/ebin_reproducible/ - lib/mix/ebin_reproducible/ - mv lib/elixir/ebin/* lib/elixir/ebin_reproducible/ - mv lib/eex/ebin/* lib/eex/ebin_reproducible/ - mv lib/iex/ebin/* lib/iex/ebin_reproducible/ - mv lib/logger/ebin/* lib/logger/ebin_reproducible/ - mv lib/mix/ebin/* lib/mix/ebin_reproducible/ - make clean - make compile - diff -r lib/elixir/ebin/ lib/elixir/ebin_reproducible/ - diff -r lib/eex/ebin/ lib/eex/ebin_reproducible/ - diff -r lib/iex/ebin/ lib/iex/ebin_reproducible/ - diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ - diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ + make check_reproducible fi; ->>>>>>> Only execute reproducible build in lastest OTP release notifications: recipients: diff --git a/Makefile b/Makefile index 4af710c8e7b..b506faeff21 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ INSTALL_PROGRAM = $(INSTALL) -m755 GIT_REVISION = $(strip $(shell git rev-parse HEAD 2> /dev/null )) GIT_TAG = $(strip $(shell head="$(call GIT_REVISION)"; git tag --points-at $$head 2> /dev/null | tail -1) ) -.PHONY: install compile erlang elixir unicode app build_plt clean_plt dialyze test clean clean_residual_files install_man clean_man docs Docs.zip Precompiled.zip zips +.PHONY: install compile erlang elixir unicode app build_plt clean_plt dialyze test check_reproducible clean clean_residual_files clean_reproducible_files install_man clean_man docs Docs.zip Precompiled.zip zips .NOTPARALLEL: compile #==> Functions @@ -113,11 +113,40 @@ install: compile done $(MAKE) install_man +check_reproducible: compile + $(Q) echo "==> Checking for reproducible builds..." + $(Q) SOURCE_DATE_EPOCH=$(shell bin/elixir -e 'IO.puts System.build_info()[:date] |> \ + DateTime.from_iso8601() |> \ + elem(1) |> \ + DateTime.to_unix()') + $(Q) $(MAKE) clean_reproducible_files + $(Q) echo "Moving files to ebin_reproducible/ ..." + $(Q) mkdir -p lib/elixir/ebin_reproducible/ \ + lib/eex/ebin_reproducible/ \ + lib/iex/ebin_reproducible/ \ + lib/logger/ebin_reproducible/ \ + lib/mix/ebin_reproducible/ + $(Q) mv lib/elixir/ebin/* lib/elixir/ebin_reproducible/ + $(Q) mv lib/eex/ebin/* lib/eex/ebin_reproducible/ + $(Q) mv lib/iex/ebin/* lib/iex/ebin_reproducible/ + $(Q) mv lib/logger/ebin/* lib/logger/ebin_reproducible/ + $(Q) mv lib/mix/ebin/* lib/mix/ebin_reproducible/ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}" $(MAKE) compile + $(Q) echo "Diffing..." + $(Q) diff -r lib/elixir/ebin/ lib/elixir/ebin_reproducible/ + $(Q) diff -r lib/eex/ebin/ lib/eex/ebin_reproducible/ + $(Q) diff -r lib/iex/ebin/ lib/iex/ebin_reproducible/ + $(Q) diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ + $(Q) diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ + $(Q) $(MAKE) clean_reproducible_files + $(Q) echo "Builds are reproducible" + clean: rm -rf ebin rm -rf lib/*/ebin rm -rf $(PARSER) $(Q) $(MAKE) clean_residual_files + $(Q) $(MAKE) clean_reproducible_files clean_elixir: $(Q) rm -f lib/*/ebin/Elixir.*.beam @@ -133,6 +162,9 @@ clean_residual_files: rm -f erl_crash.dump $(Q) $(MAKE) clean_man +clean_reproducible_files: + rm -rf lib/*/ebin_reproducible + #==> Documentation tasks LOGO_PATH = $(shell test -f ../docs/logo.png && echo "--logo ../docs/logo.png") From da635ac603cd7a76848df03d4404e4aefd02e547 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 14:38:26 +0700 Subject: [PATCH 10/19] fix --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b506faeff21..5821fb2d4e4 100644 --- a/Makefile +++ b/Makefile @@ -115,10 +115,10 @@ install: compile check_reproducible: compile $(Q) echo "==> Checking for reproducible builds..." - $(Q) SOURCE_DATE_EPOCH=$(shell bin/elixir -e 'IO.puts System.build_info()[:date] |> \ + $(Q) SOURCE_DATE_EPOCH="$(shell bin/elixir -e 'IO.puts System.build_info()[:date] |> \ DateTime.from_iso8601() |> \ elem(1) |> \ - DateTime.to_unix()') + DateTime.to_unix()')" $(Q) $(MAKE) clean_reproducible_files $(Q) echo "Moving files to ebin_reproducible/ ..." $(Q) mkdir -p lib/elixir/ebin_reproducible/ \ @@ -131,7 +131,7 @@ check_reproducible: compile $(Q) mv lib/iex/ebin/* lib/iex/ebin_reproducible/ $(Q) mv lib/logger/ebin/* lib/logger/ebin_reproducible/ $(Q) mv lib/mix/ebin/* lib/mix/ebin_reproducible/ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}" $(MAKE) compile + SOURCE_DATE_EPOCH="$(call SOURCE_DATE_EPOCH)" $(MAKE) compile $(Q) echo "Diffing..." $(Q) diff -r lib/elixir/ebin/ lib/elixir/ebin_reproducible/ $(Q) diff -r lib/eex/ebin/ lib/eex/ebin_reproducible/ From 86c19d0086d1f5a45762e55e36a42d41b353f0d7 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 14:54:35 +0700 Subject: [PATCH 11/19] export SOURCE_DATE_EPOCH in Makefile to see if it works in Travis --- .travis.yml | 14 +++++++------- Makefile | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 69e076a2102..2a9c3e827b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,14 +7,14 @@ env: - LATEST_OTP_RELEASE=false matrix: - OTP_RELEASE=OTP-20.0 - - OTP_RELEASE=OTP-20.1 - - OTP_RELEASE=OTP-20.2 - - OTP_RELEASE=OTP-20.3 - - OTP_RELEASE=OTP-21.0 - - OTP_RELEASE=OTP-21.1 + # - OTP_RELEASE=OTP-20.1 + # - OTP_RELEASE=OTP-20.2 + # - OTP_RELEASE=OTP-20.3 + # - OTP_RELEASE=OTP-21.0 + # - OTP_RELEASE=OTP-21.1 - OTP_RELEASE=OTP-21.2 LATEST_OTP_RELEASE=true - - OTP_RELEASE=maint - - OTP_RELEASE=master + # - OTP_RELEASE=maint + # - OTP_RELEASE=master matrix: fast_finish: true diff --git a/Makefile b/Makefile index 5821fb2d4e4..6aaf1d49a52 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,7 @@ install: compile check_reproducible: compile $(Q) echo "==> Checking for reproducible builds..." - $(Q) SOURCE_DATE_EPOCH="$(shell bin/elixir -e 'IO.puts System.build_info()[:date] |> \ + $(Q) export SOURCE_DATE_EPOCH="$(shell bin/elixir -e 'IO.puts System.build_info()[:date] |> \ DateTime.from_iso8601() |> \ elem(1) |> \ DateTime.to_unix()')" From 170f76c42c7f9a6c2cdb99f8e5958889b2c10b56 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 15:08:05 +0700 Subject: [PATCH 12/19] fix over fix --- Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6aaf1d49a52..464939331c9 100644 --- a/Makefile +++ b/Makefile @@ -115,10 +115,11 @@ install: compile check_reproducible: compile $(Q) echo "==> Checking for reproducible builds..." - $(Q) export SOURCE_DATE_EPOCH="$(shell bin/elixir -e 'IO.puts System.build_info()[:date] |> \ - DateTime.from_iso8601() |> \ - elem(1) |> \ - DateTime.to_unix()')" + $(Q) export SOURCE_DATE_EPOCH="$(shell bin/elixir -e \ + 'IO.puts System.build_info()[:date] |> \ + DateTime.from_iso8601() |> \ + elem(1) |> \ + DateTime.to_unix()')" $(Q) $(MAKE) clean_reproducible_files $(Q) echo "Moving files to ebin_reproducible/ ..." $(Q) mkdir -p lib/elixir/ebin_reproducible/ \ @@ -131,7 +132,7 @@ check_reproducible: compile $(Q) mv lib/iex/ebin/* lib/iex/ebin_reproducible/ $(Q) mv lib/logger/ebin/* lib/logger/ebin_reproducible/ $(Q) mv lib/mix/ebin/* lib/mix/ebin_reproducible/ - SOURCE_DATE_EPOCH="$(call SOURCE_DATE_EPOCH)" $(MAKE) compile + $(MAKE) compile $(Q) echo "Diffing..." $(Q) diff -r lib/elixir/ebin/ lib/elixir/ebin_reproducible/ $(Q) diff -r lib/eex/ebin/ lib/eex/ebin_reproducible/ From ac632a3255160a47b44d2460fb15e2d49c0ba0b2 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 15:28:08 +0700 Subject: [PATCH 13/19] implement Jose's suggestions --- .gitignore | 1 - .travis.yml | 8 ++------ Makefile | 45 ++++++++++++++++++++++----------------------- 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index d5445818359..0760a308146 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /doc/ /lib/*/ebin/ -/lib/*/ebin_reproducible/ /lib/*/_build/ /lib/*/tmp/ /lib/elixir/src/*_parser.erl diff --git a/.travis.yml b/.travis.yml index 2a9c3e827b5..c27d201145f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ sudo: false env: global: - ELIXIR_ASSERT_TIMEOUT=2000 - - LATEST_OTP_RELEASE=false matrix: - OTP_RELEASE=OTP-20.0 # - OTP_RELEASE=OTP-20.1 @@ -12,7 +11,7 @@ env: # - OTP_RELEASE=OTP-20.3 # - OTP_RELEASE=OTP-21.0 # - OTP_RELEASE=OTP-21.1 - - OTP_RELEASE=OTP-21.2 LATEST_OTP_RELEASE=true + - OTP_RELEASE=OTP-21.2 CHECK_REPRODUCIBLE=true # - OTP_RELEASE=maint # - OTP_RELEASE=master @@ -36,10 +35,7 @@ script: - dialyzer -pa lib/elixir/ebin --build_plt --output_plt elixir.plt --apps lib/elixir/ebin/elixir.beam lib/elixir/ebin/Elixir.Kernel.beam # Check for reproducible builds only in the latest OTP release - - | - if [ "${LATEST_OTP_RELEASE}" == true ]; then - make check_reproducible - fi; + - if [ -n "$CHECK_REPRODUCIBLE" ]; then make check_reproducible; fi notifications: recipients: diff --git a/Makefile b/Makefile index 464939331c9..ef0cfc88905 100644 --- a/Makefile +++ b/Makefile @@ -115,30 +115,30 @@ install: compile check_reproducible: compile $(Q) echo "==> Checking for reproducible builds..." - $(Q) export SOURCE_DATE_EPOCH="$(shell bin/elixir -e \ - 'IO.puts System.build_info()[:date] |> \ - DateTime.from_iso8601() |> \ - elem(1) |> \ - DateTime.to_unix()')" + export SOURCE_DATE_EPOCH="$(shell bin/elixir -e \ + 'IO.puts System.build_info()[:date] \ + |> DateTime.from_iso8601() \ + |> elem(1) \ + |> DateTime.to_unix()')" $(Q) $(MAKE) clean_reproducible_files $(Q) echo "Moving files to ebin_reproducible/ ..." - $(Q) mkdir -p lib/elixir/ebin_reproducible/ \ - lib/eex/ebin_reproducible/ \ - lib/iex/ebin_reproducible/ \ - lib/logger/ebin_reproducible/ \ - lib/mix/ebin_reproducible/ - $(Q) mv lib/elixir/ebin/* lib/elixir/ebin_reproducible/ - $(Q) mv lib/eex/ebin/* lib/eex/ebin_reproducible/ - $(Q) mv lib/iex/ebin/* lib/iex/ebin_reproducible/ - $(Q) mv lib/logger/ebin/* lib/logger/ebin_reproducible/ - $(Q) mv lib/mix/ebin/* lib/mix/ebin_reproducible/ - $(MAKE) compile + $(Q) mkdir -p lib/elixir/tmp/ebin_reproducible/ \ + lib/eex/tmp/ebin_reproducible/ \ + lib/iex/tmp/ebin_reproducible/ \ + lib/logger/tmp/ebin_reproducible/ \ + lib/mix/tmp/ebin_reproducible/ + $(Q) mv lib/elixir/ebin/* lib/elixir/tmp/ebin_reproducible/ + $(Q) mv lib/eex/ebin/* lib/eex/tmp/ebin_reproducible/ + $(Q) mv lib/iex/ebin/* lib/iex/tmp/ebin_reproducible/ + $(Q) mv lib/logger/ebin/* lib/logger/tmp/ebin_reproducible/ + $(Q) mv lib/mix/ebin/* lib/mix/tmp/ebin_reproducible/ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}" $(MAKE) compile $(Q) echo "Diffing..." - $(Q) diff -r lib/elixir/ebin/ lib/elixir/ebin_reproducible/ - $(Q) diff -r lib/eex/ebin/ lib/eex/ebin_reproducible/ - $(Q) diff -r lib/iex/ebin/ lib/iex/ebin_reproducible/ - $(Q) diff -r lib/logger/ebin/ lib/logger/ebin_reproducible/ - $(Q) diff -r lib/mix/ebin/ lib/mix/ebin_reproducible/ + $(Q) diff -r lib/elixir/ebin/ lib/elixir/tmp/ebin_reproducible/ + $(Q) diff -r lib/eex/ebin/ lib/eex/tmp/ebin_reproducible/ + $(Q) diff -r lib/iex/ebin/ lib/iex/tmp/ebin_reproducible/ + $(Q) diff -r lib/logger/ebin/ lib/logger/tmp/ebin_reproducible/ + $(Q) diff -r lib/mix/ebin/ lib/mix/tmp/ebin_reproducible/ $(Q) $(MAKE) clean_reproducible_files $(Q) echo "Builds are reproducible" @@ -147,7 +147,6 @@ clean: rm -rf lib/*/ebin rm -rf $(PARSER) $(Q) $(MAKE) clean_residual_files - $(Q) $(MAKE) clean_reproducible_files clean_elixir: $(Q) rm -f lib/*/ebin/Elixir.*.beam @@ -164,7 +163,7 @@ clean_residual_files: $(Q) $(MAKE) clean_man clean_reproducible_files: - rm -rf lib/*/ebin_reproducible + rm -rf lib/*/tmp/ebin_reproducible #==> Documentation tasks From 63d8bb595e6e5e8883cca9d42d32ce9a7dd906db Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sat, 26 Jan 2019 15:42:35 +0700 Subject: [PATCH 14/19] don't clean files at the very end --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index ef0cfc88905..2940786dd29 100644 --- a/Makefile +++ b/Makefile @@ -139,7 +139,6 @@ check_reproducible: compile $(Q) diff -r lib/iex/ebin/ lib/iex/tmp/ebin_reproducible/ $(Q) diff -r lib/logger/ebin/ lib/logger/tmp/ebin_reproducible/ $(Q) diff -r lib/mix/ebin/ lib/mix/tmp/ebin_reproducible/ - $(Q) $(MAKE) clean_reproducible_files $(Q) echo "Builds are reproducible" clean: From 9b43b0e8a84059f68eb397ed76f494e522caf9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 26 Jan 2019 09:52:18 +0100 Subject: [PATCH 15/19] Update Makefile --- Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 2940786dd29..d4f4b6e57f8 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ INSTALL_PROGRAM = $(INSTALL) -m755 GIT_REVISION = $(strip $(shell git rev-parse HEAD 2> /dev/null )) GIT_TAG = $(strip $(shell head="$(call GIT_REVISION)"; git tag --points-at $$head 2> /dev/null | tail -1) ) -.PHONY: install compile erlang elixir unicode app build_plt clean_plt dialyze test check_reproducible clean clean_residual_files clean_reproducible_files install_man clean_man docs Docs.zip Precompiled.zip zips +.PHONY: install compile erlang elixir unicode app build_plt clean_plt dialyze test check_reproducible clean clean_residual_files install_man clean_man docs Docs.zip Precompiled.zip zips .NOTPARALLEL: compile #==> Functions @@ -120,7 +120,7 @@ check_reproducible: compile |> DateTime.from_iso8601() \ |> elem(1) \ |> DateTime.to_unix()')" - $(Q) $(MAKE) clean_reproducible_files + $(Q) rm -rf lib/*/tmp/ebin_reproducible $(Q) echo "Moving files to ebin_reproducible/ ..." $(Q) mkdir -p lib/elixir/tmp/ebin_reproducible/ \ lib/eex/tmp/ebin_reproducible/ \ @@ -161,9 +161,6 @@ clean_residual_files: rm -f erl_crash.dump $(Q) $(MAKE) clean_man -clean_reproducible_files: - rm -rf lib/*/tmp/ebin_reproducible - #==> Documentation tasks LOGO_PATH = $(shell test -f ../docs/logo.png && echo "--logo ../docs/logo.png") From 19d75df1ce8db64c1465d58be864899559049487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 26 Jan 2019 11:51:15 +0100 Subject: [PATCH 16/19] Update Makefile --- Makefile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index d4f4b6e57f8..c841aa50a16 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,14 @@ test_$(1): compile $(1) $(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/*_test.exs"; endef +define SOURCE_DATE_EPOCH +$(shell bin/elixir -e \ + 'IO.puts System.build_info()[:date] \ + |> DateTime.from_iso8601() \ + |> elem(1) \ + |> DateTime.to_unix()') +endef + #==> Compilation tasks APP := lib/elixir/ebin/elixir.app @@ -115,11 +123,6 @@ install: compile check_reproducible: compile $(Q) echo "==> Checking for reproducible builds..." - export SOURCE_DATE_EPOCH="$(shell bin/elixir -e \ - 'IO.puts System.build_info()[:date] \ - |> DateTime.from_iso8601() \ - |> elem(1) \ - |> DateTime.to_unix()')" $(Q) rm -rf lib/*/tmp/ebin_reproducible $(Q) echo "Moving files to ebin_reproducible/ ..." $(Q) mkdir -p lib/elixir/tmp/ebin_reproducible/ \ @@ -132,7 +135,7 @@ check_reproducible: compile $(Q) mv lib/iex/ebin/* lib/iex/tmp/ebin_reproducible/ $(Q) mv lib/logger/ebin/* lib/logger/tmp/ebin_reproducible/ $(Q) mv lib/mix/ebin/* lib/mix/tmp/ebin_reproducible/ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}" $(MAKE) compile + SOURCE_DATE_EPOCH=$(call SOURCE_DATE_EPOCH) $(MAKE) compile $(Q) echo "Diffing..." $(Q) diff -r lib/elixir/ebin/ lib/elixir/tmp/ebin_reproducible/ $(Q) diff -r lib/eex/ebin/ lib/eex/tmp/ebin_reproducible/ From d29239f4969ea74d23995ba39611a74d4ade3b94 Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sun, 27 Jan 2019 02:03:08 +0700 Subject: [PATCH 17/19] Makefile: implement (read/write)_SOURCE_DATE_EPOCH --- Makefile | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c841aa50a16..6436881f415 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,8 @@ INSTALL_DATA = $(INSTALL) -m644 INSTALL_PROGRAM = $(INSTALL) -m755 GIT_REVISION = $(strip $(shell git rev-parse HEAD 2> /dev/null )) GIT_TAG = $(strip $(shell head="$(call GIT_REVISION)"; git tag --points-at $$head 2> /dev/null | tail -1) ) +SOURCE_DATE_EPOCH_PATH = lib/elixir/tmp/ebin_reproducible +SOURCE_DATE_EPOCH_FILE = $(SOURCE_DATE_EPOCH_PATH)/SOURCE_DATE_EPOCH .PHONY: install compile erlang elixir unicode app build_plt clean_plt dialyze test check_reproducible clean clean_residual_files install_man clean_man docs Docs.zip Precompiled.zip zips .NOTPARALLEL: compile @@ -46,12 +48,16 @@ test_$(1): compile $(1) $(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/*_test.exs"; endef -define SOURCE_DATE_EPOCH -$(shell bin/elixir -e \ +define write_SOURCE_DATE_EPOCH +$(shell mkdir -p $(SOURCE_DATE_EPOCH_PATH) && bin/elixir -e \ 'IO.puts System.build_info()[:date] \ |> DateTime.from_iso8601() \ |> elem(1) \ - |> DateTime.to_unix()') + |> DateTime.to_unix()' > $(SOURCE_DATE_EPOCH_FILE)) +endef + +define read_SOURCE_DATE_EPOCH +$(strip $(shell cat $(SOURCE_DATE_EPOCH_FILE))) endef #==> Compilation tasks @@ -123,8 +129,9 @@ install: compile check_reproducible: compile $(Q) echo "==> Checking for reproducible builds..." - $(Q) rm -rf lib/*/tmp/ebin_reproducible - $(Q) echo "Moving files to ebin_reproducible/ ..." + $(Q) rm -rf lib/*/tmp/ebin_reproducible/ + $(Q) echo "Moving files lib/*/tmp/ebin_reproducible/ ..." + $(call write_SOURCE_DATE_EPOCH) $(Q) mkdir -p lib/elixir/tmp/ebin_reproducible/ \ lib/eex/tmp/ebin_reproducible/ \ lib/iex/tmp/ebin_reproducible/ \ @@ -135,7 +142,7 @@ check_reproducible: compile $(Q) mv lib/iex/ebin/* lib/iex/tmp/ebin_reproducible/ $(Q) mv lib/logger/ebin/* lib/logger/tmp/ebin_reproducible/ $(Q) mv lib/mix/ebin/* lib/mix/tmp/ebin_reproducible/ - SOURCE_DATE_EPOCH=$(call SOURCE_DATE_EPOCH) $(MAKE) compile + SOURCE_DATE_EPOCH=$(call read_SOURCE_DATE_EPOCH) $(MAKE) compile $(Q) echo "Diffing..." $(Q) diff -r lib/elixir/ebin/ lib/elixir/tmp/ebin_reproducible/ $(Q) diff -r lib/eex/ebin/ lib/eex/tmp/ebin_reproducible/ From befe9cc458a4ef06d10906760990be6d34496a6d Mon Sep 17 00:00:00 2001 From: eksperimental Date: Sun, 27 Jan 2019 02:13:09 +0700 Subject: [PATCH 18/19] Add back all OTP releases to .travis.yml --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index c27d201145f..8fa2b310704 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,14 @@ env: - ELIXIR_ASSERT_TIMEOUT=2000 matrix: - OTP_RELEASE=OTP-20.0 - # - OTP_RELEASE=OTP-20.1 - # - OTP_RELEASE=OTP-20.2 - # - OTP_RELEASE=OTP-20.3 - # - OTP_RELEASE=OTP-21.0 - # - OTP_RELEASE=OTP-21.1 + - OTP_RELEASE=OTP-20.1 + - OTP_RELEASE=OTP-20.2 + - OTP_RELEASE=OTP-20.3 + - OTP_RELEASE=OTP-21.0 + - OTP_RELEASE=OTP-21.1 - OTP_RELEASE=OTP-21.2 CHECK_REPRODUCIBLE=true - # - OTP_RELEASE=maint - # - OTP_RELEASE=master + - OTP_RELEASE=maint + - OTP_RELEASE=master matrix: fast_finish: true From 8361c430b810a62a9c3cce61777ace1c7118934f Mon Sep 17 00:00:00 2001 From: eksperimental Date: Mon, 28 Jan 2019 11:29:13 +0700 Subject: [PATCH 19/19] Use uppercase in Mafile function names --- Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6436881f415..53da6494f1c 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ test_$(1): compile $(1) $(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/*_test.exs"; endef -define write_SOURCE_DATE_EPOCH +define WRITE_SOURCE_DATE_EPOCH $(shell mkdir -p $(SOURCE_DATE_EPOCH_PATH) && bin/elixir -e \ 'IO.puts System.build_info()[:date] \ |> DateTime.from_iso8601() \ @@ -56,7 +56,7 @@ $(shell mkdir -p $(SOURCE_DATE_EPOCH_PATH) && bin/elixir -e \ |> DateTime.to_unix()' > $(SOURCE_DATE_EPOCH_FILE)) endef -define read_SOURCE_DATE_EPOCH +define READ_SOURCE_DATE_EPOCH $(strip $(shell cat $(SOURCE_DATE_EPOCH_FILE))) endef @@ -130,8 +130,7 @@ install: compile check_reproducible: compile $(Q) echo "==> Checking for reproducible builds..." $(Q) rm -rf lib/*/tmp/ebin_reproducible/ - $(Q) echo "Moving files lib/*/tmp/ebin_reproducible/ ..." - $(call write_SOURCE_DATE_EPOCH) + $(call WRITE_SOURCE_DATE_EPOCH) $(Q) mkdir -p lib/elixir/tmp/ebin_reproducible/ \ lib/eex/tmp/ebin_reproducible/ \ lib/iex/tmp/ebin_reproducible/ \ @@ -142,7 +141,7 @@ check_reproducible: compile $(Q) mv lib/iex/ebin/* lib/iex/tmp/ebin_reproducible/ $(Q) mv lib/logger/ebin/* lib/logger/tmp/ebin_reproducible/ $(Q) mv lib/mix/ebin/* lib/mix/tmp/ebin_reproducible/ - SOURCE_DATE_EPOCH=$(call read_SOURCE_DATE_EPOCH) $(MAKE) compile + SOURCE_DATE_EPOCH=$(call READ_SOURCE_DATE_EPOCH) $(MAKE) compile $(Q) echo "Diffing..." $(Q) diff -r lib/elixir/ebin/ lib/elixir/tmp/ebin_reproducible/ $(Q) diff -r lib/eex/ebin/ lib/eex/tmp/ebin_reproducible/