From e627d87af99e344d67be63087111730112bd787b Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Mon, 20 Oct 2025 08:36:28 +0200 Subject: [PATCH 1/5] Rework Makefile; dependency tracking --- AGENTS.md | 24 ++++- CONTRIBUTING.md | 37 ++++--- Makefile | 201 +++++++++++++++++++++++++++--------- scripts/buildNinjaBinary.js | 1 - 4 files changed, 195 insertions(+), 68 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index de749990a1..f869f987c5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,13 +5,29 @@ This file provides guidance to AI coding assistants when working with code in th ## Quick Start: Essential Commands ```bash -# Build and test -make && make test +# Build the platform toolchain + stdlib (default target) +make + +# Build the platform toolchain + stdlib and run tests +make test + +# Format code +make format -# Format and check code -make format && make checkformat +# Check formatting +make checkformat ``` +The Makefile’s targets build on each other in this order: + +1. `yarn-install` runs automatically for targets that need JavaScript tooling (lib, playground, tests, formatting, etc.). +2. Toolchain binaries (all copied into `packages/@rescript//bin`): + - `compiler` builds the dune executables (`bsc`, `bsb_helper`, `rescript-*`, `ounit_tests`, etc.). + - `rewatch` builds the Rust-based ReScript build system and CLI. + - `ninja` bootstraps the ninja binary (part of the legacy build system). +3. `lib` (the default target) uses those toolchain outputs to build the runtime sources. +4. Test targets (`make test`, `make test-syntax`, etc.) reuse everything above. + ## ⚠️ Critical Guidelines & Common Pitfalls - **We are NOT bound by OCaml compatibility** - The ReScript compiler originated as a fork of the OCaml compiler, but we maintain our own AST and can make breaking changes. Focus on what's best for ReScript's JavaScript compilation target. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b3061cd48d..443e3b8866 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,30 +71,29 @@ You can also open this dev container with [GitHub Codespaces](https://github.com ## Building the Compiler -The compiler binaries themselves can be built directly with dune as follows: +To build the compiler, the build tools (rewatch and ninja), and the ReScript runtime/standard library, just run: ```sh -# One off build -dune build - -# Watch mode -dune build -w +make ``` -For all additional operations, a Makefile is provided: +To run tests, execute: ```sh -# Build the compiler using dune and copy the exes into the platform dir -make +make test +``` -# Build the ninja build tool -make ninja +Additional targets: -# Build the ReScript standard library using ninja and the compiler -make lib +```sh +# Build the compiler executables only +make compiler -# Run compiler tests -make test +# Build rewatch only +make rewatch + +# Build the ninja tool only +make ninja # Run syntax tests make test-syntax @@ -102,8 +101,14 @@ make test-syntax # Run syntax tests including roundtrip tests make test-syntax-roundtrip -# Populate lib/ocaml and update artifact list +# Update artifact list make artifacts + +# Format code +make format + +# Check formatting +make checkformat ``` ## Coding Style diff --git a/Makefile b/Makefile index 56b13f22c1..53b8a36010 100644 --- a/Makefile +++ b/Makefile @@ -1,102 +1,209 @@ SHELL = /bin/bash +ifeq ($(OS),Windows_NT) + PLATFORM_EXE_EXT = .exe +else + PLATFORM_EXE_EXT = +endif + +ifneq ($(OS),Windows_NT) + UNAME_S := $(shell uname -s) + UNAME_M := $(shell uname -m) +endif + +ifeq ($(OS),Windows_NT) + RESCRIPT_PLATFORM := win32-x64 +else ifeq ($(UNAME_S),Darwin) + ifeq ($(UNAME_M),arm64) + RESCRIPT_PLATFORM := darwin-arm64 + else + RESCRIPT_PLATFORM := darwin-x64 + endif +else ifeq ($(UNAME_S),Linux) + ifeq ($(UNAME_M),aarch64) + RESCRIPT_PLATFORM := linux-arm64 + else ifeq ($(UNAME_M),arm64) + RESCRIPT_PLATFORM := linux-arm64 + else + RESCRIPT_PLATFORM := linux-x64 + endif +else + $(error Unsupported platform $(UNAME_S)-$(UNAME_M)) +endif + +define COPY_EXE + cp $1 $2 + chmod 755 $2 +$(if $(filter Windows_NT,$(OS)),,strip $2) +endef + +# Directories + +BIN_DIR := packages/@rescript/$(RESCRIPT_PLATFORM)/bin +RUNTIME_DIR := packages/@rescript/runtime DUNE_BIN_DIR = ./_build/install/default/bin -build: ninja rewatch +# Build stamps + +YARN_INSTALL_STAMP := node_modules/.yarn.stamp +COMPILER_BUILD_STAMP := _build/.compiler.stamp +RUNTIME_BUILD_STAMP := _build/.runtime.stamp + +# Yarn + +WORKSPACE_PACKAGE_JSONS := $(shell find packages -path '*/lib' -prune -o -name package.json -print) +YARN_INSTALL_SOURCES := package.json yarn.lock yarn.config.cjs .yarnrc.yml $(WORKSPACE_PACKAGE_JSONS) + +yarn-install: $(YARN_INSTALL_STAMP) + +$(YARN_INSTALL_STAMP): $(YARN_INSTALL_SOURCES) + yarn install + @mkdir -p $(dir $@) + touch $@ + +# Ninja + +NINJA_SOURCES = $(wildcard ninja/src/*.cc ninja/src/*.h) $(wildcard ninja/*.py) +NINJA_EXE = $(BIN_DIR)/ninja.exe + +ninja: $(NINJA_EXE) + +ninja/ninja: $(NINJA_SOURCES) +ifeq ($(OS),Darwin) + export CXXFLAGS="-flto" +endif + cd ninja && python3 configure.py --bootstrap --verbose + +$(NINJA_EXE): ninja/ninja + $(call COPY_EXE,$<,$@) + +clean-ninja: + rm -rf $(NINJA_EXE) ninja/build.ninja ninja/build ninja/misc/__pycache__ ninja/ninja + +# Rewatch + +REWATCH_SOURCES = $(shell find rewatch/src -name '*.rs') rewatch/Cargo.toml rewatch/Cargo.lock rewatch/rust-toolchain.toml +RESCRIPT_EXE = $(BIN_DIR)/rescript.exe + +rewatch: $(RESCRIPT_EXE) + +$(RESCRIPT_EXE): rewatch/target/debug/rescript$(PLATFORM_EXE_EXT) + $(call COPY_EXE,$<,$@) + +rewatch/target/debug/rescript$(PLATFORM_EXE_EXT): $(REWATCH_SOURCES) + cargo build --manifest-path rewatch/Cargo.toml + +clean-rewatch: + cargo clean --manifest-path rewatch/Cargo.toml && rm -rf rewatch/target && rm -f $(RESCRIPT_EXE) + +# Compiler + +COMPILER_SOURCE_DIRS := compiler tests analysis tools +COMPILER_SOURCES = $(shell find $(COMPILER_SOURCE_DIRS) -type f \( -name '*.ml' -o -name '*.mli' -o -name '*.dune' -o -name dune -o -name dune-project \)) +COMPILER_BIN_NAMES := bsc bsb_helper rescript-legacy rescript-editor-analysis rescript-tools +COMPILER_EXES := $(addsuffix .exe,$(addprefix $(BIN_DIR)/,$(COMPILER_BIN_NAMES))) +COMPILER_DUNE_BINS := $(addsuffix $(PLATFORM_EXE_EXT),$(addprefix $(DUNE_BIN_DIR)/,$(COMPILER_BIN_NAMES))) + +compiler: $(COMPILER_EXES) + +define MAKE_COMPILER_COPY_RULE +$(BIN_DIR)/$(1).exe: $(DUNE_BIN_DIR)/$(1)$(PLATFORM_EXE_EXT) + $$(call COPY_EXE,$$<,$$@) +endef + +$(foreach bin,$(COMPILER_BIN_NAMES),$(eval $(call MAKE_COMPILER_COPY_RULE,$(bin)))) + +# "touch" after dune build to make sure that the binaries' timestamps are updated +# even if the actual content of the sources hasn't changed. +$(COMPILER_BUILD_STAMP): $(COMPILER_SOURCES) dune build - ./scripts/copyExes.js --compiler + touch $@ + @$(foreach bin,$(COMPILER_DUNE_BINS),touch $(bin);) -watch: - dune build -w +$(COMPILER_DUNE_BINS): $(COMPILER_BUILD_STAMP) ; -bench: - $(DUNE_BIN_DIR)/syntax_benchmarks +clean-compiler: + dune clean && rm -f $(COMPILER_EXES) $(COMPILER_BUILD_STAMP) + +# Runtime / stdlib + +RUNTIME_SOURCES := $(shell find $(RUNTIME_DIR) -path '$(RUNTIME_DIR)/lib' -prune -o -type f \( -name '*.res' -o -name '*.resi' -o -name 'rescript.json' \) -print) + +lib: $(RUNTIME_BUILD_STAMP) + +$(RUNTIME_BUILD_STAMP): $(RUNTIME_SOURCES) $(COMPILER_EXES) $(RESCRIPT_EXE) | $(YARN_INSTALL_STAMP) + yarn workspace @rescript/runtime build + @mkdir -p $(dir $@) + touch $@ -dce: - reanalyze.exe -dce-cmt _build/default/compiler +clean-lib: + yarn workspace @rescript/runtime rescript clean + rm -f $(RUNTIME_BUILD_STAMP) -rewatch: - cargo build --manifest-path rewatch/Cargo.toml --release - ./scripts/copyExes.js --rewatch +# Tests / artifacts / analysis -ninja/ninja: - ./scripts/buildNinjaBinary.js +artifacts: lib + ./scripts/updateArtifactList.js -ninja: ninja/ninja - ./scripts/copyExes.js --ninja +bench: $(COMPILER_BUILD_STAMP) + $(DUNE_BIN_DIR)/syntax_benchmarks -test: build lib +test: lib ninja | $(YARN_INSTALL_STAMP) node scripts/test.js -all -test-analysis: +test-analysis: | $(YARN_INSTALL_STAMP) make -C tests/analysis_tests clean test -test-tools: +test-tools: | $(YARN_INSTALL_STAMP) make -C tests/tools_tests clean test -test-syntax: +test-syntax: | $(YARN_INSTALL_STAMP) ./scripts/test_syntax.sh -test-syntax-roundtrip: +test-syntax-roundtrip: | $(YARN_INSTALL_STAMP) ROUNDTRIP_TEST=1 ./scripts/test_syntax.sh -test-gentype: +test-gentype: | $(YARN_INSTALL_STAMP) make -C tests/gentype_tests/typescript-react-example clean test make -C tests/gentype_tests/stdlib-no-shims clean test -test-rewatch: +test-rewatch: | $(YARN_INSTALL_STAMP) ./rewatch/tests/suite-ci.sh test-all: test test-gentype test-analysis test-tools test-rewatch -reanalyze: - reanalyze.exe -set-exit-code -all-cmt _build/default/compiler _build/default/tests -exclude-paths compiler/outcome_printer,compiler/ml,compiler/frontend,compiler/ext,compiler/depends,compiler/core,compiler/common,compiler/cmij,compiler/bsb_helper,compiler/bsb - -lib: - yarn workspace @rescript/runtime build - -artifacts: lib - ./scripts/updateArtifactList.js - # Builds the core playground bundle (without the relevant cmijs files for the runtime) -playground: +playground: | $(YARN_INSTALL_STAMP) dune build --profile browser cp -f ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js packages/playground/compiler.js # Creates all the relevant core and third party cmij files to side-load together with the playground bundle -playground-cmijs: artifacts +playground-cmijs: artifacts | $(YARN_INSTALL_STAMP) yarn workspace playground build # Builds the playground, runs some e2e tests and releases the playground to the # Cloudflare R2 (requires Rclone `rescript:` remote) -playground-release: playground playground-cmijs +playground-release: playground playground-cmijs | $(YARN_INSTALL_STAMP) yarn workspace playground test yarn workspace playground upload-bundle -format: +format: | $(YARN_INSTALL_STAMP) ./scripts/format.sh -checkformat: +checkformat: | $(YARN_INSTALL_STAMP) ./scripts/format_check.sh clean-gentype: make -C tests/gentype_tests/typescript-react-example clean make -C tests/gentype_tests/stdlib-no-shims clean -clean-rewatch: - cargo clean --manifest-path rewatch/Cargo.toml && rm -f rewatch/rewatch - -clean-lib: - yarn workspace @rescript/runtime rescript clean - -clean: clean-lib - dune clean +clean-tests: clean-gentype -clean-all: clean clean-gentype clean-rewatch +clean: clean-lib clean-compiler clean-rewatch clean-ninja dev-container: docker build -t rescript-dev-container docker -.DEFAULT_GOAL := build +.DEFAULT_GOAL := lib -.PHONY: build watch rewatch ninja bench dce test test-syntax test-syntax-roundtrip test-gentype test-analysis test-tools test-all lib playground playground-cmijs playground-release artifacts format checkformat clean-gentype clean-rewatch clean clean-all dev-container +.PHONY: yarn-install ninja rewatch compiler lib artifacts bench dce test test-analysis test-tools test-syntax test-syntax-roundtrip test-gentype test-rewatch test-all reanalyze playground playground-cmijs playground-release format checkformat clean-ninja clean-rewatch clean-compiler clean-lib clean-gentype clean-tests clean dev-container diff --git a/scripts/buildNinjaBinary.js b/scripts/buildNinjaBinary.js index dc09eaf0cc..36f7e9cf15 100755 --- a/scripts/buildNinjaBinary.js +++ b/scripts/buildNinjaBinary.js @@ -16,5 +16,4 @@ if (platform === "win32") { process.env.CXXFLAGS = "-flto"; } execSync(buildCommand, { stdio: [0, 1, 2], cwd: ninjaDir }); - execSync("strip ninja", { stdio: [0, 1, 2], cwd: ninjaDir }); } From 0c6f48a6fd5b7377bdc3ef5eedb42edba71ea82f Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Tue, 21 Oct 2025 11:37:23 +0200 Subject: [PATCH 2/5] Review change --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 53b8a36010..a894c602cc 100644 --- a/Makefile +++ b/Makefile @@ -206,4 +206,4 @@ dev-container: .DEFAULT_GOAL := lib -.PHONY: yarn-install ninja rewatch compiler lib artifacts bench dce test test-analysis test-tools test-syntax test-syntax-roundtrip test-gentype test-rewatch test-all reanalyze playground playground-cmijs playground-release format checkformat clean-ninja clean-rewatch clean-compiler clean-lib clean-gentype clean-tests clean dev-container +.PHONY: yarn-install ninja rewatch compiler lib artifacts bench test test-analysis test-tools test-syntax test-syntax-roundtrip test-gentype test-rewatch test-all playground playground-cmijs playground-release format checkformat clean-ninja clean-rewatch clean-compiler clean-lib clean-gentype clean-tests clean dev-container From 8068345517fecda2fa98b1d4b3cba965d98d0130 Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Tue, 21 Oct 2025 13:16:29 +0200 Subject: [PATCH 3/5] Restore "make build" as default target, use .yarn/install-state.gz as stamp --- AGENTS.md | 5 ++++- CONTRIBUTING.md | 9 ++++++--- Makefile | 21 ++++++++++++++------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f869f987c5..e5169ac6c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,9 +5,12 @@ This file provides guidance to AI coding assistants when working with code in th ## Quick Start: Essential Commands ```bash -# Build the platform toolchain + stdlib (default target) +# Build the platform toolchain (default target) make +# Build the platform toolchain + stdlib +make lib + # Build the platform toolchain + stdlib and run tests make test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 443e3b8866..af6812d047 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,13 +73,16 @@ You can also open this dev container with [GitHub Codespaces](https://github.com To build the compiler, the build tools (rewatch and ninja), and the ReScript runtime/standard library, just run: +Main targets: + ```sh +# Build the compiler and the build tools (rewatch and ninja) make -``` -To run tests, execute: +# Build the runtime/standard library +make lib -```sh +# Run the tests make test ``` diff --git a/Makefile b/Makefile index a894c602cc..b822966fdc 100644 --- a/Makefile +++ b/Makefile @@ -45,10 +45,19 @@ DUNE_BIN_DIR = ./_build/install/default/bin # Build stamps -YARN_INSTALL_STAMP := node_modules/.yarn.stamp -COMPILER_BUILD_STAMP := _build/.compiler.stamp +# Yarn creates `.yarn/install-state.gz` whenever dependencies are installed. +# Using that file as our stamp ensures manual `yarn install` runs are detected. +YARN_INSTALL_STAMP := .yarn/install-state.gz +# Dune updates `_build/log` for every build invocation, even when run manually. +# Treat that log file as the compiler build stamp so manual `dune build` +# keeps Make targets up to date. +COMPILER_BUILD_STAMP := _build/log RUNTIME_BUILD_STAMP := _build/.runtime.stamp +# Default target + +build: compiler rewatch ninja + # Yarn WORKSPACE_PACKAGE_JSONS := $(shell find packages -path '*/lib' -prune -o -name package.json -print) @@ -58,7 +67,6 @@ yarn-install: $(YARN_INSTALL_STAMP) $(YARN_INSTALL_STAMP): $(YARN_INSTALL_SOURCES) yarn install - @mkdir -p $(dir $@) touch $@ # Ninja @@ -133,7 +141,6 @@ lib: $(RUNTIME_BUILD_STAMP) $(RUNTIME_BUILD_STAMP): $(RUNTIME_SOURCES) $(COMPILER_EXES) $(RESCRIPT_EXE) | $(YARN_INSTALL_STAMP) yarn workspace @rescript/runtime build - @mkdir -p $(dir $@) touch $@ clean-lib: @@ -167,7 +174,7 @@ test-gentype: | $(YARN_INSTALL_STAMP) make -C tests/gentype_tests/typescript-react-example clean test make -C tests/gentype_tests/stdlib-no-shims clean test -test-rewatch: | $(YARN_INSTALL_STAMP) +test-rewatch: $(RESCRIPT_EXE) | $(YARN_INSTALL_STAMP) ./rewatch/tests/suite-ci.sh test-all: test test-gentype test-analysis test-tools test-rewatch @@ -204,6 +211,6 @@ clean: clean-lib clean-compiler clean-rewatch clean-ninja dev-container: docker build -t rescript-dev-container docker -.DEFAULT_GOAL := lib +.DEFAULT_GOAL := build -.PHONY: yarn-install ninja rewatch compiler lib artifacts bench test test-analysis test-tools test-syntax test-syntax-roundtrip test-gentype test-rewatch test-all playground playground-cmijs playground-release format checkformat clean-ninja clean-rewatch clean-compiler clean-lib clean-gentype clean-tests clean dev-container +.PHONY: yarn-install build ninja rewatch compiler lib artifacts bench test test-analysis test-tools test-syntax test-syntax-roundtrip test-gentype test-rewatch test-all playground playground-cmijs playground-release format checkformat clean-ninja clean-rewatch clean-compiler clean-lib clean-gentype clean-tests clean dev-container From 9ce2f62d15526123e101f7321a75fe1be61b220a Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Wed, 22 Oct 2025 09:19:12 +0200 Subject: [PATCH 4/5] Try to fix build --- Makefile | 9 +++++---- packages/@rescript/runtime/.gitignore | 1 + packages/@rescript/runtime/package.json | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b822966fdc..c5e274b66c 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,10 @@ YARN_INSTALL_STAMP := .yarn/install-state.gz # Treat that log file as the compiler build stamp so manual `dune build` # keeps Make targets up to date. COMPILER_BUILD_STAMP := _build/log -RUNTIME_BUILD_STAMP := _build/.runtime.stamp +# Runtime workspace touches this stamp (packages/@rescript/runtime/.buildstamp) +# after running `yarn workspace @rescript/runtime build`, which now runs `touch` +# as part of its build script. +RUNTIME_BUILD_STAMP := packages/@rescript/runtime/.buildstamp # Default target @@ -125,7 +128,6 @@ $(foreach bin,$(COMPILER_BIN_NAMES),$(eval $(call MAKE_COMPILER_COPY_RULE,$(bin) # even if the actual content of the sources hasn't changed. $(COMPILER_BUILD_STAMP): $(COMPILER_SOURCES) dune build - touch $@ @$(foreach bin,$(COMPILER_DUNE_BINS),touch $(bin);) $(COMPILER_DUNE_BINS): $(COMPILER_BUILD_STAMP) ; @@ -141,7 +143,6 @@ lib: $(RUNTIME_BUILD_STAMP) $(RUNTIME_BUILD_STAMP): $(RUNTIME_SOURCES) $(COMPILER_EXES) $(RESCRIPT_EXE) | $(YARN_INSTALL_STAMP) yarn workspace @rescript/runtime build - touch $@ clean-lib: yarn workspace @rescript/runtime rescript clean @@ -185,7 +186,7 @@ playground: | $(YARN_INSTALL_STAMP) cp -f ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js packages/playground/compiler.js # Creates all the relevant core and third party cmij files to side-load together with the playground bundle -playground-cmijs: artifacts | $(YARN_INSTALL_STAMP) +playground-cmijs: | $(YARN_INSTALL_STAMP) # should also depend on artifacts, but that causes an attempt to copy binaries for JSOO yarn workspace playground build # Builds the playground, runs some e2e tests and releases the playground to the diff --git a/packages/@rescript/runtime/.gitignore b/packages/@rescript/runtime/.gitignore index bc9f0fd609..0aeb240499 100644 --- a/packages/@rescript/runtime/.gitignore +++ b/packages/@rescript/runtime/.gitignore @@ -1,3 +1,4 @@ lib/bs lib/ocaml *.lock +.buildstamp diff --git a/packages/@rescript/runtime/package.json b/packages/@rescript/runtime/package.json index 3967e4d657..c9b4951ba8 100644 --- a/packages/@rescript/runtime/package.json +++ b/packages/@rescript/runtime/package.json @@ -40,7 +40,7 @@ }, "scripts": { "clean": "rescript clean", - "build": "rescript build", + "build": "rescript build && touch .buildstamp", "cppo": "node scripts/cppo.js" }, "devDependencies": { From 31f1c873a65289106cb502f92433c9a9b95e79ea Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Wed, 22 Oct 2025 09:55:14 +0200 Subject: [PATCH 5/5] Update AGENTS.md --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e5169ac6c3..30801092cb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,11 +24,11 @@ make checkformat The Makefile’s targets build on each other in this order: 1. `yarn-install` runs automatically for targets that need JavaScript tooling (lib, playground, tests, formatting, etc.). -2. Toolchain binaries (all copied into `packages/@rescript//bin`): +2. `build` (default target) builds the toolchain binaries (all copied into `packages/@rescript//bin`): - `compiler` builds the dune executables (`bsc`, `bsb_helper`, `rescript-*`, `ounit_tests`, etc.). - `rewatch` builds the Rust-based ReScript build system and CLI. - `ninja` bootstraps the ninja binary (part of the legacy build system). -3. `lib` (the default target) uses those toolchain outputs to build the runtime sources. +3. `lib` uses those toolchain outputs to build the runtime sources. 4. Test targets (`make test`, `make test-syntax`, etc.) reuse everything above. ## ⚠️ Critical Guidelines & Common Pitfalls