v2: finish flat AST migration (#27372)

Author: medvednikov

vlib/v2/README.md

@@ -4,8 +4,10 @@
 
 ## Flat AST
 
-`v2.ast` now includes an index-based flat AST graph for tooling and profiling:
+`v2.ast` is the default index-based AST graph used by the v2 builder for
+parsing, type checking, markused, and native SSA input:
 
 - `ast.flatten_files(files)` builds `ast.FlatAst` with contiguous `nodes` and `edges`.
 - `ast.legacy_ast_stats(files)` estimates memory/shape metrics for recursive AST files.
 - `flat.stats()` and `flat.count_reachable_nodes()` report flat graph size and reachability.
+- `V2_LEGACY_AST=1` keeps the old recursive-AST pipeline available for comparison.

vlib/v2/ast/cursor.v

@@ -73,6 +73,39 @@ pub fn (c Cursor) name_id() int {
 	return c.flat.nodes[c.id].name_id
 }
 
+// ident reads an expr_ident cursor directly into an Ident.
+pub fn (c Cursor) ident() Ident {
+	if !c.is_valid() || c.kind() != .expr_ident {
+		return Ident{}
+	}
+	return Ident{
+		pos:  c.pos()
+		name: c.name()
+	}
+}
+
+// import_stmt reads a stmt_import cursor directly into an ImportStmt.
+pub fn (c Cursor) import_stmt() ImportStmt {
+	if !c.is_valid() || c.kind() != .stmt_import {
+		return ImportStmt{}
+	}
+	mut symbols := []Expr{cap: c.edge_count()}
+	for i in 0 .. c.edge_count() {
+		sym := c.edge(i)
+		if sym.kind() == .expr_ident {
+			symbols << Expr(sym.ident())
+		} else if sym.is_valid() {
+			symbols << c.flat.decode_expr(sym.id)
+		}
+	}
+	return ImportStmt{
+		name:       c.name()
+		alias:      c.extra_str()
+		is_aliased: c.flag(flag_is_aliased)
+		symbols:    symbols
+	}
+}
+
 @[inline]
 pub fn (c Cursor) edge_count() int {
 	return c.flat.nodes[c.id].edge_count

vlib/v2/ast/flat_reader.v

@@ -93,9 +93,12 @@ fn (r &FlatReader) read_file(ff FlatFile) File {
 	}
 	mut imports := []ImportStmt{}
 	for cid in r.list_children(imports_id) {
-		s := r.read_stmt(cid)
-		if s is ImportStmt {
-			imports << s
+		c := Cursor{
+			flat: r.flat
+			id:   cid
+		}
+		if c.kind() == .stmt_import {
+			imports << c.import_stmt()
 		}
 	}
 	mut stmts := []Stmt{}
@@ -123,9 +126,12 @@ pub fn (flat &FlatAst) read_file_imports(ff FlatFile) []ImportStmt {
 	imports_id := r.edge(n, 1)
 	mut imports := []ImportStmt{}
 	for cid in r.list_children(imports_id) {
-		s := r.read_stmt(cid)
-		if s is ImportStmt {
-			imports << s
+		c := Cursor{
+			flat: r.flat
+			id:   cid
+		}
+		if c.kind() == .stmt_import {
+			imports << c.import_stmt()
 		}
 	}
 	return imports

vlib/v2/builder/builder.v

@@ -48,10 +48,10 @@ mut:
 	used_vh_for_parse                     bool
 	used_import_vh_for_parse              bool
 	used_virtual_vh_for_parse             bool
-	flat_roundtrip_enabled                bool // V2_FLAT_ROUNDTRIP=1: route parses through streaming + to_files()
-	flat_check_enabled                    bool // V2_CHECK_FLAT=1: route type-check through Checker.check_flat
-	markused_flat_enabled                 bool // V2_MARKUSED_FLAT=1: route markused through mark_used_flat shim
-	flat_ssa_enabled                      bool // V2_FLAT_SSA=1: route the (sequential) native SSA build through build_all_from_flat on the post-transform b.flat. Requires V2_CHECK_FLAT + V2_MARKUSED_FLAT so b.flat is post-transform-populated. Default off.
+	flat_roundtrip_enabled                bool // V2_FLAT_ROUNDTRIP=1: legacy comparison mode; route parses through streaming + to_files().
+	flat_check_enabled                    bool // Default on: stream parse/type-check through FlatAst. V2_LEGACY_AST=1 disables it unless V2_CHECK_FLAT=1 is set.
+	markused_flat_enabled                 bool // Default on: route markused through mark_used_flat.
+	flat_ssa_enabled                      bool // Default on: route SSA codegen through build_all_from_flat on the post-transform b.flat.
 	// flat caches the FlatAst representation of b.files. When
 	// flat_check_enabled is set, parse_batch streams directly into
 	// flat_builder so b.flat is built incrementally during parsing rather
@@ -71,15 +71,17 @@ mut:
 }
 
 pub fn new_builder(prefs &pref.Preferences) &Builder {
+	legacy_ast_enabled := os.getenv('V2_LEGACY_AST') != ''
+	flat_default_enabled := !legacy_ast_enabled
 	unsafe {
 		return &Builder{
 			pref:                   prefs
 			used_fn_keys:           map[string]bool{}
 			cached_called_fn_names: map[string]bool{}
 			flat_roundtrip_enabled: os.getenv('V2_FLAT_ROUNDTRIP') != ''
-			flat_check_enabled:     os.getenv('V2_CHECK_FLAT') != ''
-			markused_flat_enabled:  os.getenv('V2_MARKUSED_FLAT') != ''
-			flat_ssa_enabled:       os.getenv('V2_FLAT_SSA') != ''
+			flat_check_enabled:     flat_default_enabled || os.getenv('V2_CHECK_FLAT') != ''
+			markused_flat_enabled:  flat_default_enabled || os.getenv('V2_MARKUSED_FLAT') != ''
+			flat_ssa_enabled:       flat_default_enabled || os.getenv('V2_FLAT_SSA') != ''
 		}
 	}
 }
@@ -108,6 +110,20 @@ fn (b &Builder) backend_uses_markused_pruning() bool {
 	return b.pref.backend != .arm64
 }
 
+fn (b &Builder) should_build_ssa_from_flat() bool {
+	return b.flat.files.len > 0 && b.flat_ssa_enabled && b.markused_flat_enabled
+		&& b.flat_check_enabled
+}
+
+fn (b &Builder) should_keep_flat_for_codegen() bool {
+	flat_ssa_codegen := b.flat_ssa_enabled && b.markused_flat_enabled && b.flat_check_enabled
+	return match b.pref.backend {
+		.c { flat_ssa_codegen }
+		.x64, .arm64 { flat_ssa_codegen || b.native_flat_pipeline_enabled }
+		else { false }
+	}
+}
+
 fn (b &Builder) can_compile_cleanc_locally() bool {
 	if b.pref == unsafe { nil } {
 		return true
@@ -263,20 +279,26 @@ pub fn (mut b Builder) build(files []string) {
 	// transform_files_from_flat, parallel streams per-worker via
 	// to_files_range. No up-front full rehydration needed in either case.
 	//
-	// When V2_MARKUSED_FLAT is also enabled, both transform paths route
-	// through their *_to_flat wedge so the post-transform flatten lives
-	// inside the transformer call (one round-trip), avoiding a separate
-	// flatten_files() pass before mark_used_flat.
+	// Flat markused routes transform through flat-output wedges. Backends that
+	// can consume flat codegen use the direct flat-output path and drop the
+	// transformed []ast.File result; legacy backends keep the compatibility
+	// wedge that returns both flat and files.
 	mut flat_populated_by_transform := false
+	transform_flat_only := b.should_keep_flat_for_codegen()
 	if sequential_transform {
 		if use_native_flat_pipeline && !b.flat_check_enabled {
 			b.flat = trans.transform_files_to_flat_direct(b.files)
 			b.files = []ast.File{}
 			flat_populated_by_transform = true
 		} else if use_flat_markused {
-			new_flat, files_out := trans.transform_files_to_flat(&b.flat, b.files)
-			b.flat = new_flat
-			b.files = files_out
+			if transform_flat_only {
+				b.flat = trans.transform_flat_to_flat_direct(&b.flat, b.files)
+				b.files = []ast.File{}
+			} else {
+				new_flat, files_out := trans.transform_files_to_flat(&b.flat, b.files)
+				b.flat = new_flat
+				b.files = files_out
+			}
 			flat_populated_by_transform = true
 		} else if b.flat_check_enabled {
 			b.files = trans.transform_files_from_flat(&b.flat, b.files)
@@ -289,9 +311,14 @@ pub fn (mut b Builder) build(files []string) {
 			b.files = []ast.File{}
 			flat_populated_by_transform = true
 		} else if use_flat_markused {
-			new_flat, files_out := b.transform_files_parallel_to_flat(mut trans)
-			b.flat = new_flat
-			b.files = files_out
+			if transform_flat_only {
+				b.flat = trans.transform_flat_to_flat_direct(&b.flat, b.files)
+				b.files = []ast.File{}
+			} else {
+				new_flat, files_out := b.transform_files_parallel_to_flat(mut trans)
+				b.flat = new_flat
+				b.files = files_out
+			}
 			flat_populated_by_transform = true
 		} else if b.flat_check_enabled {
 			b.files = b.transform_files_parallel_from_flat(mut trans)
@@ -308,17 +335,15 @@ pub fn (mut b Builder) build(files []string) {
 		b.used_fn_keys = map[string]bool{}
 	} else {
 		mark_used_start := sw.elapsed()
-		// V2_MARKUSED_FLAT only takes effect when V2_CHECK_FLAT is also on,
-		// since b.flat is only populated when flat_check_enabled streams
-		// parses into flat_builder. Without that, b.flat is empty and the
-		// shim would walk nothing.
+		// Flat markused consumes the post-transform FlatAst. Legacy comparison
+		// mode (`V2_LEGACY_AST=1`) can still reach the AST walker unless one of
+		// the flat env flags explicitly re-enables this path.
 		//
-		// The transformer mutates b.files but does not write back into
-		// b.flat. Both sequential and parallel paths now populate b.flat
-		// as part of their *_to_flat wedge when V2_MARKUSED_FLAT is on,
-		// so the separate flatten_files() pass is gone. The branch below
-		// remains as a defensive fallback for any future code path that
-		// reaches markused without having set flat_populated_by_transform.
+		// The transformer mutates b.files but does not write back into b.flat.
+		// Both sequential and parallel paths now populate b.flat as part of
+		// their *_to_flat wedge, so the separate flatten_files() pass is gone.
+		// The branch below remains as a defensive fallback for any future code
+		// path that reaches markused without setting flat_populated_by_transform.
 		if use_flat_markused && !flat_populated_by_transform {
 			b.flat = ast.flatten_files(b.files)
 		}
@@ -340,16 +365,11 @@ pub fn (mut b Builder) build(files []string) {
 		}
 		mark_used_time := time.Duration(sw.elapsed() - mark_used_start)
 		print_time('Mark Used', mark_used_time)
-		// b.flat is unused by the legacy codegen path. Under -gc none this is a
-		// no-op for peak memory, but keep the lifetime explicit for readers.
-		// When V2_FLAT_SSA or the native flat pipeline is on, the native SSA
-		// build consumes b.flat directly (build_all_from_flat), so keep it alive
-		// through codegen.
-		if !b.flat_ssa_enabled && !b.native_flat_pipeline_enabled {
-			b.flat = ast.FlatAst{}
-		}
 		print_rss('after markused')
 	}
+	if b.flat_check_enabled && !b.should_keep_flat_for_codegen() {
+		b.flat = ast.FlatAst{}
+	}
 
 	// Generate output based on backend
 	match b.pref.backend {
@@ -601,7 +621,13 @@ fn (mut b Builder) gen_ssa_c() {
 	ssa_builder.target_os = b.pref.target_os_or_host()
 
 	mut stage_start := sw.elapsed()
-	ssa_builder.build_all(b.files)
+	mut built_from_flat := false
+	if b.should_build_ssa_from_flat() {
+		ssa_builder.build_all_from_flat(&b.flat)
+		built_from_flat = true
+	} else {
+		ssa_builder.build_all(b.files)
+	}
 	print_time('SSA Build', time.Duration(sw.elapsed() - stage_start))
 
 	// TODO: re-enable SSA optimization once the new builder is mature
@@ -618,6 +644,9 @@ fn (mut b Builder) gen_ssa_c() {
 	}
 
 	if output_name.ends_with('.c') {
+		if built_from_flat {
+			b.flat = ast.FlatAst{}
+		}
 		stage_start = sw.elapsed()
 		mut gen := c.new_gen(mod)
 		c_source := gen.gen()
@@ -633,6 +662,12 @@ fn (mut b Builder) gen_ssa_c() {
 
 	cc := if b.pref.ccompiler.len > 0 { b.pref.ccompiler } else { configured_cc(b.pref.vroot) }
 	directive_flags := b.collect_cflags_from_sources()
+	if built_from_flat {
+		// SSA has copied the program into MIR, and directive scanning has read
+		// source names from the FlatAst. Keep the later C generator/compiler
+		// working sets clear of the transformed FlatAst.
+		b.flat = ast.FlatAst{}
+	}
 	mut cc_flag_parts := []string{}
 	env_flags := configured_cflags()
 	if env_flags.trim_space() != '' {
@@ -1684,7 +1719,7 @@ fn (mut b Builder) prepare_macos_tiny_candidate_source_files() {
 	if !b.uses_macos_x64_tiny_object(.x64) {
 		return
 	}
-	b.macos_tiny_candidate_source_files = if b.flat_check_enabled {
+	b.macos_tiny_candidate_source_files = if b.flat_check_enabled && b.flat.files.len > 0 {
 		b.flat.to_files()
 	} else {
 		b.files.clone()
@@ -1998,6 +2033,12 @@ fn (b &Builder) collect_cflags_from_sources() string {
 			scan_paths << file.name
 		}
 	}
+	for ff in b.flat.files {
+		name := b.flat.file_name(ff)
+		if name != '' {
+			scan_paths << name
+		}
+	}
 	cflags_target_os := b.cflags_target_os_for_local_compile()
 	if !b.pref.skip_builtin {
 		target_os := cflags_target_os
@@ -2567,18 +2608,15 @@ fn (mut b Builder) build_native_mir_from_files(files []ast.File, arch pref.Arch,
 	}
 
 	mut stage_start := native_sw.elapsed()
-	// V2_FLAT_SSA: route the whole SSA build through the cursor-native
-	// build_all_from_flat on the post-transform b.flat (kept alive above).
-	// Sequential only (build_all_from_flat builds fn bodies in-phase). Default off.
+	// Route the whole SSA build through the cursor-native build_all_from_flat
+	// on the post-transform b.flat (kept alive above). Sequential only
+	// (build_all_from_flat builds fn bodies in-phase).
 	//
-	// b.flat is only POST-TRANSFORM when either V2_MARKUSED_FLAT has routed
-	// transform through transform_files_to_flat, or the native flat pipeline has
-	// emitted transform output directly into FlatAst. With V2_CHECK_FLAT but NOT
-	// V2_MARKUSED_FLAT, b.flat stays parse-time, so feeding it here would skip
-	// transformer rewrites.
-	build_from_flat := b.flat.files.len > 0
-		&& ((b.flat_ssa_enabled && b.markused_flat_enabled && b.flat_check_enabled)
-		|| (b.native_flat_pipeline_enabled && label == ''))
+	// b.flat is only POST-TRANSFORM when flat markused has routed transform
+	// through transform_files_to_flat, or the direct native flat pipeline has
+	// emitted transform output directly into FlatAst.
+	build_from_flat := b.should_build_ssa_from_flat()
+		|| (b.flat.files.len > 0 && b.native_flat_pipeline_enabled && label == '')
 	if build_from_flat {
 		ssa_builder.build_all_from_flat(&b.flat)
 		// SSA has copied the program into MIR; keep the FlatAst lifetime out of

vlib/v2/builder/flag_target_test.v

@@ -22,6 +22,26 @@ fn collect_cflags_for_test_source(source string, mut prefs pref.Preferences) str
 	return b.collect_cflags_from_sources()
 }
 
+fn collect_cflags_for_flat_only_test_source(source string, mut prefs pref.Preferences) string {
+	tmp_dir := os.join_path(os.vtmp_dir(), 'v2_builder_flag_target_flat_${os.getpid()}')
+	os.mkdir_all(tmp_dir) or { panic(err) }
+	defer {
+		os.rmdir_all(tmp_dir) or {}
+	}
+	source_path := os.join_path(tmp_dir, 'main.v')
+	os.write_file(source_path, source) or { panic(err) }
+	prefs.skip_builtin = true
+	mut b := new_builder(&prefs)
+	b.flat = ast.flatten_files([
+		ast.File{
+			name: source_path
+			mod:  'main'
+		},
+	])
+	b.files = []ast.File{}
+	return b.collect_cflags_from_sources()
+}
+
 fn freestanding_test_call_name(name string, user_defines []string) string {
 	return freestanding_test_call_name_with_hooks(name, user_defines, [])
 }
@@ -304,6 +324,16 @@ $if linux {
 	assert free_explicit_flags.contains('-DOPTIONAL_FREE_BLOCK')
 }
 
+fn test_collect_cflags_from_sources_reads_flat_file_names() {
+	source := 'module main
+
+#flag -DFLAT_ONLY_FLAG
+'
+	mut prefs := pref.new_preferences()
+	flags := collect_cflags_for_flat_only_test_source(source, mut prefs)
+	assert flags.contains('-DFLAT_ONLY_FLAG')
+}
+
 fn test_collect_cflags_from_sources_keeps_optional_os_flags_custom_only() {
 	source := 'module main
 

vlib/v2/builder/flat_streaming_test.v

@@ -114,7 +114,7 @@ fn test_streaming_real_source_file() {
 	assert flat_batch.signature() == flat_stream.signature(), 'real-source streaming mismatch'
 }
 
-fn test_check_flat_default_parallel_transform_keeps_user_files() {
+fn test_default_flat_parallel_transform_keeps_user_files() {
 	path := write_tmp_file('parallel_transform_main', 'module main
 
 fn main() {
@@ -126,7 +126,7 @@ fn main() {
 		os.rm(path) or {}
 		os.rm(out_path) or {}
 	}
-	cmd := 'V2_CHECK_FLAT=1 ${os.quoted_path(@VEXE)} -v2 -nocache -o ${os.quoted_path(out_path)} ${os.quoted_path(path)} 2>&1'
+	cmd := '${os.quoted_path(@VEXE)} -v2 -nocache -o ${os.quoted_path(out_path)} ${os.quoted_path(path)} 2>&1'
 	res := os.execute(cmd)
 	assert res.exit_code == 0, res.output
 	run_res := os.execute(os.quoted_path(out_path))