How to dynamically compile generated pieces of zig code?

[trenta3]

Hi! Thank you very much for this wonderful project.
Is it currently possible to programmatically compile a string representing a snippet of zig code into executable code?
Something that can be written in the code below instead of the ???? to allow the compilation of the given source.

const compile = ????
const program_source =
    \\const std = @import("std");
    \\
    \\pub fn main() anyerror!void {
    \\    std.log.info("All your codebase are belong to us.", .{});
    \\}
;
const compiled_program = compile(program_source);

My aim is to write a metacompiler and the ability to delegate the compilation to machine code to zig would be great!

[AssortedFantasy]

I'm not sure, but I think the Zig compiler code currently does something like this for testing itself so I'd look into it.
I'd start here: https://github.com/ziglang/zig/blob/master/test/cases.zig

I'm not sure how the testing works so I don't know if it fits.

[marler8997]

There was a somewhat recent commit (maybe a month ago) where there was some enhancements made to the Zig build system to make it easier to support code generation at build time.

As far as dynamically generating and compiling zig code using comptime, this has been proposed and rejected as far as I can remember. I believe the main reason for this was that the alternative of generating code before comptime (during build time) was better in most cases and good enough in the other cases. Keep in mind that having code generated in a persistent file means you can generate better compile errors and generate corresponding debug symbols that refer to it.

[trenta3]

@marler8997 I'm interested in compiling a static executable from some code at run-time, not at build time.

@AssortedFantasy Thanks for the reference, it was useful a little.
I report on what I've obtained:

build.zig
src/main.zig
src/zig --> Symbolic link to zig source code (i.e. main folder when downloading using git).

// main.zig
const std = @import("std");
const introspect = @import("zig/src/introspect.zig");
const ThreadPool = @import("zig/src/ThreadPool.zig");
const Compilation = @import("zig/src/Compilation.zig");
const Package = @import("zig/src/Package.zig");

pub fn main() anyerror!void {
    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
    var allocator = &gpa.allocator;

    const target = std.Target{
        .cpu = .{
            .arch = .x86_64,
            .model = std.Target.Cpu.Model.generic(.x86_64),
            .features = std.Target.Cpu.Feature.Set.empty_workaround(),
        },
        .os = std.Target.Os.Tag.defaultVersionRange(.linux),
        .abi = .gnuabi64,
    };

    const self_exe_path = try std.fs.selfExePathAlloc(allocator);
    var zig_lib_directory = try introspect.findZigLibDirFromSelfExe(allocator, self_exe_path);
    defer zig_lib_directory.handle.close();
    defer allocator.free(zig_lib_directory.path.?);

    var thread_pool: ThreadPool = undefined;
    try thread_pool.init(allocator);
    defer thread_pool.deinit();

    var global_tmp = std.testing.tmpDir(.{});
    defer global_tmp.cleanup();

    var cache_dir = try global_tmp.dir.makeOpenPath("zig-cache", .{});
    defer cache_dir.close();
    const tmp_dir_path = try std.fs.path.join(allocator, &[_][]const u8{ ".", "zig-cache", "tmp", &global_tmp.sub_path });
    defer allocator.free(tmp_dir_path);

    const local_cache_path = try std.fs.path.join(allocator, &[_][]const u8{ tmp_dir_path, "zig-cache" });

    const zig_cache_directory: Compilation.Directory = .{
        .handle = cache_dir,
        .path = local_cache_path,
    };

    const global_cache_directory: Compilation.Directory = .{
        .handle = cache_dir,
        .path = try std.fs.path.join(allocator, &[_][]const u8{ tmp_dir_path, "zig-cache" }),
    };
    defer allocator.free(global_cache_directory.path.?);

    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();

    const emit_directory: Compilation.Directory = .{
        .path = tmp_dir_path,
        .handle = tmp.dir,
    };
    const emit_bin: Compilation.EmitLoc = .{
        .directory = emit_directory,
        .basename = "binary",
    };

    const tmp_src_path = "tmp.zig";
    var main_pkg: Package = .{
        .root_src_directory = .{ .path = tmp_dir_path, .handle = tmp.dir },
        .root_src_path = tmp_src_path,
    };
    defer main_pkg.table.deinit(allocator);

    // TODO: Understand how to include a small source
    const comp = try Compilation.create(allocator, .{
        .local_cache_directory = zig_cache_directory,
        .global_cache_directory = global_cache_directory,
        .zig_lib_directory = zig_lib_directory,
        .thread_pool = &thread_pool,
        .root_name = "test_case",
        .target = target,
        .output_mode = .Exe,
        .is_test = false,
        .optimize_mode = .ReleaseSmall,
        .emit_bin = emit_bin,
        .emit_h = null,
        .main_pkg = &main_pkg,
        .keep_source_files_loaded = true,
        .object_format = .elf,
        .is_native_os = true,
        .is_native_abi = true,
        .link_mode = .Static,
        .use_clang = false,
        .use_lld = false,
        .link_libc = false,
        .use_llvm = false,
        .use_stage1 = false,
        .self_exe_path = std.testing.zig_exe_path, // What do we need this?
    });
    defer comp.destroy();

    try comp.makeBinFileWritable();
    try comp.update();

    std.log.info("All your codebase are belong to us.", .{});
}

// build.zig
const std = @import("std");

pub fn build(b: *std.build.Builder) void {
    // Standard target options allows the person running `zig build` to choose
    // what target to build for. Here we do not override the defaults, which
    // means any target is allowed, and the default is native. Other options
    // for restricting supported target set are available.
    const target = b.standardTargetOptions(.{});

    // Standard release options allow the person running `zig build` to select
    // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall.
    const mode = b.standardReleaseOptions();

    const exe = b.addExecutable("zigcomp", "src/main.zig");
    exe.addBuildOption(bool, "omit_stage2", false);
    exe.addBuildOption(bool, "is_stage1", false);
    exe.addBuildOption(bool, "have_llvm", false);
    exe.addBuildOption([:0]const u8, "version", "12.0.0");
    exe.addBuildOption(bool, "enable_tracy", false);
    exe.addBuildOption(bool, "skip_non_native", false);
    exe.setTarget(target);
    exe.setBuildMode(mode);
    exe.install();

    const run_cmd = exe.run();
    run_cmd.step.dependOn(b.getInstallStep());
    if (b.args) |args| {
        run_cmd.addArgs(args);
    }

    const run_step = b.step("run", "Run the app");
    run_step.dependOn(&run_cmd.step);
}

This at least builds and executes correctly, even though I've yet to understand how to make it compile a piece of source code given as a string...
I will report back if I obtain other results, and I'm well open to other suggestions.

[trenta3]

So, thank you again. I finally succeeded in compiling a string to binary using zig stage2 native compiler.
Unfortunately many things are still missing and inserting a @import("std") makes compilation impossible raising the error in this line of code.
So I think that I should wait some more time to get what I want, but I see that it is underway.