From 175dac821320ed219b18c47895b72d985f3ee64a Mon Sep 17 00:00:00 2001 From: Natan Date: Sat, 21 Oct 2023 09:26:14 -0300 Subject: [PATCH] Improve emscripten build logic --- example/lib/generator/src/main/java/Main.java | 66 +++++--- example/lib/teavm/build.gradle.kts | 2 +- .../xpenatan/jparser/builder/BuildConfig.java | 2 + .../xpenatan/jparser/builder/BuildTarget.java | 16 +- .../builder/targets/AndroidTarget.java | 6 +- .../builder/targets/EmscriptenTarget.java | 141 +++++++++++------- .../xpenatan/jparser/idl/IDLReader.java | 10 ++ 7 files changed, 161 insertions(+), 82 deletions(-) diff --git a/example/lib/generator/src/main/java/Main.java b/example/lib/generator/src/main/java/Main.java index ff631523..5c671043 100644 --- a/example/lib/generator/src/main/java/Main.java +++ b/example/lib/generator/src/main/java/Main.java @@ -2,9 +2,7 @@ import com.github.xpenatan.jparser.builder.BuildMultiTarget; import com.github.xpenatan.jparser.builder.JBuilder; import com.github.xpenatan.jparser.builder.targets.AndroidTarget; -import com.github.xpenatan.jparser.builder.targets.EmscriptenLibTarget; import com.github.xpenatan.jparser.builder.targets.EmscriptenTarget; -import com.github.xpenatan.jparser.builder.targets.IOSTarget; import com.github.xpenatan.jparser.builder.targets.WindowsTarget; import com.github.xpenatan.jparser.core.JParser; import com.github.xpenatan.jparser.core.util.FileHelper; @@ -85,6 +83,7 @@ private static void generateAndBuild() throws Exception { private static BuildMultiTarget getWindowTarget() { BuildMultiTarget multiTarget = new BuildMultiTarget(); + // Make a static library WindowsTarget windowsTarget = new WindowsTarget(); windowsTarget.isStatic = true; windowsTarget.addJNI = false; @@ -103,28 +102,53 @@ private static BuildMultiTarget getWindowTarget() { private static BuildMultiTarget getEmscriptenTarget(IDLReader idlReader) { BuildMultiTarget multiTarget = new BuildMultiTarget(); - EmscriptenLibTarget emscriptenTarget = new EmscriptenLibTarget(); - emscriptenTarget.libName = "exampleLibside"; - emscriptenTarget.headerDirs.add("-Isrc/exampleLib"); - emscriptenTarget.headerDirs.add("-includesrc/exampleLib/CustomCode.h"); - emscriptenTarget.cppIncludes.add("**/src/exampleLib/**.cpp"); - emscriptenTarget.cppFlags.add("-fPIC"); - emscriptenTarget.cppFlags.add("-sEXPORT_ALL=1"); - emscriptenTarget.linkerFlags.add("-v"); - emscriptenTarget.linkerFlags.add("-fPIC"); - emscriptenTarget.linkerFlags.add("-sSIDE_MODULE=1"); - emscriptenTarget.linkerFlags.add("-sEXPORT_ALL=1"); - emscriptenTarget.libSuffix = ".wasm"; - multiTarget.add(emscriptenTarget); - +// EmscriptenLibTarget emscriptenTarget = new EmscriptenLibTarget(); +// emscriptenTarget.libName = "exampleLibside"; +// emscriptenTarget.headerDirs.add("-Isrc/exampleLib"); +// emscriptenTarget.headerDirs.add("-includesrc/exampleLib/CustomCode.h"); +// emscriptenTarget.cppIncludes.add("**/src/exampleLib/**.cpp"); +// emscriptenTarget.cppFlags.add("-fPIC"); +// emscriptenTarget.cppFlags.add("-sEXPORT_ALL=1"); +// emscriptenTarget.linkerFlags.add("-v"); +// emscriptenTarget.linkerFlags.add("-fPIC"); +// emscriptenTarget.linkerFlags.add("-sSIDE_MODULE=1"); +// emscriptenTarget.linkerFlags.add("-sEXPORT_ALL=1"); +// emscriptenTarget.libSuffix = ".wasm"; +// multiTarget.add(emscriptenTarget); +// +// EmscriptenTarget mainTarget = new EmscriptenTarget(idlReader); +// mainTarget.headerDirs.add("-Isrc/exampleLib"); +// mainTarget.headerDirs.add("-includesrc/exampleLib/CustomCode.h"); +// mainTarget.cppFlags.add("-fPIC"); +// mainTarget.linkerFlags.add("-sMAIN_MODULE=2"); +// mainTarget.linkerFlags.add("-fPIC"); +// mainTarget.linkerFlags.add("-ERROR_ON_UNDEFINED_SYMBOLS=0"); +// mainTarget.linkerFlags.add("../../libs/emscripten/exampleLibside.wasm"); +// multiTarget.add(mainTarget); + + { + // Make a static library + EmscriptenTarget libTarget = new EmscriptenTarget(idlReader); + libTarget.isStatic = true; + libTarget.headerDirs.add("-Isrc/exampleLib"); + libTarget.headerDirs.add("-includesrc/exampleLib/CustomCode.h"); + libTarget.cppIncludes.add("**/src/exampleLib/**.cpp"); +// multiTarget.add(libTarget); + + // Only Link and make js file + EmscriptenTarget mainTarget = new EmscriptenTarget(idlReader); + mainTarget.compileGlueCode = false; + mainTarget.shouldCompile = false; + mainTarget.linkerFlags.add("-sMAIN_MODULE=2"); + mainTarget.linkerFlags.add("../../libs/emscripten/exampleLib.a"); +// multiTarget.add(mainTarget); + } + + // Compile and create a js file EmscriptenTarget mainTarget = new EmscriptenTarget(idlReader); mainTarget.headerDirs.add("-Isrc/exampleLib"); mainTarget.headerDirs.add("-includesrc/exampleLib/CustomCode.h"); - mainTarget.cppFlags.add("-fPIC"); - mainTarget.linkerFlags.add("-sMAIN_MODULE=2"); - mainTarget.linkerFlags.add("-fPIC"); - mainTarget.linkerFlags.add("-ERROR_ON_UNDEFINED_SYMBOLS=0"); - mainTarget.linkerFlags.add("../../libs/emscripten/exampleLibside.wasm"); + mainTarget.cppIncludes.add("**/src/exampleLib/**.cpp"); multiTarget.add(mainTarget); return multiTarget; diff --git a/example/lib/teavm/build.gradle.kts b/example/lib/teavm/build.gradle.kts index 0e74391a..505a196d 100644 --- a/example/lib/teavm/build.gradle.kts +++ b/example/lib/teavm/build.gradle.kts @@ -2,7 +2,7 @@ plugins { id("java") } -val emscriptenFile = "$projectDir/../generator/build/c++/libs/emscripten/" +val emscriptenFile = "$projectDir/../generator/build/c++/libs/emscripten/exampleLib.wasm.js" tasks.jar { from(emscriptenFile) diff --git a/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/BuildConfig.java b/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/BuildConfig.java index 6cfc171f..019f39f3 100644 --- a/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/BuildConfig.java +++ b/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/BuildConfig.java @@ -1,10 +1,12 @@ package com.github.xpenatan.jparser.builder; import com.github.xpenatan.jparser.core.util.CustomFileDescriptor; +import java.util.ArrayList; public class BuildConfig { public CustomFileDescriptor buildDir; public CustomFileDescriptor sourceDir; + public ArrayList additionalSourceDirs = new ArrayList<>(); public CustomFileDescriptor libDir; public String libName; diff --git a/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/BuildTarget.java b/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/BuildTarget.java index 05fd39d2..ef4e8c0e 100644 --- a/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/BuildTarget.java +++ b/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/BuildTarget.java @@ -40,8 +40,8 @@ public static boolean isUnix() { public String libName = ""; public String libDirSuffix = ""; - public boolean isCompile = true; - public boolean isLink = true; + public boolean shouldCompile = true; + public boolean shouldLink = true; public boolean addJNI = true; public boolean isStatic = false; @@ -62,15 +62,19 @@ protected boolean build(BuildConfig config) { setup(config); - ArrayList cppFiles = getCPPFiles(config.sourceDir, cppIncludes); + ArrayList cppFiles = new ArrayList<>(getCPPFiles(config.sourceDir, cppIncludes)); + for(CustomFileDescriptor sourceDir : config.additionalSourceDirs) { + ArrayList cppFiles1 = getCPPFiles(sourceDir, cppIncludes); + cppFiles.addAll(cppFiles1); + } - if(isCompile && isLink && compile(config, childTarget, cppFiles)) { + if(shouldCompile && shouldLink && compile(config, childTarget, cppFiles)) { return link(config, childTarget); } - else if(isCompile && !isLink) { + else if(shouldCompile && !shouldLink) { return compile(config, childTarget, cppFiles); } - else if(!isCompile && isLink) { + else if(!shouldCompile && shouldLink) { return link(config, childTarget); } else { diff --git a/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/targets/AndroidTarget.java b/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/targets/AndroidTarget.java index c773c5d6..6df38681 100644 --- a/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/targets/AndroidTarget.java +++ b/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/targets/AndroidTarget.java @@ -64,7 +64,11 @@ protected boolean build(BuildConfig config) { } linkerFlagsStr = linkerFlagsStr.trim(); - ArrayList cppFiles = getCPPFiles(config.sourceDir, cppIncludes); + ArrayList cppFiles = new ArrayList<>(getCPPFiles(config.sourceDir, cppIncludes)); + for(CustomFileDescriptor sourceDir : config.additionalSourceDirs) { + ArrayList cppFiles1 = getCPPFiles(sourceDir, cppIncludes); + cppFiles.addAll(cppFiles1); + } String srcFilesStr = ""; for(CustomFileDescriptor file : cppFiles) { diff --git a/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/targets/EmscriptenTarget.java b/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/targets/EmscriptenTarget.java index bea0a685..07538e89 100644 --- a/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/targets/EmscriptenTarget.java +++ b/jParser/builder/src/main/java/com/github/xpenatan/jparser/builder/targets/EmscriptenTarget.java @@ -5,13 +5,15 @@ import com.github.xpenatan.jparser.builder.JProcess; import com.github.xpenatan.jparser.core.util.CustomFileDescriptor; import com.github.xpenatan.jparser.idl.IDLReader; -import java.io.File; import java.util.ArrayList; public class EmscriptenTarget extends BuildTarget { private IDLReader idlReader; + public boolean isStatic = false; + public boolean compileGlueCode = true; + String EMSCRIPTEN_ROOT = System.getenv("EMSDK") + "/upstream/emscripten/"; String WEBIDL_BINDER_SCRIPT = EMSCRIPTEN_ROOT + "tools/webidl_binder.py"; @@ -20,8 +22,6 @@ public EmscriptenTarget(IDLReader idlReader) { this.tempBuildDir = "target/emscripten"; this.idlReader = idlReader; - long initialMemory = 64 * 1024 * 1024; - cppCompiler.clear(); linkerCompiler.clear(); @@ -32,36 +32,20 @@ public EmscriptenTarget(IDLReader idlReader) { cppCompiler.add(cppCompilerr); linkerCompiler.add(cppCompilerr); - linkerFlags.add("-O3"); - linkerFlags.add("-std=c++17"); - linkerFlags.add("--llvm-lto"); - linkerFlags.add("1"); - linkerFlags.add("-s"); - linkerFlags.add("ALLOW_MEMORY_GROWTH=1"); - linkerFlags.add("-s"); - linkerFlags.add("ALLOW_TABLE_GROWTH=1"); - linkerFlags.add("-s"); - linkerFlags.add("MODULARIZE=1"); - linkerFlags.add("-s"); - linkerFlags.add("NO_FILESYSTEM=1"); - linkerFlags.add("-s"); - linkerFlags.add("INITIAL_MEMORY=" + initialMemory); - linkerFlags.add("-s"); - linkerFlags.add("EXPORTED_FUNCTIONS=['_free','_malloc']"); - linkerFlags.add("-s"); - linkerFlags.add("EXPORTED_RUNTIME_METHODS=['UTF8ToString']"); - linkerFlags.add("-s"); - linkerFlags.add("WASM=1"); - linkerFlags.add("-s"); - linkerFlags.add("SINGLE_FILE=1"); - libSuffix = ".wasm.js"; - cppIncludes.add("**/jsglue/*.cpp"); + cppFlags.add("-c"); + cppFlags.add("-std=c++17"); + cppFlags.add("-O3"); } @Override protected boolean build(BuildConfig config) { + String libName = this.libName; + if(libName.isEmpty()) { + libName = config.libName; + } + CustomFileDescriptor childTarget = config.buildDir.child(tempBuildDir); if(childTarget.exists()) { childTarget.delete(); @@ -73,37 +57,86 @@ protected boolean build(BuildConfig config) { jsglueDir.mkdirs(); } + if(compileGlueCode) { + cppIncludes.add("**/jsglue/*.cpp"); + copyHelperClass(jsglueDir); + } + CustomFileDescriptor mergedIDLFile = mergeIDLFile(jsglueDir); + if(!createGlueCode(mergedIDLFile, jsglueDir)) { + return false; + } - CustomFileDescriptor idlHelperCPP = new CustomFileDescriptor("IDLHelper.h", CustomFileDescriptor.FileType.Classpath); - idlHelperCPP.copyTo(jsglueDir, false); + if(isStatic) { + linkerCompiler.clear(); - CustomFileDescriptor cppFile = jsglueDir.child(idlHelperCPP.name()); - headerDirs.add("-include" + cppFile.path()); + String cppCompilerr = EMSCRIPTEN_ROOT + "emar"; + if(isWindows()) { + cppCompilerr += ".bat"; + } - String jsGluePath = jsglueDir.path() + File.separator; + linkerCompiler.add(cppCompilerr); + linkerFlags.add("rcs"); + libSuffix = ".a"; + } + else { + String postPath = createPostJS(jsglueDir, libName); + long initialMemory = 64 * 1024 * 1024; + linkerFlags.add("--llvm-lto"); + linkerFlags.add("1"); + linkerFlags.add("-s"); + linkerFlags.add("ALLOW_MEMORY_GROWTH=1"); + linkerFlags.add("-s"); + linkerFlags.add("ALLOW_TABLE_GROWTH=1"); + linkerFlags.add("-s"); + linkerFlags.add("MODULARIZE=1"); + linkerFlags.add("-s"); + linkerFlags.add("NO_FILESYSTEM=1"); + linkerFlags.add("-s"); + linkerFlags.add("INITIAL_MEMORY=" + initialMemory); + linkerFlags.add("-s"); + linkerFlags.add("EXPORTED_FUNCTIONS=['_free','_malloc']"); + linkerFlags.add("-s"); + linkerFlags.add("EXPORTED_RUNTIME_METHODS=['UTF8ToString']"); + linkerFlags.add("-s"); + linkerFlags.add("WASM=1"); + linkerFlags.add("-s"); + linkerFlags.add("SINGLE_FILE=1"); + + linkerFlags.add("--post-js"); + linkerFlags.add(jsglueDir.path() + "/glue.js"); + linkerFlags.add("--extern-post-js"); + linkerFlags.add(postPath); + linkerFlags.add("-s"); + linkerFlags.add("EXPORT_NAME='" + libName + "'"); + } - CustomFileDescriptor postFile = new CustomFileDescriptor("emscripten/post.js", CustomFileDescriptor.FileType.Classpath); - String s = postFile.readString(); + return super.build(config); + } - String libName = this.libName; - if(libName.isEmpty()) { - libName = config.libName; + @Override + protected void onLink(String objFilePath, String libPath) { + if(isStatic) { + linkerCommands.addAll(linkerCompiler); + linkerCommands.addAll(linkerFlags); + linkerCommands.add(libPath); + linkerCommands.add("@" + objFilePath); + } + else { + super.onLink(objFilePath, libPath); } + } + private String createPostJS(CustomFileDescriptor jsglueDir, String libName) { + CustomFileDescriptor postFile = new CustomFileDescriptor("emscripten/post.js", CustomFileDescriptor.FileType.Classpath); + String s = postFile.readString(); s = s.replace("[MODULE_NAME]", libName); - - CustomFileDescriptor postJS = new CustomFileDescriptor(jsGluePath + "post.js"); + CustomFileDescriptor postJS = new CustomFileDescriptor(jsglueDir + "/post.js"); postJS.writeString(s, false); - String postPath = postJS.path(); - - linkerFlags.add("--post-js"); - linkerFlags.add(jsGluePath + "glue.js"); - linkerFlags.add("--extern-post-js"); - linkerFlags.add(postPath); - linkerFlags.add("-s"); - linkerFlags.add("EXPORT_NAME='" + libName + "'"); + return postJS.path(); + } + private boolean createGlueCode(CustomFileDescriptor mergedIDLFile, CustomFileDescriptor jsglueDir) { String pythonCmd = "python"; if(isUnix()) { pythonCmd = "python3"; @@ -114,13 +147,15 @@ protected boolean build(BuildConfig config) { generateGlueCommand.add(WEBIDL_BINDER_SCRIPT); generateGlueCommand.add(mergedIDLFile.toString()); generateGlueCommand.add("glue"); - if(!JProcess.startProcess(jsglueDir.file(), generateGlueCommand)) { - return false; - } - - cppFlags.add("-c"); + return JProcess.startProcess(jsglueDir.file(), generateGlueCommand); + } - return super.build(config); + private void copyHelperClass(CustomFileDescriptor jsglueDir) { + // Copy IDLHelper from base module. + CustomFileDescriptor idlHelperCPP = new CustomFileDescriptor("IDLHelper.h", CustomFileDescriptor.FileType.Classpath); + idlHelperCPP.copyTo(jsglueDir, false); + CustomFileDescriptor cppFile = jsglueDir.child(idlHelperCPP.name()); + headerDirs.add("-include" + cppFile.path()); } private CustomFileDescriptor mergeIDLFile(CustomFileDescriptor jsglueDir) { diff --git a/jParser/idl/src/main/java/com/github/xpenatan/jparser/idl/IDLReader.java b/jParser/idl/src/main/java/com/github/xpenatan/jparser/idl/IDLReader.java index 1baad7ba..ada75754 100644 --- a/jParser/idl/src/main/java/com/github/xpenatan/jparser/idl/IDLReader.java +++ b/jParser/idl/src/main/java/com/github/xpenatan/jparser/idl/IDLReader.java @@ -73,6 +73,16 @@ public static IDLReader readIDL(String idlDir) { return reader; } + public static void readIDL(IDLReader reader, String idlDir) { + try { + idlDir = new File(idlDir).getCanonicalPath() + File.separator; + } catch(IOException e) { + throw new RuntimeException("IDL file not found: " + idlDir); + } + IDLFile idlFile = parseFile(idlDir); + reader.fileArray.add(idlFile); + } + public static IDLReader readIDL(ArrayList idlDirs) { IDLReader reader = new IDLReader(); for(String idlDir : idlDirs) {