diff --git a/java/fury-core/src/main/java/io/fury/codegen/Code.java b/java/fury-core/src/main/java/io/fury/codegen/Code.java new file mode 100644 index 0000000000..dfaf7807a0 --- /dev/null +++ b/java/fury-core/src/main/java/io/fury/codegen/Code.java @@ -0,0 +1,215 @@ +/* + * Copyright 2023 The Fury authors + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fury.codegen; + +import static io.fury.codegen.Code.LiteralValue.FalseLiteral; +import static io.fury.codegen.Code.LiteralValue.TrueLiteral; + +import java.util.Objects; + +public interface Code { + + /** + * The code for a sequence of statements to evaluate the expression in a scope. If no code needs + * to be evaluated, or expression is already evaluated in a scope ( see {@link + * Expression#genCode(CodegenContext)}), thus `isNull` and `value` are already existed, the code + * should be null. + */ + class ExprCode { + private final String code; + private final ExprValue isNull; + private final ExprValue value; + + public ExprCode(String code) { + this(code, null, null); + } + + public ExprCode(ExprValue isNull, ExprValue value) { + this(null, isNull, value); + } + + /** + * Create an `ExprCode`. + * + * @param code The sequence of statements required to evaluate the expression. It should be + * null, if `isNull` and `value` are already existed, or no code needed to evaluate them + * (literals). + * @param isNull A term that holds a boolean value representing whether the expression evaluated + * to null. + * @param value A term for a (possibly primitive) value of the result of the evaluation. Not + * valid if `isNull` is set to `true`. + */ + public ExprCode(String code, ExprValue isNull, ExprValue value) { + this.code = code; + this.isNull = isNull; + this.value = value; + } + + public String code() { + return code; + } + + public ExprValue isNull() { + return isNull; + } + + public ExprValue value() { + return value; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ExprCode("); + if (code != null) { + sb.append("code=\"").append('\n').append(code).append("\n\", "); + } + sb.append("isNull=").append(isNull); + sb.append(", value=").append(value); + sb.append(')'); + return sb.toString(); + } + } + + /** Fragments of java code. */ + abstract class JavaCode { + + abstract String code(); + + @Override + public String toString() { + return code(); + } + } + + /** A typed java fragment that must be a valid java expression. */ + abstract class ExprValue extends JavaCode { + + private final Class javaType; + + public ExprValue(Class javaType) { + this.javaType = javaType; + } + + Class javaType() { + return javaType; + } + + boolean isPrimitive() { + return javaType.isPrimitive(); + } + } + + /** A java expression fragment. */ + class SimpleExprValue extends ExprValue { + + private final String expr; + + public SimpleExprValue(Class javaType, String expr) { + super(javaType); + this.expr = expr; + } + + @Override + String code() { + return String.format("(%s)", expr); + } + } + + /** A local variable java expression. */ + class VariableValue extends ExprValue { + private final String variableName; + + public VariableValue(Class javaType, String variableName) { + super(javaType); + this.variableName = variableName; + } + + @Override + String code() { + return variableName; + } + } + + /** A literal java expression. */ + class LiteralValue extends ExprValue { + static LiteralValue TrueLiteral = new LiteralValue(boolean.class, "true"); + static LiteralValue FalseLiteral = new LiteralValue(boolean.class, "false"); + + private final String value; + + public LiteralValue(Object value) { + super(value.getClass()); + this.value = value.toString(); + } + + public LiteralValue(Class javaType, String value) { + super(javaType); + this.value = value; + } + + @Override + String code() { + return value; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LiteralValue that = (LiteralValue) o; + return this.javaType() == that.javaType() && Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(value, javaType()); + } + } + + // ########################## utils ########################## + static ExprValue exprValue(Class type, String code) { + return new SimpleExprValue(type, code); + } + + static ExprValue variable(Class type, String name) { + return new VariableValue(type, name); + } + + static ExprValue isNullVariable(String name) { + return new VariableValue(boolean.class, name); + } + + static ExprValue literal(Class type, String value) { + if (type == Boolean.class || type == boolean.class) { + if ("true".equals(value)) { + return TrueLiteral; + } else if ("false".equals(value)) { + return FalseLiteral; + } else { + throw new IllegalArgumentException(value); + } + } else { + return new LiteralValue(type, value); + } + } +} diff --git a/java/fury-core/src/main/java/io/fury/codegen/CodeGenerator.java b/java/fury-core/src/main/java/io/fury/codegen/CodeGenerator.java index d490bf7ca1..c44136f71b 100644 --- a/java/fury-core/src/main/java/io/fury/codegen/CodeGenerator.java +++ b/java/fury-core/src/main/java/io/fury/codegen/CodeGenerator.java @@ -18,12 +18,33 @@ package io.fury.codegen; +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.fury.collection.MultiKeyWeakMap; +import io.fury.util.ClassLoaderUtils; +import io.fury.util.ClassLoaderUtils.ByteArrayClassLoader; import io.fury.util.LoggerFactory; import io.fury.util.StringUtils; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; +import java.util.WeakHashMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; /** - * Code generator to generate class from {@link CompileUnit}. + * Code generator will take a list of {@link CompileUnit} and compile it into a list of classes. + * + *

The compilation will be executed in a thread-pool parallel for speed. * * @author chaokunyang */ @@ -42,6 +63,239 @@ public class CodeGenerator { // The max valid length of method parameters in JVM. static final int MAX_JVM_METHOD_PARAMS_LENGTH = 255; + // FIXME The classloaders will only be reclaimed when the generated class are not be referenced. + // FIXME CodeGenerator may reference to classloader, thus cause circular reference, neither can + // be gc. + private static final WeakHashMap> sharedCodeGenerator = + new WeakHashMap<>(); + private static final MultiKeyWeakMap> sharedCodeGenerator2 = + new MultiKeyWeakMap<>(); + private static int maxPoolSize = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); + private static ListeningExecutorService compilationExecutorService; + + private ClassLoader classLoader; + private final Object classLoaderLock; + private final ConcurrentHashMap parallelCompileState; + private final ConcurrentHashMap parallelDefineStatusLock; + + public CodeGenerator(ClassLoader classLoader) { + Preconditions.checkNotNull(classLoader); + this.classLoader = classLoader; + parallelCompileState = new ConcurrentHashMap<>(); + parallelDefineStatusLock = new ConcurrentHashMap<>(); + classLoaderLock = new Object(); + } + + /** + * Compile code, return as a new classloader. If the class of a compilation unit already exists in + * previous classloader, skip the corresponding compilation unit. + * + * @param units compile units + */ + public ClassLoader compile(CompileUnit... units) { + return compile(Arrays.asList(units), compileState -> compileState.lock.lock()); + } + + public ClassLoader compile(List units, CompileCallback callback) { + List compileUnits = new ArrayList<>(); + ClassLoader parentClassLoader; + // Note: avoid deadlock between classloader lock, compiler lock, + // jit lock and class-def lock. + synchronized (classLoaderLock) { // protect classLoader. + for (CompileUnit unit : units) { + if (!classExists(classLoader, unit.getQualifiedClassName())) { + compileUnits.add(unit); + } + } + if (compileUnits.isEmpty()) { + return classLoader; + } + parentClassLoader = classLoader; + } + CompileState compileState = getCompileState(compileUnits); + callback.lock(compileState); + Map classes; + if (compileState.finished) { + classes = compileState.result; + compileState.lock.unlock(); + } else { + try { + classes = + JaninoUtils.toBytecode(parentClassLoader, compileUnits.toArray(new CompileUnit[0])); + compileState.result = classes; + compileState.finished = true; + } finally { + compileState.lock.unlock(); + } + for (Map.Entry e : classes.entrySet()) { + String key = e.getKey(); + byte[] value = e.getValue(); + LOG.info("Code stats for class {} is {}", key, JaninoUtils.getClassStats(value)); + } + } + return defineClasses(classes); + } + + /** + * Define classes in classloader, create a new classloader if classes can' be loaded into previous + * classloader. + */ + private ClassLoader defineClasses(Map classes) { + if (classes.isEmpty()) { + return getClassLoader(); + } + ClassLoader resultClassLoader = null; + boolean isByteArrayClassLoader; + synchronized (classLoaderLock) { + isByteArrayClassLoader = classLoader instanceof ByteArrayClassLoader; + if (isByteArrayClassLoader) { + resultClassLoader = classLoader; + } + } + if (isByteArrayClassLoader) { + for (Map.Entry entry : classes.entrySet()) { + String className = fullClassNameFromClassFilePath(entry.getKey()); + DefineState defineState = getDefineState(className); + // Avoid multi-compile unit classes define operation collision with single compile unit. + if (!defineState.defined) { // class not defined yet. + synchronized (defineState.lock) { + if (!defineState.defined) { // class not defined yet. + // Even if multiple compile unit is inter-dependent, they can still be defined + // separately. + ((ByteArrayClassLoader) (resultClassLoader)) + .defineClassPublic(className, entry.getValue()); + defineState.defined = true; + } + } + } + } + } else { + synchronized (classLoaderLock) { + ByteArrayClassLoader bytesClassLoader = new ByteArrayClassLoader(classes, classLoader); + for (String k : classes.keySet()) { + String className = fullClassNameFromClassFilePath(k); + DefineState defineState = getDefineState(className); + defineState.defined = true; // avoid duplicate def throws LinkError. + } + // Set up a class loader that finds and defined the generated classes. + classLoader = bytesClassLoader; + resultClassLoader = bytesClassLoader; + } + } + return resultClassLoader; + } + + public ListenableFuture[]> asyncCompile(CompileUnit... compileUnits) { + return getCompilationService() + .submit( + () -> { + ClassLoader loader = compile(compileUnits); + return Arrays.stream(compileUnits) + .map( + compileUnit -> { + try { + return (Class) loader.loadClass(compileUnit.getQualifiedClassName()); + } catch (ClassNotFoundException e) { + throw new IllegalStateException( + "Impossible because we just compiled class", e); + } + }) + .toArray(Class[]::new); + }); + } + + public static void seMaxCompilationThreadPoolSize(int maxCompilationThreadPoolSize) { + maxPoolSize = maxCompilationThreadPoolSize; + } + + public static synchronized ListeningExecutorService getCompilationService() { + if (compilationExecutorService == null) { + ThreadPoolExecutor executor = + new ThreadPoolExecutor( + maxPoolSize, + maxPoolSize, + 5L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + new ThreadFactoryBuilder().setNameFormat("fury-jit-compiler-%d").build(), + (r, e) -> LOG.warn("Task {} rejected from {}", r.toString(), e)); + // Normally task won't be rejected by executor, since we used an unbound queue. + // But when we shut down executor for debug, it'll be rejected by executor, + // in such cases we just ignore the reject exception by log it. + executor.allowCoreThreadTimeOut(true); + compilationExecutorService = MoreExecutors.listeningDecorator(executor); + } + return compilationExecutorService; + } + + public ClassLoader getClassLoader() { + synchronized (classLoaderLock) { + return classLoader; + } + } + + private CompileState getCompileState(List toCompile) { + return parallelCompileState.computeIfAbsent( + getCompileLockKey(toCompile), k -> new CompileState()); + } + + private String getCompileLockKey(List toCompile) { + if (toCompile.size() == 1) { + return toCompile.get(0).getQualifiedClassName(); + } else { + StringJoiner joiner = new StringJoiner(","); + for (CompileUnit unit : toCompile) { + joiner.add(unit.getQualifiedClassName()); + } + return joiner.toString(); + } + } + + private static class DefineState { + final Object lock; + volatile boolean defined; + + private DefineState() { + this.lock = new Object(); + } + } + + private DefineState getDefineState(String className) { + return parallelDefineStatusLock.computeIfAbsent(className, k -> new DefineState()); + } + + private boolean classExists(ClassLoader loader, String className) { + try { + loader.loadClass(className); + return true; + } catch (ClassNotFoundException e) { + return false; + } + } + + public static synchronized CodeGenerator getSharedCodeGenerator(ClassLoader... classLoaders) { + WeakReference codeGeneratorWeakRef = sharedCodeGenerator2.get(classLoaders); + CodeGenerator codeGenerator = codeGeneratorWeakRef != null ? codeGeneratorWeakRef.get() : null; + if (codeGenerator == null) { + codeGenerator = new CodeGenerator(new ClassLoaderUtils.ComposedClassLoader(classLoaders)); + sharedCodeGenerator2.put(classLoaders, new WeakReference<>(codeGenerator)); + } + return codeGenerator; + } + + public static synchronized CodeGenerator getSharedCodeGenerator(ClassLoader classLoader) { + if (classLoader == null) { + classLoader = CodeGenerator.class.getClassLoader(); + } + WeakReference codeGeneratorWeakRef = sharedCodeGenerator.get(classLoader); + CodeGenerator codeGenerator = codeGeneratorWeakRef != null ? codeGeneratorWeakRef.get() : null; + if (codeGenerator == null) { + codeGenerator = new CodeGenerator(classLoader); + sharedCodeGenerator.put(classLoader, new WeakReference<>(codeGenerator)); + } + return codeGenerator; + } + public static String getCodeDir() { return System.getProperty(CODE_DIR_KEY, System.getenv(CODE_DIR_KEY)); } @@ -55,4 +309,120 @@ static boolean deleteCodeOnExit() { } return deleteCodeOnExit; } + + public static String classFilepath(CompileUnit unit) { + return classFilepath(fullClassName(unit)); + } + + public static String classFilepath(String pkg, String className) { + return classFilepath(pkg + "." + className); + } + + public static String classFilepath(String fullClassName) { + int index = fullClassName.lastIndexOf("."); + if (index >= 0) { + return String.format( + "%s/%s.class", + fullClassName.substring(0, index).replace(".", "/"), fullClassName.substring(index + 1)); + } else { + return fullClassName + ".class"; + } + } + + public static String fullClassName(CompileUnit unit) { + return unit.pkg + "." + unit.mainClassName; + } + + public static String fullClassNameFromClassFilePath(String classFilePath) { + return classFilePath.substring(0, classFilePath.length() - ".class".length()).replace("/", "."); + } + + /** align code to have 4 spaces indent. */ + public static String alignIndent(String code) { + return alignIndent(code, 4); + } + + /** align code to have {@code numSpaces} spaces indent. */ + public static String alignIndent(String code, int numSpaces) { + if (code == null) { + return ""; + } + String[] split = code.split("\n"); + if (split.length == 1) { + return code; + } else { + StringBuilder codeBuilder = new StringBuilder(split[0]).append('\n'); + for (int i = 1; i < split.length; i++) { + for (int j = 0; j < numSpaces; j++) { + codeBuilder.append(' '); + } + codeBuilder.append(split[i]).append('\n'); + } + if (code.charAt(code.length() - 1) == '\n') { + return codeBuilder.toString(); + } else { + return codeBuilder.substring(0, codeBuilder.length() - 1); + } + } + } + + /** indent code by 4 spaces. */ + static String indent(String code) { + return indent(code, 2); + } + + /** The implementation shouldn't add redundant newline separator. */ + static String indent(String code, int numSpaces) { + if (code == null) { + return ""; + } + String[] split = code.split("\n"); + StringBuilder codeBuilder = new StringBuilder(); + for (String line : split) { + for (int i = 0; i < numSpaces; i++) { + codeBuilder.append(' '); + } + codeBuilder.append(line).append('\n'); + } + if (code.charAt(code.length() - 1) == '\n') { + return codeBuilder.toString(); + } else { + return codeBuilder.substring(0, codeBuilder.length() - 1); + } + } + + /** + * Create spaces. + * + * @param numSpaces spaces num + * @return a string of numSpaces spaces + */ + static String spaces(int numSpaces) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < numSpaces; i++) { + builder.append(' '); + } + return builder.toString(); + } + + static void appendNewlineIfNeeded(StringBuilder sb) { + if (sb.length() > 0 && sb.charAt(sb.length() - 1) != '\n') { + sb.append('\n'); + } + } + + static StringBuilder stripLastNewline(StringBuilder sb) { + int length = sb.length(); + Preconditions.checkArgument(length > 0 && sb.charAt(length - 1) == '\n'); + sb.deleteCharAt(length - 1); + return sb; + } + + static StringBuilder stripIfHasLastNewline(StringBuilder sb) { + int length = sb.length(); + if (length > 0 && sb.charAt(length - 1) == '\n') { + sb.deleteCharAt(length - 1); + } + return sb; + } } diff --git a/java/fury-core/src/main/java/io/fury/codegen/CodegenContext.java b/java/fury-core/src/main/java/io/fury/codegen/CodegenContext.java new file mode 100644 index 0000000000..7ba4e1e32b --- /dev/null +++ b/java/fury-core/src/main/java/io/fury/codegen/CodegenContext.java @@ -0,0 +1,599 @@ +/* + * Copyright 2023 The Fury authors + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fury.codegen; + +import static io.fury.codegen.Code.ExprCode; +import static io.fury.codegen.CodeGenerator.alignIndent; +import static io.fury.codegen.CodeGenerator.indent; +import static io.fury.type.TypeUtils.getArrayType; +import static io.fury.type.TypeUtils.getRawType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.reflect.TypeToken; +import io.fury.collection.Tuple2; +import io.fury.collection.Tuple3; +import io.fury.util.StringUtils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * CodegenContext can be an any scope in a class, such as class, method, local and so on. + * + *

All constructor of generated class will call {@code initialize()} to initialize object. We + * don't use instance initialize, so user can add init code which depends on used-passed + * constructor's args. + * + * @author chaokunyang + */ +public class CodegenContext { + public static Set JAVA_RESERVED_WORDS; + + static { + JAVA_RESERVED_WORDS = new HashSet<>(); + JAVA_RESERVED_WORDS.addAll( + Arrays.asList( + "abstract", + "assert", + "boolean", + "break", + "byte", + "case", + "catch", + "char", + "class", + "const", + "continue", + "default", + "do", + "double", + "else", + "enum", + "extends", + "final", + "finally", + "float", + "for", + "goto", + "if", + "implements", + "import", + "instanceof", + "int", + "interface", + "long", + "native", + "new", + "package", + "private", + "protected", + "public", + "return", + "short", + "static", + "strictfp", + "super", + "switch", + "synchronized", + "this", + "throw", + "throws", + "transient", + "try", + "void", + "volatile", + "while", + "true", + "false", + "null")); + JAVA_RESERVED_WORDS = ImmutableSet.copyOf(JAVA_RESERVED_WORDS); + } + + private static final String INITIALIZE_METHOD_NAME = "initialize"; + + Map newValNameIds = new HashMap<>(); + Set valNames = new HashSet<>(); + + /** + * State used for expression elimination/reuse. + * + *

Takes the first expression and requests it to generate a Java source code for the expression + * tree + * + *

The exprCode's code of subsequent same expression will be null, because the code is already + * added to current context + */ + Map exprState = new HashMap<>(); + + String pkg; + LinkedHashSet imports = new LinkedHashSet<>(); + String className; + String[] superClasses; + String[] interfaces; + List> fields = new ArrayList<>(); + /** + * all initCodes would be placed into a method called initialize(), which will be called by + * constructor. + */ + List initCodes = new ArrayList<>(); + + List constructors = new ArrayList<>(); + LinkedHashMap methods = new LinkedHashMap<>(); + + private CodegenContext instanceInitCtx; + + public CodegenContext() {} + + public CodegenContext(LinkedHashSet imports) { + this.imports = imports; + } + + public CodegenContext(Set valNames, LinkedHashSet imports) { + this.valNames = valNames; + this.imports = imports; + } + + /** + * Reserve name to avoid name collision for name that not created with {@link + * CodegenContext#newName(String)}. + */ + public void reserveName(String name) { + Preconditions.checkArgument(!valNames.contains(name)); + String s = newName(name); + Preconditions.checkArgument(s.equals(name)); + } + + public boolean containName(String name) { + return valNames.contains(name); + } + + /** + * If name is a java reserved word, return as if called with name "value". + * + *

Since we don't pass in TypeToken, no need to consider generics + */ + public String newName(Class clz) { + return newName(namePrefix(clz)); + } + + public String newName(Class clz, String suffix) { + String name = newName(clz); + return newName(name + suffix); + } + + /** Returns a term name that is unique within this instance of a `CodegenContext`. */ + public String newName(String name) { + newValNameIds.putIfAbsent(name, 0L); + long id = newValNameIds.get(name); + newValNameIds.put(name, id + 1); + if (id == 0) { + if (valNames.add(name)) { + return name; + } + } + String newName = String.format("%s%s", name, id); + while (valNames.contains(newName)) { + id++; + newValNameIds.put(name, id); + newName = String.format("%s%s", name, id); + } + valNames.add(newName); + return newName; + } + + /** Returns two term names that have same suffix to get more readability for generated code. */ + public String[] newNames(Class clz1, String name2) { + if (clz1.isArray()) { + return newNames("arr", name2); + } else { + String type = type(clz1); + int index = type.lastIndexOf("."); + String name; + if (index >= 0) { + name = StringUtils.uncapitalize(type.substring(index + 1)); + } else { + name = StringUtils.uncapitalize(type); + } + if (JAVA_RESERVED_WORDS.contains(name)) { + return newNames("value", name2); + } else { + return newNames(name, name2); + } + } + } + + /** + * Try to return term names that have same suffixes to get more readability for generated code. + */ + public String[] newNames(String... names) { + long id = 0; + for (String name : names) { + id = Math.max(id, newValNameIds.getOrDefault(name, 0L)); + } + for (String name : names) { + newValNameIds.put(name, id + 1); + } + if (id == 0 && Sets.intersection(valNames, Sets.newHashSet(names)).isEmpty()) { + valNames.addAll(Arrays.asList(names)); + return names; + } else { + String[] newNames = new String[names.length]; + for (int i = 0; i < names.length; i++) { + newNames[i] = String.format("%s%s", names[i], id); + while (valNames.contains(newNames[i])) { + id++; + newValNameIds.put(newNames[i], id); + newNames[i] = String.format("%s%s", names[i], id); + } + } + valNames.addAll(Arrays.asList(newNames)); + return newNames; + } + } + + public String namePrefix(Class clz) { + if (clz.isArray()) { + return "arr"; + } else { + String type = type(clz); + int index = type.lastIndexOf("."); + String name; + if (index >= 0) { + name = StringUtils.uncapitalize(type.substring(index + 1)); + } else { + name = StringUtils.uncapitalize(type); + } + + if (JAVA_RESERVED_WORDS.contains(name)) { + return "value"; + } else { + return name; + } + } + } + + /** + * Get type string. + * + * @param clz type + * @return simple name for class if type's canonical name starts with java.lang or is imported, + * return canonical name otherwise. + */ + public String type(Class clz) { + if (clz.isArray()) { + return getArrayType(clz); + } + String type = clz.getCanonicalName(); + if (type.startsWith("java.lang")) { + if (!type.substring("java.lang.".length()).contains(".")) { + return clz.getSimpleName(); + } + } + if (imports.contains(type)) { + return clz.getSimpleName(); + } else { + int index = type.lastIndexOf("."); + if (index > 0) { + // This might be package name or qualified name of outer class + String pkgOrClassName = type.substring(0, index); + if (imports.contains(pkgOrClassName + ".*")) { + return clz.getSimpleName(); + } + } + return type; + } + } + + /** return type name. since janino doesn't generics, we ignore type parameters in typeToken. */ + public String type(TypeToken typeToken) { + return type(getRawType(typeToken)); + } + + /** + * Set the generated class's package. + * + * @param pkg java package + */ + public void setPackage(String pkg) { + this.pkg = pkg; + } + + public Set getValNames() { + return valNames; + } + + public LinkedHashSet getImports() { + return imports; + } + + /** + * Import classes. + * + * @param classes classes to be imported + */ + public void addImports(Class... classes) { + for (Class clz : classes) { + imports.add(clz.getCanonicalName()); + } + } + + /** + * Add imports. + * + * @param imports import statements + */ + public void addImports(String... imports) { + this.imports.addAll(Arrays.asList(imports)); + } + + /** + * Import class. + * + *

Import class carefully, otherwise class will conflict. Only java.lang.Class is unique and + * won't conflict. + * + * @param cls class to be imported + */ + public void addImport(Class cls) { + this.imports.add(cls.getCanonicalName()); + } + + /** + * Add import. + * + *

Import class carefully, otherwise class will conflict. Only java.lang.Class is unique and + * won't conflict. + * + * @param im import statement + */ + public void addImport(String im) { + this.imports.add(im); + } + + /** + * Set class name of class to be generated. + * + * @param className class name of class to be generated + */ + public void setClassName(String className) { + this.className = className; + } + + /** + * Set super classes. + * + * @param superClasses super classes + */ + public void extendsClasses(String... superClasses) { + this.superClasses = superClasses; + } + + /** + * Set implemented interfaces. + * + * @param interfaces implemented interfaces + */ + public void implementsInterfaces(String... interfaces) { + this.interfaces = interfaces; + } + + public void addConstructor(String codeBody, Object... params) { + List> parameters = getParameters(params); + String paramsStr = + parameters.stream().map(t -> t.f0 + " " + t.f1).collect(Collectors.joining(", ")); + + StringBuilder codeBuilder = new StringBuilder(alignIndent(codeBody)).append("\n"); + for (String init : initCodes) { + codeBuilder.append(indent(init, 4)).append('\n'); + } + String constructor = + StringUtils.format( + "" + "public ${className}(${paramsStr}) {\n" + " ${codeBody}" + "}", + "className", + className, + "paramsStr", + paramsStr, + "codeBody", + codeBuilder); + constructors.add(constructor); + } + + public void addInitCode(String code) { + initCodes.add(code); + } + + public void addStaticMethod( + String methodName, String codeBody, Class returnType, Object... params) { + addMethod("public static", methodName, codeBody, returnType, params); + } + + public void addMethod(String methodName, String codeBody, Class returnType, Object... params) { + addMethod("public", methodName, codeBody, returnType, params); + } + + public void addMethod( + String modifier, String methodName, String codeBody, Class returnType, Object... params) { + List> parameters = getParameters(params); + String paramsStr = + parameters.stream().map(t -> t.f0 + " " + t.f1).collect(Collectors.joining(", ")); + String method = + StringUtils.format( + "" + + "${modifier} ${returnType} ${methodName}(${paramsStr}) {\n" + + " ${codeBody}\n" + + "}\n", + "modifier", + modifier, + "returnType", + type(returnType), + "methodName", + methodName, + "paramsStr", + paramsStr, + "codeBody", + alignIndent(codeBody)); + String signature = String.format("%s(%s)", methodName, paramsStr); + if (methods.containsKey(signature)) { + throw new IllegalStateException(String.format("Duplicated method signature: %s", signature)); + } + methods.put(signature, method); + } + + public void overrideMethod( + String methodName, String codeBody, Class returnType, Object... params) { + addMethod("@Override public final", methodName, codeBody, returnType, params); + } + + /** + * Get parameters. + * + * @param args type, value; type, value; type, value; ...... + */ + private List> getParameters(Object... args) { + Preconditions.checkArgument(args.length % 2 == 0); + List> params = new ArrayList<>(0); + for (int i = 0; i < args.length; i += 2) { + String type; + if (args[i] instanceof Class) { + type = type(((Class) args[i])); + } else { + type = args[i].toString(); + } + params.add(Tuple2.of(type, args[i + 1].toString())); + } + return params; + } + + /** + * Add a field to class. + * + * @param type type + * @param fieldName field name + * @param initExpr field init expression + */ + public void addField(Class type, String fieldName, Expression initExpr) { + addField(type(type), fieldName, initExpr); + } + + public void addField(String type, String fieldName, Expression initExpr) { + addField(type, fieldName, initExpr, true); + } + + /** + * Add a field to class. + * + * @param type type + * @param fieldName field name + * @param initExpr field init expression + */ + public void addField(String type, String fieldName, Expression initExpr, boolean isFinalField) { + if (instanceInitCtx == null) { + instanceInitCtx = new CodegenContext(valNames, imports); + } + fields.add(Tuple3.of(isFinalField, type, fieldName)); + ExprCode exprCode = initExpr.genCode(instanceInitCtx); + if (StringUtils.isNotBlank(exprCode.code())) { + initCodes.add(exprCode.code()); + } + initCodes.add(String.format("%s = %s;", fieldName, exprCode.value())); + } + + /** + * Add a field to class. + * + * @param type type + * @param fieldName field name + * @param initCode field init code + */ + public void addField(String type, String fieldName, String initCode) { + fields.add(Tuple3.of(false, type, fieldName)); + if (StringUtils.isNotBlank(initCode)) { + initCodes.add(initCode); + } + } + + /** + * Add a field to class. The init code should be placed in constructor's code + * + * @param type type + * @param fieldName field name + */ + public void addField(String type, String fieldName) { + fields.add(Tuple3.of(true, type, fieldName)); + } + + /** Generate code for class. */ + public String genCode() { + StringBuilder codeBuilder = new StringBuilder(); + + if (StringUtils.isNotBlank(pkg)) { + codeBuilder.append("package ").append(pkg).append(";\n\n"); + } + + if (!imports.isEmpty()) { + imports.forEach(clz -> codeBuilder.append("import ").append(clz).append(";\n")); + codeBuilder.append('\n'); + } + + codeBuilder.append(String.format("public final class %s ", className)); + if (superClasses != null) { + codeBuilder.append(String.format("extends %s ", String.join(", ", superClasses))); + } + if (interfaces != null) { + codeBuilder.append(String.format("implements %s ", String.join(", ", interfaces))); + } + codeBuilder.append("{\n"); + + // fields + if (!fields.isEmpty()) { + codeBuilder.append('\n'); + for (Tuple3 field : fields) { + String declare; + if (field.f0) { + declare = String.format("private final %s %s;\n", field.f1, field.f2); + } else { + declare = String.format("private %s %s;\n", field.f1, field.f2); + } + codeBuilder.append(indent(declare)); + } + } + + // constructors + if (!constructors.isEmpty()) { + codeBuilder.append('\n'); + constructors.forEach(constructor -> codeBuilder.append(indent(constructor)).append('\n')); + } + + // methods + codeBuilder.append('\n'); + methods.values().forEach(method -> codeBuilder.append(indent(method)).append('\n')); + + codeBuilder.append('}'); + return codeBuilder.toString(); + } +} diff --git a/java/fury-core/src/main/java/io/fury/codegen/CompileCallback.java b/java/fury-core/src/main/java/io/fury/codegen/CompileCallback.java new file mode 100644 index 0000000000..59986da931 --- /dev/null +++ b/java/fury-core/src/main/java/io/fury/codegen/CompileCallback.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 The Fury authors + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fury.codegen; + +/** + * Compile callback to be invoked just before compilation happen. This can be used to set up + * thread-safety or inspection. + * + * @author chaokunyang + */ +public interface CompileCallback { + /** + * This method block until lock on compileLock succeed. If lock on compileLock + * failed, then there is an opportunity to release other locks to avoid deadlock. + */ + void lock(CompileState compileState); +} diff --git a/java/fury-core/src/main/java/io/fury/codegen/CompileState.java b/java/fury-core/src/main/java/io/fury/codegen/CompileState.java new file mode 100644 index 0000000000..a5a8c979e2 --- /dev/null +++ b/java/fury-core/src/main/java/io/fury/codegen/CompileState.java @@ -0,0 +1,39 @@ +/* + * Copyright 2023 The Fury authors + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fury.codegen; + +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Compile state to store compile info such as whether finished, and if finished, store compile + * result too. + * + * @author chaokunyang + */ +public class CompileState { + public final Lock lock; + public boolean finished; + public Map result; + + public CompileState() { + this.lock = new ReentrantLock(); + } +} diff --git a/java/fury-core/src/main/java/io/fury/codegen/Expression.java b/java/fury-core/src/main/java/io/fury/codegen/Expression.java new file mode 100644 index 0000000000..64664b1591 --- /dev/null +++ b/java/fury-core/src/main/java/io/fury/codegen/Expression.java @@ -0,0 +1,76 @@ +/* + * Copyright 2023 The Fury authors + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fury.codegen; + +import static io.fury.codegen.Code.ExprCode; + +import com.google.common.reflect.TypeToken; + +/** + * An expression represents a piece of code evaluation logic which can be generated to valid java + * code. Expression can be used to compose complex code logic. + * + *

TODO refactor expression into Expression and Stmt with a common class Node. Expression will + * have a value, the stmt won't have value. If/While/For/doWhile/ForEach are statements instead of + * expression. + * + * @author chaokunyang + */ +@SuppressWarnings("UnstableApiUsage") +public interface Expression { + + /** + * Returns the Class of the result of evaluating this expression. It is invalid to query the + * type of unresolved expression (i.e., when `resolved` == false). + */ + TypeToken type(); + + /** + * If expression is already generated in this context, returned exprCode won't contains code, so + * we can reuse/elimination expression code. + */ + default ExprCode genCode(CodegenContext ctx) { + // Ctx already contains expression code, which means that the code to evaluate it has already + // been added before. In that case, we just reuse it. + ExprCode reuseExprCode = ctx.exprState.get(this); + if (reuseExprCode != null) { + return reuseExprCode; + } else { + ExprCode genCode = doGenCode(ctx); + ctx.exprState.put(this, new ExprCode(genCode.isNull(), genCode.value())); + return genCode; + } + } + + /** + * Used when Expression is requested to doGenCode. + * + * @param ctx a [[CodegenContext]] + * @return an [[ExprCode]] containing the Java source code to generate the given expression + */ + ExprCode doGenCode(CodegenContext ctx); + + default boolean nullable() { + return false; + } + + // ########################################################### + // ####################### Expressions ####################### + // ########################################################### +} diff --git a/java/fury-core/src/main/java/io/fury/collection/MultiKeyWeakMap.java b/java/fury-core/src/main/java/io/fury/collection/MultiKeyWeakMap.java index 271f84261d..c77aa3cedb 100644 --- a/java/fury-core/src/main/java/io/fury/collection/MultiKeyWeakMap.java +++ b/java/fury-core/src/main/java/io/fury/collection/MultiKeyWeakMap.java @@ -36,7 +36,7 @@ * be removed when all keys are no longer in ordinary use. More precisely, the presence of a mapping * for the given keys will not prevent the keys from being discarded by the garbage collector. * - * @param the type of keys maintained by this map + * @param the type of values maintained by this map * @see java.util.WeakHashMap * @author chaokunyang */ diff --git a/java/fury-core/src/test/java/io/fury/codegen/CodeGeneratorTest.java b/java/fury-core/src/test/java/io/fury/codegen/CodeGeneratorTest.java new file mode 100644 index 0000000000..9d252b988f --- /dev/null +++ b/java/fury-core/src/test/java/io/fury/codegen/CodeGeneratorTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2023 The Fury authors + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fury.codegen; + +import io.fury.test.bean.Foo; +import io.fury.util.ClassLoaderUtils; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class CodeGeneratorTest { + + @Test + public void classFilepath() { + String p = + CodeGenerator.classFilepath( + new CompileUnit(Foo.class.getPackage().getName(), Foo.class.getSimpleName(), "")); + Assert.assertEquals( + p, + String.format( + "%s/%s.class", + Foo.class.getPackage().getName().replace(".", "/"), Foo.class.getSimpleName())); + } + + @Test + public void fullClassName() { + CompileUnit unit = + new CompileUnit(Foo.class.getPackage().getName(), Foo.class.getSimpleName(), ""); + Assert.assertEquals(CodeGenerator.fullClassName(unit), Foo.class.getName()); + } + + @Test + public void testCompile() throws Exception { + CodeGenerator codeGenerator = CodeGenerator.getSharedCodeGenerator(getClass().getClassLoader()); + CompileUnit unit1 = + new CompileUnit( + "demo.pkg1", + "A", + ("" + + "package demo.pkg1;\n" + + "public class A {\n" + + " public static String hello() { return \"HELLO\"; }\n" + + "}")); + ClassLoader classLoader = codeGenerator.compile(unit1); + Assert.assertEquals(classLoader.loadClass("demo.pkg1.A").getSimpleName(), "A"); + Assert.assertNotEquals(classLoader, getClass().getClassLoader()); + Assert.assertEquals(classLoader.getClass(), ClassLoaderUtils.ByteArrayClassLoader.class); + } + + @Test + public void testMultiCompile() throws Exception { + CodeGenerator codeGenerator = new CodeGenerator(getClass().getClassLoader()); + CompileUnit unit1 = + new CompileUnit( + "demo.pkg1", + "A", + ("" + + "package demo.pkg1;\n" + + "import demo.pkg2.*;\n" + + "public class A {\n" + + " public static String main() { return B.hello(); }\n" + + " public static String hello() { return \"HELLO\"; }\n" + + "}")); + CompileUnit unit2 = + new CompileUnit( + "demo.pkg2", + "B", + ("" + + "package demo.pkg2;\n" + + "import demo.pkg1.*;\n" + + "public class B {\n" + + " public static String hello() { return A.hello(); }\n" + + "}")); + ClassLoader classLoader = codeGenerator.compile(unit1, unit2); + Assert.assertEquals( + "HELLO", classLoader.loadClass("demo.pkg1.A").getMethod("main").invoke(null)); + ClassLoader classLoader2 = codeGenerator.compile(unit1, unit2); + Assert.assertSame(classLoader, classLoader2); + } +} diff --git a/java/fury-core/src/test/java/io/fury/codegen/CodegenContextTest.java b/java/fury-core/src/test/java/io/fury/codegen/CodegenContextTest.java new file mode 100644 index 0000000000..89c6d58a79 --- /dev/null +++ b/java/fury-core/src/test/java/io/fury/codegen/CodegenContextTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2023 The Fury authors + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fury.codegen; + +import com.google.common.reflect.TypeToken; +import java.util.List; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class CodegenContextTest { + + public static class A { + public String f1; + } + + @Test + public void type() { + TypeToken>> typeToken = new TypeToken>>() {}; + { + CodegenContext ctx = new CodegenContext(); + ctx.addImport(List.class); + Assert.assertEquals("List", ctx.type(List.class)); + } + CodegenContext ctx = new CodegenContext(); + String type = ctx.type(typeToken); + Assert.assertEquals("java.util.List", type); + Assert.assertEquals("int[][]", ctx.type(int[][].class)); + } + + @Test + public void testTypeForInnerClass() { + CodegenContext ctx = new CodegenContext(); + Assert.assertEquals(ctx.type(A.class), A.class.getCanonicalName()); + ctx.addImport(getClass()); + Assert.assertEquals(ctx.type(A.class), A.class.getCanonicalName()); + ctx.addImport(A.class); + Assert.assertEquals(ctx.type(A.class), A.class.getSimpleName()); + } + + @Test + public void testNewName() { + { + CodegenContext ctx = new CodegenContext(); + Assert.assertEquals(ctx.newName("serializer"), "serializer"); + Assert.assertEquals(ctx.newName("serializer"), "serializer1"); + Assert.assertEquals(ctx.newName("serializer"), "serializer2"); + } + { + CodegenContext ctx = new CodegenContext(); + Assert.assertEquals(ctx.newName("serializer"), "serializer"); + // Assert.assertEquals( + // ctx.newNames(Serializer.class, "isNull"), new String[] {"serializer1", "isNull1"}); + Assert.assertEquals(ctx.newName("serializer"), "serializer1"); + } + { + CodegenContext ctx = new CodegenContext(); + Assert.assertEquals(ctx.newName("isNull"), "isNull"); + Assert.assertEquals( + ctx.newNames("serializer", "isNull"), new String[] {"serializer1", "isNull1"}); + Assert.assertEquals(ctx.newName("serializer"), "serializer2"); + } + } +}