From 79680934375a4844d64a026d9b02e58196915c7f Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Mon, 27 Nov 2017 14:16:00 +0900 Subject: [PATCH] * Move `sizeof()` and `offsetof()` data to global variables to prevent `StackOverflowError` in `JNI_OnLoad()` (issue bytedeco/javacpp-presets#331) * Propagate within `Parser` type information from macros to other macros referencing them Also fix a few more issues with `Parser` --- CHANGELOG.md | 2 + .../org/bytedeco/javacpp/tools/Generator.java | 101 +++++++++--------- .../org/bytedeco/javacpp/tools/Parser.java | 88 ++++++++++++--- .../org/bytedeco/javacpp/tools/Tokenizer.java | 17 ++- .../java/org/bytedeco/javacpp/tools/Type.java | 2 +- 5 files changed, 138 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b793a9fea..db06a9f46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ + * Move `sizeof()` and `offsetof()` data to global variables to prevent `StackOverflowError` in `JNI_OnLoad()` ([issue bytedeco/javacpp-presets#331](https://github.com/bytedeco/javacpp-presets/issues/331)) + * Propagate within `Parser` type information from macros to other macros referencing them * Add support for `JNI_OnLoad_libname()` naming scheme for iOS via new `platform.library.static=true` property * Improve the clarity of error messages on `Parser` failures * Fix `Parser` issues with multiple `typedef` declarations in a single statement diff --git a/src/main/java/org/bytedeco/javacpp/tools/Generator.java b/src/main/java/org/bytedeco/javacpp/tools/Generator.java index 6afee25dd..6eb48c99c 100644 --- a/src/main/java/org/bytedeco/javacpp/tools/Generator.java +++ b/src/main/java/org/bytedeco/javacpp/tools/Generator.java @@ -1162,52 +1162,10 @@ boolean classes(boolean handleExceptions, boolean defineAdapters, boolean conver out.println("static void " + name + "_deallocateArray(void* p) { delete[] (" + typeName[0] + typeName[1] + ")p; }"); } out.println(); - out.println("extern \"C\" {"); - if (out2 != null) { - out2.println(); - out2.println("#ifdef __cplusplus"); - out2.println("extern \"C\" {"); - out2.println("#endif"); - out2.println("JNIIMPORT int JavaCPP_init" + loadSuffix + "(int argc, const char *argv[]);"); - out.println(); - out.println("JNIEXPORT int JavaCPP_init" + loadSuffix + "(int argc, const char *argv[]) {"); - out.println("#if defined(__ANDROID__) || TARGET_OS_IPHONE"); - out.println(" return JNI_OK;"); - out.println("#else"); - out.println(" if (JavaCPP_vm != NULL) {"); - out.println(" return JNI_OK;"); - out.println(" }"); - out.println(" int err;"); - out.println(" JavaVM *vm;"); - out.println(" JNIEnv *env;"); - out.println(" int nOptions = 1 + (argc > 255 ? 255 : argc);"); - out.println(" JavaVMOption options[256] = { { NULL } };"); - out.println(" options[0].optionString = (char*)\"-Djava.class.path=" + classPath.replace('\\', '/') + "\";"); - out.println(" for (int i = 1; i < nOptions && argv != NULL; i++) {"); - out.println(" options[i].optionString = (char*)argv[i - 1];"); - out.println(" }"); - out.println(" JavaVMInitArgs vm_args = { " + JNI_VERSION + ", nOptions, options };"); - out.println(" return (err = JNI_CreateJavaVM(&vm, (void**)&env, &vm_args)) == JNI_OK && vm != NULL && (err = JNI_OnLoad" + loadSuffix + "(vm, NULL)) >= 0 ? JNI_OK : err;"); - out.println("#endif"); - out.println("}"); - } - out.println(); // XXX: JNI_OnLoad() should ideally be protected by some mutex - out.println("JNIEXPORT jint JNICALL JNI_OnLoad" + loadSuffix + "(JavaVM* vm, void* reserved) {"); - out.println(" JNIEnv* env;"); - out.println(" if (vm->GetEnv((void**)&env, " + JNI_VERSION + ") != JNI_OK) {"); - out.println(" JavaCPP_log(\"Could not get JNIEnv for " + JNI_VERSION + " inside JNI_OnLoad" + loadSuffix + "().\");"); - out.println(" return JNI_ERR;"); - out.println(" }"); - out.println(" if (JavaCPP_vm == vm) {"); - out.println(" return env->GetVersion();"); - out.println(" }"); - out.println(" JavaCPP_vm = vm;"); - out.println(" JavaCPP_haveAllocObject = env->functions->AllocObject != NULL;"); - out.println(" JavaCPP_haveNonvirtual = env->functions->CallNonvirtualVoidMethodA != NULL;"); - out.println(" const char* members[" + jclasses.size() + "][" + maxMemberSize + "] = {"); + out.println("static const char* JavaCPP_members[" + jclasses.size() + "][" + maxMemberSize + "] = {"); classIterator = jclasses.iterator(); while (classIterator.hasNext()) { - out.print(" { "); + out.print(" { "); Set m = members.get(classIterator.next()); Iterator memberIterator = m == null ? null : m.iterator(); if (memberIterator == null || !memberIterator.hasNext()) { @@ -1224,10 +1182,10 @@ boolean classes(boolean handleExceptions, boolean defineAdapters, boolean conver } } out.println(" };"); - out.println(" int offsets[" + jclasses.size() + "][" + maxMemberSize + "] = {"); + out.println("static int JavaCPP_offsets[" + jclasses.size() + "][" + maxMemberSize + "] = {"); classIterator = jclasses.iterator(); while (classIterator.hasNext()) { - out.print(" { "); + out.print(" { "); Class c = classIterator.next(); Set m = members.get(c); Iterator memberIterator = m == null ? null : m.iterator(); @@ -1255,7 +1213,7 @@ boolean classes(boolean handleExceptions, boolean defineAdapters, boolean conver } } out.println(" };"); - out.print(" int memberOffsetSizes[" + jclasses.size() + "] = { "); + out.print("static int JavaCPP_memberOffsetSizes[" + jclasses.size() + "] = { "); classIterator = jclasses.iterator(); while (classIterator.hasNext()) { Set m = members.get(classIterator.next()); @@ -1265,18 +1223,61 @@ boolean classes(boolean handleExceptions, boolean defineAdapters, boolean conver } } out.println(" };"); + out.println(); + out.println("extern \"C\" {"); + if (out2 != null) { + out2.println(); + out2.println("#ifdef __cplusplus"); + out2.println("extern \"C\" {"); + out2.println("#endif"); + out2.println("JNIIMPORT int JavaCPP_init" + loadSuffix + "(int argc, const char *argv[]);"); + out.println(); + out.println("JNIEXPORT int JavaCPP_init" + loadSuffix + "(int argc, const char *argv[]) {"); + out.println("#if defined(__ANDROID__) || TARGET_OS_IPHONE"); + out.println(" return JNI_OK;"); + out.println("#else"); + out.println(" if (JavaCPP_vm != NULL) {"); + out.println(" return JNI_OK;"); + out.println(" }"); + out.println(" int err;"); + out.println(" JavaVM *vm;"); + out.println(" JNIEnv *env;"); + out.println(" int nOptions = 1 + (argc > 255 ? 255 : argc);"); + out.println(" JavaVMOption options[256] = { { NULL } };"); + out.println(" options[0].optionString = (char*)\"-Djava.class.path=" + classPath.replace('\\', '/') + "\";"); + out.println(" for (int i = 1; i < nOptions && argv != NULL; i++) {"); + out.println(" options[i].optionString = (char*)argv[i - 1];"); + out.println(" }"); + out.println(" JavaVMInitArgs vm_args = { " + JNI_VERSION + ", nOptions, options };"); + out.println(" return (err = JNI_CreateJavaVM(&vm, (void**)&env, &vm_args)) == JNI_OK && vm != NULL && (err = JNI_OnLoad" + loadSuffix + "(vm, NULL)) >= 0 ? JNI_OK : err;"); + out.println("#endif"); + out.println("}"); + } + out.println(); // XXX: JNI_OnLoad() should ideally be protected by some mutex + out.println("JNIEXPORT jint JNICALL JNI_OnLoad" + loadSuffix + "(JavaVM* vm, void* reserved) {"); + out.println(" JNIEnv* env;"); + out.println(" if (vm->GetEnv((void**)&env, " + JNI_VERSION + ") != JNI_OK) {"); + out.println(" JavaCPP_log(\"Could not get JNIEnv for " + JNI_VERSION + " inside JNI_OnLoad" + loadSuffix + "().\");"); + out.println(" return JNI_ERR;"); + out.println(" }"); + out.println(" if (JavaCPP_vm == vm) {"); + out.println(" return env->GetVersion();"); + out.println(" }"); + out.println(" JavaCPP_vm = vm;"); + out.println(" JavaCPP_haveAllocObject = env->functions->AllocObject != NULL;"); + out.println(" JavaCPP_haveNonvirtual = env->functions->CallNonvirtualVoidMethodA != NULL;"); out.println(" jmethodID putMemberOffsetMID = JavaCPP_getStaticMethodID(env, " + jclasses.index(Loader.class) + ", \"putMemberOffset\", \"(Ljava/lang/String;Ljava/lang/String;I)Ljava/lang/Class;\");"); out.println(" if (putMemberOffsetMID == NULL) {"); out.println(" return JNI_ERR;"); out.println(" }"); out.println(" for (int i = 0; i < " + jclasses.size() + " && !env->ExceptionCheck(); i++) {"); - out.println(" for (int j = 0; j < memberOffsetSizes[i] && !env->ExceptionCheck(); j++) {"); + out.println(" for (int j = 0; j < JavaCPP_memberOffsetSizes[i] && !env->ExceptionCheck(); j++) {"); out.println(" if (env->PushLocalFrame(3) == 0) {"); out.println(" jvalue args[3];"); out.println(" args[0].l = env->NewStringUTF(JavaCPP_classNames[i]);"); - out.println(" args[1].l = members[i][j] == NULL ? NULL : env->NewStringUTF(members[i][j]);"); - out.println(" args[2].i = offsets[i][j];"); + out.println(" args[1].l = JavaCPP_members[i][j] == NULL ? NULL : env->NewStringUTF(JavaCPP_members[i][j]);"); + out.println(" args[2].i = JavaCPP_offsets[i][j];"); out.println(" jclass cls = (jclass)env->CallStaticObjectMethodA(JavaCPP_getClass(env, " + jclasses.index(Loader.class) + "), putMemberOffsetMID, args);"); out.println(" if (cls == NULL || env->ExceptionCheck()) {"); diff --git a/src/main/java/org/bytedeco/javacpp/tools/Parser.java b/src/main/java/org/bytedeco/javacpp/tools/Parser.java index d191f5525..8452fb36a 100644 --- a/src/main/java/org/bytedeco/javacpp/tools/Parser.java +++ b/src/main/java/org/bytedeco/javacpp/tools/Parser.java @@ -512,11 +512,13 @@ Type type(Context context, boolean definition) throws ParserException { } } else if (token.match(Token.FRIEND)) { type.friend = true; + } else if (token.match(Token.TYPEDEF)) { + type.typedef = true; } else if (token.match(Token.VIRTUAL)) { type.virtual = true; } else if (token.match(Token.AUTO, Token.ENUM, Token.EXPLICIT, Token.EXTERN, Token.INLINE, Token.CLASS, Token.FINAL, Token.INTERFACE, Token.__INTERFACE, Token.MUTABLE, Token.NAMESPACE, Token.STRUCT, Token.UNION, - Token.TYPEDEF, Token.TYPENAME, Token.USING, Token.REGISTER, Token.THREAD_LOCAL, Token.VOLATILE)) { + Token.TYPENAME, Token.USING, Token.REGISTER, Token.THREAD_LOCAL, Token.VOLATILE)) { token = tokens.next(); continue; } else if (token.match((Object[])infoMap.getFirst("basic/types").cppTypes) && (type.cppName.length() == 0 || type.simple)) { @@ -721,6 +723,7 @@ Declarator declarator(Context context, String defaultName, int infoNumber, boole if (type == null) { return null; } + typedef |= type.typedef; // pick the requested identifier out of the statement in the case of multiple variable declaractions int count = 0, number = 0; @@ -818,6 +821,12 @@ Declarator declarator(Context context, String defaultName, int infoNumber, boole Declaration definition = new Declaration(); boolean fieldPointer = false; Attribute convention = null; + for (Attribute a : attributes) { + if (a.annotation && a.javaName.length() == 0 && a.arguments.length() == 0) { + // we may have a calling convention for function pointers + convention = a; + } + } if (tokens.get().match('(') || (typedef && tokens.get(1).match('('))) { // probably a function pointer declaration if (tokens.get().match('(')) { @@ -851,6 +860,7 @@ Declarator declarator(Context context, String defaultName, int infoNumber, boole definition.text += "@Namespace(\"" + dcl.cppName + "\") "; } else if (convention != null || dcl.cppName.length() > 0) { definition.text += "@Convention(\"" + (convention != null ? convention.cppName : dcl.cppName) + "\") "; + convention = null; } dcl.cppName = ""; } else if (token.match('[')) { @@ -1163,6 +1173,9 @@ Declarator declarator(Context context, String defaultName, int infoNumber, boole if (dcl.parameters != null && indirections2 == 0 && !typedef) { dcl.signature += dcl.parameters.signature; } else { + if (convention != null) { + definition.text += "@Convention(\"" + convention.cppName + "\") "; + } String cppType = ""; if (dcl.type != null) { cppType += dcl.type.cppName; @@ -1493,6 +1506,7 @@ Attribute attribute() throws ParserException { } String body() throws ParserException { + String text = ""; if (!tokens.get().match('{')) { return null; } @@ -1505,9 +1519,12 @@ String body() throws ParserException { } else if (token.match('}')) { count--; } + if (count > 0) { + text += token.spacing + token; + } } tokens.raw = false; - return ""; + return text; } Parameters parameters(Context context, int infoNumber, boolean useDefaults) throws ParserException { @@ -1920,7 +1937,7 @@ boolean variable(Context context, DeclarationList declList) throws ParserExcepti cppName = context.namespace + "::" + cppName; } Info info = infoMap.getFirst(cppName); - if (info != null && info.skip) { + if (dcl.cppName.length() == 0 || (info != null && info.skip)) { decl.text = spacing; declList.add(decl); while (!tokens.get().match(Token.EOF, ';')) { @@ -2144,6 +2161,7 @@ boolean macro(Context context, DeclarationList declList) throws ParserException } if (!found) { decl.text += "public static native " + dcl.type.annotations + dcl.type.javaName + " " + macroName + dcl.parameters.list + ";\n"; + decl.signature = dcl.signature; } else if (found && n > 0) { break; } @@ -2153,6 +2171,7 @@ boolean macro(Context context, DeclarationList declList) throws ParserException (info.cppTypes == null || info.cppTypes.length == 1)))) { // declare as a static final variable String value = ""; + String cppType = "int"; String type = "int"; String cat = ""; tokens.index = beginIndex + 1; @@ -2160,20 +2179,29 @@ boolean macro(Context context, DeclarationList declList) throws ParserException boolean translate = true; for (Token token = tokens.get(); tokens.index < lastIndex; token = tokens.next()) { if (token.match(Token.STRING)) { - type = "String"; cat = " + "; break; + cppType = "const char*"; type = "String"; cat = " + "; break; } else if (token.match(Token.FLOAT)) { - type = "double"; cat = ""; break; + cppType = "double"; type = "double"; cat = ""; break; } else if (token.match(Token.INTEGER) && token.value.endsWith("L")) { - type = "long"; cat = ""; break; + cppType = "long long"; type = "long"; cat = ""; break; } else if ((prevToken.match(Token.IDENTIFIER, '>') && token.match(Token.IDENTIFIER, '(')) || token.match('{', '}')) { translate = false; + } else if (token.match(Token.IDENTIFIER)) { + // get types for the values of the macro + Info info2 = infoMap.getFirst(token.value); + if (info == null && info2 != null && info2.cppTypes != null) { + info = info2; + } } prevToken = token; } if (info != null) { - if (info.cppTypes != null) { + if (info.cppTypes != null && info.cppTypes.length > 0) { Declarator dcl = new Parser(this, info.cppTypes[0]).declarator(context, null, -1, false, 0, false, true); - type = dcl.type.annotations + dcl.type.javaName; + if (!dcl.type.javaName.equals("int")) { + cppType = dcl.type.cppName; + type = dcl.type.annotations + (info.pointerTypes != null ? info.pointerTypes[0] : dcl.type.javaName);; + } } for (int i = 0; i < info.cppNames.length; i++) { if (macroName.equals(info.cppNames[i]) && info.javaNames != null) { @@ -2186,16 +2214,34 @@ boolean macro(Context context, DeclarationList declList) throws ParserException tokens.index = beginIndex + 1; if (translate) { for (Token token = tokens.get(); tokens.index < lastIndex; token = tokens.next()) { - value += token.spacing + token + (tokens.index + 1 < lastIndex ? cat : ""); + value += token.spacing; + if (type.equals("String") && token.match("L")) { + // strip unnecessary prefixes from strings + continue; + } + value += token + (tokens.index + 1 < lastIndex && token.value.trim().length() > 0 ? cat : ""); } value = translate(value); if (type.equals("int")) { if (value.contains("(String)")) { - type = "String"; + cppType = "const char*"; type = "String"; } else if (value.contains("(float)") || value.contains("(double)")) { - type = "double"; + cppType = "double"; type = "double"; } else if (value.contains("(long)")) { - type = "long"; + cppType = "long long"; type = "long"; + } else { + try { + String trimmedValue = value.trim(); + long longValue = Long.parseLong(trimmedValue); + if (longValue > Integer.MAX_VALUE && (longValue >>> 32) == 0) { + // probably some unsigned value, so let's just cast to int + value = value.substring(0, value.length() - trimmedValue.length()) + "(int)" + trimmedValue + "L"; + } else if (longValue > Integer.MAX_VALUE || longValue < Integer.MIN_VALUE) { + cppType = "long long"; type = "long"; value += "L"; + } + } catch (NumberFormatException e) { + // leave as int? + } } } } else { @@ -2215,6 +2261,10 @@ boolean macro(Context context, DeclarationList declList) throws ParserException decl.text += "public static final " + type + " " + macroName + " =" + value + ";\n"; } decl.signature = macroName; + if (info == null || !Arrays.asList(info.cppNames).contains(macroName)) { + // save the C++ type (and Java type via pointerTypes) to propagate to other macros referencing this one + infoMap.put(new Info(macroName).define(true).cppTypes(cppType).pointerTypes(type).translate(translate)); + } } if (info != null && info.javaText != null) { decl.text = info.javaText; @@ -2457,14 +2507,16 @@ boolean group(Context context, DeclarationList declList) throws ParserException tokens.next(); } } - if (!tokens.get().match('{', ';')) { + if (!tokens.get().match('{', ',', ';')) { tokens.index = backIndex; return false; } int startIndex = tokens.index; List variables = new ArrayList(); String originalName = type.cppName; - if (body() != null && !tokens.get().match(';')) { + String body = body(); + boolean hasBody = body != null && body.length() > 0; + if (!tokens.get().match(';')) { if (typedef) { Token token = tokens.get(); while (!token.match(';', Token.EOF)) { @@ -2475,11 +2527,13 @@ boolean group(Context context, DeclarationList declList) throws ParserException } if (token.match(Token.IDENTIFIER)) { String name2 = token.value; - // use typedef name, unless it's something weird like a pointer + // use first typedef name, unless it's something weird like a pointer if (indirections > 0) { infoMap.put(new Info(name2).cast().valueTypes(name).pointerTypes("PointerPointer")); } else { - name = type.javaName = type.cppName = token.value; + if (type.cppName.equals(originalName)) { + name = type.javaName = type.cppName = token.value; + } infoMap.put(new Info(name2).cast().pointerTypes(name)); } } @@ -2545,7 +2599,7 @@ boolean group(Context context, DeclarationList declList) throws ParserException } decl.signature = type.javaName; tokens.index = startIndex; - if (name.length() > 0 && (tokens.get().match(';') || tokens.get(2).match(';'))) { + if (name.length() > 0 && !hasBody) { // incomplete type (forward or friend declaration) if (!tokens.get().match(';')) { tokens.next(); diff --git a/src/main/java/org/bytedeco/javacpp/tools/Tokenizer.java b/src/main/java/org/bytedeco/javacpp/tools/Tokenizer.java index 417e769a9..a23b3cde3 100644 --- a/src/main/java/org/bytedeco/javacpp/tools/Tokenizer.java +++ b/src/main/java/org/bytedeco/javacpp/tools/Tokenizer.java @@ -173,8 +173,8 @@ public Token nextToken() throws IOException { int prevc = 0; boolean exp = false, large = false, unsigned = false, hex = false; while ((c = readChar()) != -1 && (Character.isDigit(c) || c == '.' || c == '-' || c == '+' || - (c >= 'a' && c <= 'f') || c == 'l' || c == 'u' || c == 'x' || - (c >= 'A' && c <= 'F') || c == 'L' || c == 'U' || c == 'X')) { + (c >= 'a' && c <= 'f') || c == 'i' || c == 'l' || c == 'u' || c == 'x' || + (c >= 'A' && c <= 'F') || c == 'I' || c == 'L' || c == 'U' || c == 'X')) { switch (c) { case '.': token.type = Token.FLOAT; break; case 'e': case 'E': exp = true; break; @@ -193,8 +193,17 @@ public Token nextToken() throws IOException { if (token.type == Token.INTEGER && !large) { try { long high = Long.decode(buffer.toString()) >> 32; - large = high != 0 && high != 0xFFFFFFFF; - } catch (NumberFormatException e) { /* not an integer? */ } + large = high != 0 && high != -1; + } catch (NumberFormatException e) { + // set as large, for things like 0x8000000000000000L + if (buffer.length() >= 16) { + large = true; + } + } + } + if (buffer.toString().endsWith("i64")) { + buffer.setLength(buffer.length() - 3); + large = true; } if (token.type == Token.INTEGER && (large || (unsigned && !hex))) { buffer.append('L'); diff --git a/src/main/java/org/bytedeco/javacpp/tools/Type.java b/src/main/java/org/bytedeco/javacpp/tools/Type.java index e94480669..f569a1cdf 100644 --- a/src/main/java/org/bytedeco/javacpp/tools/Type.java +++ b/src/main/java/org/bytedeco/javacpp/tools/Type.java @@ -33,7 +33,7 @@ class Type { int indirections = 0; boolean anonymous = false, constPointer = false, constValue = false, constructor = false, destructor = false, operator = false, simple = false, staticMember = false, - reference = false, value = false, friend = false, virtual = false; + reference = false, value = false, friend = false, typedef = false, virtual = false; String annotations = "", cppName = "", javaName = "", javaNames[] = null; Type[] arguments = null; Attribute[] attributes = null;