From 618617454e8527b83b729b93505f12d7cbb8a740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Mon, 11 May 2026 22:12:36 +0800 Subject: [PATCH 01/58] fix format for tasks --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3f7548ecfd..801df22a97 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,7 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th - Reject semantic hacks. Do not bypass broken semantics by deleting cases, simplifying callers, adding coercion hooks, or using workaround fallbacks; fix the underlying bug and prove it with focused tests. - Protect hot paths. Avoid per-call allocations, callback objects, result tuples or records, unnecessary runtime branches, and wrapper-class substitutions in hot codec/runtime paths; prefer conditional imports and allocation-free concrete implementations where they fit the language. - Keep public APIs minimal. Public APIs must match user ownership and mental model, not internal implementation details; generated flows stay type-owned, while manual serializer registration stays explicit. -- Use semantic naming only. Name things after protocol or domain concepts, not history, runtime origin, or workaround style; avoid vague names such as `Internal`, `java_style_*`, `Runtime`, `Session`, `Plan`, or `Binding` when they do not name the real concept. +- Use semantic naming only. Name things after protocol or domain concepts, not history, runtime origin, or workaround style; avoid vague names such as `Internal`, `java_style_*`, `Runtime`, `Session`, `Plan`, or `Binding` when they do not name the real concept. Never name a class or method with a `Plan` suffix; use the real domain concept instead. - Keep one implementation path. Do not keep parallel helpers, serializers, harnesses, wrappers, or registration flows for the same concept; extend the existing owner path instead of inventing another one. - Follow current scope exactly. The latest explicit user instruction overrides earlier plans, and when scope narrows, remove leaked out-of-scope edits immediately. - Preserve user corrections. When a user corrects code behavior, ownership, invariants, or review feedback in a way that should prevent repeat mistakes, encode the corrected rule where future agents will see it: prefer the nearest source comment for non-obvious code invariants, or the owning docs/spec for user-visible or protocol behavior. If the correction changes API usage, defaults, generated output, tests, or cross-runtime behavior, update the matching docs, examples, or source comments in the same task so future agents do not repeat the violation. Keep the note concise, English-only, and avoid comments that merely restate obvious code. @@ -61,7 +61,7 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th - Do not replace existing C, C++, Cython, unsafe, or other low-level optimized paths with simpler high-level implementations just to make a refactor easier. - If a refactor accidentally changes logic or implementation strategy, revert that part and re-implement the refactor around the existing logic. - Use English only in code, comments, and documentation. -- After editing any Markdown file, run `prettier --write ` on each changed Markdown file before finishing. +- After editing Markdown files outside `tasks/`, run `prettier --write ` on each changed Markdown file before finishing. Do not format Markdown under `tasks/`. - Add comments only when behavior is hard to understand or an algorithm is non-obvious. - Do not remove existing code comments unless they are stale, misleading, redundant, or no longer necessary after the change. - Only add tests that verify internal behaviors or fix specific bugs; do not create unnecessary tests unless requested. From e5fda6fcb8eeb35efcacdb11664cbcce8836de05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 00:21:08 +0800 Subject: [PATCH 02/58] feat(java): add ForyStruct static serializers --- .github/workflows/ci.yml | 36 + docs/guide/java/android-support.md | 8 + docs/guide/java/configuration.md | 3 +- docs/guide/java/graalvm-support.md | 7 + docs/guide/java/index.md | 1 + docs/guide/java/static-struct-serializers.md | 120 +++ java/fory-annotation-processor/pom.xml | 69 ++ .../processing/ForyStructProcessor.java | 789 ++++++++++++++++++ .../annotation/processing/SourceField.java | 142 ++++ .../annotation/processing/SourceStruct.java | 62 ++ .../annotation/processing/SourceTypeNode.java | 99 +++ .../StaticSerializerSourceWriter.java | 504 +++++++++++ .../javax.annotation.processing.Processor | 1 + .../processing/ForyStructProcessorTest.java | 515 ++++++++++++ .../org/apache/fory/builder/CodecUtils.java | 19 + .../CompatibleMetaSharedCodecBuilder.java | 38 + .../org/apache/fory/builder/Generated.java | 24 + .../fory/builder/MetaSharedCodecBuilder.java | 42 +- .../java/org/apache/fory/meta/FieldTypes.java | 52 +- .../fory/meta/NativeTypeDefEncoder.java | 69 +- .../java/org/apache/fory/meta/TypeDef.java | 42 +- .../org/apache/fory/meta/TypeDefEncoder.java | 44 +- .../apache/fory/resolver/ClassResolver.java | 10 +- .../apache/fory/resolver/TypeResolver.java | 159 +++- .../apache/fory/resolver/XtypeResolver.java | 3 +- .../apache/fory/serializer/FieldGroups.java | 4 + .../StaticGeneratedStructSerializer.java | 294 +++++++ .../fory/serializer/struct/Fingerprint.java | 11 +- .../java/org/apache/fory/type/Descriptor.java | 91 +- .../apache/fory/type/DescriptorBuilder.java | 15 + .../org/apache/fory/type/ForyFieldPolicy.java | 89 ++ .../org/apache/fory/type/GenericType.java | 29 +- .../apache/fory/type/TypeAnnotationUtils.java | 48 ++ .../java/org/apache/fory/type/TypeUtils.java | 25 +- .../main/java/org/apache/fory/type/Types.java | 15 +- .../fory-core/native-image.properties | 2 + .../org/apache/fory/xlang/XlangTestBase.java | 2 + java/pom.xml | 1 + 38 files changed, 3321 insertions(+), 163 deletions(-) create mode 100644 docs/guide/java/static-struct-serializers.md create mode 100644 java/fory-annotation-processor/pom.xml create mode 100644 java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java create mode 100644 java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java create mode 100644 java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java create mode 100644 java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceTypeNode.java create mode 100644 java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java create mode 100644 java/fory-annotation-processor/src/main/resources/META-INF/services/javax.annotation.processing.Processor create mode 100644 java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/builder/CompatibleMetaSharedCodecBuilder.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/type/ForyFieldPolicy.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62b133d73b..9792b0e5ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1235,6 +1235,42 @@ jobs: - name: Run Go IDL Tests run: ./integration_tests/idl_tests/run_go_tests.sh + android_go_xlang: + name: Android Go Xlang Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.21" + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: "temurin" + - name: Cache Maven local repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Run Android Go Xlang Tests + env: + FORY_ANDROID_ENABLED: "1" + FORY_GO_JAVA_CI: "1" + ENABLE_FORY_DEBUG_OUTPUT: "1" + run: | + cd java + mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true + cd fory-core + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.GoXlangTest + android_serializer: name: Android Serializer JVM Round Trip Test runs-on: ubuntu-latest diff --git a/docs/guide/java/android-support.md b/docs/guide/java/android-support.md index c8f316313a..4b22457fae 100644 --- a/docs/guide/java/android-support.md +++ b/docs/guide/java/android-support.md @@ -71,6 +71,14 @@ warning. Explicit `withCodegen(false)` and platform-default disabled codegen do Android runtime codegen entry points must fail before Janino, class definition, generated accessor definition, or generated serializer loading starts. +`@ForyStruct` build-time static serializers are allowed on Android because they are compiled by +javac before the app is built. Android loads them by deterministic generated class name when present, +including compatible-mode reads. This lookup is not gated on an Android-only public API, does not +enable Janino or runtime class definition, and uses the same wire protocol as ordinary JVM Fory. +For reflection metadata, Fory checks whether the runtime actually exposes `Field#getAnnotatedType()` +before using type-use annotations; older Android runtimes that do not expose it must use generated +static descriptors for nested type-use metadata. + ## ByteBuffer On Android, `BaseFory#deserialize(ByteBuffer)` copies the remaining input bytes into a Fory-owned diff --git a/docs/guide/java/configuration.md b/docs/guide/java/configuration.md index 1a0342aabb..6d60590869 100644 --- a/docs/guide/java/configuration.md +++ b/docs/guide/java/configuration.md @@ -43,7 +43,7 @@ This page documents all configuration options available through `ForyBuilder`. | `scopedMetaShareEnabled` | Scoped meta share focuses on a single serialization process. Metadata created or identified during this process is exclusive to it and is not shared with by other serializations. | `true` if compatible mode is enabled, otherwise false. | | `metaCompressor` | Set a compressor for meta compression. Note that the passed MetaCompressor should be thread-safe. By default, a `Deflater` based compressor `DeflaterMetaCompressor` will be used. Users can pass other compressor such as `zstd` for better compression rate. | `DeflaterMetaCompressor` | | `deserializeUnknownClass` | Enables or disables deserialization/skipping of data for non-existent or unknown classes. | `true` if compatible mode is enabled, otherwise false. | -| `codeGenEnabled` | Disabling may result in faster initial serialization but slower subsequent serializations. When unset, codegen defaults to enabled on ordinary JVMs and disabled on Android and GraalVM native image. Explicit `withCodegen(true)` on Android or GraalVM native image is accepted, but final build configuration forces interpreter serializers and emits a warning. | `true` on ordinary JVMs; `false` on Android and GraalVM native image | +| `codeGenEnabled` | Disabling may result in faster initial serialization but slower subsequent serializations. When unset, codegen defaults to enabled on ordinary JVMs and disabled on Android and GraalVM native image. Explicit `withCodegen(true)` on Android or GraalVM native image is accepted, but final build configuration forces interpreter serializers and emits a warning. If a build-time `@ForyStruct` static serializer is available, ordinary JVM `withCodegen(false)` and Android use it instead of the interpreter object serializer. | `true` on ordinary JVMs; `false` on Android and GraalVM native image | | `asyncCompilationEnabled` | If enabled, serialization uses interpreter mode first and switches to JIT serialization after async serializer JIT for a class is finished. This option is forced off on Android and GraalVM native image because runtime code generation is unavailable there. | `false` | | `scalaOptimizationEnabled` | Enables or disables Scala-specific serialization optimization. | `false` | | `copyRef` | When disabled, the copy performance will be better. But fory deep copy will ignore circular and shared reference. Same reference of an object graph will be copied into different objects in one `Fory#copy`. | `false` | @@ -77,3 +77,4 @@ Fory fory = Fory.builder() - [Schema Evolution](schema-evolution.md) - Compatible mode and meta sharing - [Compression](compression.md) - Int, long, and array compression details - [Type Registration](type-registration.md) - Class registration options +- [Static Struct Serializers](static-struct-serializers.md) - Build-time `@ForyStruct` serializers for `codegen=false` and Android diff --git a/docs/guide/java/graalvm-support.md b/docs/guide/java/graalvm-support.md index aa3e2527c3..a25a5f816d 100644 --- a/docs/guide/java/graalvm-support.md +++ b/docs/guide/java/graalvm-support.md @@ -38,6 +38,13 @@ Fory generates serialization code at GraalVM build time when you: Note: Fory's `asyncCompilationEnabled` option is automatically disabled for GraalVM native image since runtime JIT is not supported. +`@ForyStruct` annotation-processor static serializers are not the GraalVM native-image serializer +path. Native image builds use the Fory GraalVM registry: equal local and remote `TypeDef` hashes use +the existing meta-shared generated serializer, while mismatched remote schemas use a build-time +generated read-only compatible serializer cached by remote `TypeDef` id. Runtime native images load +those cached classes from the registry and do not attempt annotation-generated static serializer +lookup, Janino, or runtime class definition. + ## Basic Usage ### Step 0: Add the GraalVM Support Dependency diff --git a/docs/guide/java/index.md b/docs/guide/java/index.md index 59cca958fc..ea9f13bcf1 100644 --- a/docs/guide/java/index.md +++ b/docs/guide/java/index.md @@ -210,4 +210,5 @@ ThreadSafeFory threadLocalFory = Fory.builder() - [Type Registration](type-registration.md) - Class registration and security - [Custom Serializers](custom-serializers.md) - Implement custom serializers - [Cross-Language Serialization](cross-language.md) - Serialize data for other languages +- [Static Struct Serializers](static-struct-serializers.md) - Build-time serializers for `@ForyStruct` - [GraalVM Support](graalvm-support.md) - Build-time serializer compilation for native images diff --git a/docs/guide/java/static-struct-serializers.md b/docs/guide/java/static-struct-serializers.md new file mode 100644 index 0000000000..c424d05103 --- /dev/null +++ b/docs/guide/java/static-struct-serializers.md @@ -0,0 +1,120 @@ +--- +title: Static Struct Serializers +sidebar_position: 15 +id: static_struct_serializers +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +Build-time static serializers are generated for Java classes annotated with `@ForyStruct`. They +provide a non-JIT serializer path for ordinary JVM runtimes using `ForyBuilder#withCodegen(false)` +and for Android, where runtime bytecode generation is disabled. + +## Enabling The Processor + +Add the annotation processor to the Java compile configuration. The generated code depends only on +`fory-core` at runtime. `fory-core` itself does not depend on the processor; applications opt in by +placing `fory-annotation-processor` on their build's annotation-processor path. + +```xml + + + org.apache.fory + fory-annotation-processor + ${fory.version} + + +``` + +Annotate serializable structs with `@ForyStruct`: + +```java +import org.apache.fory.annotation.ForyStruct; + +@ForyStruct +public class Order { + public long id; + public String note; + + public Order() {} +} +``` + +The processor emits a public top-level serializer in the same package. For `Order`, the generated +class is `Order__ForyStaticSerializer__`. For a static member type `Outer.Inner`, the generated +top-level class is `Outer$Inner__ForyStaticSerializer__`; the processor does not modify the +enclosing class and does not generate inner serializer classes. + +## Runtime Selection + +Static serializers are used when available on: + +- ordinary JVMs with `ForyBuilder#withCodegen(false)`. +- Android runtimes, because runtime code generation is disabled there. +- compatible-mode meta-share reads when a generated static serializer exists for the target struct. + +Ordinary JVM `codegen=true` keeps the runtime-generated serializer precedence. Static serializer +lookup is deterministic by generated class name and does not scan the classpath. + +GraalVM native image does not use annotation-processor-generated static serializer classes. Native +image builds use the GraalVM registry path: matching local and remote `TypeDef` hashes use the +existing meta-shared generated serializer, and mismatched hashes use a build-time generated +read-only compatible serializer cached by remote `TypeDef` id. + +## Field Access Rules + +The processor never falls back to reflection for private serialized fields. + +- Public, protected, and package-private fields can be accessed directly when Java package access + allows the generated same-package serializer to use them. +- Private serialized fields must have accessible non-private getter and setter methods, or be + excluded with `transient` or Fory `@Ignore`. +- Public, protected, and package-private getter/setter methods are accepted when they are accessible + from the generated serializer package. +- Final fields are rejected for normal classes because generated read and copy methods must assign + them. Use records for constructor-based immutable structs. + +For records, generated serializers use public record accessors and construct values through the +canonical record constructor. Ignored record components are skipped by serialization and copy, and +their constructor arguments use Java default values during generated read/copy. + +## Generated Metadata + +Generated serializers expose descriptor metadata through +`StaticGeneratedStructSerializer#getDescriptors()`. The descriptor list is a static immutable list +owned by the generated serializer. Each descriptor carries: + +- field name and declaring-class identity. +- `@ForyField` id, nullable, reference-tracking, and dynamic-field semantics. +- a `TypeRef` tree with nested type arguments, array component type, and `TypeExtMeta` for nested + `TYPE_USE` metadata such as `@Ref`, `@UInt8Type`, and `@UInt16Type`. + +The runtime uses these descriptors to build schema metadata instead of reading nested +`Field#getAnnotatedType()` information at runtime. This keeps Android and JVM wire protocol unified +while avoiding Android reflection gaps. + +## Compatible Reads + +Generated static serializers include normal read/write/copy methods and a compatible read method. +The compatible path consumes remote schema metadata, matches remote fields to local fields, skips +unknown fields, and preserves Java defaults for missing fields. + +Field matching assigns dense generated matched ids for the generated branch table. Those ids are +local dispatch ids only; they are not `@ForyField.id` values and are not wire ids. Remote field order +still controls payload consumption. + +The same-schema fast path is used only when the remote schema hash equals the local schema hash and +the struct has no nested struct fields that require compatible layouts. diff --git a/java/fory-annotation-processor/pom.xml b/java/fory-annotation-processor/pom.xml new file mode 100644 index 0000000000..e14f499bcb --- /dev/null +++ b/java/fory-annotation-processor/pom.xml @@ -0,0 +1,69 @@ + + + + + org.apache.fory + fory-parent + 0.18.0-SNAPSHOT + + 4.0.0 + + fory-annotation-processor + + + Apache Fory annotation processor for build-time @ForyStruct serializer generation. + + + + 8 + 8 + ${basedir}/.. + + + + + org.apache.fory + fory-core + ${project.version} + test + + + org.testng + testng + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + none + + + + + diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java new file mode 100644 index 0000000000..d47e42c67d --- /dev/null +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -0,0 +1,789 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.annotation.processing; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.Filer; +import javax.annotation.processing.Messager; +import javax.annotation.processing.ProcessingEnvironment; +import javax.annotation.processing.RoundEnvironment; +import javax.annotation.processing.SupportedAnnotationTypes; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.AnnotationMirror; +import javax.lang.model.element.AnnotationValue; +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.Modifier; +import javax.lang.model.element.NestingKind; +import javax.lang.model.element.PackageElement; +import javax.lang.model.element.RecordComponentElement; +import javax.lang.model.element.TypeElement; +import javax.lang.model.element.VariableElement; +import javax.lang.model.type.ArrayType; +import javax.lang.model.type.DeclaredType; +import javax.lang.model.type.TypeKind; +import javax.lang.model.type.TypeMirror; +import javax.lang.model.type.TypeVariable; +import javax.lang.model.type.WildcardType; +import javax.lang.model.util.ElementFilter; +import javax.lang.model.util.Elements; +import javax.tools.Diagnostic; +import javax.tools.JavaFileObject; + +@SupportedAnnotationTypes("org.apache.fory.annotation.ForyStruct") +public final class ForyStructProcessor extends AbstractProcessor { + private static final String ARRAY_TYPE = "org.apache.fory.annotation.ArrayType"; + private static final String BFLOAT16_TYPE = "org.apache.fory.annotation.BFloat16Type"; + private static final String EXPOSE = "org.apache.fory.annotation.Expose"; + private static final String FLOAT16_TYPE = "org.apache.fory.annotation.Float16Type"; + private static final String FORY_FIELD = "org.apache.fory.annotation.ForyField"; + private static final String FORY_STRUCT = "org.apache.fory.annotation.ForyStruct"; + private static final String IGNORE = "org.apache.fory.annotation.Ignore"; + private static final String INT32_TYPE = "org.apache.fory.annotation.Int32Type"; + private static final String INT64_TYPE = "org.apache.fory.annotation.Int64Type"; + private static final String INT8_TYPE = "org.apache.fory.annotation.Int8Type"; + private static final String REF = "org.apache.fory.annotation.Ref"; + private static final String UINT16_TYPE = "org.apache.fory.annotation.UInt16Type"; + private static final String UINT32_TYPE = "org.apache.fory.annotation.UInt32Type"; + private static final String UINT64_TYPE = "org.apache.fory.annotation.UInt64Type"; + private static final String UINT8_TYPE = "org.apache.fory.annotation.UInt8Type"; + + private final Set processed = new HashSet<>(); + private final Map generatedTypes = new HashMap<>(); + private Messager messager; + private Filer filer; + private Elements elements; + private javax.lang.model.util.Types types; + + @Override + public synchronized void init(ProcessingEnvironment processingEnv) { + super.init(processingEnv); + messager = processingEnv.getMessager(); + filer = processingEnv.getFiler(); + elements = processingEnv.getElementUtils(); + types = processingEnv.getTypeUtils(); + } + + @Override + public SourceVersion getSupportedSourceVersion() { + return SourceVersion.latestSupported(); + } + + @Override + public boolean process(Set annotations, RoundEnvironment roundEnv) { + TypeElement foryStruct = elements.getTypeElement(FORY_STRUCT); + if (foryStruct == null) { + return false; + } + for (Element element : roundEnv.getElementsAnnotatedWith(foryStruct)) { + if (!(element instanceof TypeElement)) { + continue; + } + TypeElement type = (TypeElement) element; + String binaryName = elements.getBinaryName(type).toString(); + if (!processed.add(binaryName)) { + continue; + } + try { + SourceStruct struct = buildStruct(type); + if (struct != null) { + emit(struct, type); + } + } catch (InvalidStructException e) { + messager.printMessage(Diagnostic.Kind.ERROR, e.getMessage(), e.element); + } catch (RuntimeException e) { + messager.printMessage( + Diagnostic.Kind.ERROR, + "Failed to generate Fory static serializer for " + binaryName + ": " + e.getMessage(), + type); + } + } + return true; + } + + private SourceStruct buildStruct(TypeElement type) { + if (type.getModifiers().contains(Modifier.PRIVATE)) { + throw new InvalidStructException("@ForyStruct classes must not be private", type); + } + NestingKind nestingKind = type.getNestingKind(); + if (nestingKind == NestingKind.LOCAL || nestingKind == NestingKind.ANONYMOUS) { + throw new InvalidStructException( + "@ForyStruct local and anonymous classes are unsupported", type); + } + if (nestingKind == NestingKind.MEMBER && !type.getModifiers().contains(Modifier.STATIC)) { + throw new InvalidStructException("@ForyStruct member classes must be static", type); + } + + PackageElement packageElement = elements.getPackageOf(type); + String packageName = + packageElement.isUnnamed() ? "" : packageElement.getQualifiedName().toString(); + String binaryName = elements.getBinaryName(type).toString(); + String serializerName = + binaryName.substring(packageName.isEmpty() ? 0 : packageName.length() + 1) + + "__ForyStaticSerializer__"; + String qualifiedSerializerName = + packageName.isEmpty() ? serializerName : packageName + "." + serializerName; + TypeElement existing = elements.getTypeElement(qualifiedSerializerName); + if (existing != null && !existing.equals(type)) { + throw new InvalidStructException( + "Generated serializer name collides with existing type " + qualifiedSerializerName, type); + } + TypeElement previous = generatedTypes.put(qualifiedSerializerName, type); + if (previous != null && !previous.equals(type)) { + throw new InvalidStructException( + "Generated serializer name " + + qualifiedSerializerName + + " is ambiguous for " + + elements.getBinaryName(previous) + + " and " + + binaryName, + type); + } + + boolean record = isRecord(type); + List fields = record ? recordComponentFields(type) : serializableFields(type); + List sourceFields = new ArrayList<>(fields.size()); + List recordConstructorFields = new ArrayList<>(); + Map fieldIds = new HashMap<>(); + if (record) { + int serializedId = 0; + for (VariableElement field : fields) { + boolean serialized = isSerializableRecordField(field, type); + int id = serialized ? serializedId++ : -1; + SourceField sourceField = buildField(id, type, packageName, field, true, serialized); + recordConstructorFields.add(sourceField); + if (serialized) { + validateForyFieldId(binaryName, fieldIds, field); + sourceFields.add(sourceField); + } + } + } else { + for (int i = 0; i < fields.size(); i++) { + VariableElement field = fields.get(i); + validateForyFieldId(binaryName, fieldIds, field); + SourceField sourceField = buildField(i, type, packageName, field, false, true); + sourceFields.add(sourceField); + recordConstructorFields.add(sourceField); + } + } + return new SourceStruct( + packageName, + canonicalName(type.asType()), + serializerName, + record, + sourceFields, + recordConstructorFields); + } + + private void validateForyFieldId( + String binaryName, Map fieldIds, VariableElement field) { + ForyFieldMeta foryField = foryField(field); + if (foryField.hasPolicy && foryField.id >= 0) { + VariableElement previousField = fieldIds.put(foryField.id, field); + if (previousField != null) { + throw new InvalidStructException( + "Duplicate @ForyField id " + foryField.id + " in " + binaryName, field); + } + } + } + + private void emit(SourceStruct struct, TypeElement originatingType) { + try { + JavaFileObject file = + filer.createSourceFile(struct.qualifiedSerializerName(), originatingType); + try (Writer writer = file.openWriter()) { + writer.write(new StaticSerializerSourceWriter(struct).write()); + } + } catch (IOException e) { + throw new InvalidStructException( + "Failed to write generated serializer: " + e, originatingType); + } + } + + private SourceField buildField( + int id, + TypeElement owner, + String generatedPackage, + VariableElement field, + boolean record, + boolean serialized) { + Set modifiers = field.getModifiers(); + if (!record && modifiers.contains(Modifier.FINAL)) { + throw new InvalidStructException( + "Static serializers cannot assign final field " + + field.getSimpleName() + + "; use a record component or mark the field @Ignore/transient", + field); + } + SourceTypeNode typeNode = buildTypeNode(field.asType()); + String erasedType = canonicalName(types.erasure(field.asType())); + String declaringClass = + elements.getBinaryName((TypeElement) field.getEnclosingElement()).toString(); + ForyFieldMeta foryField = foryField(field); + + SourceField.AccessKind readKind; + SourceField.AccessKind writeKind; + String readAccess; + String writeAccess; + if (record) { + readKind = SourceField.AccessKind.METHOD; + writeKind = SourceField.AccessKind.METHOD; + readAccess = field.getSimpleName().toString(); + writeAccess = null; + } else if (isAccessibleFromGenerated(field, generatedPackage)) { + readKind = SourceField.AccessKind.FIELD; + writeKind = SourceField.AccessKind.FIELD; + readAccess = field.getSimpleName().toString(); + writeAccess = readAccess; + } else { + ExecutableElement getter = findGetter(owner, field, generatedPackage); + ExecutableElement setter = findSetter(owner, field, generatedPackage); + if (getter == null || setter == null) { + throw new InvalidStructException( + "Field " + + field.getSimpleName() + + " is not directly accessible from the generated serializer. Add accessible " + + "non-private getter/setter methods or mark it @Ignore/transient.", + field); + } + readKind = SourceField.AccessKind.METHOD; + writeKind = SourceField.AccessKind.METHOD; + readAccess = getter.getSimpleName().toString(); + writeAccess = setter.getSimpleName().toString(); + } + return new SourceField( + id, + field.getSimpleName().toString(), + erasedType, + typeNode, + reflectionModifiers(modifiers), + declaringClass, + serialized, + hasAnnotation(field, ARRAY_TYPE), + readKind, + readAccess, + writeKind, + writeAccess, + foryField.hasPolicy, + foryField.id, + foryField.hasPolicy ? foryField.nullable : !typeNode.primitive, + foryField.hasPolicy && foryField.ref, + foryField.dynamic); + } + + private List serializableFields(TypeElement type) { + List hierarchy = hierarchy(type); + List fields = new ArrayList<>(); + for (int i = hierarchy.size() - 1; i >= 0; i--) { + TypeElement current = hierarchy.get(i); + List declaredFields = ElementFilter.fieldsIn(current.getEnclosedElements()); + boolean haveExpose = false; + boolean haveIgnore = false; + for (VariableElement field : declaredFields) { + haveExpose |= hasAnnotation(field, EXPOSE); + haveIgnore |= hasAnnotation(field, IGNORE); + if (haveExpose && haveIgnore) { + throw new InvalidStructException( + "Fields of a class must not mix @Expose and @Ignore", field); + } + } + for (VariableElement field : declaredFields) { + Set modifiers = field.getModifiers(); + if (modifiers.contains(Modifier.STATIC) || modifiers.contains(Modifier.TRANSIENT)) { + continue; + } + if (haveExpose) { + if (hasAnnotation(field, EXPOSE)) { + fields.add(field); + } + } else if (!hasAnnotation(field, IGNORE)) { + fields.add(field); + } + } + } + return fields; + } + + private List recordComponentFields(TypeElement type) { + Map fieldsByName = new LinkedHashMap<>(); + for (VariableElement field : ElementFilter.fieldsIn(type.getEnclosedElements())) { + fieldsByName.put(field.getSimpleName().toString(), field); + } + List fields = new ArrayList<>(); + for (RecordComponentElement component : type.getRecordComponents()) { + VariableElement field = fieldsByName.get(component.getSimpleName().toString()); + if (field != null) { + fields.add(field); + } + } + return fields; + } + + private boolean isSerializableRecordField(VariableElement field, TypeElement owner) { + if (field.getModifiers().contains(Modifier.TRANSIENT)) { + return false; + } + if (hasAnnotation(field, IGNORE)) { + return false; + } + ExecutableElement accessor = findRecordAccessor(owner, field); + return accessor == null || !hasAnnotation(accessor, IGNORE); + } + + private ExecutableElement findRecordAccessor(TypeElement owner, VariableElement field) { + String name = field.getSimpleName().toString(); + for (ExecutableElement method : ElementFilter.methodsIn(owner.getEnclosedElements())) { + if (method.getSimpleName().contentEquals(name) && method.getParameters().isEmpty()) { + return method; + } + } + return null; + } + + private List hierarchy(TypeElement type) { + List hierarchy = new ArrayList<>(); + TypeElement current = type; + while (current != null && !current.getQualifiedName().contentEquals("java.lang.Object")) { + hierarchy.add(current); + TypeMirror superclass = current.getSuperclass(); + if (superclass == null || superclass.getKind() == TypeKind.NONE) { + break; + } + Element element = types.asElement(superclass); + current = element instanceof TypeElement ? (TypeElement) element : null; + } + return hierarchy; + } + + private ExecutableElement findGetter( + TypeElement owner, VariableElement field, String generatedPackage) { + String name = field.getSimpleName().toString(); + String suffix = Character.toUpperCase(name.charAt(0)) + name.substring(1); + List candidates = new ArrayList<>(); + candidates.add("get" + suffix); + if (field.asType().getKind() == TypeKind.BOOLEAN) { + candidates.add("is" + suffix); + } + for (ExecutableElement method : methods(owner)) { + if (!candidates.contains(method.getSimpleName().toString())) { + continue; + } + if (!method.getParameters().isEmpty() || method.getReturnType().getKind() == TypeKind.VOID) { + continue; + } + if (!isAccessibleFromGenerated(method, generatedPackage)) { + continue; + } + if (types.isAssignable(method.getReturnType(), field.asType())) { + return method; + } + } + return null; + } + + private ExecutableElement findSetter( + TypeElement owner, VariableElement field, String generatedPackage) { + String name = field.getSimpleName().toString(); + String suffix = Character.toUpperCase(name.charAt(0)) + name.substring(1); + String setterName = "set" + suffix; + for (ExecutableElement method : methods(owner)) { + if (!method.getSimpleName().contentEquals(setterName)) { + continue; + } + if (method.getParameters().size() != 1 || method.getReturnType().getKind() != TypeKind.VOID) { + continue; + } + if (!isAccessibleFromGenerated(method, generatedPackage)) { + continue; + } + if (types.isAssignable(field.asType(), method.getParameters().get(0).asType())) { + return method; + } + } + return null; + } + + private List methods(TypeElement owner) { + List methods = new ArrayList<>(); + for (TypeElement type : hierarchy(owner)) { + methods.addAll(ElementFilter.methodsIn(type.getEnclosedElements())); + } + return methods; + } + + private boolean isAccessibleFromGenerated(Element element, String generatedPackage) { + Set modifiers = element.getModifiers(); + if (modifiers.contains(Modifier.PUBLIC)) { + return true; + } + if (modifiers.contains(Modifier.PRIVATE)) { + return false; + } + return elements.getPackageOf(element).getQualifiedName().contentEquals(generatedPackage); + } + + private boolean isRecord(TypeElement type) { + return type.getKind().name().equals("RECORD"); + } + + private int reflectionModifiers(Set modifiers) { + int value = 0; + if (modifiers.contains(Modifier.PUBLIC)) { + value |= java.lang.reflect.Modifier.PUBLIC; + } + if (modifiers.contains(Modifier.PROTECTED)) { + value |= java.lang.reflect.Modifier.PROTECTED; + } + if (modifiers.contains(Modifier.PRIVATE)) { + value |= java.lang.reflect.Modifier.PRIVATE; + } + if (modifiers.contains(Modifier.STATIC)) { + value |= java.lang.reflect.Modifier.STATIC; + } + if (modifiers.contains(Modifier.FINAL)) { + value |= java.lang.reflect.Modifier.FINAL; + } + if (modifiers.contains(Modifier.TRANSIENT)) { + value |= java.lang.reflect.Modifier.TRANSIENT; + } + if (modifiers.contains(Modifier.VOLATILE)) { + value |= java.lang.reflect.Modifier.VOLATILE; + } + return value; + } + + private SourceTypeNode buildTypeNode(TypeMirror type) { + TypeKind kind = type.getKind(); + if (kind == TypeKind.TYPEVAR) { + TypeVariable typeVariable = (TypeVariable) type; + return buildTypeNode(typeVariable.getUpperBound()); + } + if (kind == TypeKind.WILDCARD) { + WildcardType wildcard = (WildcardType) type; + TypeMirror bound = wildcard.getExtendsBound(); + return buildTypeNode( + bound == null ? elements.getTypeElement("java.lang.Object").asType() : bound); + } + List arguments = new ArrayList<>(); + SourceTypeNode componentType = null; + if (kind == TypeKind.ARRAY) { + componentType = buildTypeNode(((ArrayType) type).getComponentType()); + } else if (type instanceof DeclaredType) { + for (TypeMirror argument : ((DeclaredType) type).getTypeArguments()) { + arguments.add(buildTypeNode(argument)); + } + } + String rawType = canonicalName(types.erasure(type)); + String extMeta = typeExtMetaExpression(type, rawType); + boolean primitive = kind.isPrimitive(); + boolean nestedStruct = isForyStructType(type); + return new SourceTypeNode( + rawType, typeName(type), extMeta, arguments, componentType, primitive, nestedStruct); + } + + private boolean isForyStructType(TypeMirror type) { + TypeMirror erased = types.erasure(type); + Element element = types.asElement(erased); + return element instanceof TypeElement && hasAnnotation(element, FORY_STRUCT); + } + + private String typeExtMetaExpression(TypeMirror type, String rawType) { + String typeId = scalarTypeId(type, rawType); + AnnotationMirror refMirror = annotationMirror(type, REF); + if (typeId == null && refMirror == null) { + return null; + } + return "meta(" + + (typeId == null ? "Types.UNKNOWN" : typeId) + + ", true, " + + booleanValue(refMirror, "enable", true) + + ")"; + } + + private String scalarTypeId(TypeMirror type, String rawType) { + if (hasTypeAnnotation(type, INT8_TYPE)) { + return rawType.equals("byte[]") ? "Types.INT8_ARRAY" : "Types.INT8"; + } + if (hasTypeAnnotation(type, UINT8_TYPE)) { + return rawType.equals("byte[]") ? "Types.UINT8_ARRAY" : "Types.UINT8"; + } + if (hasTypeAnnotation(type, UINT16_TYPE)) { + return rawType.equals("short[]") ? "Types.UINT16_ARRAY" : "Types.UINT16"; + } + AnnotationMirror uint32Mirror = typeAnnotationMirror(type, UINT32_TYPE); + if (uint32Mirror != null) { + String encoding = int32Encoding(uint32Mirror); + if (rawType.equals("int[]")) { + return "Types.UINT32_ARRAY"; + } + return "FIXED".equals(encoding) ? "Types.UINT32" : "Types.VAR_UINT32"; + } + AnnotationMirror uint64Mirror = typeAnnotationMirror(type, UINT64_TYPE); + if (uint64Mirror != null) { + String encoding = int64Encoding(uint64Mirror); + if (rawType.equals("long[]")) { + return "Types.UINT64_ARRAY"; + } + if ("FIXED".equals(encoding)) { + return "Types.UINT64"; + } + return "TAGGED".equals(encoding) ? "Types.TAGGED_UINT64" : "Types.VAR_UINT64"; + } + AnnotationMirror int32Mirror = typeAnnotationMirror(type, INT32_TYPE); + if (int32Mirror != null) { + String encoding = int32Encoding(int32Mirror); + return "FIXED".equals(encoding) ? "Types.INT32" : "Types.VARINT32"; + } + AnnotationMirror int64Mirror = typeAnnotationMirror(type, INT64_TYPE); + if (int64Mirror != null) { + String encoding = int64Encoding(int64Mirror); + if ("FIXED".equals(encoding)) { + return "Types.INT64"; + } + return "TAGGED".equals(encoding) ? "Types.TAGGED_INT64" : "Types.VARINT64"; + } + if (hasTypeAnnotation(type, FLOAT16_TYPE)) { + return "Types.FLOAT16_ARRAY"; + } + if (hasTypeAnnotation(type, BFLOAT16_TYPE)) { + return "Types.BFLOAT16_ARRAY"; + } + return null; + } + + private boolean hasTypeAnnotation(TypeMirror type, String annotationName) { + return typeAnnotationMirror(type, annotationName) != null; + } + + private AnnotationMirror typeAnnotationMirror(TypeMirror type, String annotationName) { + AnnotationMirror mirror = annotationMirror(type, annotationName); + if (mirror != null || type.getKind() != TypeKind.ARRAY) { + return mirror; + } + TypeMirror componentType = ((ArrayType) type).getComponentType(); + if (!componentType.getKind().isPrimitive()) { + return null; + } + return annotationMirror(componentType, annotationName); + } + + private AnnotationMirror annotationMirror(TypeMirror type, String annotationName) { + for (AnnotationMirror mirror : type.getAnnotationMirrors()) { + Element element = mirror.getAnnotationType().asElement(); + if (element instanceof TypeElement + && ((TypeElement) element).getQualifiedName().contentEquals(annotationName)) { + return mirror; + } + } + return null; + } + + private AnnotationMirror annotationMirror(Element element, String annotationName) { + for (AnnotationMirror mirror : element.getAnnotationMirrors()) { + Element annotationElement = mirror.getAnnotationType().asElement(); + if (annotationElement instanceof TypeElement + && ((TypeElement) annotationElement).getQualifiedName().contentEquals(annotationName)) { + return mirror; + } + } + return null; + } + + private boolean hasAnnotation(Element element, String annotationName) { + return annotationMirror(element, annotationName) != null; + } + + private boolean booleanValue(AnnotationMirror mirror, String name, boolean defaultValue) { + if (mirror == null) { + return defaultValue; + } + for (Map.Entry entry : + mirror.getElementValues().entrySet()) { + if (entry.getKey().getSimpleName().contentEquals(name)) { + return (Boolean) entry.getValue().getValue(); + } + } + return defaultValue; + } + + private String int32Encoding(AnnotationMirror mirror) { + return enumValue(mirror, "encoding", "VARINT"); + } + + private String int64Encoding(AnnotationMirror mirror) { + return enumValue(mirror, "encoding", "VARINT"); + } + + private String enumValue(AnnotationMirror mirror, String name, String defaultValue) { + if (mirror == null) { + return defaultValue; + } + for (Map.Entry entry : + mirror.getElementValues().entrySet()) { + if (entry.getKey().getSimpleName().contentEquals(name)) { + return String.valueOf(entry.getValue().getValue()); + } + } + return defaultValue; + } + + private ForyFieldMeta foryField(VariableElement field) { + AnnotationMirror mirror = annotationMirror(field, FORY_FIELD); + if (mirror == null) { + return ForyFieldMeta.NONE; + } + Map values = + elements.getElementValuesWithDefaults(mirror); + int id = -1; + boolean nullable = !field.asType().getKind().isPrimitive(); + boolean ref = false; + String dynamic = "AUTO"; + for (Map.Entry entry : + values.entrySet()) { + String name = entry.getKey().getSimpleName().toString(); + Object value = entry.getValue().getValue(); + if ("id".equals(name)) { + id = ((Number) value).intValue(); + } else if ("nullable".equals(name)) { + nullable = (Boolean) value; + } else if ("ref".equals(name)) { + ref = (Boolean) value; + } else if ("dynamic".equals(name)) { + dynamic = String.valueOf(value); + } + } + return new ForyFieldMeta(true, id, nullable, ref, dynamic); + } + + private String canonicalName(TypeMirror type) { + if (type.getKind().isPrimitive()) { + return primitiveName(type.getKind()); + } + if (type.getKind() == TypeKind.ARRAY) { + return canonicalName(((ArrayType) type).getComponentType()) + "[]"; + } + TypeMirror erased = types.erasure(type); + Element element = types.asElement(erased); + if (element instanceof TypeElement) { + return ((TypeElement) element).getQualifiedName().toString(); + } + return erased.toString().toLowerCase(Locale.ROOT); + } + + private String typeName(TypeMirror type) { + TypeKind kind = type.getKind(); + if (kind.isPrimitive()) { + return primitiveName(kind); + } + if (kind == TypeKind.ARRAY) { + return typeName(((ArrayType) type).getComponentType()) + "[]"; + } + if (kind == TypeKind.TYPEVAR) { + return typeName(((TypeVariable) type).getUpperBound()); + } + if (kind == TypeKind.WILDCARD) { + TypeMirror bound = ((WildcardType) type).getExtendsBound(); + return bound == null ? Object.class.getName() : typeName(bound); + } + TypeMirror erased = types.erasure(type); + Element element = types.asElement(erased); + String rawType = + element instanceof TypeElement + ? ((TypeElement) element).getQualifiedName().toString() + : erased.toString(); + if (!(type instanceof DeclaredType)) { + return rawType; + } + List arguments = ((DeclaredType) type).getTypeArguments(); + if (arguments.isEmpty()) { + return rawType; + } + StringBuilder builder = new StringBuilder(rawType).append("<"); + for (int i = 0; i < arguments.size(); i++) { + if (i > 0) { + builder.append(", "); + } + builder.append(typeName(arguments.get(i))); + } + return builder.append(">").toString(); + } + + private String primitiveName(TypeKind kind) { + switch (kind) { + case BOOLEAN: + return "boolean"; + case BYTE: + return "byte"; + case CHAR: + return "char"; + case SHORT: + return "short"; + case INT: + return "int"; + case LONG: + return "long"; + case FLOAT: + return "float"; + case DOUBLE: + return "double"; + case VOID: + return "void"; + default: + throw new IllegalArgumentException("Not a primitive kind: " + kind); + } + } + + private static final class InvalidStructException extends RuntimeException { + final Element element; + + InvalidStructException(String message, Element element) { + super(message); + this.element = element; + } + } + + private static final class ForyFieldMeta { + static final ForyFieldMeta NONE = new ForyFieldMeta(false, -1, false, false, "AUTO"); + + final boolean hasPolicy; + final int id; + final boolean nullable; + final boolean ref; + final String dynamic; + + ForyFieldMeta(boolean hasPolicy, int id, boolean nullable, boolean ref, String dynamic) { + this.hasPolicy = hasPolicy; + this.id = id; + this.nullable = nullable; + this.ref = ref; + this.dynamic = dynamic; + } + } +} diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java new file mode 100644 index 0000000000..bd9eb11f97 --- /dev/null +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.annotation.processing; + +final class SourceField { + enum AccessKind { + FIELD, + METHOD + } + + final int id; + final String name; + final String erasedType; + final SourceTypeNode typeNode; + final int modifiers; + final String declaringClass; + final boolean serialized; + final boolean arrayType; + final AccessKind readAccessKind; + final String readAccess; + final AccessKind writeAccessKind; + final String writeAccess; + final boolean hasForyFieldPolicy; + final int foryFieldId; + final boolean nullable; + final boolean trackingRef; + final String dynamic; + + SourceField( + int id, + String name, + String erasedType, + SourceTypeNode typeNode, + int modifiers, + String declaringClass, + boolean serialized, + boolean arrayType, + AccessKind readAccessKind, + String readAccess, + AccessKind writeAccessKind, + String writeAccess, + boolean hasForyFieldPolicy, + int foryFieldId, + boolean nullable, + boolean trackingRef, + String dynamic) { + this.id = id; + this.name = name; + this.erasedType = erasedType; + this.typeNode = typeNode; + this.modifiers = modifiers; + this.declaringClass = declaringClass; + this.serialized = serialized; + this.arrayType = arrayType; + this.readAccessKind = readAccessKind; + this.readAccess = readAccess; + this.writeAccessKind = writeAccessKind; + this.writeAccess = writeAccess; + this.hasForyFieldPolicy = hasForyFieldPolicy; + this.foryFieldId = foryFieldId; + this.nullable = nullable; + this.trackingRef = trackingRef; + this.dynamic = dynamic; + } + + String readExpression(String target) { + if (readAccessKind == AccessKind.METHOD) { + return target + "." + readAccess + "()"; + } + return target + "." + readAccess; + } + + String writeStatement(String target, String valueExpression) { + if (writeAccessKind == AccessKind.METHOD) { + return target + "." + writeAccess + "(" + valueExpression + ");"; + } + return target + "." + writeAccess + " = " + valueExpression + ";"; + } + + String defaultValue() { + switch (erasedType) { + case "boolean": + return "false"; + case "byte": + return "(byte) 0"; + case "char": + return "(char) 0"; + case "short": + return "(short) 0"; + case "int": + return "0"; + case "long": + return "0L"; + case "float": + return "0.0f"; + case "double": + return "0.0d"; + default: + return "null"; + } + } + + String castExpression(String valueExpression) { + switch (erasedType) { + case "boolean": + return "((Boolean) " + valueExpression + ").booleanValue()"; + case "byte": + return "((Byte) " + valueExpression + ").byteValue()"; + case "char": + return "((Character) " + valueExpression + ").charValue()"; + case "short": + return "((Short) " + valueExpression + ").shortValue()"; + case "int": + return "((Integer) " + valueExpression + ").intValue()"; + case "long": + return "((Long) " + valueExpression + ").longValue()"; + case "float": + return "((Float) " + valueExpression + ").floatValue()"; + case "double": + return "((Double) " + valueExpression + ").doubleValue()"; + default: + return "(" + erasedType + ") " + valueExpression; + } + } +} diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java new file mode 100644 index 0000000000..925ba55172 --- /dev/null +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.annotation.processing; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +final class SourceStruct { + final String packageName; + final String typeName; + final String serializerName; + final boolean record; + final boolean hasNestedCompatibleStructFields; + final List fields; + final List recordConstructorFields; + + SourceStruct( + String packageName, + String typeName, + String serializerName, + boolean record, + List fields, + List recordConstructorFields) { + this.packageName = packageName; + this.typeName = typeName; + this.serializerName = serializerName; + this.record = record; + this.fields = Collections.unmodifiableList(new ArrayList<>(fields)); + this.recordConstructorFields = + Collections.unmodifiableList(new ArrayList<>(recordConstructorFields)); + boolean hasNestedStruct = false; + for (SourceField field : fields) { + hasNestedStruct |= field.typeNode.hasNestedStruct(); + } + this.hasNestedCompatibleStructFields = hasNestedStruct; + } + + String qualifiedSerializerName() { + if (packageName.isEmpty()) { + return serializerName; + } + return packageName + "." + serializerName; + } +} diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceTypeNode.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceTypeNode.java new file mode 100644 index 0000000000..80804eedab --- /dev/null +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceTypeNode.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.annotation.processing; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +final class SourceTypeNode { + final String rawType; + final String typeName; + final String typeExtMeta; + final List typeArguments; + final SourceTypeNode componentType; + final boolean primitive; + final boolean nestedStruct; + + SourceTypeNode( + String rawType, + String typeName, + String typeExtMeta, + List typeArguments, + SourceTypeNode componentType, + boolean primitive, + boolean nestedStruct) { + this.rawType = rawType; + this.typeName = typeName; + this.typeExtMeta = typeExtMeta; + this.typeArguments = + typeArguments == null + ? Collections.emptyList() + : Collections.unmodifiableList(new ArrayList<>(typeArguments)); + this.componentType = componentType; + this.primitive = primitive; + this.nestedStruct = nestedStruct; + } + + String typeRefExpression() { + if (typeExtMeta == null && typeArguments.isEmpty() && componentType == null) { + return "TypeRef.of(" + rawType + ".class)"; + } + return "TypeRef.of(" + + rawType + + ".class, " + + (typeExtMeta == null ? "null" : typeExtMeta) + + ", " + + typeArgumentsExpression() + + ", " + + (componentType == null ? "null" : componentType.typeRefExpression()) + + ")"; + } + + private String typeArgumentsExpression() { + if (typeArguments.isEmpty()) { + return "null"; + } + StringBuilder builder = new StringBuilder("Arrays.>asList("); + for (int i = 0; i < typeArguments.size(); i++) { + if (i > 0) { + builder.append(", "); + } + builder.append(typeArguments.get(i).typeRefExpression()); + } + builder.append(')'); + return builder.toString(); + } + + boolean hasNestedStruct() { + if (nestedStruct) { + return true; + } + if (componentType != null && componentType.hasNestedStruct()) { + return true; + } + for (SourceTypeNode typeArgument : typeArguments) { + if (typeArgument.hasNestedStruct()) { + return true; + } + } + return false; + } +} diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java new file mode 100644 index 0000000000..f3a2a354db --- /dev/null +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -0,0 +1,504 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.annotation.processing; + +final class StaticSerializerSourceWriter { + private final SourceStruct struct; + private final StringBuilder builder = new StringBuilder(16384); + + StaticSerializerSourceWriter(SourceStruct struct) { + this.struct = struct; + } + + String write() { + writeHeader(); + writeClassStart(); + writeDescriptors(); + writeConstructors(); + writeSerializerMethods(); + writeSchemaConsistentRead(); + writeWriteGroups(); + writeReadGroups(); + writeCompatibleRead(); + writeCopy(); + writeDescriptorHelpers(); + builder.append("}\n"); + return builder.toString(); + } + + private void writeHeader() { + if (!struct.packageName.isEmpty()) { + builder.append("package ").append(struct.packageName).append(";\n\n"); + } + builder.append("import java.util.ArrayList;\n"); + builder.append("import java.util.Arrays;\n"); + builder.append("import java.util.Collections;\n"); + builder.append("import java.util.List;\n"); + builder.append("import org.apache.fory.annotation.ForyField;\n"); + builder.append("import org.apache.fory.context.CopyContext;\n"); + builder.append("import org.apache.fory.context.ReadContext;\n"); + builder.append("import org.apache.fory.context.WriteContext;\n"); + builder.append("import org.apache.fory.memory.MemoryBuffer;\n"); + builder.append("import org.apache.fory.meta.TypeDef;\n"); + builder.append("import org.apache.fory.meta.TypeExtMeta;\n"); + builder.append("import org.apache.fory.reflect.TypeRef;\n"); + builder.append("import org.apache.fory.resolver.TypeResolver;\n"); + builder.append("import org.apache.fory.serializer.FieldGroups;\n"); + builder.append("import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo;\n"); + builder.append("import org.apache.fory.serializer.StaticGeneratedStructSerializer;\n"); + builder.append("import org.apache.fory.type.Descriptor;\n"); + builder.append("import org.apache.fory.type.ForyFieldPolicy;\n"); + builder.append("import org.apache.fory.type.Types;\n\n"); + } + + private void writeClassStart() { + builder.append("@SuppressWarnings({\"unchecked\", \"rawtypes\"})\n"); + builder + .append("public final class ") + .append(struct.serializerName) + .append(" extends StaticGeneratedStructSerializer<") + .append(struct.typeName) + .append("> {\n"); + builder + .append(" private static final boolean HAS_NESTED_COMPATIBLE_STRUCT_FIELDS = ") + .append(struct.hasNestedCompatibleStructFields) + .append(";\n"); + builder.append(" private static final List DESCRIPTORS = buildDescriptors();\n\n"); + builder.append(" private final SerializationFieldInfo[] buildInFields;\n"); + builder.append(" private final int[] buildInFieldIds;\n"); + builder.append(" private final SerializationFieldInfo[] containerFields;\n"); + builder.append(" private final int[] containerFieldIds;\n"); + builder.append(" private final SerializationFieldInfo[] otherFields;\n"); + builder.append(" private final int[] otherFieldIds;\n"); + builder.append(" private final SerializationFieldInfo[] fieldsById;\n"); + builder.append(" private final int classVersionHash;\n"); + builder.append(" private final boolean sameSchemaCompatible;\n\n"); + } + + private void writeDescriptors() { + builder.append(" private static List buildDescriptors() {\n"); + builder + .append(" ArrayList descriptors = new ArrayList(") + .append(struct.fields.size()) + .append(");\n"); + for (SourceField field : struct.fields) { + builder + .append(" descriptors.add(new Descriptor(") + .append(field.typeNode.typeRefExpression()) + .append(", \"") + .append(escape(field.typeNode.typeName)) + .append("\", \"") + .append(escape(field.name)) + .append("\", ") + .append(field.modifiers) + .append(", \"") + .append(escape(field.declaringClass)) + .append("\", ") + .append(foryFieldPolicyExpression(field)) + .append(", ") + .append(field.arrayType) + .append("));\n"); + } + builder.append(" return Collections.unmodifiableList(descriptors);\n"); + builder.append(" }\n\n"); + builder.append(" @Override\n"); + builder.append(" public List getDescriptors() {\n"); + builder.append(" return DESCRIPTORS;\n"); + builder.append(" }\n\n"); + } + + private String foryFieldPolicyExpression(SourceField field) { + if (!field.hasForyFieldPolicy) { + return "null"; + } + return "ForyFieldPolicy.of(" + + field.foryFieldId + + ", " + + field.nullable + + ", " + + field.trackingRef + + ", ForyField.Dynamic." + + field.dynamic + + ")"; + } + + private void writeConstructors() { + builder + .append(" public ") + .append(struct.serializerName) + .append("(TypeResolver typeResolver, Class type) {\n"); + builder.append(" super(typeResolver, type);\n"); + writeConstructorBody("false"); + builder.append(" }\n\n"); + builder + .append(" public ") + .append(struct.serializerName) + .append("(TypeResolver typeResolver, Class type, TypeDef typeDef) {\n"); + builder.append(" super(typeResolver, type, typeDef, DESCRIPTORS);\n"); + writeConstructorBody( + "typeDef != null && !HAS_NESTED_COMPATIBLE_STRUCT_FIELDS && typeDef.getId() == TypeDef.buildTypeDef(typeResolver, type).getId()"); + builder.append(" }\n\n"); + } + + private void writeConstructorBody(String sameSchemaExpression) { + builder.append(" FieldGroups fieldGroups = buildFieldGroups(DESCRIPTORS);\n"); + builder.append(" this.buildInFields = fieldGroups.buildInFields;\n"); + builder.append(" this.buildInFieldIds = localFieldIds(buildInFields, DESCRIPTORS);\n"); + builder.append(" this.containerFields = fieldGroups.containerFields;\n"); + builder.append(" this.containerFieldIds = localFieldIds(containerFields, DESCRIPTORS);\n"); + builder.append(" this.otherFields = fieldGroups.userTypeFields;\n"); + builder.append(" this.otherFieldIds = localFieldIds(otherFields, DESCRIPTORS);\n"); + builder.append(" this.fieldsById = new SerializationFieldInfo[DESCRIPTORS.size()];\n"); + builder.append(" SerializationFieldInfo[] allFields = fieldGroups.allFields;\n"); + builder.append(" int[] allFieldIds = localFieldIds(allFields, DESCRIPTORS);\n"); + builder.append(" for (int i = 0; i < allFields.length; i++) {\n"); + builder.append(" this.fieldsById[allFieldIds[i]] = allFields[i];\n"); + builder.append(" }\n"); + builder.append( + " this.classVersionHash = typeResolver.checkClassVersion() ? computeClassVersionHash(DESCRIPTORS) : 0;\n"); + builder.append(" this.sameSchemaCompatible = ").append(sameSchemaExpression).append(";\n"); + } + + private void writeSerializerMethods() { + builder.append(" @Override\n"); + builder + .append(" public void write(WriteContext writeContext, ") + .append(struct.typeName) + .append(" value) {\n"); + builder.append(" MemoryBuffer buffer = writeContext.getBuffer();\n"); + builder.append(" if (typeResolver.checkClassVersion()) {\n"); + builder.append(" buffer.writeInt32(classVersionHash);\n"); + builder.append(" }\n"); + builder.append(" writeBuildInFields(writeContext, value);\n"); + builder.append(" writeContainerFields(writeContext, value);\n"); + builder.append(" writeOtherFields(writeContext, value);\n"); + builder.append(" }\n\n"); + builder.append(" @Override\n"); + builder + .append(" public ") + .append(struct.typeName) + .append(" read(ReadContext readContext) {\n"); + builder.append(" if (typeDef != null) {\n"); + builder.append( + " return sameSchemaCompatible ? readSchemaConsistent(readContext) : readCompatible(readContext);\n"); + builder.append(" }\n"); + builder.append(" return readSchemaConsistent(readContext);\n"); + builder.append(" }\n\n"); + } + + private void writeSchemaConsistentRead() { + builder + .append(" private ") + .append(struct.typeName) + .append(" readSchemaConsistent(ReadContext readContext) {\n"); + builder.append(" MemoryBuffer buffer = readContext.getBuffer();\n"); + builder.append(" if (typeResolver.checkClassVersion()) {\n"); + builder.append(" checkClassVersion(buffer.readInt32(), classVersionHash);\n"); + builder.append(" }\n"); + if (struct.record) { + for (SourceField field : struct.fields) { + builder + .append(" ") + .append(field.erasedType) + .append(" field") + .append(field.id) + .append(" = ") + .append(field.defaultValue()) + .append(";\n"); + } + builder.append(" Object[] values = new Object[DESCRIPTORS.size()];\n"); + builder.append(" readBuildInRecordFields(readContext, values);\n"); + builder.append(" readContainerRecordFields(readContext, values);\n"); + builder.append(" readOtherRecordFields(readContext, values);\n"); + for (SourceField field : struct.fields) { + builder + .append(" field") + .append(field.id) + .append(" = ") + .append(field.castExpression("values[" + field.id + "]")) + .append(";\n"); + } + builder.append(" return new ").append(struct.typeName).append("("); + appendRecordConstructorArguments("field"); + builder.append(");\n"); + } else { + builder.append(" ").append(struct.typeName).append(" value = newBean();\n"); + builder.append(" readContext.reference(value);\n"); + builder.append(" readBuildInFields(readContext, value);\n"); + builder.append(" readContainerFields(readContext, value);\n"); + builder.append(" readOtherFields(readContext, value);\n"); + builder.append(" return value;\n"); + } + builder.append(" }\n\n"); + } + + private void writeWriteGroups() { + writeWriteGroup("BuildIn", "buildInFields", "buildInFieldIds", "writeBuildInFieldValue"); + writeWriteGroup( + "Container", "containerFields", "containerFieldIds", "writeContainerFieldValue"); + writeWriteGroup("Other", "otherFields", "otherFieldIds", "writeOtherFieldValue"); + } + + private void writeWriteGroup( + String groupName, String fieldsName, String idsName, String helperName) { + builder + .append(" private void write") + .append(groupName) + .append("Fields(WriteContext writeContext, ") + .append(struct.typeName) + .append(" value) {\n"); + builder.append(" for (int i = 0; i < ").append(fieldsName).append(".length; i++) {\n"); + builder.append(" SerializationFieldInfo fieldInfo = ").append(fieldsName).append("[i];\n"); + builder.append(" switch (").append(idsName).append("[i]) {\n"); + for (SourceField field : struct.fields) { + builder.append(" case ").append(field.id).append(":\n"); + builder + .append(" ") + .append(helperName) + .append("(writeContext, fieldInfo, ") + .append(field.readExpression("value")) + .append(");\n"); + builder.append(" break;\n"); + } + builder.append(" default:\n"); + builder + .append(" throw new IllegalStateException(\"Unknown generated field id \" + ") + .append(idsName) + .append("[i]);\n"); + builder.append(" }\n"); + builder.append(" }\n"); + builder.append(" }\n\n"); + } + + private void writeReadGroups() { + if (struct.record) { + writeReadRecordGroup("BuildIn", "buildInFields", "buildInFieldIds", "readBuildInFieldValue"); + writeReadRecordGroup( + "Container", "containerFields", "containerFieldIds", "readContainerFieldValue"); + writeReadRecordGroup("Other", "otherFields", "otherFieldIds", "readOtherFieldValue"); + } else { + writeReadBeanGroup("BuildIn", "buildInFields", "buildInFieldIds", "readBuildInFieldValue"); + writeReadBeanGroup( + "Container", "containerFields", "containerFieldIds", "readContainerFieldValue"); + writeReadBeanGroup("Other", "otherFields", "otherFieldIds", "readOtherFieldValue"); + } + } + + private void writeReadBeanGroup( + String groupName, String fieldsName, String idsName, String helperName) { + builder + .append(" private void read") + .append(groupName) + .append("Fields(ReadContext readContext, ") + .append(struct.typeName) + .append(" value) {\n"); + builder.append(" for (int i = 0; i < ").append(fieldsName).append(".length; i++) {\n"); + builder.append(" SerializationFieldInfo fieldInfo = ").append(fieldsName).append("[i];\n"); + builder + .append(" Object fieldValue = ") + .append(helperName) + .append("(readContext, fieldInfo);\n"); + builder.append(" switch (").append(idsName).append("[i]) {\n"); + for (SourceField field : struct.fields) { + builder.append(" case ").append(field.id).append(":\n"); + builder + .append(" ") + .append(field.writeStatement("value", field.castExpression("fieldValue"))) + .append("\n"); + builder.append(" break;\n"); + } + builder.append(" default:\n"); + builder + .append(" throw new IllegalStateException(\"Unknown generated field id \" + ") + .append(idsName) + .append("[i]);\n"); + builder.append(" }\n"); + builder.append(" }\n"); + builder.append(" }\n\n"); + } + + private void writeReadRecordGroup( + String groupName, String fieldsName, String idsName, String helperName) { + builder + .append(" private void read") + .append(groupName) + .append("RecordFields(ReadContext readContext, Object[] values) {\n"); + builder.append(" for (int i = 0; i < ").append(fieldsName).append(".length; i++) {\n"); + builder.append(" SerializationFieldInfo fieldInfo = ").append(fieldsName).append("[i];\n"); + builder + .append(" values[") + .append(idsName) + .append("[i]] = ") + .append(helperName) + .append("(readContext, fieldInfo);\n"); + builder.append(" }\n"); + builder.append(" }\n\n"); + } + + private void writeCompatibleRead() { + builder.append(" @Override\n"); + builder + .append(" public ") + .append(struct.typeName) + .append(" readCompatible(ReadContext readContext) {\n"); + builder.append(" if (sameSchemaCompatible) {\n"); + builder.append(" return readSchemaConsistent(readContext);\n"); + builder.append(" }\n"); + if (struct.record) { + for (SourceField field : struct.fields) { + builder + .append(" ") + .append(field.erasedType) + .append(" field") + .append(field.id) + .append(" = ") + .append(field.defaultValue()) + .append(";\n"); + } + builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); + builder.append(" RemoteFieldInfo remoteField = remoteFields.get(i);\n"); + builder.append(" switch (matchedId(remoteField)) {\n"); + for (SourceField field : struct.fields) { + builder.append(" case ").append(field.id).append(":\n"); + builder + .append(" field") + .append(field.id) + .append(" = ") + .append(field.castExpression("readRemoteField(readContext, remoteField)")) + .append(";\n"); + builder.append(" break;\n"); + } + builder.append(" default:\n"); + builder.append(" skipField(readContext, remoteField);\n"); + builder.append(" }\n"); + builder.append(" }\n"); + builder.append(" return new ").append(struct.typeName).append("("); + appendRecordConstructorArguments("field"); + builder.append(");\n"); + } else { + builder.append(" ").append(struct.typeName).append(" value = newBean();\n"); + builder.append(" readContext.reference(value);\n"); + builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); + builder.append(" RemoteFieldInfo remoteField = remoteFields.get(i);\n"); + builder.append(" switch (matchedId(remoteField)) {\n"); + for (SourceField field : struct.fields) { + builder.append(" case ").append(field.id).append(":\n"); + builder + .append(" ") + .append( + field.writeStatement( + "value", field.castExpression("readRemoteField(readContext, remoteField)"))) + .append("\n"); + builder.append(" break;\n"); + } + builder.append(" default:\n"); + builder.append(" skipField(readContext, remoteField);\n"); + builder.append(" }\n"); + builder.append(" }\n"); + builder.append(" return value;\n"); + } + builder.append(" }\n\n"); + } + + private void writeCopy() { + builder.append(" @Override\n"); + builder + .append(" public ") + .append(struct.typeName) + .append(" copy(CopyContext copyContext, ") + .append(struct.typeName) + .append(" value) {\n"); + builder.append(" if (immutable) {\n"); + builder.append(" return value;\n"); + builder.append(" }\n"); + if (struct.record) { + for (SourceField field : struct.fields) { + builder + .append(" ") + .append(field.erasedType) + .append(" field") + .append(field.id) + .append(" = ") + .append( + field.castExpression( + "copyFieldValue(copyContext, " + + field.readExpression("value") + + ", fieldsById[" + + field.id + + "])")) + .append(";\n"); + } + builder + .append(" ") + .append(struct.typeName) + .append(" copied = new ") + .append(struct.typeName) + .append("("); + appendRecordConstructorArguments("field"); + builder.append(");\n"); + builder.append(" copyContext.reference(value, copied);\n"); + builder.append(" return copied;\n"); + } else { + builder.append(" ").append(struct.typeName).append(" copied = newBean();\n"); + builder.append(" copyContext.reference(value, copied);\n"); + for (SourceField field : struct.fields) { + builder + .append(" ") + .append( + field.writeStatement( + "copied", + field.castExpression( + "copyFieldValue(copyContext, " + + field.readExpression("value") + + ", fieldsById[" + + field.id + + "])"))) + .append("\n"); + } + builder.append(" return copied;\n"); + } + builder.append(" }\n\n"); + } + + private void writeDescriptorHelpers() { + builder.append( + " private static TypeExtMeta meta(int typeId, boolean nullable, boolean trackingRef) {\n"); + builder.append(" return TypeExtMeta.of(typeId, nullable, trackingRef);\n"); + builder.append(" }\n"); + } + + private void appendRecordConstructorArguments(String prefix) { + for (int i = 0; i < struct.recordConstructorFields.size(); i++) { + if (i > 0) { + builder.append(", "); + } + SourceField field = struct.recordConstructorFields.get(i); + if (field.serialized) { + builder.append(prefix).append(field.id); + } else { + builder.append(field.defaultValue()); + } + } + } + + private static String escape(String value) { + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } +} diff --git a/java/fory-annotation-processor/src/main/resources/META-INF/services/javax.annotation.processing.Processor b/java/fory-annotation-processor/src/main/resources/META-INF/services/javax.annotation.processing.Processor new file mode 100644 index 0000000000..9375d6ca64 --- /dev/null +++ b/java/fory-annotation-processor/src/main/resources/META-INF/services/javax.annotation.processing.Processor @@ -0,0 +1 @@ +org.apache.fory.annotation.processing.ForyStructProcessor diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java new file mode 100644 index 0000000000..d5aae45d77 --- /dev/null +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -0,0 +1,515 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.annotation.processing; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import javax.tools.Diagnostic; +import javax.tools.DiagnosticCollector; +import javax.tools.JavaCompiler; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; +import org.apache.fory.Fory; +import org.apache.fory.builder.CodecUtils; +import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.context.MetaReadContext; +import org.apache.fory.context.MetaWriteContext; +import org.apache.fory.meta.TypeDef; +import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.serializer.Serializer; +import org.apache.fory.serializer.Serializers; +import org.apache.fory.serializer.StaticGeneratedStructSerializer; +import org.apache.fory.type.Descriptor; +import org.apache.fory.type.Types; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ForyStructProcessorTest { + @Test + public void testStaticSerializerSelectedWithCodegenDisabled() throws Exception { + CompilationResult result = + compile( + "test.SimpleStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class SimpleStruct {\n" + + " public int id;\n" + + " public String name;\n" + + " public SimpleStruct() {}\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.SimpleStruct"); + Class serializerType = loader.loadClass("test.SimpleStruct__ForyStaticSerializer__"); + Assert.assertTrue(StaticGeneratedStructSerializer.class.isAssignableFrom(serializerType)); + + Object value = type.getConstructor().newInstance(); + setField(type, value, "id", 7); + setField(type, value, "name", "fory"); + + Fory fory = + Fory.builder() + .withClassLoader(loader) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + Object serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); + Assert.assertEquals(serializer.getClass().getName(), serializerType.getName()); + Object roundTrip = fory.deserialize(fory.serialize(value)); + Assert.assertEquals(getField(type, roundTrip, "id"), 7); + Assert.assertEquals(getField(type, roundTrip, "name"), "fory"); + Object copied = fory.copy(value); + Assert.assertEquals(getField(type, copied, "id"), 7); + Assert.assertEquals(getField(type, copied, "name"), "fory"); + } + } + + @Test + public void testPrivateFieldUsesAccessibleAccessors() throws Exception { + CompilationResult result = + compile( + "test.PrivateStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class PrivateStruct {\n" + + " private int id;\n" + + " private String name;\n" + + " public PrivateStruct() {}\n" + + " int getId() { return id; }\n" + + " void setId(int id) { this.id = id; }\n" + + " protected String getName() { return name; }\n" + + " protected void setName(String name) { this.name = name; }\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.PrivateStruct"); + Object value = type.getConstructor().newInstance(); + invoke(type, value, "setId", int.class, 8); + invoke(type, value, "setName", String.class, "static"); + Fory fory = + Fory.builder() + .withClassLoader(loader) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + Object roundTrip = fory.deserialize(fory.serialize(value)); + Assert.assertEquals(invoke(type, roundTrip, "getId"), 8); + Assert.assertEquals(invoke(type, roundTrip, "getName"), "static"); + } + } + + @Test + public void testPrivateFieldWithoutAccessorsFailsCompilation() throws Exception { + CompilationResult result = + compile( + "test.BadStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class BadStruct {\n" + + " private int id;\n" + + " public BadStruct() {}\n" + + "}\n"); + Assert.assertFalse(result.success); + Assert.assertTrue(result.diagnostics().contains("getter/setter"), result.diagnostics()); + } + + @Test + public void testPrivateStructFailsCompilation() throws Exception { + CompilationResult result = + compile( + "test.PrivateOwner", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "public class PrivateOwner {\n" + + " @ForyStruct private static class HiddenStruct {\n" + + " int id;\n" + + " HiddenStruct() {}\n" + + " }\n" + + "}\n"); + Assert.assertFalse(result.success); + Assert.assertTrue(result.diagnostics().contains("must not be private"), result.diagnostics()); + } + + @Test + public void testDuplicateForyFieldIdFailsCompilation() throws Exception { + CompilationResult result = + compile( + "test.DuplicateIdStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyField;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class DuplicateIdStruct {\n" + + " @ForyField(id = 1) public int left;\n" + + " @ForyField(id = 1) public int right;\n" + + " public DuplicateIdStruct() {}\n" + + "}\n"); + Assert.assertFalse(result.success); + Assert.assertTrue( + result.diagnostics().contains("Duplicate @ForyField id 1"), result.diagnostics()); + } + + @Test + public void testInnerTypeGeneratedAsTopLevelBinaryTail() throws Exception { + CompilationResult result = + compile( + "test.Outer", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "public class Outer {\n" + + " @ForyStruct public static class Inner {\n" + + " public int id;\n" + + " public Inner() {}\n" + + " }\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class serializer = loader.loadClass("test.Outer$Inner__ForyStaticSerializer__"); + Assert.assertTrue(StaticGeneratedStructSerializer.class.isAssignableFrom(serializer)); + } + } + + @Test + public void testGeneratedNameCollisionFailsCompilation() throws Exception { + CompilationResult result = + compile( + "test.Outer", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "public class Outer {\n" + + " @ForyStruct public static class Inner {\n" + + " public int id;\n" + + " public Inner() {}\n" + + " }\n" + + "}\n" + + "class Outer$Inner__ForyStaticSerializer__ {}\n"); + Assert.assertFalse(result.success); + Assert.assertTrue(result.diagnostics().contains("collides"), result.diagnostics()); + } + + @Test + public void testGeneratedDescriptorsCarryNestedTypeMetadata() throws Exception { + CompilationResult result = + compile( + "test.MetadataStruct", + "package test;\n" + + "import java.util.List;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "import org.apache.fory.annotation.Ref;\n" + + "import org.apache.fory.annotation.UInt16Type;\n" + + "@ForyStruct public class MetadataStruct {\n" + + " public List<@Ref String> names;\n" + + " public @UInt16Type int code;\n" + + " public MetadataStruct() {}\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.MetadataStruct"); + Class serializerType = loader.loadClass("test.MetadataStruct__ForyStaticSerializer__"); + Fory fory = + Fory.builder() + .withClassLoader(loader) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + StaticGeneratedStructSerializer serializer = + (StaticGeneratedStructSerializer) + serializerType + .getConstructor(org.apache.fory.resolver.TypeResolver.class, Class.class) + .newInstance(fory.getTypeResolver(), type); + Descriptor names = descriptor(serializer.getDescriptors(), "names"); + Assert.assertTrue(names.getTypeRef().hasExplicitTypeArguments()); + Assert.assertTrue(names.getTypeRef().getTypeArguments().get(0).hasTypeExtMeta()); + Assert.assertTrue( + names.getTypeRef().getTypeArguments().get(0).getTypeExtMeta().trackingRef()); + Descriptor code = descriptor(serializer.getDescriptors(), "code"); + Assert.assertTrue(code.getTypeRef().hasTypeExtMeta()); + Assert.assertEquals(code.getTypeRef().getTypeExtMeta().typeId(), Types.UINT16); + } + } + + @Test + public void testRecordReadAndCopyUseCanonicalConstructor() throws Exception { + CompilationResult result = + compile( + "test.RecordStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "import org.apache.fory.annotation.Ignore;\n" + + "@ForyStruct public record RecordStruct(int id, String ignored, String name) {\n" + + " @Ignore public String ignored() { return ignored; }\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.RecordStruct"); + Object value = + type.getConstructor(int.class, String.class, String.class) + .newInstance(5, "skip", "record"); + Fory fory = + Fory.builder() + .withClassLoader(loader) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + Object roundTrip = fory.deserialize(fory.serialize(value)); + Assert.assertEquals(invoke(type, roundTrip, "id"), 5); + Assert.assertNull(invoke(type, roundTrip, "ignored")); + Assert.assertEquals(invoke(type, roundTrip, "name"), "record"); + Object copied = fory.copy(value); + Assert.assertEquals(invoke(type, copied, "id"), 5); + Assert.assertNull(invoke(type, copied, "ignored")); + Assert.assertEquals(invoke(type, copied, "name"), "record"); + } + } + + @Test + public void testCompatibleReadUsesGeneratedSerializer() throws Exception { + CompilationResult writerResult = + compile( + "test.EvolvingStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyField;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class EvolvingStruct {\n" + + " @ForyField(id = 1) public int id;\n" + + " @ForyField(id = 2, nullable = true) public String name;\n" + + " public EvolvingStruct() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.EvolvingStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyField;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class EvolvingStruct {\n" + + " @ForyField(id = 1) public int id;\n" + + " @ForyField(id = 2, nullable = true) public String name;\n" + + " @ForyField(id = 3, nullable = true) public String added = \"default\";\n" + + " public EvolvingStruct() {}\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.EvolvingStruct"); + Object value = writerType.getConstructor().newInstance(); + setField(writerType, value, "id", 42); + setField(writerType, value, "name", "old"); + Fory writer = + Fory.builder() + .withClassLoader(writerLoader) + .withCodegen(false) + .withMetaShare(true) + .withScopedMetaShare(false) + .withCompatible(true) + .requireClassRegistration(false) + .build(); + writer.setMetaWriteContext(new MetaWriteContext()); + byte[] bytes = writer.serialize(value); + + Class readerType = readerLoader.loadClass("test.EvolvingStruct"); + Fory reader = + Fory.builder() + .withClassLoader(readerLoader) + .withCodegen(false) + .withMetaShare(true) + .withScopedMetaShare(false) + .withCompatible(true) + .requireClassRegistration(false) + .build(); + reader.setMetaReadContext(new MetaReadContext()); + Object roundTrip = reader.deserialize(bytes); + Assert.assertSame(roundTrip.getClass(), readerType); + Assert.assertEquals(getField(readerType, roundTrip, "id"), 42); + Assert.assertEquals(getField(readerType, roundTrip, "name"), "old"); + Assert.assertEquals(getField(readerType, roundTrip, "added"), "default"); + Object serializer = reader.getTypeResolver().getTypeInfo(readerType).getSerializer(); + Assert.assertTrue(serializer instanceof StaticGeneratedStructSerializer); + } + } + + @Test + public void testGraalvmCompatibleMetaSharedGeneratorIsReadOnly() throws Exception { + if (AndroidSupport.IS_ANDROID) { + return; + } + CompilationResult writerResult = + compile( + "test.NativeImageStruct", + "package test;\n" + + "public class NativeImageStruct {\n" + + " public int id;\n" + + " public NativeImageStruct() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.NativeImageStruct", + "package test;\n" + + "public class NativeImageStruct {\n" + + " public int id;\n" + + " public String added = \"default\";\n" + + " public NativeImageStruct() {}\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Fory writer = + Fory.builder() + .withClassLoader(writerLoader) + .withCodegen(false) + .withMetaShare(true) + .withScopedMetaShare(false) + .withCompatible(true) + .requireClassRegistration(false) + .build(); + Fory reader = + Fory.builder() + .withClassLoader(readerLoader) + .withCodegen(true) + .withMetaShare(true) + .withScopedMetaShare(false) + .withCompatible(true) + .requireClassRegistration(false) + .build(); + Class writerType = writerLoader.loadClass("test.NativeImageStruct"); + Class readerType = readerLoader.loadClass("test.NativeImageStruct"); + TypeDef remoteTypeDef = TypeDef.buildTypeDef(writer.getTypeResolver(), writerType); + Assert.assertNotEquals( + remoteTypeDef.getId(), + TypeDef.buildTypeDef(reader.getTypeResolver(), readerType).getId()); + Class> serializerClass = + CodecUtils.loadOrGenCompatibleMetaSharedCodecClass( + reader.getTypeResolver(), (Class) readerType, remoteTypeDef); + Assert.assertTrue( + GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(serializerClass)); + Serializer serializer = + Serializers.newSerializer( + reader.getTypeResolver(), (Class) readerType, serializerClass); + Assert.assertThrows( + UnsupportedOperationException.class, + () -> + serializer.write( + reader.getWriteContext(), readerType.getConstructor().newInstance())); + } + } + + private static CompilationResult compile(String typeName, String source) throws IOException { + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + Assert.assertNotNull(compiler, "Tests require a JDK compiler"); + Path root = Files.createTempDirectory("fory-processor-test"); + Path sourceRoot = root.resolve("src"); + Path classRoot = root.resolve("classes"); + Files.createDirectories(sourceRoot); + Files.createDirectories(classRoot); + Path sourceFile = sourceRoot.resolve(typeName.replace('.', '/') + ".java"); + Files.createDirectories(sourceFile.getParent()); + Files.write(sourceFile, source.getBytes(StandardCharsets.UTF_8)); + DiagnosticCollector diagnostics = new DiagnosticCollector<>(); + try (StandardJavaFileManager fileManager = + compiler.getStandardFileManager(diagnostics, null, StandardCharsets.UTF_8)) { + Iterable units = + fileManager.getJavaFileObjectsFromFiles(Collections.singletonList(sourceFile.toFile())); + List options = + Arrays.asList( + "-classpath", + System.getProperty("java.class.path"), + "-d", + classRoot.toString(), + "-s", + root.resolve("generated").toString()); + JavaCompiler.CompilationTask task = + compiler.getTask(null, fileManager, diagnostics, options, null, units); + task.setProcessors(Collections.singletonList(new ForyStructProcessor())); + return new CompilationResult(classRoot, task.call(), diagnostics.getDiagnostics()); + } + } + + private static void setField(Class type, Object target, String name, Object value) + throws Exception { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } + + private static Object getField(Class type, Object target, String name) throws Exception { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } + + private static Object invoke(Class type, Object target, String name) throws Exception { + java.lang.reflect.Method method = type.getDeclaredMethod(name); + method.setAccessible(true); + return method.invoke(target); + } + + private static void invoke( + Class type, Object target, String name, Class parameterType, Object value) + throws Exception { + java.lang.reflect.Method method = type.getDeclaredMethod(name, parameterType); + method.setAccessible(true); + method.invoke(target, value); + } + + private static Descriptor descriptor(List descriptors, String name) { + for (Descriptor descriptor : descriptors) { + if (descriptor.getName().equals(name)) { + return descriptor; + } + } + throw new AssertionError("Missing descriptor " + name); + } + + private static final class CompilationResult { + final Path classRoot; + final boolean success; + final List> diagnostics; + + CompilationResult( + Path classRoot, boolean success, List> diagnostics) { + this.classRoot = classRoot; + this.success = success; + this.diagnostics = new ArrayList<>(diagnostics); + } + + URLClassLoader classLoader() throws IOException { + URL[] urls = {classRoot.toUri().toURL()}; + return new URLClassLoader(urls, ForyStructProcessorTest.class.getClassLoader()); + } + + String diagnostics() { + StringBuilder builder = new StringBuilder(); + for (Diagnostic diagnostic : diagnostics) { + builder.append(diagnostic).append('\n'); + } + return builder.toString(); + } + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index 68a0c64d18..76225b6fab 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -73,6 +73,25 @@ public static Class> loadOrGenMetaSharedCodecClass( .asyncVisitFory(f -> loadOrGenMetaSharedCodecClass(f, cls, typeDef)); } + public static Class> loadOrGenCompatibleMetaSharedCodecClass( + Fory fory, Class cls, TypeDef typeDef) { + Preconditions.checkNotNull(fory); + return loadSerializer( + "loadOrGenCompatibleMetaSharedCodecClass", + cls, + fory, + () -> + loadOrGenCodecClass( + cls, fory, new CompatibleMetaSharedCodecBuilder(TypeRef.of(cls), fory, typeDef))); + } + + public static Class> loadOrGenCompatibleMetaSharedCodecClass( + TypeResolver typeResolver, Class cls, TypeDef typeDef) { + return typeResolver + .getJITContext() + .asyncVisitFory(f -> loadOrGenCompatibleMetaSharedCodecClass(f, cls, typeDef)); + } + /** * Load or generate a JIT serializer class for single-layer meta-shared serialization. * diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleMetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleMetaSharedCodecBuilder.java new file mode 100644 index 0000000000..4884b423be --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleMetaSharedCodecBuilder.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.builder; + +import org.apache.fory.Fory; +import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.meta.TypeDef; +import org.apache.fory.reflect.TypeRef; + +/** Builds GraalVM read-only compatible serializers for remote {@link TypeDef} schemas. */ +public final class CompatibleMetaSharedCodecBuilder extends MetaSharedCodecBuilder { + public CompatibleMetaSharedCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { + super( + beanType, + fory, + typeDef, + GeneratedCompatibleMetaSharedSerializer.class, + false, + "CompatibleMetaShared"); + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java index c79e4c57aa..e487ccfe71 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import org.apache.fory.context.CopyContext; import org.apache.fory.context.WriteContext; import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.ReflectionUtils; @@ -117,6 +118,29 @@ public void write(WriteContext writeContext, Object value) { } } + /** + * Base class for GraalVM build-time compatible read serializers for remote {@link TypeDef} + * schemas. + */ + abstract class GeneratedCompatibleMetaSharedSerializer extends GeneratedSerializer + implements Generated { + public GeneratedCompatibleMetaSharedSerializer(TypeResolver typeResolver, Class cls) { + super(typeResolver, cls); + } + + @Override + public void write(WriteContext writeContext, Object value) { + throw new UnsupportedOperationException( + "GraalVM compatible meta-shared serializers are read-only"); + } + + @Override + public Object copy(CopyContext copyContext, Object value) { + throw new UnsupportedOperationException( + "GraalVM compatible meta-shared serializers do not implement copy"); + } + } + /** * Base class for layer serializers with meta shared by {@link TypeDef}. Unlike {@link * GeneratedMetaSharedSerializer}, this serializer only handles fields from a single class layer diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index 5ece041dc7..a657b3af84 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -30,6 +30,7 @@ import java.util.SortedMap; import java.util.concurrent.ConcurrentHashMap; import org.apache.fory.Fory; +import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; import org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; @@ -81,13 +82,27 @@ public class MetaSharedCodecBuilder extends ObjectCodecBuilder { private final TypeDef typeDef; private final String defaultValueLanguage; private final DefaultValueUtils.DefaultValueField[] defaultValueFields; + private final boolean writeDelegate; + private final String codecKind; public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { - super(beanType, fory, GeneratedMetaSharedSerializer.class); + this(beanType, fory, typeDef, GeneratedMetaSharedSerializer.class, true, "MetaShared"); + } + + protected MetaSharedCodecBuilder( + TypeRef beanType, + Fory fory, + TypeDef typeDef, + Class parentSerializerClass, + boolean writeDelegate, + String codecKind) { + super(beanType, fory, parentSerializerClass); Preconditions.checkArgument( !fory.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); this.typeDef = typeDef; + this.writeDelegate = writeDelegate; + this.codecKind = codecKind; DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(typeDef, beanClass)); List sortedDescriptors = grouper.getSortedDescriptors(); if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { @@ -146,7 +161,7 @@ protected String codecSuffix() { id = idGenerator.computeIfAbsent(typeDef.getId(), k -> idGenerator.size()); } } - return "MetaShared" + id; + return codecKind + id; } @Override @@ -162,8 +177,7 @@ public String genCode() { StringUtils.format( "" + "super(${typeResolver}, ${cls});\n" - + "this.${generatedTypeResolver} = (${generatedTypeResolverType}) ${typeResolver};\n" - + "${serializer} = ${builderClass}.setCodegenSerializer(${typeResolver}, ${cls}, this);\n", + + "this.${generatedTypeResolver} = (${generatedTypeResolverType}) ${typeResolver};\n", "typeResolver", CONSTRUCTOR_TYPE_RESOLVER_NAME, "generatedTypeResolver", @@ -171,11 +185,20 @@ public String genCode() { "generatedTypeResolverType", ctx.type(concreteTypeResolverType), "cls", - POJO_CLASS_TYPE_NAME, - "builderClass", - MetaSharedCodecBuilder.class.getName(), - "serializer", - SERIALIZER_FIELD_NAME); + POJO_CLASS_TYPE_NAME); + if (writeDelegate) { + constructorCode += + StringUtils.format( + "${serializer} = ${builderClass}.setCodegenSerializer(${typeResolver}, ${cls}, this);\n", + "serializer", + SERIALIZER_FIELD_NAME, + "builderClass", + MetaSharedCodecBuilder.class.getName(), + "typeResolver", + CONSTRUCTOR_TYPE_RESOLVER_NAME, + "cls", + POJO_CLASS_TYPE_NAME); + } ctx.clearExprState(); Expression decodeExpr = buildDecodeExpression(); String decodeCode = decodeExpr.genCode(ctx).code(); @@ -208,6 +231,7 @@ public String genCode() { protected void addCommonImports() { super.addCommonImports(); ctx.addImport(GeneratedMetaSharedSerializer.class); + ctx.addImport(GeneratedCompatibleMetaSharedSerializer.class); } // Invoked by JIT. diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index ddd23631db..0894ec311b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -32,8 +32,6 @@ import java.lang.reflect.Array; import java.lang.reflect.Field; import java.util.Objects; -import org.apache.fory.annotation.ArrayType; -import org.apache.fory.annotation.ForyField; import org.apache.fory.collection.BFloat16List; import org.apache.fory.collection.BoolList; import org.apache.fory.collection.Float16List; @@ -77,8 +75,8 @@ static boolean useFieldType(Class parsedType, Descriptor descriptor) { } if (parsedType.isArray()) { Tuple2, Integer> info = getArrayComponentInfo(parsedType); - Field field = descriptor.getField(); - if (!field.getType().isArray() || getArrayDimensions(field.getType()) != info.f1) { + Class rawType = descriptor.getRawType(); + if (!rawType.isArray() || getArrayDimensions(rawType) != info.f1) { return false; } return info.f0.isEnum(); @@ -91,26 +89,34 @@ public static FieldType buildFieldType(TypeResolver resolver, Field field) { Preconditions.checkNotNull(field); TypeRef typeRef = TypeUtils.getFieldTypeRef(field); GenericType genericType = resolver.buildGenericType(typeRef); - return buildFieldType(resolver, field, genericType); + return buildFieldType(resolver, new Descriptor(field, typeRef, null, null), genericType); + } + + /** Build field type from a descriptor, including generated descriptor TypeRef metadata. */ + public static FieldType buildFieldType(TypeResolver resolver, Descriptor descriptor) { + Preconditions.checkNotNull(descriptor); + GenericType genericType = resolver.buildGenericType(descriptor.getTypeRef()); + return buildFieldType(resolver, descriptor, genericType); } /** Build field type from generics, nested generics will be extracted too. */ private static FieldType buildFieldType( - TypeResolver resolver, Field field, GenericType genericType) { + TypeResolver resolver, Descriptor descriptor, GenericType genericType) { Preconditions.checkNotNull(genericType); + Field field = descriptor == null ? null : descriptor.getField(); Class rawType = genericType.getCls(); boolean isXlang = resolver.isCrossLanguage(); // Get type ID for both xlang and native mode // This supports unsigned types and field-configurable compression in both modes int typeId; - Annotation typeAnnotation = field == null ? null : Descriptor.getAnnotation(field); + Annotation typeAnnotation = descriptor == null ? null : descriptor.getTypeAnnotation(); boolean primitiveList = TypeUtils.isPrimitiveListClass(rawType); - boolean primitiveListArray = field != null && field.isAnnotationPresent(ArrayType.class); + boolean primitiveListArray = descriptor != null && TypeAnnotationUtils.isArrayType(descriptor); boolean boxedListArray = isXlang - && field != null + && descriptor != null && !primitiveList - && TypeAnnotationUtils.isBoxedListArrayType(field); + && TypeAnnotationUtils.isBoxedListArrayType(descriptor); int primitiveListElementTypeId = primitiveList ? TypeAnnotationUtils.getPrimitiveListElementTypeId(typeAnnotation, rawType, isXlang) @@ -124,21 +130,26 @@ private static FieldType buildFieldType( ? TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType) : Types.LIST; } else if (boxedListArray) { - typeId = TypeAnnotationUtils.getBoxedListArrayTypeId(field); + typeId = TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor); } else if (primitiveListElementTypeId != Types.UNKNOWN) { typeId = TypeAnnotationUtils.getPrimitiveListTypeId(typeAnnotation, rawType); } else if (TypeUtils.unwrap(rawType).isPrimitive()) { if (field != null) { - typeId = Types.getDescriptorTypeId(resolver, field); + typeId = + descriptor == null + ? Types.getDescriptorTypeId(resolver, field) + : Types.getDescriptorTypeId(resolver, descriptor); } else { typeId = Types.getTypeId(resolver, rawType); } - } else if (rawType.isArray() && rawType.getComponentType().isPrimitive() && field != null) { + } else if (rawType.isArray() + && rawType.getComponentType().isPrimitive() + && descriptor != null) { // For primitive arrays with type annotations, use getDescriptorTypeId to parse annotation. // This allows @UInt8Type etc. to override the default byte[] bytes schema. - typeId = Types.getDescriptorTypeId(resolver, field); - } else if (typeAnnotation != null && rawType.isArray() && field != null) { - typeId = Types.getDescriptorTypeId(resolver, field); + typeId = Types.getDescriptorTypeId(resolver, descriptor); + } else if (typeAnnotation != null && rawType.isArray() && descriptor != null) { + typeId = Types.getDescriptorTypeId(resolver, descriptor); } else { TypeInfo info = isXlang && rawType == Object.class ? null : resolver.getTypeInfo(rawType, false); @@ -195,12 +206,9 @@ private static FieldType buildFieldType( } // Apply @ForyField annotation if present - if (field != null) { - ForyField foryField = field.getAnnotation(ForyField.class); - if (foryField != null) { - nullable = foryField.nullable(); - trackingRef = foryField.ref(); - } + if (descriptor != null && descriptor.hasForyFieldPolicy()) { + nullable = descriptor.isNullable(); + trackingRef = descriptor.isTrackingRef(); } boolean isUnionType = Types.isUnionType(typeId); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java index 4ef3979d9b..66e674a775 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java @@ -34,7 +34,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; import org.apache.fory.collection.Tuple2; import org.apache.fory.memory.MemoryBuffer; @@ -45,6 +44,7 @@ import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; +import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; import org.apache.fory.util.MurmurHash3; @@ -72,67 +72,65 @@ private static boolean needsUserTypeId(int typeId) { } } - static List buildFields(TypeResolver typeResolver, Class cls, boolean resolveParent) { + static List buildDescriptors( + TypeResolver typeResolver, Class cls, boolean resolveParent) { DescriptorGrouper descriptorGrouper = typeResolver.getFieldDescriptorGrouper(cls, resolveParent, false, IDENTITY_DESCRIPTOR); - List fields = new ArrayList<>(); - descriptorGrouper - .getPrimitiveDescriptors() - .forEach(descriptor -> fields.add(descriptor.getField())); - descriptorGrouper - .getBoxedDescriptors() - .forEach(descriptor -> fields.add(descriptor.getField())); - descriptorGrouper - .getBuildInDescriptors() - .forEach(descriptor -> fields.add(descriptor.getField())); + List descriptors = new ArrayList<>(); + descriptorGrouper.getPrimitiveDescriptors().forEach(descriptors::add); + descriptorGrouper.getBoxedDescriptors().forEach(descriptors::add); + descriptorGrouper.getBuildInDescriptors().forEach(descriptors::add); // Order must match ObjectSerializer serialization order: buildIn, container, other - descriptorGrouper - .getCollectionDescriptors() - .forEach(descriptor -> fields.add(descriptor.getField())); - descriptorGrouper.getMapDescriptors().forEach(descriptor -> fields.add(descriptor.getField())); - descriptorGrouper - .getOtherDescriptors() - .forEach(descriptor -> fields.add(descriptor.getField())); - return fields; + descriptorGrouper.getCollectionDescriptors().forEach(descriptors::add); + descriptorGrouper.getMapDescriptors().forEach(descriptors::add); + descriptorGrouper.getOtherDescriptors().forEach(descriptors::add); + return descriptors; } public static List buildFieldsInfo(ClassResolver resolver, Class cls) { - return buildFieldsInfo(resolver, buildFields(resolver, cls, true)); + return buildFieldsInfoFromDescriptors(resolver, buildDescriptors(resolver, cls, true)); } public static List buildFieldsInfo(TypeResolver resolver, List fields) { + List descriptors = new ArrayList<>(fields.size()); + for (Field field : fields) { + descriptors.add(new Descriptor(field, TypeUtils.getFieldTypeRef(field), null, null)); + } + return buildFieldsInfoFromDescriptors(resolver, descriptors); + } + + public static List buildFieldsInfoFromDescriptors( + TypeResolver resolver, List descriptors) { List fieldInfos = new ArrayList<>(); Set usedTagIds = new HashSet<>(); - for (Field field : fields) { - // Check for @ForyField annotation to extract tag ID - ForyField foryField = field.getAnnotation(ForyField.class); - FieldType fieldType = FieldTypes.buildFieldType(resolver, field); + for (Descriptor descriptor : descriptors) { + FieldType fieldType = FieldTypes.buildFieldType(resolver, descriptor); FieldInfo fieldInfo; - if (foryField != null) { - int tagId = foryField.id(); + if (descriptor.hasForyFieldPolicy()) { + int tagId = descriptor.getForyFieldId(); if (tagId >= 0) { if (!usedTagIds.add(tagId)) { throw new IllegalArgumentException( "Duplicate tag id: " + tagId + ", field: " - + field + + descriptor.getName() + ", class: " - + field.getDeclaringClass()); + + descriptor.getDeclaringClass()); } // Create FieldInfo with tag ID for optimized serialization fieldInfo = new FieldInfo( - field.getDeclaringClass().getName(), field.getName(), fieldType, (short) tagId); + descriptor.getDeclaringClass(), descriptor.getName(), fieldType, (short) tagId); } else { // tagId == -1 means opt-out, use field name fieldInfo = - new FieldInfo(field.getDeclaringClass().getName(), field.getName(), fieldType); + new FieldInfo(descriptor.getDeclaringClass(), descriptor.getName(), fieldType); } } else { // No annotation, use field name - fieldInfo = new FieldInfo(field.getDeclaringClass().getName(), field.getName(), fieldType); + fieldInfo = new FieldInfo(descriptor.getDeclaringClass(), descriptor.getName(), fieldType); } fieldInfos.add(fieldInfo); } @@ -144,6 +142,13 @@ static TypeDef buildTypeDef(ClassResolver classResolver, Class type, List type, List descriptors) { + return buildTypeDefWithFieldInfos( + classResolver, type, buildFieldsInfoFromDescriptors(classResolver, descriptors)); + } + public static TypeDef buildTypeDefWithFieldInfos( ClassResolver classResolver, Class type, List fieldInfos) { boolean hasFieldMetadata = !fieldInfos.isEmpty(); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java index 2747c9318b..1602ffe42c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java @@ -19,7 +19,7 @@ package org.apache.fory.meta; -import static org.apache.fory.meta.NativeTypeDefEncoder.buildFields; +import static org.apache.fory.meta.NativeTypeDefEncoder.buildDescriptors; import java.io.ObjectStreamClass; import java.io.Serializable; @@ -371,8 +371,18 @@ public List getDescriptors(TypeResolver resolver, Class cls) { this, cls, () -> buildDescriptors(resolver, cls)); } + public List getDescriptors( + TypeResolver resolver, Class cls, Collection localDescriptors) { + return buildDescriptors(resolver, cls, localDescriptors); + } + private List buildDescriptors(TypeResolver resolver, Class cls) { Collection fieldDescriptors = resolver.getFieldDescriptors(cls, true); + return buildDescriptors(resolver, cls, fieldDescriptors); + } + + private List buildDescriptors( + TypeResolver resolver, Class cls, Collection fieldDescriptors) { Map descriptorsMap = new HashMap<>(); Map fieldIdToDescriptorMap = new HashMap<>(); @@ -381,20 +391,18 @@ private List buildDescriptors(TypeResolver resolver, Class cls) { if (descriptorsMap.put(fullName, descriptor) != null) { throw new IllegalStateException("Duplicate key"); } - if (descriptor.getForyField() != null) { - int fieldId = descriptor.getForyField().id(); - if (fieldId >= 0) { - if (fieldIdToDescriptorMap.containsKey((short) fieldId)) { - throw new IllegalArgumentException( - "Duplicate field id " - + fieldId - + " for field " - + descriptor.getName() - + " in class " - + cls.getName()); - } - fieldIdToDescriptorMap.put((short) fieldId, descriptor); + if (descriptor.hasForyFieldId()) { + int fieldId = descriptor.getForyFieldId(); + if (fieldIdToDescriptorMap.containsKey((short) fieldId)) { + throw new IllegalArgumentException( + "Duplicate field id " + + fieldId + + " for field " + + descriptor.getName() + + " in class " + + cls.getName()); } + fieldIdToDescriptorMap.put((short) fieldId, descriptor); } } List descriptors = new ArrayList<>(fieldsInfo.size()); @@ -426,8 +434,10 @@ public static TypeDef buildTypeDef(TypeResolver resolver, Class cls, boolean if (resolver.isCrossLanguage()) { return TypeDefEncoder.buildTypeDef((XtypeResolver) resolver, cls); } - return NativeTypeDefEncoder.buildTypeDef( - (ClassResolver) resolver, cls, buildFields(resolver, cls, resolveParent)); + return NativeTypeDefEncoder.buildTypeDefFromDescriptors( + (ClassResolver) resolver, + cls, + NativeTypeDefEncoder.buildDescriptors(resolver, cls, resolveParent)); } /** Build class definition from fields of class. */ diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java index 4bed7c13d6..22a3a156d8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java @@ -32,7 +32,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.fory.annotation.ForyField; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; @@ -43,6 +42,7 @@ import org.apache.fory.resolver.XtypeResolver; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; +import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; @@ -63,43 +63,53 @@ static TypeDef buildTypeDef(XtypeResolver resolver, Class type) { DescriptorGrouper descriptorGrouper = resolver.getFieldDescriptorGrouper(type, true, false, IDENTITY_DESCRIPTOR); TypeInfo typeInfo = resolver.getTypeInfo(type); - List fields; + List descriptors; int typeId = typeInfo.getTypeId(); if (Types.isStructType(typeId)) { - fields = - descriptorGrouper.getSortedDescriptors().stream() - .map(Descriptor::getField) - .collect(Collectors.toList()); + descriptors = descriptorGrouper.getSortedDescriptors(); } else { - fields = new ArrayList<>(); + descriptors = new ArrayList<>(); } - return buildTypeDefWithFieldInfos(resolver, type, buildFieldsInfo(resolver, type, fields)); + return buildTypeDefWithFieldInfos( + resolver, type, buildFieldsInfoFromDescriptors(resolver, type, descriptors)); } static List buildFieldsInfo(TypeResolver resolver, Class type, List fields) { + List descriptors = new ArrayList<>(fields.size()); + for (Field field : fields) { + descriptors.add(new Descriptor(field, TypeUtils.getFieldTypeRef(field), null, null)); + } + return buildFieldsInfoFromDescriptors(resolver, type, descriptors); + } + + static List buildFieldsInfoFromDescriptors( + TypeResolver resolver, Class type, List descriptors) { Set usedTagIds = new HashSet<>(); - return fields.stream() + return descriptors.stream() .map( - field -> { - ForyField foryField = field.getAnnotation(ForyField.class); - FieldType fieldType = FieldTypes.buildFieldType(resolver, field); - if (foryField != null) { - int tagId = foryField.id(); + descriptor -> { + FieldType fieldType = FieldTypes.buildFieldType(resolver, descriptor); + if (descriptor.hasForyFieldPolicy()) { + int tagId = descriptor.getForyFieldId(); if (tagId >= 0) { if (!usedTagIds.add(tagId)) { throw new IllegalArgumentException( "Duplicate tag id " + tagId + " for field " - + field.getName() + + descriptor.getName() + " in class " + type.getName()); } - return new FieldInfo(type.getName(), field.getName(), fieldType, (short) tagId); + return new FieldInfo( + descriptor.getDeclaringClass(), + descriptor.getName(), + fieldType, + (short) tagId); } // tagId == -1 means use field name, fall through to create regular FieldInfo } - return new FieldInfo(type.getName(), field.getName(), fieldType); + return new FieldInfo(descriptor.getDeclaringClass(), descriptor.getName(), fieldType); }) .collect(Collectors.toList()); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 9d2276e523..fdfc15bc27 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -70,7 +70,6 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.fory.ForyCopyable; import org.apache.fory.annotation.CodegenInvoke; -import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.CodecUtils; import org.apache.fory.builder.JITContext; @@ -977,9 +976,8 @@ public int getUserTypeIdForTypeDef(Class cls) { @Override public boolean isMonomorphic(Descriptor descriptor) { - ForyField foryField = descriptor.getForyField(); - if (foryField != null) { - switch (foryField.dynamic()) { + if (descriptor.hasForyFieldPolicy()) { + switch (descriptor.getMorphic()) { case TRUE: return false; case FALSE: @@ -1546,8 +1544,8 @@ public Class getObjectSerializerClass( if (codegen) { LOG.info("Object of type {} can't be serialized by jit", cls); } - // Always use ObjectSerializer for both modes - return ObjectSerializer.class; + Class serializerClass = getStaticGeneratedStructSerializerClass(cls); + return serializerClass == null ? ObjectSerializer.class : serializerClass; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index ad24d2d734..dcd639a7aa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -22,6 +22,7 @@ import static org.apache.fory.type.Types.INVALID_USER_TYPE_ID; import java.lang.reflect.AnnotatedType; +import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Member; import java.lang.reflect.Type; @@ -44,6 +45,7 @@ import org.apache.fory.annotation.ForyStruct; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.CodecUtils; +import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; import org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer; import org.apache.fory.builder.Generated.GeneratedObjectSerializer; import org.apache.fory.builder.JITContext; @@ -84,6 +86,7 @@ import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.SerializerFactory; import org.apache.fory.serializer.Serializers; +import org.apache.fory.serializer.StaticGeneratedStructSerializer; import org.apache.fory.serializer.UnknownClass; import org.apache.fory.serializer.UnknownClass.UnknownEmptyStruct; import org.apache.fory.serializer.UnknownClass.UnknownStruct; @@ -1028,29 +1031,39 @@ private TypeInfo getMetaSharedTypeInfo(TypeDef typeDef, Class clz) { Class sc = getMetaSharedDeserializerClassFromGraalvmRegistry(cls, typeDef); if (sc == null) { - if (AndroidSupport.IS_ANDROID) { + if (GraalvmSupport.isGraalBuildTime() && config.isCodeGenEnabled()) { + sc = loadGraalvmMetaSharedDeserializerClass(cls, typeDef); + } else if (AndroidSupport.IS_ANDROID || !config.isCodeGenEnabled()) { + sc = getStaticGeneratedStructSerializerClass(cls); + } + if (sc == null && AndroidSupport.IS_ANDROID) { sc = MetaSharedSerializer.class; - } else if (GraalvmSupport.isGraalRuntime()) { + } else if (sc == null && GraalvmSupport.isGraalRuntime()) { sc = MetaSharedSerializer.class; LOG.warn( "Can't generate class at runtime in graalvm for class def {}, use {} instead", typeDef, sc); - } else { + } else if (sc == null && config.isCodeGenEnabled()) { sc = jitContext.registerSerializerJITCallback( () -> MetaSharedSerializer.class, () -> CodecUtils.loadOrGenMetaSharedCodecClass(this, cls, typeDef), c -> typeInfo.setSerializer(this, Serializers.newSerializer(this, cls, c))); + } else if (sc == null) { + sc = MetaSharedSerializer.class; } } if (GraalvmSupport.isGraalBuildTime() - && GeneratedMetaSharedSerializer.class.isAssignableFrom(sc)) { + && (GeneratedMetaSharedSerializer.class.isAssignableFrom(sc) + || GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(sc))) { getGraalvmClassRegistry().putIfAbsentDeserializerClass(typeDef.getId(), sc); typeInfo.setSerializer(this, new MetaSharedSerializer(this, cls, typeDef)); return typeInfo; } - if (sc == MetaSharedSerializer.class) { + if (StaticGeneratedStructSerializer.class.isAssignableFrom(sc)) { + typeInfo.setSerializer(this, newStaticGeneratedStructSerializer(sc, cls, typeDef)); + } else if (sc == MetaSharedSerializer.class) { typeInfo.setSerializer(this, new MetaSharedSerializer(this, cls, typeDef)); } else { typeInfo.setSerializer(this, Serializers.newSerializer(this, cls, sc)); @@ -1058,6 +1071,14 @@ private TypeInfo getMetaSharedTypeInfo(TypeDef typeDef, Class clz) { return typeInfo; } + private Class loadGraalvmMetaSharedDeserializerClass( + Class cls, TypeDef typeDef) { + if (typeDef.getId() == TypeDef.buildTypeDef(this, cls).getId()) { + return CodecUtils.loadOrGenMetaSharedCodecClass(this, cls, typeDef); + } + return CodecUtils.loadOrGenCompatibleMetaSharedCodecClass(this, cls, typeDef); + } + protected int buildUnregisteredTypeId(Class cls, Serializer serializer) { if (cls.isEnum()) { return Types.NAMED_ENUM; @@ -1084,7 +1105,8 @@ protected static boolean isStructSerializer(Serializer serializer) { || serializer instanceof GeneratedMetaSharedSerializer || serializer instanceof LazyInitBeanSerializer || serializer instanceof ObjectSerializer - || serializer instanceof MetaSharedSerializer; + || serializer instanceof MetaSharedSerializer + || serializer instanceof StaticGeneratedStructSerializer; } protected static boolean isStructSerializerClass(Class serializerClass) { @@ -1092,7 +1114,8 @@ protected static boolean isStructSerializerClass(Class ser || GeneratedMetaSharedSerializer.class.isAssignableFrom(serializerClass) || LazyInitBeanSerializer.class.isAssignableFrom(serializerClass) || ObjectSerializer.class.isAssignableFrom(serializerClass) - || MetaSharedSerializer.class.isAssignableFrom(serializerClass); + || MetaSharedSerializer.class.isAssignableFrom(serializerClass) + || StaticGeneratedStructSerializer.class.isAssignableFrom(serializerClass); } protected TypeDef readTypeDef(MemoryBuffer buffer, long header) { @@ -1311,8 +1334,8 @@ public Class getObjectSerializerClass( } } } else { - // Always use ObjectSerializer for both modes - return ObjectSerializer.class; + Class serializerClass = getStaticGeneratedStructSerializerClass(cls); + return serializerClass == null ? ObjectSerializer.class : serializerClass; } } @@ -1427,19 +1450,34 @@ private DescriptorGrouper buildDescriptorGrouper( } private List buildFieldDescriptors(Class clz, boolean searchParent) { + List staticDescriptors = getStaticGeneratedStructDescriptors(clz); + if (staticDescriptors != null) { + return buildFieldDescriptors(clz, searchParent, staticDescriptors); + } SortedMap allDescriptors = getAllDescriptorsMap(clz, searchParent); List result = new ArrayList<>(allDescriptors.size()); + List descriptors = new ArrayList<>(allDescriptors.size()); + for (Map.Entry entry : allDescriptors.entrySet()) { + Member member = entry.getKey(); + if (member instanceof Field) { + descriptors.add(entry.getValue()); + } + } + return buildFieldDescriptors(clz, searchParent, descriptors); + } + + private List buildFieldDescriptors( + Class clz, boolean searchParent, List descriptors) { + List result = new ArrayList<>(descriptors.size()); boolean globalRefTracking = trackingRef(); boolean isXlang = isCrossLanguage(); - for (Map.Entry entry : allDescriptors.entrySet()) { - Member member = entry.getKey(); - Descriptor descriptor = entry.getValue(); - if (!(member instanceof Field)) { + for (Descriptor descriptor : descriptors) { + if (!searchParent && !descriptor.getDeclaringClass().equals(clz.getName())) { continue; } - boolean hasForyField = descriptor.getForyField() != null; + boolean hasForyField = descriptor.hasForyFieldPolicy(); // Compute the final isTrackingRef value: // For xlang mode: "Reference tracking is disabled by default" (xlang spec) // - Only enable ref tracking if explicitly set via @ForyField(ref=true) @@ -1474,6 +1512,84 @@ private List buildFieldDescriptors(Class clz, boolean searchParen return result; } + protected final Class getStaticGeneratedStructSerializerClass( + Class cls) { + if (GraalvmSupport.isGraalBuildTime() || GraalvmSupport.isGraalRuntime()) { + return null; + } + if (!cls.isAnnotationPresent(ForyStruct.class)) { + return null; + } + String generatedName = cls.getName() + "__ForyStaticSerializer__"; + Class serializerClass = loadStaticGeneratedStructSerializerClass(cls, generatedName); + if (serializerClass == null) { + return null; + } + if (!StaticGeneratedStructSerializer.class.isAssignableFrom(serializerClass)) { + throw new ForyException( + "Generated static serializer " + + generatedName + + " for " + + cls.getName() + + " does not extend " + + StaticGeneratedStructSerializer.class.getName()); + } + return (Class) serializerClass.asSubclass(Serializer.class); + } + + private Class loadStaticGeneratedStructSerializerClass(Class cls, String generatedName) { + ClassLoader classLoader = cls.getClassLoader(); + Class serializerClass = loadStaticGeneratedStructSerializerClass(generatedName, classLoader); + if (serializerClass != null || classLoader == extRegistry.classLoader) { + return serializerClass; + } + return loadStaticGeneratedStructSerializerClass(generatedName, extRegistry.classLoader); + } + + private Class loadStaticGeneratedStructSerializerClass( + String generatedName, ClassLoader classLoader) { + try { + return Class.forName(generatedName, false, classLoader); + } catch (ClassNotFoundException e) { + return null; + } catch (LinkageError e) { + throw new ForyException("Failed to load generated static serializer " + generatedName, e); + } + } + + protected final StaticGeneratedStructSerializer newStaticGeneratedStructSerializer( + Class serializerClass, Class cls, TypeDef typeDef) { + try { + Constructor constructor = + serializerClass.getDeclaredConstructor(TypeResolver.class, Class.class, TypeDef.class); + constructor.setAccessible(true); + return (StaticGeneratedStructSerializer) constructor.newInstance(this, cls, typeDef); + } catch (NoSuchMethodException e) { + throw new ForyException( + "Generated static serializer " + + serializerClass.getName() + + " must define constructor (TypeResolver, Class, TypeDef)", + e); + } catch (ReflectiveOperationException e) { + throw new ForyException( + "Failed to create generated static serializer " + + serializerClass.getName() + + " for " + + cls.getName(), + e); + } + } + + private List getStaticGeneratedStructDescriptors(Class cls) { + Class serializerClass = getStaticGeneratedStructSerializerClass(cls); + if (serializerClass == null) { + return null; + } + StaticGeneratedStructSerializer serializer = + (StaticGeneratedStructSerializer) Serializers.newSerializer(this, cls, serializerClass); + return serializer.getDescriptors(); + } + /** * Gets the sort key for a field descriptor. * @@ -1485,9 +1601,8 @@ private List buildFieldDescriptors(Class clz, boolean searchParen * @return the sort key (tag ID as string or snake_case name) */ protected static String getFieldSortKey(Descriptor descriptor) { - ForyField foryField = descriptor.getForyField(); - if (foryField != null && foryField.id() >= 0) { - return String.valueOf(foryField.id()); + if (descriptor.hasForyFieldId()) { + return String.valueOf(descriptor.getForyFieldId()); } String name = descriptor.getName(); if (name != null && name.startsWith("$tag")) { @@ -1525,9 +1640,8 @@ protected static boolean hasFieldSortId(Descriptor descriptor) { } private static Integer getFieldSortId(Descriptor descriptor) { - ForyField foryField = descriptor.getForyField(); - if (foryField != null && foryField.id() >= 0) { - return foryField.id(); + if (descriptor.hasForyFieldId()) { + return descriptor.getForyFieldId(); } String name = descriptor.getName(); if (name != null && name.startsWith("$tag")) { @@ -1621,10 +1735,9 @@ private boolean isFieldNullable(Descriptor descriptor) { if (isCrossLanguage()) { // For xlang mode: apply xlang defaults // This must match what TypeDefEncoder.buildFieldType uses for TypeDef metadata - ForyField foryField = descriptor.getForyField(); - if (foryField != null) { + if (descriptor.hasForyFieldPolicy()) { // Use explicit annotation value - return foryField.nullable(); + return descriptor.isNullable(); } // Default for xlang: false for all non-primitives, except Optional types return TypeUtils.isOptionalType(rawType); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 9f3b76fc5f..766412c00b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -611,8 +611,7 @@ public boolean isRegisteredByName(Class cls) { @Override public boolean isMonomorphic(Descriptor descriptor) { - ForyField foryField = descriptor.getForyField(); - ForyField.Dynamic dynamic = foryField != null ? foryField.dynamic() : ForyField.Dynamic.AUTO; + ForyField.Dynamic dynamic = descriptor.getMorphic(); switch (dynamic) { case TRUE: return false; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 8c0f3ced76..a662af9f9f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -152,6 +152,10 @@ public static FieldGroups buildFieldInfos(TypeResolver typeResolver, DescriptorG return new FieldGroups(allBuildIn, containerFields, otherFields); } + static SerializationFieldInfo buildFieldInfo(TypeResolver typeResolver, Descriptor descriptor) { + return new SerializationFieldInfo(typeResolver, descriptor); + } + public static final class SerializationFieldInfo { public final Descriptor descriptor; public final Class type; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java new file mode 100644 index 0000000000..6892cb8b84 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.fory.annotation.Internal; +import org.apache.fory.context.CopyContext; +import org.apache.fory.context.ReadContext; +import org.apache.fory.context.RefReader; +import org.apache.fory.context.WriteContext; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.meta.FieldInfo; +import org.apache.fory.meta.TypeDef; +import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; +import org.apache.fory.type.Descriptor; +import org.apache.fory.type.DescriptorGrouper; +import org.apache.fory.util.StringUtils; + +/** Base class used by javac-generated {@code @ForyStruct} serializers. */ +@Internal +public abstract class StaticGeneratedStructSerializer extends AbstractObjectSerializer { + protected static final int UNKNOWN_FIELD = -1; + + protected final TypeDef typeDef; + protected final List remoteFields; + + @SuppressWarnings("unchecked") + public StaticGeneratedStructSerializer(TypeResolver typeResolver, Class type) { + super(typeResolver, (Class) type); + this.typeDef = null; + this.remoteFields = Collections.emptyList(); + } + + @SuppressWarnings("unchecked") + protected StaticGeneratedStructSerializer( + TypeResolver typeResolver, Class type, TypeDef typeDef, List descriptors) { + super(typeResolver, (Class) type); + this.typeDef = typeDef; + this.remoteFields = + typeDef == null ? Collections.emptyList() : buildRemoteFields(typeDef, descriptors); + } + + @Override + public abstract void write(WriteContext writeContext, T value); + + @Override + public abstract T read(ReadContext readContext); + + @Override + public abstract T copy(CopyContext copyContext, T value); + + public abstract List getDescriptors(); + + public abstract T readCompatible(ReadContext readContext); + + protected final FieldGroups buildFieldGroups(List descriptors) { + DescriptorGrouper grouper = + FieldGroups.buildDescriptorGrouper( + typeResolver, descriptors, false, descriptor -> descriptor); + return FieldGroups.buildFieldInfos(typeResolver, grouper); + } + + protected final int[] localFieldIds( + SerializationFieldInfo[] fieldInfos, List descriptors) { + Map localIds = new HashMap<>(); + for (int i = 0; i < descriptors.size(); i++) { + Descriptor descriptor = descriptors.get(i); + localIds.put(fieldKey(descriptor), i); + } + int[] ids = new int[fieldInfos.length]; + for (int i = 0; i < fieldInfos.length; i++) { + Integer id = localIds.get(fieldKey(fieldInfos[i].descriptor)); + if (id == null) { + throw new IllegalStateException( + "Generated descriptor is not part of local descriptor list: " + + fieldInfos[i].descriptor); + } + ids[i] = id; + } + return ids; + } + + protected final void writeBuildInFieldValue( + WriteContext writeContext, SerializationFieldInfo fieldInfo, Object fieldValue) { + AbstractObjectSerializer.writeBuildInFieldValue( + writeContext, + typeResolver, + writeContext.getRefWriter(), + fieldInfo, + writeContext.getBuffer(), + fieldValue); + } + + protected final void writeContainerFieldValue( + WriteContext writeContext, SerializationFieldInfo fieldInfo, Object fieldValue) { + AbstractObjectSerializer.writeContainerFieldValue( + writeContext, + typeResolver, + writeContext.getRefWriter(), + writeContext.getGenerics(), + fieldInfo, + writeContext.getBuffer(), + fieldValue); + } + + protected final void writeOtherFieldValue( + WriteContext writeContext, SerializationFieldInfo fieldInfo, Object fieldValue) { + AbstractObjectSerializer.writeField( + writeContext, + typeResolver, + writeContext.getRefWriter(), + fieldInfo, + writeContext.getBuffer(), + fieldValue); + } + + protected final Object readBuildInFieldValue( + ReadContext readContext, SerializationFieldInfo fieldInfo) { + return AbstractObjectSerializer.readBuildInFieldValue( + readContext, typeResolver, readContext.getRefReader(), fieldInfo, readContext.getBuffer()); + } + + protected final Object readContainerFieldValue( + ReadContext readContext, SerializationFieldInfo fieldInfo) { + return AbstractObjectSerializer.readContainerFieldValue( + readContext, + typeResolver, + readContext.getRefReader(), + readContext.getGenerics(), + fieldInfo, + readContext.getBuffer()); + } + + protected final Object readOtherFieldValue( + ReadContext readContext, SerializationFieldInfo fieldInfo) { + return AbstractObjectSerializer.readField( + readContext, typeResolver, readContext.getRefReader(), fieldInfo, readContext.getBuffer()); + } + + protected final Object readRemoteField(ReadContext readContext, RemoteFieldInfo remoteField) { + if (remoteField.compatibleCollectionArrayReadAction != null) { + return CompatibleCollectionArrayReader.read( + readContext, + remoteField.serializationFieldInfo.refMode, + remoteField.compatibleCollectionArrayReadAction); + } + return readField(readContext, remoteField.serializationFieldInfo); + } + + protected final void skipField(ReadContext readContext, RemoteFieldInfo remoteField) { + skipField( + readContext, + readContext.getRefReader(), + remoteField.serializationFieldInfo, + readContext.getBuffer()); + } + + protected final void skipField( + ReadContext readContext, + RefReader refReader, + SerializationFieldInfo fieldInfo, + MemoryBuffer buffer) { + FieldSkipper.skipField(readContext, typeResolver, refReader, fieldInfo, buffer); + } + + protected final int matchedId(RemoteFieldInfo remoteField) { + return remoteField.matchedId; + } + + protected final Object copyFieldValue( + CopyContext copyContext, Object fieldValue, SerializationFieldInfo fieldInfo) { + return copyContext.copyObject(fieldValue, fieldInfo.dispatchId); + } + + protected final int computeClassVersionHash(List descriptors) { + return ObjectSerializer.computeStructHash( + typeResolver, + FieldGroups.buildDescriptorGrouper( + typeResolver, descriptors, false, descriptor -> descriptor)); + } + + protected final void checkClassVersion(int readHash, int classVersionHash) { + ObjectSerializer.checkClassVersion(type, readHash, classVersionHash); + } + + private Object readField(ReadContext readContext, SerializationFieldInfo fieldInfo) { + if (typeResolver.isCollectionDescriptor(fieldInfo.descriptor) + || typeResolver.isMap(fieldInfo.type)) { + return readContainerFieldValue(readContext, fieldInfo); + } + if (typeResolver.isBuildIn(fieldInfo.descriptor) + || typeResolver.usesPrimitiveFieldOrdering(fieldInfo.descriptor)) { + return readBuildInFieldValue(readContext, fieldInfo); + } + return readOtherFieldValue(readContext, fieldInfo); + } + + private List buildRemoteFields( + TypeDef remoteTypeDef, List localDescriptors) { + List remoteFieldInfos = remoteTypeDef.getFieldsInfo(); + List remoteDescriptors = + remoteTypeDef.getDescriptors(typeResolver, type, localDescriptors); + Map fieldIds = new HashMap<>(); + Map fields = new HashMap<>(); + for (int i = 0; i < localDescriptors.size(); i++) { + Descriptor descriptor = localDescriptors.get(i); + if (descriptor.hasForyFieldId()) { + fieldIds.put((short) descriptor.getForyFieldId(), i); + } + fields.put(fieldKey(descriptor), i); + } + List remoteFields = new ArrayList<>(remoteFieldInfos.size()); + for (int i = 0; i < remoteFieldInfos.size(); i++) { + FieldInfo fieldInfo = remoteFieldInfos.get(i); + Descriptor descriptor = remoteDescriptors.get(i); + int matchedId = matchField(fieldInfo, fieldIds, fields); + SerializationFieldInfo serializationFieldInfo = + FieldGroups.buildFieldInfo(typeResolver, descriptor); + remoteFields.add( + new RemoteFieldInfo( + typeResolver, matchedId, fieldInfo, descriptor, serializationFieldInfo)); + } + return Collections.unmodifiableList(remoteFields); + } + + private int matchField( + FieldInfo fieldInfo, Map fieldIds, Map fields) { + Integer localId; + if (fieldInfo.hasFieldId()) { + localId = fieldIds.get(fieldInfo.getFieldId()); + } else { + String key = fieldInfo.getDefinedClass() + "." + fieldInfo.getFieldName(); + localId = fields.get(key); + if (localId == null && typeResolver.isCrossLanguage()) { + localId = + fields.get( + fieldInfo.getDefinedClass() + + "." + + StringUtils.lowerCamelToLowerUnderscore(fieldInfo.getFieldName())); + } + } + return localId == null ? UNKNOWN_FIELD : localId; + } + + private static String fieldKey(Descriptor descriptor) { + return descriptor.getDeclaringClass() + "." + descriptor.getName(); + } + + /** Remote field metadata consumed by generated compatible read methods. */ + @Internal + protected static final class RemoteFieldInfo { + private final int matchedId; + private final FieldInfo fieldInfo; + private final Descriptor descriptor; + private final SerializationFieldInfo serializationFieldInfo; + private final CompatibleCollectionArrayReader.ReadAction compatibleCollectionArrayReadAction; + + private RemoteFieldInfo( + TypeResolver typeResolver, + int matchedId, + FieldInfo fieldInfo, + Descriptor descriptor, + SerializationFieldInfo serializationFieldInfo) { + this.matchedId = matchedId; + this.fieldInfo = fieldInfo; + this.descriptor = descriptor; + this.serializationFieldInfo = serializationFieldInfo; + this.compatibleCollectionArrayReadAction = + CompatibleCollectionArrayReader.readAction(typeResolver, descriptor); + } + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java index fb0bfb231b..77f8adde62 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java @@ -104,9 +104,8 @@ public static String computeStructFingerprint( // Get field identifier: tag ID if configured, otherwise snake_case name String fieldIdentifier; int fieldId = -1; - ForyField foryField = descriptor.getForyField(); - if (foryField != null && foryField.id() >= 0) { - fieldId = foryField.id(); + if (descriptor.hasForyFieldId()) { + fieldId = descriptor.getForyFieldId(); fieldIdentifier = String.valueOf(fieldId); } else { fieldIdentifier = descriptor.getSnakeCaseName(); @@ -115,7 +114,7 @@ public static String computeStructFingerprint( // Get ref flag from @ForyField annotation only (compile-time info) // If annotation is absent or ref not explicitly set to true, ref is 0 // This allows fingerprint to be computed at compile time for C++/Rust - char ref = (foryField != null && foryField.ref()) ? '1' : '0'; + char ref = (descriptor.hasForyFieldPolicy() && descriptor.isTrackingRef()) ? '1' : '0'; // Get nullable flag: // - Primitives are always non-nullable @@ -128,8 +127,8 @@ public static String computeStructFingerprint( } else if (resolver.isCrossLanguage()) { // For xlang: nullable defaults to false, except for Optional types, boxed types // If @ForyField annotation is present, use its nullable value - if (foryField != null) { - nullable = foryField.nullable() ? '1' : '0'; + if (descriptor.hasForyFieldPolicy()) { + nullable = descriptor.isNullable() ? '1' : '0'; } else { // Default: Optional types, boxed primitives are nullable nullable = (TypeUtils.isOptionalType(rawType) || TypeUtils.isBoxed(rawType)) ? '1' : '0'; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java index f856f42529..ccb432e654 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java @@ -44,6 +44,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.fory.annotation.ArrayType; import org.apache.fory.annotation.BFloat16Type; import org.apache.fory.annotation.Expose; import org.apache.fory.annotation.Float16Type; @@ -101,7 +102,9 @@ public static void clearDescriptorCache() { private final Method readMethod; private final Method writeMethod; private final ForyField foryField; + private final ForyFieldPolicy foryFieldPolicy; private final Annotation typeAnnotation; + private final boolean arrayType; private boolean nullable; // trackingRef should only be true if explicitly set to true via @ForyField(ref=true) // If no annotation or ref not specified, trackingRef stays false and type-based tracking applies @@ -121,11 +124,13 @@ public Descriptor(Field field, TypeRef typeRef, Method readMethod, Method wri this.writeMethod = writeMethod; this.typeRef = typeRef; this.foryField = this.field.getAnnotation(ForyField.class); + this.foryFieldPolicy = ForyFieldPolicy.from(foryField); typeAnnotation = getAnnotation(field); + arrayType = field.isAnnotationPresent(ArrayType.class); if (!typeRef.isPrimitive()) { - this.nullable = foryField == null || foryField.nullable(); + this.nullable = foryFieldPolicy == null || foryFieldPolicy.nullable(); } - this.trackingRef = foryField != null && foryField.ref(); + this.trackingRef = foryFieldPolicy != null && foryFieldPolicy.trackingRef(); } public Descriptor( @@ -145,11 +150,51 @@ public Descriptor( this.readMethod = null; this.writeMethod = null; this.foryField = null; + this.foryFieldPolicy = null; typeAnnotation = null; + arrayType = false; this.nullable = nullable; this.trackingRef = trackingRef; } + public Descriptor( + TypeRef typeRef, + String typeName, + String name, + int modifier, + String declaringClass, + ForyFieldPolicy foryFieldPolicy) { + this(typeRef, typeName, name, modifier, declaringClass, foryFieldPolicy, false); + } + + public Descriptor( + TypeRef typeRef, + String typeName, + String name, + int modifier, + String declaringClass, + ForyFieldPolicy foryFieldPolicy, + boolean arrayType) { + this.field = null; + this.typeName = typeName; + this.name = name; + this.modifier = modifier; + this.declaringClass = declaringClass; + this.typeRef = typeRef; + this.readMethod = null; + this.writeMethod = null; + this.foryField = null; + this.foryFieldPolicy = foryFieldPolicy; + typeAnnotation = null; + this.arrayType = arrayType; + if (typeRef.isPrimitive()) { + this.nullable = false; + } else { + this.nullable = foryFieldPolicy == null || foryFieldPolicy.nullable(); + } + this.trackingRef = foryFieldPolicy != null && foryFieldPolicy.trackingRef(); + } + private Descriptor(Field field, Method readMethod) { this.field = field; // Compute typeRef from field's generic type to include generic info @@ -163,11 +208,13 @@ private Descriptor(Field field, Method readMethod) { this.readMethod = readMethod; this.writeMethod = null; this.foryField = this.field.getAnnotation(ForyField.class); + this.foryFieldPolicy = ForyFieldPolicy.from(foryField); typeAnnotation = getAnnotation(field); + arrayType = field.isAnnotationPresent(ArrayType.class); if (!field.getType().isPrimitive()) { - this.nullable = foryField == null || foryField.nullable(); + this.nullable = foryFieldPolicy == null || foryFieldPolicy.nullable(); } - this.trackingRef = foryField != null && foryField.ref(); + this.trackingRef = foryFieldPolicy != null && foryFieldPolicy.trackingRef(); } private Descriptor(Method readMethod) { @@ -182,12 +229,14 @@ private Descriptor(Method readMethod) { this.readMethod = readMethod; this.writeMethod = null; this.foryField = readMethod.getAnnotation(ForyField.class); + this.foryFieldPolicy = ForyFieldPolicy.from(foryField); typeAnnotation = getTypeUseAnnotation(readMethod.getAnnotatedReturnType(), readMethod.getName()); + arrayType = readMethod.isAnnotationPresent(ArrayType.class); if (!readMethod.getReturnType().isPrimitive()) { - this.nullable = foryField == null || foryField.nullable(); + this.nullable = foryFieldPolicy == null || foryFieldPolicy.nullable(); } - this.trackingRef = foryField != null && foryField.ref(); + this.trackingRef = foryFieldPolicy != null && foryFieldPolicy.trackingRef(); } public Descriptor(DescriptorBuilder builder) { @@ -201,7 +250,13 @@ public Descriptor(DescriptorBuilder builder) { this.writeMethod = builder.writeMethod; this.trackingRef = builder.trackingRef; this.foryField = this.field == null ? null : this.field.getAnnotation(ForyField.class); + this.foryFieldPolicy = + builder.foryFieldPolicy != null ? builder.foryFieldPolicy : ForyFieldPolicy.from(foryField); typeAnnotation = field == null ? null : getAnnotation(field); + arrayType = + builder.arrayType + || (field != null && field.isAnnotationPresent(ArrayType.class)) + || (readMethod != null && readMethod.isAnnotationPresent(ArrayType.class)); // Use builder.nullable directly - this is set by DescriptorBuilder.nullable() // and should be respected, especially for xlang compatible mode where remote // TypeDef's nullable flag may differ from local field's nullable @@ -281,14 +336,30 @@ public ForyField getForyField() { return foryField; } + public ForyFieldPolicy getForyFieldPolicy() { + return foryFieldPolicy; + } + + public boolean hasForyFieldPolicy() { + return foryFieldPolicy != null; + } + + public boolean hasForyFieldId() { + return foryFieldPolicy != null && foryFieldPolicy.id() >= 0; + } + + public int getForyFieldId() { + return foryFieldPolicy == null ? -1 : foryFieldPolicy.id(); + } + /** * Returns the morphic setting for this field. * * @return the morphic setting from @ForyField annotation, or AUTO if not specified */ public ForyField.Dynamic getMorphic() { - if (foryField != null) { - return foryField.dynamic(); + if (foryFieldPolicy != null) { + return foryFieldPolicy.dynamic(); } return ForyField.Dynamic.AUTO; } @@ -297,6 +368,10 @@ public Annotation getTypeAnnotation() { return typeAnnotation; } + public boolean isArrayType() { + return arrayType; + } + /** Try not use {@link TypeRef#getRawType()} since it's expensive. */ public Class getRawType() { Class type = this.type; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java index 56a1afafd1..4152b3ccc3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java @@ -37,6 +37,8 @@ public class DescriptorBuilder { Method readMethod; Method writeMethod; ForyField foryField; + ForyFieldPolicy foryFieldPolicy; + boolean arrayType; boolean nullable; boolean trackingRef; FieldConverter fieldConverter; @@ -52,6 +54,8 @@ public DescriptorBuilder(Descriptor descriptor) { this.readMethod = descriptor.getReadMethod(); this.writeMethod = descriptor.getWriteMethod(); this.foryField = descriptor.getForyField(); + this.foryFieldPolicy = descriptor.getForyFieldPolicy(); + this.arrayType = descriptor.isArrayType(); this.nullable = descriptor.isNullable(); this.trackingRef = descriptor.isTrackingRef(); this.fieldConverter = descriptor.getFieldConverter(); @@ -114,6 +118,17 @@ public DescriptorBuilder trackingRef(boolean trackingRef) { public DescriptorBuilder foryField(ForyField foryField) { this.foryField = foryField; + this.foryFieldPolicy = ForyFieldPolicy.from(foryField); + return this; + } + + public DescriptorBuilder foryFieldPolicy(ForyFieldPolicy foryFieldPolicy) { + this.foryFieldPolicy = foryFieldPolicy; + return this; + } + + public DescriptorBuilder arrayType(boolean arrayType) { + this.arrayType = arrayType; return this; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/ForyFieldPolicy.java b/java/fory-core/src/main/java/org/apache/fory/type/ForyFieldPolicy.java new file mode 100644 index 0000000000..c5cba6e4a6 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/type/ForyFieldPolicy.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.type; + +import java.util.Objects; +import org.apache.fory.annotation.ForyField; +import org.apache.fory.annotation.Internal; + +/** Descriptor-owned representation of {@link ForyField} values for generated descriptors. */ +@Internal +public final class ForyFieldPolicy { + private final int id; + private final boolean nullable; + private final boolean trackingRef; + private final ForyField.Dynamic dynamic; + + public static ForyFieldPolicy of( + int id, boolean nullable, boolean trackingRef, ForyField.Dynamic dynamic) { + return new ForyFieldPolicy(id, nullable, trackingRef, dynamic); + } + + public static ForyFieldPolicy from(ForyField field) { + if (field == null) { + return null; + } + return of(field.id(), field.nullable(), field.ref(), field.dynamic()); + } + + private ForyFieldPolicy( + int id, boolean nullable, boolean trackingRef, ForyField.Dynamic dynamic) { + this.id = id; + this.nullable = nullable; + this.trackingRef = trackingRef; + this.dynamic = Objects.requireNonNull(dynamic); + } + + public int id() { + return id; + } + + public boolean nullable() { + return nullable; + } + + public boolean trackingRef() { + return trackingRef; + } + + public ForyField.Dynamic dynamic() { + return dynamic; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ForyFieldPolicy)) { + return false; + } + ForyFieldPolicy that = (ForyFieldPolicy) o; + return id == that.id + && nullable == that.nullable + && trackingRef == that.trackingRef + && dynamic == that.dynamic; + } + + @Override + public int hashCode() { + return Objects.hash(id, nullable, trackingRef, dynamic); + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/type/GenericType.java b/java/fory-core/src/main/java/org/apache/fory/type/GenericType.java index 45acd4365d..3ac7739356 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/GenericType.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/GenericType.java @@ -130,26 +130,33 @@ public static GenericType build(Type type, Predicate finalPredicate) { public static GenericType build(TypeRef typeRef, Predicate finalPredicate) { Type type = typeRef.getType(); + if (typeRef.hasExplicitTypeArguments()) { + List> explicitTypeArguments = typeRef.getTypeArguments(); + List list = new ArrayList<>(explicitTypeArguments.size()); + for (TypeRef explicitTypeArgument : explicitTypeArguments) { + list.add(GenericType.build(explicitTypeArgument, finalPredicate)); + } + GenericType[] genericTypes = list.toArray(new GenericType[0]); + return new GenericType(typeRef, finalPredicate.test(type), genericTypes); + } + if (typeRef.isArray()) { + TypeRef explicitComponentType = typeRef.getComponentType(); + return new GenericType( + typeRef, + finalPredicate.test(type), + GenericType.build(explicitComponentType, finalPredicate)); + } if (type instanceof ParameterizedType) { // List, List, Map>, SomeClass Type[] actualTypeArguments = ((ParameterizedType) type).getActualTypeArguments(); - List> explicitTypeArguments = - typeRef.hasExplicitTypeArguments() ? typeRef.getTypeArguments() : null; List list = new ArrayList<>(); for (int i = 0; i < actualTypeArguments.length; i++) { - GenericType build = - explicitTypeArguments != null - ? GenericType.build(explicitTypeArguments.get(i), finalPredicate) - : GenericType.build(actualTypeArguments[i], finalPredicate); - list.add(build); + list.add(GenericType.build(actualTypeArguments[i], finalPredicate)); } GenericType[] genericTypes = list.toArray(new GenericType[0]); return new GenericType(typeRef, finalPredicate.test(type), genericTypes); } else if (type instanceof GenericArrayType) { // List[] or T[] - TypeRef componentType = - typeRef.getComponentType() != null - ? typeRef.getComponentType() - : TypeRef.of(((GenericArrayType) type).getGenericComponentType()); + TypeRef componentType = TypeRef.of(((GenericArrayType) type).getGenericComponentType()); return new GenericType(typeRef, finalPredicate.test(type), build(componentType)); } else if (type instanceof Class && ((Class) type).isArray()) { TypeRef componentType = typeRef.getComponentType(); diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java index e3df014dfd..79ed581942 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java @@ -297,6 +297,32 @@ public static boolean isBoxedListArrayType(Field field) { return true; } + public static boolean isBoxedListArrayType(Descriptor descriptor) { + Field field = descriptor.getField(); + if (field != null) { + return isBoxedListArrayType(field); + } + if (!descriptor.isArrayType()) { + return false; + } + Class rawType = descriptor.getRawType(); + if (TypeUtils.isPrimitiveListClass(rawType)) { + return false; + } + if (!List.class.isAssignableFrom(rawType)) { + if (Collection.class.isAssignableFrom(rawType)) { + throw new IllegalArgumentException( + "@ArrayType can only be applied to Fory primitive-list carriers or ordered " + + "java.util.List fields, but got " + + rawType.getName()); + } + throw new IllegalArgumentException( + "@ArrayType can only be applied to Fory primitive-list carriers or ordered " + + "java.util.List fields; primitive arrays already use array schema"); + } + return true; + } + public static int getBoxedListArrayTypeId(Field field) { validateBoxedListArrayType(field); TypeRef elementTypeRef = TypeUtils.getElementType(TypeUtils.getFieldTypeRef(field)); @@ -310,6 +336,25 @@ public static int getBoxedListArrayTypeId(Field field) { return typeId; } + public static int getBoxedListArrayTypeId(Descriptor descriptor) { + Field field = descriptor.getField(); + if (field != null) { + return getBoxedListArrayTypeId(field); + } + if (!isBoxedListArrayType(descriptor)) { + return Types.UNKNOWN; + } + TypeRef elementTypeRef = TypeUtils.getElementType(descriptor.getTypeRef()); + int typeId = getArrayTypeIdFromElementType(elementTypeRef); + if (typeId == Types.UNKNOWN) { + throw new IllegalArgumentException( + "@ArrayType List field " + + descriptor.getName() + + " must use a bool, numeric, float16, or bfloat16 element type"); + } + return typeId; + } + public static int getArrayTypeIdFromElementType(TypeRef elementTypeRef) { TypeExtMeta extMeta = elementTypeRef.getTypeExtMeta(); if (extMeta != null && extMeta.typeId() != Types.UNKNOWN) { @@ -503,6 +548,9 @@ public static TypeRef getPrimitiveListElementTypeRef( } public static boolean isArrayType(Descriptor descriptor) { + if (descriptor.isArrayType()) { + return true; + } if (descriptor.getField() != null && descriptor.getField().isAnnotationPresent(ArrayType.class)) { return true; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index 07a0d24747..a1e6a4a56c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -83,7 +83,6 @@ import org.apache.fory.collection.UInt64List; import org.apache.fory.collection.UInt8List; import org.apache.fory.meta.TypeExtMeta; -import org.apache.fory.platform.AndroidSupport; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeParameter; import org.apache.fory.reflect.TypeRef; @@ -98,6 +97,7 @@ public class TypeUtils { private static final String SQL_DATE_CLASS_NAME = "java.sql.Date"; private static final String SQL_TIME_CLASS_NAME = "java.sql.Time"; private static final String SQL_TIMESTAMP_CLASS_NAME = "java.sql.Timestamp"; + private static final boolean FIELD_ANNOTATED_TYPE_SUPPORTED = isFieldAnnotatedTypeSupported(); public static final String JAVA_BOOLEAN = "boolean"; public static final String JAVA_BYTE = "byte"; @@ -542,7 +542,7 @@ public static TypeRef getMultiDimensionArrayElementType(TypeRef type) { /** Returns element type of iterable. */ public static TypeRef getElementType(TypeRef typeRef) { - if (typeRef.hasTypeExtMeta() && typeRef.hasExplicitTypeArguments()) { + if (typeRef.hasExplicitTypeArguments()) { List> typeArguments = typeRef.getTypeArguments(); if (typeArguments.size() == 1) { Class rawType = getRawType(typeRef); @@ -631,17 +631,26 @@ public static void applyRefTrackingOverride( } public static TypeRef getFieldTypeRef(Field field) { - if (AndroidSupport.IS_ANDROID) { - return TypeRef.of(field.getGenericType()); + if (FIELD_ANNOTATED_TYPE_SUPPORTED) { + return TypeRef.of(field.getAnnotatedType()); } - return TypeRef.of(field.getAnnotatedType()); + return TypeRef.of(field.getGenericType()); } public static AnnotatedType getFieldAnnotatedType(Field field) { - if (AndroidSupport.IS_ANDROID) { - return null; + if (FIELD_ANNOTATED_TYPE_SUPPORTED) { + return field.getAnnotatedType(); + } + return null; + } + + private static boolean isFieldAnnotatedTypeSupported() { + try { + Field.class.getMethod("getAnnotatedType"); + return true; + } catch (NoSuchMethodException | LinkageError e) { + return false; } - return field.getAnnotatedType(); } public static void applyFieldRefTrackingOverride( diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Types.java b/java/fory-core/src/main/java/org/apache/fory/type/Types.java index 5ef681ccba..f669603eb9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Types.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Types.java @@ -410,8 +410,8 @@ public static int getDescriptorTypeId(TypeResolver resolver, Field field) { } public static int getDescriptorTypeId(TypeResolver resolver, Descriptor d) { - if (TypeAnnotationUtils.isBoxedListArrayType(d.getField())) { - return TypeAnnotationUtils.getBoxedListArrayTypeId(d.getField()); + if (TypeAnnotationUtils.isBoxedListArrayType(d)) { + return TypeAnnotationUtils.getBoxedListArrayTypeId(d); } TypeRef typeRef = d.getTypeRef(); TypeExtMeta extMeta = typeRef.getTypeExtMeta(); @@ -419,6 +419,17 @@ public static int getDescriptorTypeId(TypeResolver resolver, Descriptor d) { return extMeta.typeId(); } else { Class rawType = typeRef.getRawType(); + TypeRef componentType = typeRef.getComponentType(); + if (rawType.isArray() && componentType != null) { + TypeExtMeta componentMeta = componentType.getTypeExtMeta(); + if (componentMeta != null && componentMeta.typeId() != Types.UNKNOWN) { + int arrayTypeId = + TypeAnnotationUtils.getArrayTypeIdFromElementTypeId(componentMeta.typeId()); + if (arrayTypeId != Types.UNKNOWN) { + return arrayTypeId; + } + } + } Annotation typeAnnotation = d.getTypeAnnotation(); if (TypeUtils.isPrimitiveListClass(rawType)) { if (TypeAnnotationUtils.isArrayType(d)) { diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 96eea48464..281165eeca 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -157,6 +157,8 @@ Args=--initialize-at-build-time=org.apache.fory.collection.BiMap,\ org.apache.fory.builder.JITContext,\ org.apache.fory.builder.ObjectCodecBuilder,\ org.apache.fory.builder.MetaSharedCodecBuilder,\ + org.apache.fory.builder.CompatibleMetaSharedCodecBuilder,\ + org.apache.fory.builder.Generated$GeneratedCompatibleMetaSharedSerializer,\ org.apache.fory.builder.CodecUtils,\ org.apache.fory.resolver.RefMode,\ org.apache.fory.serializer.FieldGroups$SerializationFieldInfo,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index d72094cd2a..0d500528d3 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -1259,6 +1259,8 @@ public void testStructWithMap(boolean enableCodegen) throws java.io.IOException static class MyStruct { int id; + public MyStruct() {} + public MyStruct(int id) { this.id = id; } diff --git a/java/pom.xml b/java/pom.xml index 018ad33caf..b3000ffa10 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -60,6 +60,7 @@ fory-core + fory-annotation-processor fory-extensions fory-test-core From 897581208ce1dd193f7363557edf55b583b0ecbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 00:26:01 +0800 Subject: [PATCH 03/58] docs(java): document annotation processor usage --- java/fory-annotation-processor/README.md | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 java/fory-annotation-processor/README.md diff --git a/java/fory-annotation-processor/README.md b/java/fory-annotation-processor/README.md new file mode 100644 index 0000000000..820abf34d3 --- /dev/null +++ b/java/fory-annotation-processor/README.md @@ -0,0 +1,26 @@ +# Fory Annotation Processor + +`fory-annotation-processor` generates build-time static serializers for Java classes annotated with +`@ForyStruct`. + +For ordinary JVM applications, prefer Fory's runtime generated serializers. Runtime generation is +optimized for the active JVM and is usually more efficient than javac-generated static serializers. +That is the normal high-performance path for server and desktop JVM deployments. + +Use this annotation processor when runtime source generation, bytecode generation, or dynamic class +loading is not acceptable. The main target is Android, where runtime code generation is disabled. +It can also be useful for restricted JVM environments that forbid loading generated bytecode at +runtime. + +## Ownership + +The processor is an opt-in build tool. Applications add it to their annotation-processor path; it +emits Java source that references `fory-core` runtime APIs. `fory-core` does not depend on this +module, and the processor implementation does not need a compile-time dependency on `fory-core`. + +Generated serializers are public top-level classes in the same package as the annotated type. The +runtime loads them deterministically by name when static serializers are available and runtime +codegen is disabled. + +For the full user-facing guide, see +[`docs/guide/java/static-struct-serializers.md`](../../docs/guide/java/static-struct-serializers.md). From c0e864280dbd2c6b36e49fad2bf3a8899c2db47c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 00:29:43 +0800 Subject: [PATCH 04/58] refactor(java): rename field metadata spec --- .../annotation/processing/SourceField.java | 6 +-- .../StaticSerializerSourceWriter.java | 10 ++-- .../java/org/apache/fory/meta/FieldTypes.java | 2 +- .../fory/meta/NativeTypeDefEncoder.java | 2 +- .../org/apache/fory/meta/TypeDefEncoder.java | 2 +- .../apache/fory/resolver/ClassResolver.java | 2 +- .../apache/fory/resolver/TypeResolver.java | 4 +- .../fory/serializer/struct/Fingerprint.java | 4 +- .../java/org/apache/fory/type/Descriptor.java | 53 +++++++++---------- .../apache/fory/type/DescriptorBuilder.java | 10 ++-- .../{ForyFieldPolicy.java => FieldSpec.java} | 15 +++--- 11 files changed, 54 insertions(+), 56 deletions(-) rename java/fory-core/src/main/java/org/apache/fory/type/{ForyFieldPolicy.java => FieldSpec.java} (84%) diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java index bd9eb11f97..6a128cd6a3 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java @@ -37,7 +37,7 @@ enum AccessKind { final String readAccess; final AccessKind writeAccessKind; final String writeAccess; - final boolean hasForyFieldPolicy; + final boolean hasFieldSpec; final int foryFieldId; final boolean nullable; final boolean trackingRef; @@ -56,7 +56,7 @@ enum AccessKind { String readAccess, AccessKind writeAccessKind, String writeAccess, - boolean hasForyFieldPolicy, + boolean hasFieldSpec, int foryFieldId, boolean nullable, boolean trackingRef, @@ -73,7 +73,7 @@ enum AccessKind { this.readAccess = readAccess; this.writeAccessKind = writeAccessKind; this.writeAccess = writeAccess; - this.hasForyFieldPolicy = hasForyFieldPolicy; + this.hasFieldSpec = hasFieldSpec; this.foryFieldId = foryFieldId; this.nullable = nullable; this.trackingRef = trackingRef; diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index f3a2a354db..5144f10012 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -64,7 +64,7 @@ private void writeHeader() { builder.append("import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo;\n"); builder.append("import org.apache.fory.serializer.StaticGeneratedStructSerializer;\n"); builder.append("import org.apache.fory.type.Descriptor;\n"); - builder.append("import org.apache.fory.type.ForyFieldPolicy;\n"); + builder.append("import org.apache.fory.type.FieldSpec;\n"); builder.append("import org.apache.fory.type.Types;\n\n"); } @@ -111,7 +111,7 @@ private void writeDescriptors() { .append(", \"") .append(escape(field.declaringClass)) .append("\", ") - .append(foryFieldPolicyExpression(field)) + .append(fieldSpecExpression(field)) .append(", ") .append(field.arrayType) .append("));\n"); @@ -124,11 +124,11 @@ private void writeDescriptors() { builder.append(" }\n\n"); } - private String foryFieldPolicyExpression(SourceField field) { - if (!field.hasForyFieldPolicy) { + private String fieldSpecExpression(SourceField field) { + if (!field.hasFieldSpec) { return "null"; } - return "ForyFieldPolicy.of(" + return "FieldSpec.of(" + field.foryFieldId + ", " + field.nullable diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 0894ec311b..997b8b463b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -206,7 +206,7 @@ private static FieldType buildFieldType( } // Apply @ForyField annotation if present - if (descriptor != null && descriptor.hasForyFieldPolicy()) { + if (descriptor != null && descriptor.hasFieldSpec()) { nullable = descriptor.isNullable(); trackingRef = descriptor.isTrackingRef(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java index 66e674a775..923a2f4bba 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java @@ -107,7 +107,7 @@ public static List buildFieldsInfoFromDescriptors( FieldType fieldType = FieldTypes.buildFieldType(resolver, descriptor); FieldInfo fieldInfo; - if (descriptor.hasForyFieldPolicy()) { + if (descriptor.hasFieldSpec()) { int tagId = descriptor.getForyFieldId(); if (tagId >= 0) { if (!usedTagIds.add(tagId)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java index 22a3a156d8..eb0dfafff2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java @@ -89,7 +89,7 @@ static List buildFieldsInfoFromDescriptors( .map( descriptor -> { FieldType fieldType = FieldTypes.buildFieldType(resolver, descriptor); - if (descriptor.hasForyFieldPolicy()) { + if (descriptor.hasFieldSpec()) { int tagId = descriptor.getForyFieldId(); if (tagId >= 0) { if (!usedTagIds.add(tagId)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index fdfc15bc27..9d591cef33 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -976,7 +976,7 @@ public int getUserTypeIdForTypeDef(Class cls) { @Override public boolean isMonomorphic(Descriptor descriptor) { - if (descriptor.hasForyFieldPolicy()) { + if (descriptor.hasFieldSpec()) { switch (descriptor.getMorphic()) { case TRUE: return false; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index dcd639a7aa..be8b219f6c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1477,7 +1477,7 @@ private List buildFieldDescriptors( if (!searchParent && !descriptor.getDeclaringClass().equals(clz.getName())) { continue; } - boolean hasForyField = descriptor.hasForyFieldPolicy(); + boolean hasForyField = descriptor.hasFieldSpec(); // Compute the final isTrackingRef value: // For xlang mode: "Reference tracking is disabled by default" (xlang spec) // - Only enable ref tracking if explicitly set via @ForyField(ref=true) @@ -1735,7 +1735,7 @@ private boolean isFieldNullable(Descriptor descriptor) { if (isCrossLanguage()) { // For xlang mode: apply xlang defaults // This must match what TypeDefEncoder.buildFieldType uses for TypeDef metadata - if (descriptor.hasForyFieldPolicy()) { + if (descriptor.hasFieldSpec()) { // Use explicit annotation value return descriptor.isNullable(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java index 77f8adde62..ea08f56505 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java @@ -114,7 +114,7 @@ public static String computeStructFingerprint( // Get ref flag from @ForyField annotation only (compile-time info) // If annotation is absent or ref not explicitly set to true, ref is 0 // This allows fingerprint to be computed at compile time for C++/Rust - char ref = (descriptor.hasForyFieldPolicy() && descriptor.isTrackingRef()) ? '1' : '0'; + char ref = (descriptor.hasFieldSpec() && descriptor.isTrackingRef()) ? '1' : '0'; // Get nullable flag: // - Primitives are always non-nullable @@ -127,7 +127,7 @@ public static String computeStructFingerprint( } else if (resolver.isCrossLanguage()) { // For xlang: nullable defaults to false, except for Optional types, boxed types // If @ForyField annotation is present, use its nullable value - if (descriptor.hasForyFieldPolicy()) { + if (descriptor.hasFieldSpec()) { nullable = descriptor.isNullable() ? '1' : '0'; } else { // Default: Optional types, boxed primitives are nullable diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java index ccb432e654..cfe529016e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java @@ -102,7 +102,7 @@ public static void clearDescriptorCache() { private final Method readMethod; private final Method writeMethod; private final ForyField foryField; - private final ForyFieldPolicy foryFieldPolicy; + private final FieldSpec fieldSpec; private final Annotation typeAnnotation; private final boolean arrayType; private boolean nullable; @@ -124,13 +124,13 @@ public Descriptor(Field field, TypeRef typeRef, Method readMethod, Method wri this.writeMethod = writeMethod; this.typeRef = typeRef; this.foryField = this.field.getAnnotation(ForyField.class); - this.foryFieldPolicy = ForyFieldPolicy.from(foryField); + this.fieldSpec = FieldSpec.from(foryField); typeAnnotation = getAnnotation(field); arrayType = field.isAnnotationPresent(ArrayType.class); if (!typeRef.isPrimitive()) { - this.nullable = foryFieldPolicy == null || foryFieldPolicy.nullable(); + this.nullable = fieldSpec == null || fieldSpec.nullable(); } - this.trackingRef = foryFieldPolicy != null && foryFieldPolicy.trackingRef(); + this.trackingRef = fieldSpec != null && fieldSpec.trackingRef(); } public Descriptor( @@ -150,7 +150,7 @@ public Descriptor( this.readMethod = null; this.writeMethod = null; this.foryField = null; - this.foryFieldPolicy = null; + this.fieldSpec = null; typeAnnotation = null; arrayType = false; this.nullable = nullable; @@ -163,8 +163,8 @@ public Descriptor( String name, int modifier, String declaringClass, - ForyFieldPolicy foryFieldPolicy) { - this(typeRef, typeName, name, modifier, declaringClass, foryFieldPolicy, false); + FieldSpec fieldSpec) { + this(typeRef, typeName, name, modifier, declaringClass, fieldSpec, false); } public Descriptor( @@ -173,7 +173,7 @@ public Descriptor( String name, int modifier, String declaringClass, - ForyFieldPolicy foryFieldPolicy, + FieldSpec fieldSpec, boolean arrayType) { this.field = null; this.typeName = typeName; @@ -184,15 +184,15 @@ public Descriptor( this.readMethod = null; this.writeMethod = null; this.foryField = null; - this.foryFieldPolicy = foryFieldPolicy; + this.fieldSpec = fieldSpec; typeAnnotation = null; this.arrayType = arrayType; if (typeRef.isPrimitive()) { this.nullable = false; } else { - this.nullable = foryFieldPolicy == null || foryFieldPolicy.nullable(); + this.nullable = fieldSpec == null || fieldSpec.nullable(); } - this.trackingRef = foryFieldPolicy != null && foryFieldPolicy.trackingRef(); + this.trackingRef = fieldSpec != null && fieldSpec.trackingRef(); } private Descriptor(Field field, Method readMethod) { @@ -208,13 +208,13 @@ private Descriptor(Field field, Method readMethod) { this.readMethod = readMethod; this.writeMethod = null; this.foryField = this.field.getAnnotation(ForyField.class); - this.foryFieldPolicy = ForyFieldPolicy.from(foryField); + this.fieldSpec = FieldSpec.from(foryField); typeAnnotation = getAnnotation(field); arrayType = field.isAnnotationPresent(ArrayType.class); if (!field.getType().isPrimitive()) { - this.nullable = foryFieldPolicy == null || foryFieldPolicy.nullable(); + this.nullable = fieldSpec == null || fieldSpec.nullable(); } - this.trackingRef = foryFieldPolicy != null && foryFieldPolicy.trackingRef(); + this.trackingRef = fieldSpec != null && fieldSpec.trackingRef(); } private Descriptor(Method readMethod) { @@ -229,14 +229,14 @@ private Descriptor(Method readMethod) { this.readMethod = readMethod; this.writeMethod = null; this.foryField = readMethod.getAnnotation(ForyField.class); - this.foryFieldPolicy = ForyFieldPolicy.from(foryField); + this.fieldSpec = FieldSpec.from(foryField); typeAnnotation = getTypeUseAnnotation(readMethod.getAnnotatedReturnType(), readMethod.getName()); arrayType = readMethod.isAnnotationPresent(ArrayType.class); if (!readMethod.getReturnType().isPrimitive()) { - this.nullable = foryFieldPolicy == null || foryFieldPolicy.nullable(); + this.nullable = fieldSpec == null || fieldSpec.nullable(); } - this.trackingRef = foryFieldPolicy != null && foryFieldPolicy.trackingRef(); + this.trackingRef = fieldSpec != null && fieldSpec.trackingRef(); } public Descriptor(DescriptorBuilder builder) { @@ -250,8 +250,7 @@ public Descriptor(DescriptorBuilder builder) { this.writeMethod = builder.writeMethod; this.trackingRef = builder.trackingRef; this.foryField = this.field == null ? null : this.field.getAnnotation(ForyField.class); - this.foryFieldPolicy = - builder.foryFieldPolicy != null ? builder.foryFieldPolicy : ForyFieldPolicy.from(foryField); + this.fieldSpec = builder.fieldSpec != null ? builder.fieldSpec : FieldSpec.from(foryField); typeAnnotation = field == null ? null : getAnnotation(field); arrayType = builder.arrayType @@ -336,20 +335,20 @@ public ForyField getForyField() { return foryField; } - public ForyFieldPolicy getForyFieldPolicy() { - return foryFieldPolicy; + public FieldSpec getFieldSpec() { + return fieldSpec; } - public boolean hasForyFieldPolicy() { - return foryFieldPolicy != null; + public boolean hasFieldSpec() { + return fieldSpec != null; } public boolean hasForyFieldId() { - return foryFieldPolicy != null && foryFieldPolicy.id() >= 0; + return fieldSpec != null && fieldSpec.id() >= 0; } public int getForyFieldId() { - return foryFieldPolicy == null ? -1 : foryFieldPolicy.id(); + return fieldSpec == null ? -1 : fieldSpec.id(); } /** @@ -358,8 +357,8 @@ public int getForyFieldId() { * @return the morphic setting from @ForyField annotation, or AUTO if not specified */ public ForyField.Dynamic getMorphic() { - if (foryFieldPolicy != null) { - return foryFieldPolicy.dynamic(); + if (fieldSpec != null) { + return fieldSpec.dynamic(); } return ForyField.Dynamic.AUTO; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java index 4152b3ccc3..aefdb79208 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java @@ -37,7 +37,7 @@ public class DescriptorBuilder { Method readMethod; Method writeMethod; ForyField foryField; - ForyFieldPolicy foryFieldPolicy; + FieldSpec fieldSpec; boolean arrayType; boolean nullable; boolean trackingRef; @@ -54,7 +54,7 @@ public DescriptorBuilder(Descriptor descriptor) { this.readMethod = descriptor.getReadMethod(); this.writeMethod = descriptor.getWriteMethod(); this.foryField = descriptor.getForyField(); - this.foryFieldPolicy = descriptor.getForyFieldPolicy(); + this.fieldSpec = descriptor.getFieldSpec(); this.arrayType = descriptor.isArrayType(); this.nullable = descriptor.isNullable(); this.trackingRef = descriptor.isTrackingRef(); @@ -118,12 +118,12 @@ public DescriptorBuilder trackingRef(boolean trackingRef) { public DescriptorBuilder foryField(ForyField foryField) { this.foryField = foryField; - this.foryFieldPolicy = ForyFieldPolicy.from(foryField); + this.fieldSpec = FieldSpec.from(foryField); return this; } - public DescriptorBuilder foryFieldPolicy(ForyFieldPolicy foryFieldPolicy) { - this.foryFieldPolicy = foryFieldPolicy; + public DescriptorBuilder fieldSpec(FieldSpec fieldSpec) { + this.fieldSpec = fieldSpec; return this; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/ForyFieldPolicy.java b/java/fory-core/src/main/java/org/apache/fory/type/FieldSpec.java similarity index 84% rename from java/fory-core/src/main/java/org/apache/fory/type/ForyFieldPolicy.java rename to java/fory-core/src/main/java/org/apache/fory/type/FieldSpec.java index c5cba6e4a6..64983789d8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/ForyFieldPolicy.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/FieldSpec.java @@ -25,26 +25,25 @@ /** Descriptor-owned representation of {@link ForyField} values for generated descriptors. */ @Internal -public final class ForyFieldPolicy { +public final class FieldSpec { private final int id; private final boolean nullable; private final boolean trackingRef; private final ForyField.Dynamic dynamic; - public static ForyFieldPolicy of( + public static FieldSpec of( int id, boolean nullable, boolean trackingRef, ForyField.Dynamic dynamic) { - return new ForyFieldPolicy(id, nullable, trackingRef, dynamic); + return new FieldSpec(id, nullable, trackingRef, dynamic); } - public static ForyFieldPolicy from(ForyField field) { + public static FieldSpec from(ForyField field) { if (field == null) { return null; } return of(field.id(), field.nullable(), field.ref(), field.dynamic()); } - private ForyFieldPolicy( - int id, boolean nullable, boolean trackingRef, ForyField.Dynamic dynamic) { + private FieldSpec(int id, boolean nullable, boolean trackingRef, ForyField.Dynamic dynamic) { this.id = id; this.nullable = nullable; this.trackingRef = trackingRef; @@ -72,10 +71,10 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof ForyFieldPolicy)) { + if (!(o instanceof FieldSpec)) { return false; } - ForyFieldPolicy that = (ForyFieldPolicy) o; + FieldSpec that = (FieldSpec) o; return id == that.id && nullable == that.nullable && trackingRef == that.trackingRef From d63c99a44eaa7dc533cdd8cbc2d1a9cfbb4bc4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 00:37:00 +0800 Subject: [PATCH 05/58] refactor(java): keep ForyField options in Descriptor --- .../processing/ForyStructProcessor.java | 14 +-- .../annotation/processing/SourceField.java | 6 +- .../StaticSerializerSourceWriter.java | 26 ++--- .../java/org/apache/fory/meta/FieldTypes.java | 2 +- .../fory/meta/NativeTypeDefEncoder.java | 2 +- .../org/apache/fory/meta/TypeDefEncoder.java | 2 +- .../apache/fory/resolver/ClassResolver.java | 2 +- .../apache/fory/resolver/TypeResolver.java | 4 +- .../fory/serializer/struct/Fingerprint.java | 4 +- .../java/org/apache/fory/type/Descriptor.java | 97 +++++++++++++------ .../apache/fory/type/DescriptorBuilder.java | 24 +++-- .../java/org/apache/fory/type/FieldSpec.java | 88 ----------------- 12 files changed, 111 insertions(+), 160 deletions(-) delete mode 100644 java/fory-core/src/main/java/org/apache/fory/type/FieldSpec.java diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index d47e42c67d..fbc41b939d 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -205,7 +205,7 @@ boolean record = isRecord(type); private void validateForyFieldId( String binaryName, Map fieldIds, VariableElement field) { ForyFieldMeta foryField = foryField(field); - if (foryField.hasPolicy && foryField.id >= 0) { + if (foryField.hasForyField && foryField.id >= 0) { VariableElement previousField = fieldIds.put(foryField.id, field); if (previousField != null) { throw new InvalidStructException( @@ -291,10 +291,10 @@ private SourceField buildField( readAccess, writeKind, writeAccess, - foryField.hasPolicy, + foryField.hasForyField, foryField.id, - foryField.hasPolicy ? foryField.nullable : !typeNode.primitive, - foryField.hasPolicy && foryField.ref, + foryField.hasForyField ? foryField.nullable : !typeNode.primitive, + foryField.hasForyField && foryField.ref, foryField.dynamic); } @@ -772,14 +772,14 @@ private static final class InvalidStructException extends RuntimeException { private static final class ForyFieldMeta { static final ForyFieldMeta NONE = new ForyFieldMeta(false, -1, false, false, "AUTO"); - final boolean hasPolicy; + final boolean hasForyField; final int id; final boolean nullable; final boolean ref; final String dynamic; - ForyFieldMeta(boolean hasPolicy, int id, boolean nullable, boolean ref, String dynamic) { - this.hasPolicy = hasPolicy; + ForyFieldMeta(boolean hasForyField, int id, boolean nullable, boolean ref, String dynamic) { + this.hasForyField = hasForyField; this.id = id; this.nullable = nullable; this.ref = ref; diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java index 6a128cd6a3..005586483d 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceField.java @@ -37,7 +37,7 @@ enum AccessKind { final String readAccess; final AccessKind writeAccessKind; final String writeAccess; - final boolean hasFieldSpec; + final boolean hasForyField; final int foryFieldId; final boolean nullable; final boolean trackingRef; @@ -56,7 +56,7 @@ enum AccessKind { String readAccess, AccessKind writeAccessKind, String writeAccess, - boolean hasFieldSpec, + boolean hasForyField, int foryFieldId, boolean nullable, boolean trackingRef, @@ -73,7 +73,7 @@ enum AccessKind { this.readAccess = readAccess; this.writeAccessKind = writeAccessKind; this.writeAccess = writeAccess; - this.hasFieldSpec = hasFieldSpec; + this.hasForyField = hasForyField; this.foryFieldId = foryFieldId; this.nullable = nullable; this.trackingRef = trackingRef; diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 5144f10012..0536855222 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -64,7 +64,6 @@ private void writeHeader() { builder.append("import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo;\n"); builder.append("import org.apache.fory.serializer.StaticGeneratedStructSerializer;\n"); builder.append("import org.apache.fory.type.Descriptor;\n"); - builder.append("import org.apache.fory.type.FieldSpec;\n"); builder.append("import org.apache.fory.type.Types;\n\n"); } @@ -111,7 +110,15 @@ private void writeDescriptors() { .append(", \"") .append(escape(field.declaringClass)) .append("\", ") - .append(fieldSpecExpression(field)) + .append(field.hasForyField) + .append(", ") + .append(field.foryFieldId) + .append(", ") + .append(field.nullable) + .append(", ") + .append(field.trackingRef) + .append(", ForyField.Dynamic.") + .append(field.dynamic) .append(", ") .append(field.arrayType) .append("));\n"); @@ -124,21 +131,6 @@ private void writeDescriptors() { builder.append(" }\n\n"); } - private String fieldSpecExpression(SourceField field) { - if (!field.hasFieldSpec) { - return "null"; - } - return "FieldSpec.of(" - + field.foryFieldId - + ", " - + field.nullable - + ", " - + field.trackingRef - + ", ForyField.Dynamic." - + field.dynamic - + ")"; - } - private void writeConstructors() { builder .append(" public ") diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 997b8b463b..38f0a4b7fb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -206,7 +206,7 @@ private static FieldType buildFieldType( } // Apply @ForyField annotation if present - if (descriptor != null && descriptor.hasFieldSpec()) { + if (descriptor != null && descriptor.hasForyField()) { nullable = descriptor.isNullable(); trackingRef = descriptor.isTrackingRef(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java index 923a2f4bba..12b7bf7169 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java @@ -107,7 +107,7 @@ public static List buildFieldsInfoFromDescriptors( FieldType fieldType = FieldTypes.buildFieldType(resolver, descriptor); FieldInfo fieldInfo; - if (descriptor.hasFieldSpec()) { + if (descriptor.hasForyField()) { int tagId = descriptor.getForyFieldId(); if (tagId >= 0) { if (!usedTagIds.add(tagId)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java index eb0dfafff2..4b3ab040b4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java @@ -89,7 +89,7 @@ static List buildFieldsInfoFromDescriptors( .map( descriptor -> { FieldType fieldType = FieldTypes.buildFieldType(resolver, descriptor); - if (descriptor.hasFieldSpec()) { + if (descriptor.hasForyField()) { int tagId = descriptor.getForyFieldId(); if (tagId >= 0) { if (!usedTagIds.add(tagId)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 9d591cef33..a38ad799ba 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -976,7 +976,7 @@ public int getUserTypeIdForTypeDef(Class cls) { @Override public boolean isMonomorphic(Descriptor descriptor) { - if (descriptor.hasFieldSpec()) { + if (descriptor.hasForyField()) { switch (descriptor.getMorphic()) { case TRUE: return false; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index be8b219f6c..c1fa96918b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1477,7 +1477,7 @@ private List buildFieldDescriptors( if (!searchParent && !descriptor.getDeclaringClass().equals(clz.getName())) { continue; } - boolean hasForyField = descriptor.hasFieldSpec(); + boolean hasForyField = descriptor.hasForyField(); // Compute the final isTrackingRef value: // For xlang mode: "Reference tracking is disabled by default" (xlang spec) // - Only enable ref tracking if explicitly set via @ForyField(ref=true) @@ -1735,7 +1735,7 @@ private boolean isFieldNullable(Descriptor descriptor) { if (isCrossLanguage()) { // For xlang mode: apply xlang defaults // This must match what TypeDefEncoder.buildFieldType uses for TypeDef metadata - if (descriptor.hasFieldSpec()) { + if (descriptor.hasForyField()) { // Use explicit annotation value return descriptor.isNullable(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java index ea08f56505..dbf35e8bfe 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java @@ -114,7 +114,7 @@ public static String computeStructFingerprint( // Get ref flag from @ForyField annotation only (compile-time info) // If annotation is absent or ref not explicitly set to true, ref is 0 // This allows fingerprint to be computed at compile time for C++/Rust - char ref = (descriptor.hasFieldSpec() && descriptor.isTrackingRef()) ? '1' : '0'; + char ref = (descriptor.hasForyField() && descriptor.isTrackingRef()) ? '1' : '0'; // Get nullable flag: // - Primitives are always non-nullable @@ -127,7 +127,7 @@ public static String computeStructFingerprint( } else if (resolver.isCrossLanguage()) { // For xlang: nullable defaults to false, except for Optional types, boxed types // If @ForyField annotation is present, use its nullable value - if (descriptor.hasFieldSpec()) { + if (descriptor.hasForyField()) { nullable = descriptor.isNullable() ? '1' : '0'; } else { // Default: Optional types, boxed primitives are nullable diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java index cfe529016e..a189578537 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java @@ -102,7 +102,9 @@ public static void clearDescriptorCache() { private final Method readMethod; private final Method writeMethod; private final ForyField foryField; - private final FieldSpec fieldSpec; + private final boolean hasForyField; + private final int foryFieldId; + private final ForyField.Dynamic dynamic; private final Annotation typeAnnotation; private final boolean arrayType; private boolean nullable; @@ -124,13 +126,15 @@ public Descriptor(Field field, TypeRef typeRef, Method readMethod, Method wri this.writeMethod = writeMethod; this.typeRef = typeRef; this.foryField = this.field.getAnnotation(ForyField.class); - this.fieldSpec = FieldSpec.from(foryField); + this.hasForyField = foryField != null; + this.foryFieldId = hasForyField ? foryField.id() : -1; + this.dynamic = hasForyField ? foryField.dynamic() : ForyField.Dynamic.AUTO; typeAnnotation = getAnnotation(field); arrayType = field.isAnnotationPresent(ArrayType.class); if (!typeRef.isPrimitive()) { - this.nullable = fieldSpec == null || fieldSpec.nullable(); + this.nullable = !hasForyField || foryField.nullable(); } - this.trackingRef = fieldSpec != null && fieldSpec.trackingRef(); + this.trackingRef = hasForyField && foryField.ref(); } public Descriptor( @@ -150,7 +154,9 @@ public Descriptor( this.readMethod = null; this.writeMethod = null; this.foryField = null; - this.fieldSpec = null; + this.hasForyField = false; + this.foryFieldId = -1; + this.dynamic = ForyField.Dynamic.AUTO; typeAnnotation = null; arrayType = false; this.nullable = nullable; @@ -163,8 +169,23 @@ public Descriptor( String name, int modifier, String declaringClass, - FieldSpec fieldSpec) { - this(typeRef, typeName, name, modifier, declaringClass, fieldSpec, false); + boolean hasForyField, + int foryFieldId, + boolean nullable, + boolean trackingRef, + ForyField.Dynamic dynamic) { + this( + typeRef, + typeName, + name, + modifier, + declaringClass, + hasForyField, + foryFieldId, + nullable, + trackingRef, + dynamic, + false); } public Descriptor( @@ -173,7 +194,11 @@ public Descriptor( String name, int modifier, String declaringClass, - FieldSpec fieldSpec, + boolean hasForyField, + int foryFieldId, + boolean nullable, + boolean trackingRef, + ForyField.Dynamic dynamic, boolean arrayType) { this.field = null; this.typeName = typeName; @@ -184,15 +209,17 @@ public Descriptor( this.readMethod = null; this.writeMethod = null; this.foryField = null; - this.fieldSpec = fieldSpec; + this.hasForyField = hasForyField; + this.foryFieldId = hasForyField ? foryFieldId : -1; + this.dynamic = hasForyField ? Objects.requireNonNull(dynamic) : ForyField.Dynamic.AUTO; typeAnnotation = null; this.arrayType = arrayType; if (typeRef.isPrimitive()) { this.nullable = false; } else { - this.nullable = fieldSpec == null || fieldSpec.nullable(); + this.nullable = nullable; } - this.trackingRef = fieldSpec != null && fieldSpec.trackingRef(); + this.trackingRef = hasForyField && trackingRef; } private Descriptor(Field field, Method readMethod) { @@ -208,13 +235,15 @@ private Descriptor(Field field, Method readMethod) { this.readMethod = readMethod; this.writeMethod = null; this.foryField = this.field.getAnnotation(ForyField.class); - this.fieldSpec = FieldSpec.from(foryField); + this.hasForyField = foryField != null; + this.foryFieldId = hasForyField ? foryField.id() : -1; + this.dynamic = hasForyField ? foryField.dynamic() : ForyField.Dynamic.AUTO; typeAnnotation = getAnnotation(field); arrayType = field.isAnnotationPresent(ArrayType.class); if (!field.getType().isPrimitive()) { - this.nullable = fieldSpec == null || fieldSpec.nullable(); + this.nullable = !hasForyField || foryField.nullable(); } - this.trackingRef = fieldSpec != null && fieldSpec.trackingRef(); + this.trackingRef = hasForyField && foryField.ref(); } private Descriptor(Method readMethod) { @@ -229,14 +258,16 @@ private Descriptor(Method readMethod) { this.readMethod = readMethod; this.writeMethod = null; this.foryField = readMethod.getAnnotation(ForyField.class); - this.fieldSpec = FieldSpec.from(foryField); + this.hasForyField = foryField != null; + this.foryFieldId = hasForyField ? foryField.id() : -1; + this.dynamic = hasForyField ? foryField.dynamic() : ForyField.Dynamic.AUTO; typeAnnotation = getTypeUseAnnotation(readMethod.getAnnotatedReturnType(), readMethod.getName()); arrayType = readMethod.isAnnotationPresent(ArrayType.class); if (!readMethod.getReturnType().isPrimitive()) { - this.nullable = fieldSpec == null || fieldSpec.nullable(); + this.nullable = !hasForyField || foryField.nullable(); } - this.trackingRef = fieldSpec != null && fieldSpec.trackingRef(); + this.trackingRef = hasForyField && foryField.ref(); } public Descriptor(DescriptorBuilder builder) { @@ -249,8 +280,19 @@ public Descriptor(DescriptorBuilder builder) { this.readMethod = builder.readMethod; this.writeMethod = builder.writeMethod; this.trackingRef = builder.trackingRef; - this.foryField = this.field == null ? null : this.field.getAnnotation(ForyField.class); - this.fieldSpec = builder.fieldSpec != null ? builder.fieldSpec : FieldSpec.from(foryField); + this.foryField = + builder.foryField != null + ? builder.foryField + : (this.field == null ? null : this.field.getAnnotation(ForyField.class)); + if (builder.hasForyField) { + this.hasForyField = true; + this.foryFieldId = builder.foryFieldId; + this.dynamic = Objects.requireNonNull(builder.dynamic); + } else { + this.hasForyField = foryField != null; + this.foryFieldId = hasForyField ? foryField.id() : -1; + this.dynamic = hasForyField ? foryField.dynamic() : ForyField.Dynamic.AUTO; + } typeAnnotation = field == null ? null : getAnnotation(field); arrayType = builder.arrayType @@ -335,20 +377,20 @@ public ForyField getForyField() { return foryField; } - public FieldSpec getFieldSpec() { - return fieldSpec; + public boolean hasForyField() { + return hasForyField; } - public boolean hasFieldSpec() { - return fieldSpec != null; + public ForyField.Dynamic getForyFieldDynamic() { + return dynamic; } public boolean hasForyFieldId() { - return fieldSpec != null && fieldSpec.id() >= 0; + return hasForyField && foryFieldId >= 0; } public int getForyFieldId() { - return fieldSpec == null ? -1 : fieldSpec.id(); + return foryFieldId; } /** @@ -357,10 +399,7 @@ public int getForyFieldId() { * @return the morphic setting from @ForyField annotation, or AUTO if not specified */ public ForyField.Dynamic getMorphic() { - if (fieldSpec != null) { - return fieldSpec.dynamic(); - } - return ForyField.Dynamic.AUTO; + return dynamic; } public Annotation getTypeAnnotation() { diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java index aefdb79208..308cb8ce09 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java @@ -37,7 +37,9 @@ public class DescriptorBuilder { Method readMethod; Method writeMethod; ForyField foryField; - FieldSpec fieldSpec; + boolean hasForyField; + int foryFieldId = -1; + ForyField.Dynamic dynamic = ForyField.Dynamic.AUTO; boolean arrayType; boolean nullable; boolean trackingRef; @@ -54,7 +56,9 @@ public DescriptorBuilder(Descriptor descriptor) { this.readMethod = descriptor.getReadMethod(); this.writeMethod = descriptor.getWriteMethod(); this.foryField = descriptor.getForyField(); - this.fieldSpec = descriptor.getFieldSpec(); + this.hasForyField = descriptor.hasForyField(); + this.foryFieldId = descriptor.getForyFieldId(); + this.dynamic = descriptor.getForyFieldDynamic(); this.arrayType = descriptor.isArrayType(); this.nullable = descriptor.isNullable(); this.trackingRef = descriptor.isTrackingRef(); @@ -118,12 +122,16 @@ public DescriptorBuilder trackingRef(boolean trackingRef) { public DescriptorBuilder foryField(ForyField foryField) { this.foryField = foryField; - this.fieldSpec = FieldSpec.from(foryField); - return this; - } - - public DescriptorBuilder fieldSpec(FieldSpec fieldSpec) { - this.fieldSpec = fieldSpec; + this.hasForyField = foryField != null; + if (hasForyField) { + this.foryFieldId = foryField.id(); + this.nullable = foryField.nullable(); + this.trackingRef = foryField.ref(); + this.dynamic = foryField.dynamic(); + } else { + this.foryFieldId = -1; + this.dynamic = ForyField.Dynamic.AUTO; + } return this; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/FieldSpec.java b/java/fory-core/src/main/java/org/apache/fory/type/FieldSpec.java deleted file mode 100644 index 64983789d8..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/type/FieldSpec.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.type; - -import java.util.Objects; -import org.apache.fory.annotation.ForyField; -import org.apache.fory.annotation.Internal; - -/** Descriptor-owned representation of {@link ForyField} values for generated descriptors. */ -@Internal -public final class FieldSpec { - private final int id; - private final boolean nullable; - private final boolean trackingRef; - private final ForyField.Dynamic dynamic; - - public static FieldSpec of( - int id, boolean nullable, boolean trackingRef, ForyField.Dynamic dynamic) { - return new FieldSpec(id, nullable, trackingRef, dynamic); - } - - public static FieldSpec from(ForyField field) { - if (field == null) { - return null; - } - return of(field.id(), field.nullable(), field.ref(), field.dynamic()); - } - - private FieldSpec(int id, boolean nullable, boolean trackingRef, ForyField.Dynamic dynamic) { - this.id = id; - this.nullable = nullable; - this.trackingRef = trackingRef; - this.dynamic = Objects.requireNonNull(dynamic); - } - - public int id() { - return id; - } - - public boolean nullable() { - return nullable; - } - - public boolean trackingRef() { - return trackingRef; - } - - public ForyField.Dynamic dynamic() { - return dynamic; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof FieldSpec)) { - return false; - } - FieldSpec that = (FieldSpec) o; - return id == that.id - && nullable == that.nullable - && trackingRef == that.trackingRef - && dynamic == that.dynamic; - } - - @Override - public int hashCode() { - return Objects.hash(id, nullable, trackingRef, dynamic); - } -} From 7f807f4b9d1ce8da047dac3bf58090a5329b61d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 01:20:14 +0800 Subject: [PATCH 06/58] feat(java): add static compatible meta readers --- docs/guide/java/graalvm-support.md | 6 +- docs/guide/java/static-struct-serializers.md | 4 +- .../StaticSerializerSourceWriter.java | 144 +++++- .../processing/ForyStructProcessorTest.java | 64 ++- .../org/apache/fory/builder/CodecUtils.java | 2 +- .../CompatibleMetaSharedCodecBuilder.java | 38 -- .../org/apache/fory/builder/Generated.java | 23 +- .../fory/builder/MetaSharedCodecBuilder.java | 44 +- .../builder/StaticCompatibleCodecBuilder.java | 463 ++++++++++++++++++ .../apache/fory/platform/GraalvmSupport.java | 13 + .../apache/fory/resolver/ClassResolver.java | 5 +- .../apache/fory/resolver/TypeResolver.java | 15 +- .../StaticGeneratedStructSerializer.java | 9 + 13 files changed, 712 insertions(+), 118 deletions(-) delete mode 100644 java/fory-core/src/main/java/org/apache/fory/builder/CompatibleMetaSharedCodecBuilder.java create mode 100644 java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java diff --git a/docs/guide/java/graalvm-support.md b/docs/guide/java/graalvm-support.md index a25a5f816d..eae6f3df9d 100644 --- a/docs/guide/java/graalvm-support.md +++ b/docs/guide/java/graalvm-support.md @@ -41,9 +41,9 @@ Note: Fory's `asyncCompilationEnabled` option is automatically disabled for Graa `@ForyStruct` annotation-processor static serializers are not the GraalVM native-image serializer path. Native image builds use the Fory GraalVM registry: equal local and remote `TypeDef` hashes use the existing meta-shared generated serializer, while mismatched remote schemas use a build-time -generated read-only compatible serializer cached by remote `TypeDef` id. Runtime native images load -those cached classes from the registry and do not attempt annotation-generated static serializer -lookup, Janino, or runtime class definition. +generated read-only compatible serializer cached by local Java class. Runtime native images load +that cached class from the registry, pass the current remote `TypeDef` into its constructor, and do +not attempt annotation-generated static serializer lookup, Janino, or runtime class definition. ## Basic Usage diff --git a/docs/guide/java/static-struct-serializers.md b/docs/guide/java/static-struct-serializers.md index c424d05103..5bdd331f7f 100644 --- a/docs/guide/java/static-struct-serializers.md +++ b/docs/guide/java/static-struct-serializers.md @@ -72,7 +72,9 @@ lookup is deterministic by generated class name and does not scan the classpath. GraalVM native image does not use annotation-processor-generated static serializer classes. Native image builds use the GraalVM registry path: matching local and remote `TypeDef` hashes use the existing meta-shared generated serializer, and mismatched hashes use a build-time generated -read-only compatible serializer cached by remote `TypeDef` id. +read-only compatible serializer cached by local Java class. At runtime, the compatible serializer +constructor receives the current remote `TypeDef` and derives the remote-field layout from that +metadata. ## Field Access Rules diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 0536855222..3b62ad29c7 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -20,6 +20,8 @@ package org.apache.fory.annotation.processing; final class StaticSerializerSourceWriter { + private static final int DISPATCH_GROUP_SIZE = 8; + private final SourceStruct struct; private final StringBuilder builder = new StringBuilder(16384); @@ -354,33 +356,30 @@ private void writeCompatibleRead() { builder.append(" return readSchemaConsistent(readContext);\n"); builder.append(" }\n"); if (struct.record) { + builder.append(" Object[] values = new Object[DESCRIPTORS.size()];\n"); for (SourceField field : struct.fields) { builder - .append(" ") - .append(field.erasedType) - .append(" field") + .append(" values[") .append(field.id) - .append(" = ") + .append("] = ") .append(field.defaultValue()) .append(";\n"); } builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); builder.append(" RemoteFieldInfo remoteField = remoteFields.get(i);\n"); - builder.append(" switch (matchedId(remoteField)) {\n"); + builder.append( + " readCompatibleRecordField(readContext, values, remoteField, matchedId(remoteField));\n"); + builder.append(" }\n"); for (SourceField field : struct.fields) { - builder.append(" case ").append(field.id).append(":\n"); builder - .append(" field") + .append(" ") + .append(field.erasedType) + .append(" field") .append(field.id) .append(" = ") - .append(field.castExpression("readRemoteField(readContext, remoteField)")) + .append(field.castExpression("values[" + field.id + "]")) .append(";\n"); - builder.append(" break;\n"); } - builder.append(" default:\n"); - builder.append(" skipField(readContext, remoteField);\n"); - builder.append(" }\n"); - builder.append(" }\n"); builder.append(" return new ").append(struct.typeName).append("("); appendRecordConstructorArguments("field"); builder.append(");\n"); @@ -389,24 +388,115 @@ private void writeCompatibleRead() { builder.append(" readContext.reference(value);\n"); builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); builder.append(" RemoteFieldInfo remoteField = remoteFields.get(i);\n"); - builder.append(" switch (matchedId(remoteField)) {\n"); - for (SourceField field : struct.fields) { - builder.append(" case ").append(field.id).append(":\n"); - builder - .append(" ") - .append( - field.writeStatement( - "value", field.castExpression("readRemoteField(readContext, remoteField)"))) - .append("\n"); - builder.append(" break;\n"); - } - builder.append(" default:\n"); - builder.append(" skipField(readContext, remoteField);\n"); - builder.append(" }\n"); + builder.append( + " readCompatibleField(readContext, value, remoteField, matchedId(remoteField));\n"); builder.append(" }\n"); builder.append(" return value;\n"); } builder.append(" }\n\n"); + writeCompatibleDispatchMethods(); + } + + private void writeCompatibleDispatchMethods() { + int groupCount = (struct.fields.size() + DISPATCH_GROUP_SIZE - 1) / DISPATCH_GROUP_SIZE; + if (struct.record) { + writeCompatibleDispatchRouter("readCompatibleRecordField", true, groupCount); + for (int group = 0; group < groupCount; group++) { + writeCompatibleRecordDispatchGroup(group); + } + } else { + writeCompatibleDispatchRouter("readCompatibleField", false, groupCount); + for (int group = 0; group < groupCount; group++) { + writeCompatibleBeanDispatchGroup(group); + } + } + } + + private void writeCompatibleDispatchRouter(String methodName, boolean record, int groupCount) { + builder.append(" private void ").append(methodName).append("("); + appendCompatibleDispatchParameters(record); + builder.append(") {\n"); + for (int group = 0; group < groupCount; group++) { + int upperBound = Math.min(struct.fields.size(), (group + 1) * DISPATCH_GROUP_SIZE); + if (group == 0) { + builder.append(" if (matchedId >= 0 && matchedId < ").append(upperBound).append(") {\n"); + } else { + builder.append(" if (matchedId < ").append(upperBound).append(") {\n"); + } + builder.append(" ").append(methodName).append(group).append("("); + appendCompatibleDispatchArguments(record); + builder.append(");\n"); + builder.append(" return;\n"); + builder.append(" }\n"); + } + builder.append(" skipField(readContext, remoteField);\n"); + builder.append(" }\n\n"); + } + + private void writeCompatibleBeanDispatchGroup(int group) { + int start = group * DISPATCH_GROUP_SIZE; + int end = Math.min(struct.fields.size(), start + DISPATCH_GROUP_SIZE); + builder.append(" private void readCompatibleField").append(group).append("("); + appendCompatibleDispatchParameters(false); + builder.append(") {\n"); + builder.append(" switch (matchedId) {\n"); + for (int i = start; i < end; i++) { + SourceField field = struct.fields.get(i); + builder.append(" case ").append(field.id).append(":\n"); + builder + .append(" ") + .append( + field.writeStatement( + "value", field.castExpression("readRemoteField(readContext, remoteField)"))) + .append("\n"); + builder.append(" return;\n"); + } + builder.append(" default:\n"); + builder.append(" skipField(readContext, remoteField);\n"); + builder.append(" }\n"); + builder.append(" }\n\n"); + } + + private void writeCompatibleRecordDispatchGroup(int group) { + int start = group * DISPATCH_GROUP_SIZE; + int end = Math.min(struct.fields.size(), start + DISPATCH_GROUP_SIZE); + builder.append(" private void readCompatibleRecordField").append(group).append("("); + appendCompatibleDispatchParameters(true); + builder.append(") {\n"); + builder.append(" switch (matchedId) {\n"); + for (int i = start; i < end; i++) { + SourceField field = struct.fields.get(i); + builder.append(" case ").append(field.id).append(":\n"); + builder + .append(" values[") + .append(field.id) + .append("] = readRemoteField(readContext, remoteField);\n"); + builder.append(" return;\n"); + } + builder.append(" default:\n"); + builder.append(" skipField(readContext, remoteField);\n"); + builder.append(" }\n"); + builder.append(" }\n\n"); + } + + private void appendCompatibleDispatchParameters(boolean record) { + builder.append("ReadContext readContext, "); + if (record) { + builder.append("Object[] values, "); + } else { + builder.append(struct.typeName).append(" value, "); + } + builder.append("RemoteFieldInfo remoteField, int matchedId"); + } + + private void appendCompatibleDispatchArguments(boolean record) { + builder.append("readContext, "); + if (record) { + builder.append("values, "); + } else { + builder.append("value, "); + } + builder.append("remoteField, matchedId"); } private void writeCopy() { diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index d5aae45d77..571aacc17a 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -20,6 +20,7 @@ package org.apache.fory.annotation.processing; import java.io.IOException; +import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.net.URL; import java.net.URLClassLoader; @@ -41,10 +42,11 @@ import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; +import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.TypeDef; import org.apache.fory.platform.AndroidSupport; +import org.apache.fory.serializer.MetaSharedSerializer; import org.apache.fory.serializer.Serializer; -import org.apache.fory.serializer.Serializers; import org.apache.fory.serializer.StaticGeneratedStructSerializer; import org.apache.fory.type.Descriptor; import org.apache.fory.type.Types; @@ -266,6 +268,10 @@ public void testRecordReadAndCopyUseCanonicalConstructor() throws Exception { + " @Ignore public String ignored() { return ignored; }\n" + "}\n"); Assert.assertTrue(result.success, result.diagnostics()); + String generatedSource = + result.generatedSource("test/RecordStruct__ForyStaticSerializer__.java"); + Assert.assertTrue(generatedSource.contains("private void readCompatibleRecordField0(")); + Assert.assertTrue(generatedSource.contains("switch (matchedId)")); try (URLClassLoader loader = result.classLoader()) { Class type = loader.loadClass("test.RecordStruct"); Object value = @@ -315,6 +321,13 @@ public void testCompatibleReadUsesGeneratedSerializer() throws Exception { + "}\n"); Assert.assertTrue(writerResult.success, writerResult.diagnostics()); Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + String generatedSource = + readerResult.generatedSource("test/EvolvingStruct__ForyStaticSerializer__.java"); + Assert.assertTrue( + generatedSource.contains( + "readCompatibleField(readContext, value, remoteField, matchedId(remoteField))")); + Assert.assertTrue(generatedSource.contains("private void readCompatibleField0(")); + Assert.assertTrue(generatedSource.contains("switch (matchedId)")); try (URLClassLoader writerLoader = writerResult.classLoader(); URLClassLoader readerLoader = readerResult.classLoader()) { Class writerType = writerLoader.loadClass("test.EvolvingStruct"); @@ -355,7 +368,7 @@ public void testCompatibleReadUsesGeneratedSerializer() throws Exception { } @Test - public void testGraalvmCompatibleMetaSharedGeneratorIsReadOnly() throws Exception { + public void testGraalvmStaticCompatibleSerializerReadsRuntimeRemoteTypeDef() throws Exception { if (AndroidSupport.IS_ANDROID) { return; } @@ -365,6 +378,7 @@ public void testGraalvmCompatibleMetaSharedGeneratorIsReadOnly() throws Exceptio "package test;\n" + "public class NativeImageStruct {\n" + " public int id;\n" + + " public int legacy;\n" + " public NativeImageStruct() {}\n" + "}\n"); CompilationResult readerResult = @@ -409,9 +423,36 @@ public void testGraalvmCompatibleMetaSharedGeneratorIsReadOnly() throws Exceptio reader.getTypeResolver(), (Class) readerType, remoteTypeDef); Assert.assertTrue( GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(serializerClass)); + Constructor> constructor = + serializerClass.getConstructor( + org.apache.fory.resolver.TypeResolver.class, Class.class, TypeDef.class); Serializer serializer = - Serializers.newSerializer( - reader.getTypeResolver(), (Class) readerType, serializerClass); + constructor.newInstance( + reader.getTypeResolver(), (Class) readerType, remoteTypeDef); + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "id", 42); + setField(writerType, writerValue, "legacy", 99); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(128); + MetaSharedSerializer writerSerializer = + new MetaSharedSerializer<>( + writer.getTypeResolver(), (Class) writerType, remoteTypeDef); + writer.getWriteContext().prepare(buffer, null); + try { + writerSerializer.write(writer.getWriteContext(), writerValue); + } finally { + writer.getWriteContext().reset(); + } + buffer.readerIndex(0); + reader.getReadContext().prepare(buffer, null, false); + Object result; + try { + result = serializer.read(reader.getReadContext()); + } finally { + reader.getReadContext().reset(); + } + Assert.assertSame(result.getClass(), readerType); + Assert.assertEquals(getField(readerType, result, "id"), 42); + Assert.assertEquals(getField(readerType, result, "added"), "default"); Assert.assertThrows( UnsupportedOperationException.class, () -> @@ -447,7 +488,8 @@ private static CompilationResult compile(String typeName, String source) throws JavaCompiler.CompilationTask task = compiler.getTask(null, fileManager, diagnostics, options, null, units); task.setProcessors(Collections.singletonList(new ForyStructProcessor())); - return new CompilationResult(classRoot, task.call(), diagnostics.getDiagnostics()); + return new CompilationResult( + classRoot, root.resolve("generated"), task.call(), diagnostics.getDiagnostics()); } } @@ -489,12 +531,17 @@ private static Descriptor descriptor(List descriptors, String name) private static final class CompilationResult { final Path classRoot; + final Path generatedRoot; final boolean success; final List> diagnostics; CompilationResult( - Path classRoot, boolean success, List> diagnostics) { + Path classRoot, + Path generatedRoot, + boolean success, + List> diagnostics) { this.classRoot = classRoot; + this.generatedRoot = generatedRoot; this.success = success; this.diagnostics = new ArrayList<>(diagnostics); } @@ -504,6 +551,11 @@ URLClassLoader classLoader() throws IOException { return new URLClassLoader(urls, ForyStructProcessorTest.class.getClassLoader()); } + String generatedSource(String relativePath) throws IOException { + return new String( + Files.readAllBytes(generatedRoot.resolve(relativePath)), StandardCharsets.UTF_8); + } + String diagnostics() { StringBuilder builder = new StringBuilder(); for (Diagnostic diagnostic : diagnostics) { diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index 76225b6fab..2bb4665f40 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -82,7 +82,7 @@ public static Class> loadOrGenCompatibleMetaSharedCo fory, () -> loadOrGenCodecClass( - cls, fory, new CompatibleMetaSharedCodecBuilder(TypeRef.of(cls), fory, typeDef))); + cls, fory, new StaticCompatibleCodecBuilder(TypeRef.of(cls), fory, typeDef))); } public static Class> loadOrGenCompatibleMetaSharedCodecClass( diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleMetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleMetaSharedCodecBuilder.java deleted file mode 100644 index 4884b423be..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleMetaSharedCodecBuilder.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.builder; - -import org.apache.fory.Fory; -import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; -import org.apache.fory.meta.TypeDef; -import org.apache.fory.reflect.TypeRef; - -/** Builds GraalVM read-only compatible serializers for remote {@link TypeDef} schemas. */ -public final class CompatibleMetaSharedCodecBuilder extends MetaSharedCodecBuilder { - public CompatibleMetaSharedCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { - super( - beanType, - fory, - typeDef, - GeneratedCompatibleMetaSharedSerializer.class, - false, - "CompatibleMetaShared"); - } -} diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java index e487ccfe71..e63bc95fbb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java @@ -21,9 +21,11 @@ import java.lang.reflect.Field; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import org.apache.fory.context.CopyContext; +import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.ReflectionUtils; @@ -31,6 +33,8 @@ import org.apache.fory.serializer.AbstractObjectSerializer; import org.apache.fory.serializer.MetaSharedLayerSerializerBase; import org.apache.fory.serializer.Serializer; +import org.apache.fory.serializer.StaticGeneratedStructSerializer; +import org.apache.fory.type.Descriptor; import org.apache.fory.util.Preconditions; /** @@ -118,14 +122,17 @@ public void write(WriteContext writeContext, Object value) { } } - /** - * Base class for GraalVM build-time compatible read serializers for remote {@link TypeDef} - * schemas. - */ - abstract class GeneratedCompatibleMetaSharedSerializer extends GeneratedSerializer - implements Generated { - public GeneratedCompatibleMetaSharedSerializer(TypeResolver typeResolver, Class cls) { - super(typeResolver, cls); + /** Base class for GraalVM build-time compatible read serializers. */ + abstract class GeneratedCompatibleMetaSharedSerializer + extends StaticGeneratedStructSerializer implements Generated { + public GeneratedCompatibleMetaSharedSerializer( + TypeResolver typeResolver, Class cls, TypeDef typeDef, List descriptors) { + super(typeResolver, cls, typeDef, descriptors); + } + + @Override + public final Object read(ReadContext readContext) { + return readCompatible(readContext); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index a657b3af84..3fdc1c6dbc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -30,7 +30,6 @@ import java.util.SortedMap; import java.util.concurrent.ConcurrentHashMap; import org.apache.fory.Fory; -import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; import org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; @@ -82,27 +81,13 @@ public class MetaSharedCodecBuilder extends ObjectCodecBuilder { private final TypeDef typeDef; private final String defaultValueLanguage; private final DefaultValueUtils.DefaultValueField[] defaultValueFields; - private final boolean writeDelegate; - private final String codecKind; public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { - this(beanType, fory, typeDef, GeneratedMetaSharedSerializer.class, true, "MetaShared"); - } - - protected MetaSharedCodecBuilder( - TypeRef beanType, - Fory fory, - TypeDef typeDef, - Class parentSerializerClass, - boolean writeDelegate, - String codecKind) { - super(beanType, fory, parentSerializerClass); + super(beanType, fory, GeneratedMetaSharedSerializer.class); Preconditions.checkArgument( !fory.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); this.typeDef = typeDef; - this.writeDelegate = writeDelegate; - this.codecKind = codecKind; DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(typeDef, beanClass)); List sortedDescriptors = grouper.getSortedDescriptors(); if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { @@ -161,7 +146,7 @@ protected String codecSuffix() { id = idGenerator.computeIfAbsent(typeDef.getId(), k -> idGenerator.size()); } } - return codecKind + id; + return "MetaShared" + id; } @Override @@ -186,19 +171,17 @@ public String genCode() { ctx.type(concreteTypeResolverType), "cls", POJO_CLASS_TYPE_NAME); - if (writeDelegate) { - constructorCode += - StringUtils.format( - "${serializer} = ${builderClass}.setCodegenSerializer(${typeResolver}, ${cls}, this);\n", - "serializer", - SERIALIZER_FIELD_NAME, - "builderClass", - MetaSharedCodecBuilder.class.getName(), - "typeResolver", - CONSTRUCTOR_TYPE_RESOLVER_NAME, - "cls", - POJO_CLASS_TYPE_NAME); - } + constructorCode += + StringUtils.format( + "${serializer} = ${builderClass}.setCodegenSerializer(${typeResolver}, ${cls}, this);\n", + "serializer", + SERIALIZER_FIELD_NAME, + "builderClass", + MetaSharedCodecBuilder.class.getName(), + "typeResolver", + CONSTRUCTOR_TYPE_RESOLVER_NAME, + "cls", + POJO_CLASS_TYPE_NAME); ctx.clearExprState(); Expression decodeExpr = buildDecodeExpression(); String decodeCode = decodeExpr.genCode(ctx).code(); @@ -231,7 +214,6 @@ public String genCode() { protected void addCommonImports() { super.addCommonImports(); ctx.addImport(GeneratedMetaSharedSerializer.class); - ctx.addImport(GeneratedCompatibleMetaSharedSerializer.class); } // Invoked by JIT. diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java new file mode 100644 index 0000000000..0be893ae3a --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -0,0 +1,463 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.builder; + +import static org.apache.fory.type.TypeUtils.OBJECT_TYPE; + +import java.util.Collections; +import java.util.List; +import org.apache.fory.Fory; +import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.codegen.Code; +import org.apache.fory.codegen.CodeGenerator; +import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.Invoke; +import org.apache.fory.codegen.Expression.Reference; +import org.apache.fory.config.ForyBuilder; +import org.apache.fory.context.ReadContext; +import org.apache.fory.meta.TypeDef; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.MetaSharedSerializer; +import org.apache.fory.type.Descriptor; +import org.apache.fory.util.Preconditions; +import org.apache.fory.util.StringUtils; +import org.apache.fory.util.record.RecordComponent; +import org.apache.fory.util.record.RecordUtils; + +/** + * Builds GraalVM read-only compatible serializers that bind the runtime remote {@link TypeDef} in + * the serializer constructor. + * + *

The generated class is keyed by the local Java class, not by a fixed remote schema. Its + * constructor receives the runtime remote {@link TypeDef}; {@link + * org.apache.fory.serializer.StaticGeneratedStructSerializer} derives the remote field list from + * that definition and the generated local descriptors. + * + * @see ForyBuilder#withMetaShare + * @see MetaSharedSerializer + */ +public final class StaticCompatibleCodecBuilder extends ObjectCodecBuilder { + private static final int DISPATCH_GROUP_SIZE = 8; + + private final List localDescriptors; + + public StaticCompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { + super(beanType, fory, GeneratedCompatibleMetaSharedSerializer.class); + Preconditions.checkArgument( + !fory.getConfig().checkClassVersion(), + "Class version check should be disabled when compatible mode is enabled."); + localDescriptors = Collections.unmodifiableList(Descriptor.getDescriptors(beanClass)); + } + + @Override + protected String codecSuffix() { + return "CompatibleMetaShared"; + } + + @Override + public String genCode() { + ctx.setPackage(CodeGenerator.getPackage(beanClass)); + String className = codecClassName(beanClass); + ctx.setClassName(className); + ctx.extendsClasses(ctx.type(parentSerializerClass)); + ctx.reserveName(POJO_CLASS_TYPE_NAME); + ctx.addImports(List.class, TypeDef.class, Descriptor.class); + String constructorCode = + StringUtils.format( + "" + + "super(${typeResolver}, ${cls}, ${typeDef}, Descriptor.getDescriptors(${cls}));\n" + + "this.${generatedTypeResolver} = (${generatedTypeResolverType}) ${typeResolver};\n", + "typeResolver", + CONSTRUCTOR_TYPE_RESOLVER_NAME, + "cls", + POJO_CLASS_TYPE_NAME, + "typeDef", + "_f_typeDef", + "generatedTypeResolver", + TYPE_RESOLVER_NAME, + "generatedTypeResolverType", + ctx.type(concreteTypeResolverType)); + ctx.addConstructor( + constructorCode, + TypeResolver.class, + CONSTRUCTOR_TYPE_RESOLVER_NAME, + Class.class, + POJO_CLASS_TYPE_NAME, + TypeDef.class, + "_f_typeDef"); + ctx.addMethod("getDescriptors", "return Descriptor.getDescriptors(type);", List.class); + ctx.overrideMethod( + "readCompatible", + isRecord ? genRecordCompatibleRead() : genObjectCompatibleRead(), + Object.class, + ReadContext.class, + READ_CONTEXT_NAME); + genDispatchMethods(); + return ctx.genCode(); + } + + @Override + protected void addCommonImports() { + super.addCommonImports(); + ctx.addImport(GeneratedCompatibleMetaSharedSerializer.class); + } + + @Override + public Expression buildEncodeExpression() { + throw new IllegalStateException("unreachable"); + } + + @Override + public Expression buildDecodeExpression() { + throw new IllegalStateException("unreachable"); + } + + private String genObjectCompatibleRead() { + ctx.clearExprState(); + Code.ExprCode beanCode = newBean().genCode(ctx); + String bean = beanCode.value().toString(); + StringBuilder code = new StringBuilder(); + if (StringUtils.isNotBlank(beanCode.code())) { + code.append(beanCode.code()).append('\n'); + } + code.append("if (") + .append(READ_CONTEXT_NAME) + .append(".hasPreservedRefId()) {\n") + .append(" ") + .append(READ_CONTEXT_NAME) + .append(".reference(") + .append(bean) + .append(");\n") + .append("}\n") + .append("for (int _f_i = 0; _f_i < remoteFields.size(); _f_i++) {\n") + .append(" RemoteFieldInfo _f_remoteField = (RemoteFieldInfo) remoteFields.get(_f_i);\n") + .append(" readMatchedField(") + .append(READ_CONTEXT_NAME) + .append(", ") + .append(bean) + .append(", _f_remoteField, matchedId(_f_remoteField));\n") + .append("}\n") + .append("return ") + .append(bean) + .append(';'); + return code.toString(); + } + + private String genRecordCompatibleRead() { + RecordComponent[] components = RecordUtils.getRecordComponents(beanClass); + StringBuilder code = new StringBuilder(); + String recordValues; + if (recordCtrAccessible) { + recordValues = "_f_recordValues"; + code.append("Object[] ") + .append(recordValues) + .append(" = new Object[") + .append(components.length) + .append("];\n"); + for (int i = 0; i < components.length; i++) { + String defaultValue = boxedDefaultValue(components[i].getType()); + if (defaultValue != null) { + code.append(recordValues) + .append("[") + .append(i) + .append("] = ") + .append(defaultValue) + .append(";\n"); + } + } + } else { + ctx.clearExprState(); + Code.ExprCode recordValuesCode = buildComponentsArray().genCode(ctx); + recordValues = recordValuesCode.value().toString(); + if (StringUtils.isNotBlank(recordValuesCode.code())) { + code.append(recordValuesCode.code()).append('\n'); + } + } + code.append("for (int _f_i = 0; _f_i < remoteFields.size(); _f_i++) {\n") + .append(" RemoteFieldInfo _f_remoteField = (RemoteFieldInfo) remoteFields.get(_f_i);\n") + .append(" readMatchedRecordField(") + .append(READ_CONTEXT_NAME) + .append(", ") + .append(recordValues) + .append(", _f_remoteField, matchedId(_f_remoteField));\n") + .append("}\n"); + if (recordCtrAccessible) { + code.append("return new ") + .append(ctx.type(beanClass)) + .append("(") + .append(recordConstructorArgs(components, recordValues)) + .append(");"); + } else { + ctx.clearExprState(); + Reference values = new Reference(recordValues, objectArrayTypeRef, false); + Code.ExprCode newRecord = + new Invoke(getObjectCreator(beanClass), "newInstanceWithArguments", OBJECT_TYPE, values) + .genCode(ctx); + if (StringUtils.isNotBlank(newRecord.code())) { + code.append(newRecord.code()).append('\n'); + } + code.append("return ").append(newRecord.value()).append(';'); + } + return code.toString(); + } + + private void genDispatchMethods() { + String valueType = + ctx.sourcePublicAccessible(beanClass) ? ctx.type(beanClass) : ctx.type(Object.class); + TypeRef valueTypeRef = ctx.sourcePublicAccessible(beanClass) ? beanType : OBJECT_TYPE; + int groupCount = (localDescriptors.size() + DISPATCH_GROUP_SIZE - 1) / DISPATCH_GROUP_SIZE; + if (isRecord) { + ctx.addMethod( + "private", + "readMatchedRecordField", + genDispatchRouter("readMatchedRecordField", groupCount), + void.class, + ReadContext.class, + READ_CONTEXT_NAME, + Object[].class, + "_f_recordValues", + "RemoteFieldInfo", + "_f_remoteField", + int.class, + "_f_matchedId"); + for (int group = 0; group < groupCount; group++) { + ctx.addMethod( + "private", + "readMatchedRecordField" + group, + genRecordDispatchGroup(group), + void.class, + ReadContext.class, + READ_CONTEXT_NAME, + Object[].class, + "_f_recordValues", + "RemoteFieldInfo", + "_f_remoteField", + int.class, + "_f_matchedId"); + } + return; + } + ctx.addMethod( + "private", + "readMatchedField", + genDispatchRouter("readMatchedField", groupCount), + void.class, + ReadContext.class, + READ_CONTEXT_NAME, + valueType, + "_f_value", + "RemoteFieldInfo", + "_f_remoteField", + int.class, + "_f_matchedId"); + for (int group = 0; group < groupCount; group++) { + ctx.addMethod( + "private", + "readMatchedField" + group, + genObjectDispatchGroup(group, valueTypeRef), + void.class, + ReadContext.class, + READ_CONTEXT_NAME, + valueType, + "_f_value", + "RemoteFieldInfo", + "_f_remoteField", + int.class, + "_f_matchedId"); + } + } + + private String genDispatchRouter(String methodPrefix, int groupCount) { + StringBuilder code = new StringBuilder(); + for (int group = 0; group < groupCount; group++) { + int upperBound = Math.min(localDescriptors.size(), (group + 1) * DISPATCH_GROUP_SIZE); + if (group == 0) { + code.append("if (_f_matchedId >= 0 && _f_matchedId < ").append(upperBound).append(") {\n"); + } else { + code.append("if (_f_matchedId < ").append(upperBound).append(") {\n"); + } + code.append(" ") + .append(methodPrefix) + .append(group) + .append("(") + .append(READ_CONTEXT_NAME) + .append(", "); + code.append(isRecord ? "_f_recordValues" : "_f_value"); + code.append(", _f_remoteField, _f_matchedId);\n").append(" return;\n").append("}\n"); + } + code.append("skipField(").append(READ_CONTEXT_NAME).append(", _f_remoteField);"); + return code.toString(); + } + + private String genObjectDispatchGroup(int group, TypeRef valueTypeRef) { + int start = group * DISPATCH_GROUP_SIZE; + int end = Math.min(localDescriptors.size(), start + DISPATCH_GROUP_SIZE); + StringBuilder code = new StringBuilder("switch (_f_matchedId) {\n"); + for (int i = start; i < end; i++) { + Descriptor descriptor = localDescriptors.get(i); + code.append(" case ") + .append(i) + .append(": {\n") + .append(" Object _f_fieldValue = readRemoteField(") + .append(READ_CONTEXT_NAME) + .append(", _f_remoteField);\n") + .append(" if (hasFieldConverter(_f_remoteField)) {\n") + .append(" setConvertedField(_f_value, _f_fieldValue, _f_remoteField);\n") + .append(" } else {\n") + .append(indent(genSetFieldCode(descriptor, valueTypeRef), 6)) + .append('\n') + .append(" }\n") + .append(" return;\n") + .append(" }\n"); + } + code.append(" default:\n") + .append(" skipField(") + .append(READ_CONTEXT_NAME) + .append(", _f_remoteField);\n") + .append("}\n"); + return code.toString(); + } + + private String genRecordDispatchGroup(int group) { + int start = group * DISPATCH_GROUP_SIZE; + int end = Math.min(localDescriptors.size(), start + DISPATCH_GROUP_SIZE); + StringBuilder code = new StringBuilder("switch (_f_matchedId) {\n"); + for (int i = start; i < end; i++) { + Descriptor descriptor = localDescriptors.get(i); + Integer componentIndex = recordReversedMapping.get(descriptor.getName()); + if (componentIndex == null) { + continue; + } + code.append(" case ") + .append(i) + .append(": {\n") + .append(" if (hasFieldConverter(_f_remoteField)) {\n") + .append(" readRemoteField(") + .append(READ_CONTEXT_NAME) + .append(", _f_remoteField);\n") + .append(" } else {\n") + .append(" _f_recordValues[") + .append(componentIndex) + .append("] = readRemoteField(") + .append(READ_CONTEXT_NAME) + .append(", _f_remoteField);\n") + .append(" }\n") + .append(" return;\n") + .append(" }\n"); + } + code.append(" default:\n") + .append(" skipField(") + .append(READ_CONTEXT_NAME) + .append(", _f_remoteField);\n") + .append("}\n"); + return code.toString(); + } + + private String genSetFieldCode(Descriptor descriptor, TypeRef valueTypeRef) { + ctx.clearExprState(); + Reference value = new Reference("_f_value", valueTypeRef, false); + Reference fieldValue = new Reference("_f_fieldValue", OBJECT_TYPE, false); + Expression setField = + setFieldValue(value, descriptor, tryInlineCast(fieldValue, descriptor.getTypeRef())); + Code.ExprCode setCode = setField.genCode(ctx); + String code = ctx.optimizeMethodCode(setCode.code()); + return code == null ? "" : code; + } + + private String recordConstructorArgs(RecordComponent[] components, String recordValues) { + StringBuilder args = new StringBuilder(); + for (int i = 0; i < components.length; i++) { + if (i > 0) { + args.append(", "); + } + args.append(castRecordComponent(recordValues + "[" + i + "]", components[i].getType())); + } + return args.toString(); + } + + private String castRecordComponent(String value, Class type) { + if (!type.isPrimitive()) { + return "(" + ctx.type(type) + ") " + value; + } + return "(" + ctx.type(boxedType(type)) + ") " + value; + } + + private String boxedDefaultValue(Class type) { + if (!type.isPrimitive()) { + return null; + } + if (type == boolean.class) { + return "false"; + } + if (type == char.class) { + return "'\\0'"; + } + if (type == long.class) { + return "0L"; + } + if (type == float.class) { + return "0.0f"; + } + if (type == double.class) { + return "0.0d"; + } + if (type == byte.class) { + return "(byte) 0"; + } + if (type == short.class) { + return "(short) 0"; + } + return "0"; + } + + private static Class boxedType(Class type) { + if (type == boolean.class) { + return Boolean.class; + } + if (type == byte.class) { + return Byte.class; + } + if (type == char.class) { + return Character.class; + } + if (type == short.class) { + return Short.class; + } + if (type == int.class) { + return Integer.class; + } + if (type == long.class) { + return Long.class; + } + if (type == float.class) { + return Float.class; + } + if (type == double.class) { + return Double.class; + } + return type; + } + + private static String indent(String code, int spaces) { + String prefix = String.join("", Collections.nCopies(spaces, " ")); + return prefix + code.replace("\n", "\n" + prefix); + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java index 524126e411..573b379f83 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java @@ -381,6 +381,7 @@ public static class GraalvmClassRegistry { private final Map, Class> serializerClassMap; private final Map, Class> objectSerializerClassMap; private final Map> deserializerClassMap; + private final Map, Class> compatibleDeserializerClassMap; private final Map> layerSerializerClassMap; private GraalvmClassRegistry() { @@ -388,6 +389,7 @@ private GraalvmClassRegistry() { serializerClassMap = new ConcurrentHashMap<>(); objectSerializerClassMap = new ConcurrentHashMap<>(); deserializerClassMap = new ConcurrentHashMap<>(); + compatibleDeserializerClassMap = new ConcurrentHashMap<>(); layerSerializerClassMap = new ConcurrentHashMap<>(); } @@ -445,6 +447,15 @@ public Map> getDeserializerClasses() { return Collections.unmodifiableMap(deserializerClassMap); } + public Class getCompatibleDeserializerClass(Class cls) { + return getRegisteredClassValue(compatibleDeserializerClassMap, cls); + } + + public void putCompatibleDeserializerClass( + Class cls, Class serializerClass) { + compatibleDeserializerClassMap.put(cls, serializerClass); + } + public Class getLayerSerializerClass(long typeDefId) { return layerSerializerClassMap.get(typeDefId); } @@ -459,6 +470,7 @@ public Set> getRegisteredSerializerClasses() { serializerClasses.addAll(serializerClassMap.values()); serializerClasses.addAll(objectSerializerClassMap.values()); serializerClasses.addAll(deserializerClassMap.values()); + serializerClasses.addAll(compatibleDeserializerClassMap.values()); return serializerClasses; } @@ -466,6 +478,7 @@ public void clear() { serializerClassMap.clear(); objectSerializerClassMap.clear(); deserializerClassMap.clear(); + compatibleDeserializerClassMap.clear(); layerSerializerClassMap.clear(); resolvers.clear(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index a38ad799ba..e04897b2d5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1765,7 +1765,7 @@ private boolean needsGraalvmObjectSerializerClass( private Class getMetaSharedDeserializerClassForGraalvmBuild( Class cls, TypeDef typeDef) { Class serializerClass = - getMetaSharedDeserializerClassFromGraalvmRegistry(cls, typeDef); + getGraalvmClassRegistry().getDeserializerClass(typeDef.getId()); if (serializerClass != null) { return serializerClass; } @@ -1792,6 +1792,9 @@ private void registerGraalvmSerializerClass(Class cls) { getGraalvmClassRegistry() .putDeserializerClass( typeDef.getId(), getMetaSharedDeserializerClassForGraalvmBuild(cls, typeDef)); + getGraalvmClassRegistry() + .putCompatibleDeserializerClass( + cls, CodecUtils.loadOrGenCompatibleMetaSharedCodecClass(this, cls, typeDef)); } typeInfoCache = NIL_TYPE_INFO; if (RecordUtils.isRecord(cls)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index c1fa96918b..99866e73e8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1055,12 +1055,17 @@ private TypeInfo getMetaSharedTypeInfo(TypeDef typeDef, Class clz) { } } if (GraalvmSupport.isGraalBuildTime() - && (GeneratedMetaSharedSerializer.class.isAssignableFrom(sc) - || GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(sc))) { + && GeneratedMetaSharedSerializer.class.isAssignableFrom(sc)) { getGraalvmClassRegistry().putIfAbsentDeserializerClass(typeDef.getId(), sc); typeInfo.setSerializer(this, new MetaSharedSerializer(this, cls, typeDef)); return typeInfo; } + if (GraalvmSupport.isGraalBuildTime() + && GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(sc)) { + getGraalvmClassRegistry().putCompatibleDeserializerClass(cls, sc); + typeInfo.setSerializer(this, new MetaSharedSerializer(this, cls, typeDef)); + return typeInfo; + } if (StaticGeneratedStructSerializer.class.isAssignableFrom(sc)) { typeInfo.setSerializer(this, newStaticGeneratedStructSerializer(sc, cls, typeDef)); } else if (sc == MetaSharedSerializer.class) { @@ -1933,6 +1938,12 @@ protected final Class getMetaSharedDeserializerClassFromGr if (deserializerClass != null) { return deserializerClass; } + deserializerClass = registry.getCompatibleDeserializerClass(cls); + if (deserializerClass != null + && (!GraalvmSupport.isGraalBuildTime() + || typeDef.getId() != TypeDef.buildTypeDef(this, cls).getId())) { + return deserializerClass; + } if (!registry.hasResolvers()) { return null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 6892cb8b84..8827a2ffab 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -189,6 +189,15 @@ protected final int matchedId(RemoteFieldInfo remoteField) { return remoteField.matchedId; } + protected final boolean hasFieldConverter(RemoteFieldInfo remoteField) { + return remoteField.serializationFieldInfo.fieldConverter != null; + } + + protected final void setConvertedField( + Object targetObject, Object fieldValue, RemoteFieldInfo remoteField) { + remoteField.serializationFieldInfo.fieldConverter.set(targetObject, fieldValue); + } + protected final Object copyFieldValue( CopyContext copyContext, Object fieldValue, SerializationFieldInfo fieldInfo) { return copyContext.copyObject(fieldValue, fieldInfo.dispatchId); From 1a7cee0e88656721ea354eb9c7368ecfb8bccd3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 01:29:55 +0800 Subject: [PATCH 07/58] docs(java): rename static generated serializer guide --- docs/guide/java/configuration.md | 2 +- docs/guide/java/graalvm-support.md | 9 ++---- docs/guide/java/index.md | 2 +- ...ers.md => static-generated-serializers.md} | 30 ++++++++----------- java/fory-annotation-processor/README.md | 11 +++---- 5 files changed, 24 insertions(+), 30 deletions(-) rename docs/guide/java/{static-struct-serializers.md => static-generated-serializers.md} (79%) diff --git a/docs/guide/java/configuration.md b/docs/guide/java/configuration.md index 6d60590869..e7c09b123e 100644 --- a/docs/guide/java/configuration.md +++ b/docs/guide/java/configuration.md @@ -77,4 +77,4 @@ Fory fory = Fory.builder() - [Schema Evolution](schema-evolution.md) - Compatible mode and meta sharing - [Compression](compression.md) - Int, long, and array compression details - [Type Registration](type-registration.md) - Class registration options -- [Static Struct Serializers](static-struct-serializers.md) - Build-time `@ForyStruct` serializers for `codegen=false` and Android +- [Static Generated Serializers](static-generated-serializers.md) - Annotation-processor static generated serializers for `@ForyStruct`, `codegen=false`, and Android diff --git a/docs/guide/java/graalvm-support.md b/docs/guide/java/graalvm-support.md index eae6f3df9d..50cc90e727 100644 --- a/docs/guide/java/graalvm-support.md +++ b/docs/guide/java/graalvm-support.md @@ -38,12 +38,9 @@ Fory generates serialization code at GraalVM build time when you: Note: Fory's `asyncCompilationEnabled` option is automatically disabled for GraalVM native image since runtime JIT is not supported. -`@ForyStruct` annotation-processor static serializers are not the GraalVM native-image serializer -path. Native image builds use the Fory GraalVM registry: equal local and remote `TypeDef` hashes use -the existing meta-shared generated serializer, while mismatched remote schemas use a build-time -generated read-only compatible serializer cached by local Java class. Runtime native images load -that cached class from the registry, pass the current remote `TypeDef` into its constructor, and do -not attempt annotation-generated static serializer lookup, Janino, or runtime class definition. +`@ForyStruct` annotation-processor static generated serializers are not the GraalVM native-image +serializer path. GraalVM native images use Fory's native-image build-time serializer generation +instead. ## Basic Usage diff --git a/docs/guide/java/index.md b/docs/guide/java/index.md index ea9f13bcf1..3e4d29d493 100644 --- a/docs/guide/java/index.md +++ b/docs/guide/java/index.md @@ -210,5 +210,5 @@ ThreadSafeFory threadLocalFory = Fory.builder() - [Type Registration](type-registration.md) - Class registration and security - [Custom Serializers](custom-serializers.md) - Implement custom serializers - [Cross-Language Serialization](cross-language.md) - Serialize data for other languages -- [Static Struct Serializers](static-struct-serializers.md) - Build-time serializers for `@ForyStruct` +- [Static Generated Serializers](static-generated-serializers.md) - Annotation-processor static generated serializers for `@ForyStruct` - [GraalVM Support](graalvm-support.md) - Build-time serializer compilation for native images diff --git a/docs/guide/java/static-struct-serializers.md b/docs/guide/java/static-generated-serializers.md similarity index 79% rename from docs/guide/java/static-struct-serializers.md rename to docs/guide/java/static-generated-serializers.md index 5bdd331f7f..9f8c5d80c2 100644 --- a/docs/guide/java/static-struct-serializers.md +++ b/docs/guide/java/static-generated-serializers.md @@ -1,7 +1,7 @@ --- -title: Static Struct Serializers +title: Static Generated Serializers sidebar_position: 15 -id: static_struct_serializers +id: static_generated_serializers license: | Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with @@ -19,9 +19,9 @@ license: | limitations under the License. --- -Build-time static serializers are generated for Java classes annotated with `@ForyStruct`. They -provide a non-JIT serializer path for ordinary JVM runtimes using `ForyBuilder#withCodegen(false)` -and for Android, where runtime bytecode generation is disabled. +The Fory Java annotation processor generates build-time static generated serializers for classes +annotated with `@ForyStruct`. They provide a non-JIT serializer path for ordinary JVM runtimes using +`ForyBuilder#withCodegen(false)` and for Android, where runtime bytecode generation is disabled. ## Enabling The Processor @@ -60,21 +60,17 @@ enclosing class and does not generate inner serializer classes. ## Runtime Selection -Static serializers are used when available on: +Annotation-processor static generated serializers are used when available on: - ordinary JVMs with `ForyBuilder#withCodegen(false)`. - Android runtimes, because runtime code generation is disabled there. - compatible-mode meta-share reads when a generated static serializer exists for the target struct. -Ordinary JVM `codegen=true` keeps the runtime-generated serializer precedence. Static serializer -lookup is deterministic by generated class name and does not scan the classpath. +Ordinary JVM `codegen=true` keeps the runtime-generated serializer precedence. Static generated +serializer lookup is deterministic by generated class name and does not scan the classpath. -GraalVM native image does not use annotation-processor-generated static serializer classes. Native -image builds use the GraalVM registry path: matching local and remote `TypeDef` hashes use the -existing meta-shared generated serializer, and mismatched hashes use a build-time generated -read-only compatible serializer cached by local Java class. At runtime, the compatible serializer -constructor receives the current remote `TypeDef` and derives the remote-field layout from that -metadata. +GraalVM native image does not use annotation-processor-generated static serializer classes. Use the +GraalVM guide for native-image build-time serializer generation. ## Field Access Rules @@ -110,9 +106,9 @@ while avoiding Android reflection gaps. ## Compatible Reads -Generated static serializers include normal read/write/copy methods and a compatible read method. -The compatible path consumes remote schema metadata, matches remote fields to local fields, skips -unknown fields, and preserves Java defaults for missing fields. +Annotation-processor static generated serializers include normal read/write/copy methods and a +compatible read method. The compatible path consumes remote schema metadata, matches remote fields +to local fields, skips unknown fields, and preserves Java defaults for missing fields. Field matching assigns dense generated matched ids for the generated branch table. Those ids are local dispatch ids only; they are not `@ForyField.id` values and are not wire ids. Remote field order diff --git a/java/fory-annotation-processor/README.md b/java/fory-annotation-processor/README.md index 820abf34d3..1de33460ed 100644 --- a/java/fory-annotation-processor/README.md +++ b/java/fory-annotation-processor/README.md @@ -1,11 +1,12 @@ # Fory Annotation Processor -`fory-annotation-processor` generates build-time static serializers for Java classes annotated with -`@ForyStruct`. +`fory-annotation-processor` generates build-time static generated serializers for Java classes +annotated with `@ForyStruct`. For ordinary JVM applications, prefer Fory's runtime generated serializers. Runtime generation is -optimized for the active JVM and is usually more efficient than javac-generated static serializers. -That is the normal high-performance path for server and desktop JVM deployments. +optimized for the active JVM and is usually more efficient than annotation-processor static +generated serializers. That is the normal high-performance path for server and desktop JVM +deployments. Use this annotation processor when runtime source generation, bytecode generation, or dynamic class loading is not acceptable. The main target is Android, where runtime code generation is disabled. @@ -23,4 +24,4 @@ runtime loads them deterministically by name when static serializers are availab codegen is disabled. For the full user-facing guide, see -[`docs/guide/java/static-struct-serializers.md`](../../docs/guide/java/static-struct-serializers.md). +[`docs/guide/java/static-generated-serializers.md`](../../docs/guide/java/static-generated-serializers.md). From 1bc4cf235de49b5ac5ed9aeb1470e45b7911261c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 01:46:05 +0800 Subject: [PATCH 08/58] test(java): cover graalvm compatible schema reads --- .../CompatibleDifferentSchemaExample.java | 111 ++++++++++++++++++ .../java/org/apache/fory/graalvm/Main.java | 1 + .../graalvm_tests/native-image.properties | 1 + .../fory-core/native-image.properties | 3 +- 4 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/CompatibleDifferentSchemaExample.java diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/CompatibleDifferentSchemaExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/CompatibleDifferentSchemaExample.java new file mode 100644 index 0000000000..597e515d10 --- /dev/null +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/CompatibleDifferentSchemaExample.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.graalvm; + +import org.apache.fory.Fory; +import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.context.ReadContext; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.platform.GraalvmSupport; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.serializer.Serializer; +import org.apache.fory.util.Preconditions; + +/** Native-image regression test for compatible reads from a peer with a different schema. */ +public class CompatibleDifferentSchemaExample { + private static final int SCHEMA_ID = 1001; + + private static final Fory WRITER = createWriter(); + private static final Fory READER = createReader(); + + private static Fory createWriter() { + Fory fory = + Fory.builder() + .withName(CompatibleDifferentSchemaExample.class.getName() + ".writer") + .withCompatible(true) + .requireClassRegistration(true) + .build(); + fory.register(WriterSchema.class, SCHEMA_ID); + fory.ensureSerializersCompiled(); + return fory; + } + + private static Fory createReader() { + Fory fory = + Fory.builder() + .withName(CompatibleDifferentSchemaExample.class.getName() + ".reader") + .withCompatible(true) + .requireClassRegistration(true) + .build(); + fory.register(ReaderSchema.class, SCHEMA_ID); + fory.ensureSerializersCompiled(); + return fory; + } + + public static void main(String[] args) { + WriterSchema writerValue = new WriterSchema(); + writerValue.id = 42; + writerValue.name = "writer"; + writerValue.legacy = 99; + + byte[] bytes = WRITER.serialize(writerValue); + if (GraalvmSupport.isGraalRuntime()) { + Serializer serializer = readSerializerForTarget(READER, bytes, ReaderSchema.class); + Preconditions.checkArgument( + serializer instanceof GeneratedCompatibleMetaSharedSerializer, + "Expected GraalVM generated compatible serializer, got %s", + serializer.getClass()); + } + + ReaderSchema result = READER.deserialize(bytes, ReaderSchema.class); + Preconditions.checkArgument(result.id == 42); + Preconditions.checkArgument("writer".equals(result.name)); + Preconditions.checkArgument("reader-default".equals(result.added)); + System.out.println("CompatibleDifferentSchemaExample succeed"); + } + + private static Serializer readSerializerForTarget( + Fory fory, byte[] bytes, Class targetClass) { + MemoryBuffer buffer = MemoryUtils.wrap(bytes); + buffer.readByte(); + ReadContext readContext = fory.getReadContext(); + readContext.prepare(buffer, null, false); + try { + readContext.getRefReader().tryPreserveRefId(buffer); + TypeInfo typeInfo = fory.getTypeResolver().readTypeInfo(readContext, targetClass); + return typeInfo.getSerializer(); + } finally { + readContext.reset(); + } + } + + public static final class WriterSchema { + public int id; + public String name; + public int legacy; + } + + public static final class ReaderSchema { + public int id; + public String name; + public String added = "reader-default"; + } +} diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java index 021f5c8336..f84eeb50b5 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java @@ -29,6 +29,7 @@ public static void main(String[] args) throws Throwable { XlangExample.main(args); CompatibleExample.main(args); ScopedCompatibleExample.main(args); + CompatibleDifferentSchemaExample.main(args); RecordExample.main(args); CompatibleRecordExample.main(args); RecordExample2.main(args); diff --git a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties index a39144f2b4..aee7d41368 100644 --- a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties +++ b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties @@ -22,6 +22,7 @@ Args=-H:+ReportExceptionStackTraces \ org.apache.fory.graalvm.XlangExample,\ org.apache.fory.graalvm.CompatibleExample,\ org.apache.fory.graalvm.ScopedCompatibleExample,\ + org.apache.fory.graalvm.CompatibleDifferentSchemaExample,\ org.apache.fory.graalvm.record.RecordExample,\ org.apache.fory.graalvm.record.CompatibleRecordExample,\ org.apache.fory.graalvm.record.RecordExample2,\ diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 281165eeca..99b763f971 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -17,7 +17,8 @@ # https://www.graalvm.org/latest/reference-manual/native-image/dynamic-features/Reflection/#unsafe-accesses : # The unsafe offset get on build time may be different from runtime -Args=--initialize-at-build-time=org.apache.fory.collection.BiMap,\ +Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ + org.apache.fory.collection.BiMap,\ org.apache.fory.collection.BiMap$Inverse,\ org.apache.fory.collection.CacheBuilder$LocalCache,\ org.apache.fory.collection.ReferenceConcurrentMap,\ From 6f00ebdd367e19b5b3fa4b7cbd8c3ec2f10cadac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 01:56:09 +0800 Subject: [PATCH 09/58] feat(java): add ForyStruct evolution policy --- compiler/fory_compiler/generators/java.py | 5 ++- .../tests/test_generated_code.py | 23 +++++++++++ docs/compiler/generated-code.md | 12 ++++++ docs/guide/java/schema-evolution.md | 13 +++++-- .../apache/fory/annotation/ForyStruct.java | 22 ++++++++--- .../apache/fory/resolver/ClassResolver.java | 2 +- .../apache/fory/resolver/TypeResolver.java | 29 +++++++++++--- .../apache/fory/resolver/XtypeResolver.java | 6 +-- .../apache/fory/resolver/TypeInfoTest.java | 38 ++++++++++++++++++- .../org/apache/fory/xlang/XlangTestBase.java | 3 +- 10 files changed, 132 insertions(+), 21 deletions(-) diff --git a/compiler/fory_compiler/generators/java.py b/compiler/fory_compiler/generators/java.py index 04de2bd456..e46aba3d05 100644 --- a/compiler/fory_compiler/generators/java.py +++ b/compiler/fory_compiler/generators/java.py @@ -436,7 +436,7 @@ def generate_message_file(self, message: Message) -> GeneratedFile: if comment: lines.append(comment) if not self.get_effective_evolving(message): - lines.append("@ForyStruct(evolving = false)") + lines.append("@ForyStruct(evolving = Evolution.DISABLED)") lines.append(f"public class {message.name} {{") # Generate nested enums as static inner classes @@ -590,6 +590,7 @@ def collect_message_imports(self, message: Message, imports: Set[str]): if not self.get_effective_evolving(message): imports.add("org.apache.fory.annotation.ForyStruct") + imports.add("org.apache.fory.annotation.ForyStruct.Evolution") # Add imports for equals/hashCode imports.add("java.util.Objects") @@ -1042,7 +1043,7 @@ def generate_nested_message( if comment: lines.append(comment) if not self.get_effective_evolving(message): - lines.append("@ForyStruct(evolving = false)") + lines.append("@ForyStruct(evolving = Evolution.DISABLED)") lines.append(f"public static class {message.name} {{") # Generate nested enums diff --git a/compiler/fory_compiler/tests/test_generated_code.py b/compiler/fory_compiler/tests/test_generated_code.py index 263c023cf3..79a201f5ff 100644 --- a/compiler/fory_compiler/tests/test_generated_code.py +++ b/compiler/fory_compiler/tests/test_generated_code.py @@ -610,6 +610,29 @@ def test_java_unsigned_carriers_and_integer_encoding_annotations(): ) +def test_java_evolving_false_generation_uses_struct_evolution_enum(): + schema = parse_fdl( + dedent( + """ + package gen; + + message Stable [evolving=false] { + string name = 1; + + message NestedStable [evolving=false] { + int32 id = 1; + } + } + """ + ) + ) + java_output = render_files(generate_files(schema, JavaGenerator)) + assert "import org.apache.fory.annotation.ForyStruct;" in java_output + assert "import org.apache.fory.annotation.ForyStruct.Evolution;" in java_output + assert java_output.count("@ForyStruct(evolving = Evolution.DISABLED)") == 2 + assert "@ForyStruct(evolving = false)" not in java_output + + def test_java_nested_integer_annotations_in_generic_containers(): schema = parse_fdl( dedent( diff --git a/docs/compiler/generated-code.md b/docs/compiler/generated-code.md index 73746b877f..a47be8a511 100644 --- a/docs/compiler/generated-code.md +++ b/docs/compiler/generated-code.md @@ -154,6 +154,18 @@ public class Person { } ``` +When a message or inherited schema option sets `evolving=false`, the Java generator emits +`@ForyStruct(evolving = Evolution.DISABLED)` and imports `ForyStruct.Evolution` so the generated +class uses fixed-schema struct encoding: + +```java +import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.ForyStruct.Evolution; + +@ForyStruct(evolving = Evolution.DISABLED) +public class StableMessage { ... } +``` + Unions generate classes extending `org.apache.fory.type.union.Union`: ```java diff --git a/docs/guide/java/schema-evolution.md b/docs/guide/java/schema-evolution.md index d2db111388..f79613fd2b 100644 --- a/docs/guide/java/schema-evolution.md +++ b/docs/guide/java/schema-evolution.md @@ -48,14 +48,21 @@ System.out.println(fory.deserialize(bytes)); This compatible mode involves serializing class metadata into the serialized output. Despite Fory's use of sophisticated compression techniques to minimize overhead, there is still some additional space cost associated with class metadata. -### Disable Evolution for Stable Classes +### Per-Class Evolution Policy -If a class schema is stable and will not change, you can opt out of schema evolution on a per-class basis to avoid compatible metadata overhead. Annotate the class with `@ForyStruct(evolving = false)` to force `STRUCT/NAMED_STRUCT` type IDs even when Compatible mode is enabled. +`@ForyStruct` can set a per-class evolution policy: + +- `Evolution.INHERIT`: follow the Fory instance's compatible/meta-share configuration. This is the default. +- `Evolution.ENABLED`: require schema evolution metadata for this class. Registration or type resolution fails if the Fory instance cannot emit that metadata. +- `Evolution.DISABLED`: force fixed-schema `STRUCT/NAMED_STRUCT` encoding even when compatible metadata is otherwise enabled. + +If a class schema is stable and will not change, opt out of schema evolution on that class to avoid compatible metadata overhead: ```java import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.ForyStruct.Evolution; -@ForyStruct(evolving = false) +@ForyStruct(evolving = Evolution.DISABLED) public class StableMessage { public int id; public String name; diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java index 5e7980fc42..00f75732de 100644 --- a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java +++ b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java @@ -30,12 +30,24 @@ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) public @interface ForyStruct { + enum Evolution { + /** Follow Fory global compatible/meta-share options. */ + INHERIT, + + /** Require schema evolution metadata for this struct. */ + ENABLED, + + /** Require fixed-schema struct encoding for this struct. */ + DISABLED + } + /** - * Whether the annotated type should use schema evolution in compatible mode. + * Per-struct schema evolution policy. * - *

When {@code true} (default), compatible mode uses COMPATIBLE_STRUCT/NAMED_COMPATIBLE_STRUCT - * to include schema metadata for evolution. When {@code false}, STRUCT/NAMED_STRUCT is used to - * avoid that overhead. + *

{@link Evolution#INHERIT} follows the Fory instance's compatible/meta-share configuration. + * {@link Evolution#ENABLED} requires that configuration to emit schema evolution metadata for + * this struct. {@link Evolution#DISABLED} uses fixed-schema struct encoding even when compatible + * metadata is otherwise enabled. */ - boolean evolving() default true; + Evolution evolving() default Evolution.INHERIT; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index e04897b2d5..9bcb250058 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -687,7 +687,7 @@ private int buildUserTypeId(Class cls, Serializer serializer) { } else if (serializer != null && !isStructSerializer(serializer)) { return Types.EXT; } else { - return metaContextShareEnabled && isStructEvolving(cls) + return useStructEvolution(cls, metaContextShareEnabled) ? Types.COMPATIBLE_STRUCT : Types.STRUCT; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 99866e73e8..ff74c43b29 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -43,6 +43,7 @@ import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.ForyStruct.Evolution; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.CodecUtils; import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; @@ -1091,18 +1092,36 @@ protected int buildUnregisteredTypeId(Class cls, Serializer serializer) { if (serializer != null && !isStructSerializer(serializer)) { return Types.NAMED_EXT; } - if (isCompatible() && isStructEvolving(cls)) { - return metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; + if (useStructEvolution(cls, isCompatible() && metaContextShareEnabled)) { + return Types.NAMED_COMPATIBLE_STRUCT; } return Types.NAMED_STRUCT; } - protected boolean isStructEvolving(Class cls) { - if (cls == null) { + protected boolean useStructEvolution(Class cls, boolean inheritedEvolutionEnabled) { + Evolution evolution = getStructEvolution(cls); + if (evolution == Evolution.DISABLED) { + return false; + } + if (inheritedEvolutionEnabled) { return true; } + if (evolution == Evolution.ENABLED) { + throw new IllegalStateException( + String.format( + "Class %s is annotated with @ForyStruct(evolving = ENABLED), but this Fory " + + "instance is not configured to write schema evolution metadata", + cls.getName())); + } + return false; + } + + private Evolution getStructEvolution(Class cls) { + if (cls == null) { + return Evolution.INHERIT; + } ForyStruct annotation = cls.getAnnotation(ForyStruct.class); - return annotation == null || annotation.evolving(); + return annotation == null ? Evolution.INHERIT : annotation.evolving(); } protected static boolean isStructSerializer(Serializer serializer) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 766412c00b..de82a7446e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -227,7 +227,7 @@ public void register(Class type, long userTypeId) { typeId = Types.ENUM; } else { int structTypeId = - shareMeta && isStructEvolving(type) ? Types.COMPATIBLE_STRUCT : Types.STRUCT; + useStructEvolution(type, shareMeta) ? Types.COMPATIBLE_STRUCT : Types.STRUCT; if (serializer != null) { if (isStructType(serializer)) { typeId = structTypeId; @@ -274,7 +274,7 @@ public void register(Class type, String namespace, String typeName) { if (isStructType(serializer)) { xtypeId = (short) - (shareMeta && isStructEvolving(type) + (useStructEvolution(type, shareMeta) ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); } else if (serializer instanceof EnumSerializer) { @@ -288,7 +288,7 @@ public void register(Class type, String namespace, String typeName) { } else { xtypeId = (short) - (shareMeta && isStructEvolving(type) + (useStructEvolution(type, shareMeta) ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); } diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java index 41753942da..2e2c240e22 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java @@ -25,6 +25,7 @@ import org.apache.fory.Fory; import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.ForyStruct.Evolution; import org.apache.fory.type.Types; import org.testng.annotations.Test; @@ -33,7 +34,12 @@ public static class EvolvingStruct { public int id; } - @ForyStruct(evolving = false) + @ForyStruct(evolving = Evolution.ENABLED) + public static class ExplicitEvolvingStruct { + public int id; + } + + @ForyStruct(evolving = Evolution.DISABLED) public static class FixedStruct { public int id; } @@ -50,13 +56,18 @@ public void testEncodePackageNameAndTypeName() { public void testStructEvolvingOverride() { Fory fory = Fory.builder().withXlang(true).withCompatible(true).build(); fory.register(EvolvingStruct.class, "test", "EvolvingStruct"); + fory.register(ExplicitEvolvingStruct.class, "test", "ExplicitEvolvingStruct"); fory.register(FixedStruct.class, "test", "FixedStruct"); TypeInfo evolvingInfo = fory.getTypeResolver().getTypeInfo(EvolvingStruct.class, false); + TypeInfo explicitEvolvingInfo = + fory.getTypeResolver().getTypeInfo(ExplicitEvolvingStruct.class, false); TypeInfo fixedInfo = fory.getTypeResolver().getTypeInfo(FixedStruct.class, false); assertNotNull(evolvingInfo); + assertNotNull(explicitEvolvingInfo); assertNotNull(fixedInfo); assertEquals(evolvingInfo.getTypeId(), Types.NAMED_COMPATIBLE_STRUCT); + assertEquals(explicitEvolvingInfo.getTypeId(), Types.NAMED_COMPATIBLE_STRUCT); assertEquals(fixedInfo.getTypeId(), Types.NAMED_STRUCT); EvolvingStruct evolving = new EvolvingStruct(); @@ -71,4 +82,29 @@ public void testStructEvolvingOverride() { assertEquals(fory.deserialize(evolvingPayload, EvolvingStruct.class).id, evolving.id); assertEquals(fory.deserialize(fixedPayload, FixedStruct.class).id, fixed.id); } + + @Test + public void testStructEvolvingOverrideForRegisteredClasses() { + Fory fory = Fory.builder().withXlang(false).withCompatible(true).build(); + fory.register(EvolvingStruct.class, 100); + fory.register(ExplicitEvolvingStruct.class, 101); + fory.register(FixedStruct.class, 102); + + assertEquals( + fory.getTypeResolver().getTypeInfo(EvolvingStruct.class, false).getTypeId(), + Types.COMPATIBLE_STRUCT); + assertEquals( + fory.getTypeResolver().getTypeInfo(ExplicitEvolvingStruct.class, false).getTypeId(), + Types.COMPATIBLE_STRUCT); + assertEquals( + fory.getTypeResolver().getTypeInfo(FixedStruct.class, false).getTypeId(), Types.STRUCT); + } + + @Test( + expectedExceptions = IllegalStateException.class, + expectedExceptionsMessageRegExp = ".*schema evolution metadata.*") + public void testStructEvolutionEnabledRequiresMetadata() { + Fory fory = Fory.builder().withXlang(false).withCompatible(false).build(); + fory.register(ExplicitEvolvingStruct.class, "test", "ExplicitEvolvingStruct"); + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index 0d500528d3..517f4e6a3a 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -40,6 +40,7 @@ import org.apache.fory.annotation.ArrayType; import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.ForyStruct.Evolution; import org.apache.fory.annotation.Int32Type; import org.apache.fory.annotation.Int64Type; import org.apache.fory.annotation.Int8Type; @@ -195,7 +196,7 @@ protected static class EvolvingOverrideStruct { } @Data - @ForyStruct(evolving = false) + @ForyStruct(evolving = Evolution.DISABLED) protected static class FixedOverrideStruct { String f1; } From 0d0b06a7c345fa5009e25bae2761e502b92da601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 02:32:16 +0800 Subject: [PATCH 10/58] fix(java): keep xlang TypeDef fields root-owned --- .../src/main/java/org/apache/fory/meta/TypeDefEncoder.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java index 4b3ab040b4..c2fd0edf23 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java @@ -102,14 +102,11 @@ static List buildFieldsInfoFromDescriptors( + type.getName()); } return new FieldInfo( - descriptor.getDeclaringClass(), - descriptor.getName(), - fieldType, - (short) tagId); + type.getName(), descriptor.getName(), fieldType, (short) tagId); } // tagId == -1 means use field name, fall through to create regular FieldInfo } - return new FieldInfo(descriptor.getDeclaringClass(), descriptor.getName(), fieldType); + return new FieldInfo(type.getName(), descriptor.getName(), fieldType); }) .collect(Collectors.toList()); } From 3566f3700a57777a4f4a33c35871fb037d5fea27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 03:39:13 +0800 Subject: [PATCH 11/58] fix(java): keep annotation processor JDK 11 compatible --- .../processing/ForyStructProcessor.java | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index fbc41b939d..41a213a765 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -43,7 +43,6 @@ import javax.lang.model.element.Modifier; import javax.lang.model.element.NestingKind; import javax.lang.model.element.PackageElement; -import javax.lang.model.element.RecordComponentElement; import javax.lang.model.element.TypeElement; import javax.lang.model.element.VariableElement; import javax.lang.model.type.ArrayType; @@ -337,7 +336,7 @@ private List recordComponentFields(TypeElement type) { fieldsByName.put(field.getSimpleName().toString(), field); } List fields = new ArrayList<>(); - for (RecordComponentElement component : type.getRecordComponents()) { + for (Element component : recordComponents(type)) { VariableElement field = fieldsByName.get(component.getSimpleName().toString()); if (field != null) { fields.add(field); @@ -346,6 +345,32 @@ private List recordComponentFields(TypeElement type) { return fields; } + private List recordComponents(TypeElement type) { + // Keep the processor artifact compilable on JDK 11 while still using record components + // when a newer compiler model provides them. + Object components; + try { + components = TypeElement.class.getMethod("getRecordComponents").invoke(type); + } catch (NoSuchMethodException e) { + throw new InvalidStructException( + "Record @ForyStruct processing requires a compiler with record component support", type); + } catch (ReflectiveOperationException e) { + throw new InvalidStructException("Failed to inspect record components: " + e, type); + } + if (!(components instanceof List)) { + throw new InvalidStructException("Unexpected record component model for " + type, type); + } + List componentList = (List) components; + List componentElements = new ArrayList<>(componentList.size()); + for (Object component : componentList) { + if (!(component instanceof Element)) { + throw new InvalidStructException("Unexpected record component model for " + type, type); + } + componentElements.add((Element) component); + } + return componentElements; + } + private boolean isSerializableRecordField(VariableElement field, TypeElement owner) { if (field.getModifiers().contains(Modifier.TRANSIENT)) { return false; From 497ff8e00de61b2d22ab5b03e39fe750235b7d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 04:34:14 +0800 Subject: [PATCH 12/58] fix(java): preserve processor type-use metadata on Java 8 --- .../processing/ForyStructProcessor.java | 267 +++++++++++++++--- .../processing/ForyStructProcessorTest.java | 26 +- 2 files changed, 254 insertions(+), 39 deletions(-) diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index 41a213a765..c9b0628dbb 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -21,7 +21,9 @@ import java.io.IOException; import java.io.Writer; +import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -80,6 +82,9 @@ public final class ForyStructProcessor extends AbstractProcessor { private Filer filer; private Elements elements; private javax.lang.model.util.Types types; + // JDK 8 javac can omit nested TYPE_USE annotations from TypeMirror, so recover them from + // javac's public tree API reflectively while keeping this processor targetable to Java 8. + private Object trees; @Override public synchronized void init(ProcessingEnvironment processingEnv) { @@ -88,6 +93,18 @@ public synchronized void init(ProcessingEnvironment processingEnv) { filer = processingEnv.getFiler(); elements = processingEnv.getElementUtils(); types = processingEnv.getTypeUtils(); + try { + ClassLoader javacLoader = processingEnv.getClass().getClassLoader(); + Class treesClass = + Class.forName( + "com.sun.source.util.Trees", + false, + javacLoader == null ? ClassLoader.getSystemClassLoader() : javacLoader); + trees = + treesClass.getMethod("instance", ProcessingEnvironment.class).invoke(null, processingEnv); + } catch (ReflectiveOperationException | IllegalArgumentException e) { + trees = null; + } } @Override @@ -241,7 +258,7 @@ private SourceField buildField( + "; use a record component or mark the field @Ignore/transient", field); } - SourceTypeNode typeNode = buildTypeNode(field.asType()); + SourceTypeNode typeNode = buildFieldTypeNode(field); String erasedType = canonicalName(types.erasure(field.asType())); String declaringClass = elements.getBinaryName((TypeElement) field.getEnclosingElement()).toString(); @@ -504,8 +521,32 @@ private int reflectionModifiers(Set modifiers) { return value; } + private SourceTypeNode buildFieldTypeNode(VariableElement field) { + return buildTypeNode(field.asType(), typeTree(field)); + } + + private Object typeTree(VariableElement field) { + if (trees == null) { + return null; + } + Object path = invoke(trees, "getPath", new Class[] {Element.class}, field); + if (path == null) { + return null; + } + Object leaf = invoke(path, "getLeaf"); + if (!isInstance("com.sun.source.tree.VariableTree", leaf)) { + return null; + } + return invoke(leaf, "getType"); + } + private SourceTypeNode buildTypeNode(TypeMirror type) { + return buildTypeNode(type, null); + } + + private SourceTypeNode buildTypeNode(TypeMirror type, Object tree) { TypeKind kind = type.getKind(); + TypeTreeInfo treeInfo = typeTreeInfo(tree); if (kind == TypeKind.TYPEVAR) { TypeVariable typeVariable = (TypeVariable) type; return buildTypeNode(typeVariable.getUpperBound()); @@ -519,60 +560,75 @@ private SourceTypeNode buildTypeNode(TypeMirror type) { List arguments = new ArrayList<>(); SourceTypeNode componentType = null; if (kind == TypeKind.ARRAY) { - componentType = buildTypeNode(((ArrayType) type).getComponentType()); + componentType = + buildTypeNode(((ArrayType) type).getComponentType(), treeInfo.arrayComponentTree()); } else if (type instanceof DeclaredType) { + List argumentTrees = treeInfo.typeArgumentTrees(); + int index = 0; for (TypeMirror argument : ((DeclaredType) type).getTypeArguments()) { - arguments.add(buildTypeNode(argument)); + Object argumentTree = index < argumentTrees.size() ? argumentTrees.get(index) : null; + arguments.add(buildTypeNode(argument, argumentTree)); + index++; } } String rawType = canonicalName(types.erasure(type)); - String extMeta = typeExtMetaExpression(type, rawType); + String extMeta = typeExtMetaExpression(type, rawType, treeInfo.annotations); boolean primitive = kind.isPrimitive(); boolean nestedStruct = isForyStructType(type); return new SourceTypeNode( rawType, typeName(type), extMeta, arguments, componentType, primitive, nestedStruct); } + private TypeTreeInfo typeTreeInfo(Object tree) { + List annotations = Collections.emptyList(); + Object current = tree; + while (isInstance("com.sun.source.tree.AnnotatedTypeTree", current)) { + annotations = listValue(invoke(current, "getAnnotations")); + current = invoke(current, "getUnderlyingType"); + } + return new TypeTreeInfo(annotations, current); + } + private boolean isForyStructType(TypeMirror type) { TypeMirror erased = types.erasure(type); Element element = types.asElement(erased); return element instanceof TypeElement && hasAnnotation(element, FORY_STRUCT); } - private String typeExtMetaExpression(TypeMirror type, String rawType) { - String typeId = scalarTypeId(type, rawType); - AnnotationMirror refMirror = annotationMirror(type, REF); - if (typeId == null && refMirror == null) { + private String typeExtMetaExpression(TypeMirror type, String rawType, List treeAnnotations) { + String typeId = scalarTypeId(type, rawType, treeAnnotations); + TypeUseAnnotation ref = typeUseAnnotation(type, treeAnnotations, REF); + if (typeId == null && ref == null) { return null; } return "meta(" + (typeId == null ? "Types.UNKNOWN" : typeId) + ", true, " - + booleanValue(refMirror, "enable", true) + + booleanValue(ref, "enable", true) + ")"; } - private String scalarTypeId(TypeMirror type, String rawType) { - if (hasTypeAnnotation(type, INT8_TYPE)) { + private String scalarTypeId(TypeMirror type, String rawType, List treeAnnotations) { + if (hasTypeAnnotation(type, treeAnnotations, INT8_TYPE)) { return rawType.equals("byte[]") ? "Types.INT8_ARRAY" : "Types.INT8"; } - if (hasTypeAnnotation(type, UINT8_TYPE)) { + if (hasTypeAnnotation(type, treeAnnotations, UINT8_TYPE)) { return rawType.equals("byte[]") ? "Types.UINT8_ARRAY" : "Types.UINT8"; } - if (hasTypeAnnotation(type, UINT16_TYPE)) { + if (hasTypeAnnotation(type, treeAnnotations, UINT16_TYPE)) { return rawType.equals("short[]") ? "Types.UINT16_ARRAY" : "Types.UINT16"; } - AnnotationMirror uint32Mirror = typeAnnotationMirror(type, UINT32_TYPE); - if (uint32Mirror != null) { - String encoding = int32Encoding(uint32Mirror); + TypeUseAnnotation uint32 = typeUseAnnotation(type, treeAnnotations, UINT32_TYPE); + if (uint32 != null) { + String encoding = int32Encoding(uint32); if (rawType.equals("int[]")) { return "Types.UINT32_ARRAY"; } return "FIXED".equals(encoding) ? "Types.UINT32" : "Types.VAR_UINT32"; } - AnnotationMirror uint64Mirror = typeAnnotationMirror(type, UINT64_TYPE); - if (uint64Mirror != null) { - String encoding = int64Encoding(uint64Mirror); + TypeUseAnnotation uint64 = typeUseAnnotation(type, treeAnnotations, UINT64_TYPE); + if (uint64 != null) { + String encoding = int64Encoding(uint64); if (rawType.equals("long[]")) { return "Types.UINT64_ARRAY"; } @@ -581,30 +637,45 @@ private String scalarTypeId(TypeMirror type, String rawType) { } return "TAGGED".equals(encoding) ? "Types.TAGGED_UINT64" : "Types.VAR_UINT64"; } - AnnotationMirror int32Mirror = typeAnnotationMirror(type, INT32_TYPE); - if (int32Mirror != null) { - String encoding = int32Encoding(int32Mirror); + TypeUseAnnotation int32 = typeUseAnnotation(type, treeAnnotations, INT32_TYPE); + if (int32 != null) { + String encoding = int32Encoding(int32); return "FIXED".equals(encoding) ? "Types.INT32" : "Types.VARINT32"; } - AnnotationMirror int64Mirror = typeAnnotationMirror(type, INT64_TYPE); - if (int64Mirror != null) { - String encoding = int64Encoding(int64Mirror); + TypeUseAnnotation int64 = typeUseAnnotation(type, treeAnnotations, INT64_TYPE); + if (int64 != null) { + String encoding = int64Encoding(int64); if ("FIXED".equals(encoding)) { return "Types.INT64"; } return "TAGGED".equals(encoding) ? "Types.TAGGED_INT64" : "Types.VARINT64"; } - if (hasTypeAnnotation(type, FLOAT16_TYPE)) { + if (hasTypeAnnotation(type, treeAnnotations, FLOAT16_TYPE)) { return "Types.FLOAT16_ARRAY"; } - if (hasTypeAnnotation(type, BFLOAT16_TYPE)) { + if (hasTypeAnnotation(type, treeAnnotations, BFLOAT16_TYPE)) { return "Types.BFLOAT16_ARRAY"; } return null; } - private boolean hasTypeAnnotation(TypeMirror type, String annotationName) { - return typeAnnotationMirror(type, annotationName) != null; + private boolean hasTypeAnnotation( + TypeMirror type, List treeAnnotations, String annotationName) { + return typeUseAnnotation(type, treeAnnotations, annotationName) != null; + } + + private TypeUseAnnotation typeUseAnnotation( + TypeMirror type, List treeAnnotations, String annotationName) { + AnnotationMirror mirror = typeAnnotationMirror(type, annotationName); + if (mirror != null) { + return new TypeUseAnnotation(mirror, null); + } + for (Object annotationTree : treeAnnotations) { + if (isAnnotationTree(annotationTree, annotationName)) { + return new TypeUseAnnotation(null, annotationTree); + } + } + return null; } private AnnotationMirror typeAnnotationMirror(TypeMirror type, String annotationName) { @@ -619,6 +690,20 @@ private AnnotationMirror typeAnnotationMirror(TypeMirror type, String annotation return annotationMirror(componentType, annotationName); } + private boolean isAnnotationTree(Object annotationTree, String annotationName) { + Object annotationType = invoke(annotationTree, "getAnnotationType"); + if (annotationType == null) { + return false; + } + String treeName = annotationType.toString(); + return treeName.equals(annotationName) || treeName.equals(simpleName(annotationName)); + } + + private String simpleName(String className) { + int index = className.lastIndexOf('.'); + return index < 0 ? className : className.substring(index + 1); + } + private AnnotationMirror annotationMirror(TypeMirror type, String annotationName) { for (AnnotationMirror mirror : type.getAnnotationMirrors()) { Element element = mirror.getAnnotationType().asElement(); @@ -645,10 +730,14 @@ private boolean hasAnnotation(Element element, String annotationName) { return annotationMirror(element, annotationName) != null; } - private boolean booleanValue(AnnotationMirror mirror, String name, boolean defaultValue) { - if (mirror == null) { + private boolean booleanValue(TypeUseAnnotation annotation, String name, boolean defaultValue) { + if (annotation == null) { return defaultValue; } + if (annotation.mirror == null) { + return Boolean.parseBoolean(treeAnnotationValue(annotation.tree, name, defaultValue)); + } + AnnotationMirror mirror = annotation.mirror; for (Map.Entry entry : mirror.getElementValues().entrySet()) { if (entry.getKey().getSimpleName().contentEquals(name)) { @@ -658,18 +747,22 @@ private boolean booleanValue(AnnotationMirror mirror, String name, boolean defau return defaultValue; } - private String int32Encoding(AnnotationMirror mirror) { - return enumValue(mirror, "encoding", "VARINT"); + private String int32Encoding(TypeUseAnnotation annotation) { + return enumValue(annotation, "encoding", "VARINT"); } - private String int64Encoding(AnnotationMirror mirror) { - return enumValue(mirror, "encoding", "VARINT"); + private String int64Encoding(TypeUseAnnotation annotation) { + return enumValue(annotation, "encoding", "VARINT"); } - private String enumValue(AnnotationMirror mirror, String name, String defaultValue) { - if (mirror == null) { + private String enumValue(TypeUseAnnotation annotation, String name, String defaultValue) { + if (annotation == null) { return defaultValue; } + if (annotation.mirror == null) { + return enumConstant(treeAnnotationValue(annotation.tree, name, defaultValue)); + } + AnnotationMirror mirror = annotation.mirror; for (Map.Entry entry : mirror.getElementValues().entrySet()) { if (entry.getKey().getSimpleName().contentEquals(name)) { @@ -679,6 +772,106 @@ private String enumValue(AnnotationMirror mirror, String name, String defaultVal return defaultValue; } + private String treeAnnotationValue(Object annotationTree, String name, Object defaultValue) { + for (Object argument : listValue(invoke(annotationTree, "getArguments"))) { + Object valueTree = argument; + if (isInstance("com.sun.source.tree.AssignmentTree", argument)) { + Object variable = invoke(argument, "getVariable"); + if (variable == null || !variable.toString().equals(name)) { + continue; + } + valueTree = invoke(argument, "getExpression"); + } + return valueTree.toString(); + } + return String.valueOf(defaultValue); + } + + private String enumConstant(String value) { + int index = value.lastIndexOf('.'); + return index < 0 ? value : value.substring(index + 1); + } + + private static boolean isInstance(String className, Object value) { + if (value == null) { + return false; + } + return hasType(value.getClass(), className); + } + + private static boolean hasType(Class type, String className) { + if (type == null) { + return false; + } + if (type.getName().equals(className)) { + return true; + } + for (Class interfaceType : type.getInterfaces()) { + if (hasType(interfaceType, className)) { + return true; + } + } + return hasType(type.getSuperclass(), className); + } + + private static Object invoke(Object target, String methodName) { + return invoke(target, methodName, new Class[0]); + } + + private static Object invoke( + Object target, String methodName, Class[] parameterTypes, Object... args) { + if (target == null) { + return null; + } + try { + Method method = target.getClass().getMethod(methodName, parameterTypes); + return method.invoke(target, args); + } catch (ReflectiveOperationException e) { + return null; + } + } + + private static List listValue(Object value) { + if (value instanceof List) { + return (List) value; + } + return Collections.emptyList(); + } + + private static final class TypeTreeInfo { + final List annotations; + final Object tree; + + TypeTreeInfo(List annotations, Object tree) { + this.annotations = annotations; + this.tree = tree; + } + + Object arrayComponentTree() { + if (isInstance("com.sun.source.tree.ArrayTypeTree", tree)) { + return invoke(tree, "getType"); + } + return null; + } + + List typeArgumentTrees() { + if (isInstance("com.sun.source.tree.ParameterizedTypeTree", tree)) { + return listValue(invoke(tree, "getTypeArguments")); + } + return Collections.emptyList(); + } + } + + private static final class TypeUseAnnotation { + final AnnotationMirror mirror; + final Object tree; + + TypeUseAnnotation(AnnotationMirror mirror, Object tree) { + this.mirror = mirror; + this.tree = tree; + } + } + private ForyFieldMeta foryField(VariableElement field) { AnnotationMirror mirror = annotationMirror(field, FORY_FIELD); if (mirror == null) { diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 571aacc17a..e5bcea15a3 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -51,6 +51,7 @@ import org.apache.fory.type.Descriptor; import org.apache.fory.type.Types; import org.testng.Assert; +import org.testng.SkipException; import org.testng.annotations.Test; public class ForyStructProcessorTest { @@ -258,6 +259,7 @@ public void testGeneratedDescriptorsCarryNestedTypeMetadata() throws Exception { @Test public void testRecordReadAndCopyUseCanonicalConstructor() throws Exception { + assumeRecordSupport(); CompilationResult result = compile( "test.RecordStruct", @@ -467,8 +469,10 @@ private static CompilationResult compile(String typeName, String source) throws Path root = Files.createTempDirectory("fory-processor-test"); Path sourceRoot = root.resolve("src"); Path classRoot = root.resolve("classes"); + Path generatedRoot = root.resolve("generated"); Files.createDirectories(sourceRoot); Files.createDirectories(classRoot); + Files.createDirectories(generatedRoot); Path sourceFile = sourceRoot.resolve(typeName.replace('.', '/') + ".java"); Files.createDirectories(sourceFile.getParent()); Files.write(sourceFile, source.getBytes(StandardCharsets.UTF_8)); @@ -484,15 +488,33 @@ private static CompilationResult compile(String typeName, String source) throws "-d", classRoot.toString(), "-s", - root.resolve("generated").toString()); + generatedRoot.toString()); JavaCompiler.CompilationTask task = compiler.getTask(null, fileManager, diagnostics, options, null, units); task.setProcessors(Collections.singletonList(new ForyStructProcessor())); return new CompilationResult( - classRoot, root.resolve("generated"), task.call(), diagnostics.getDiagnostics()); + classRoot, generatedRoot, task.call(), diagnostics.getDiagnostics()); } } + private static void assumeRecordSupport() { + if (javaSpecificationVersion() < 16) { + throw new SkipException("Record source tests require JDK 16 or newer"); + } + } + + private static int javaSpecificationVersion() { + String version = System.getProperty("java.specification.version"); + if (version.startsWith("1.")) { + version = version.substring(2); + } + int dotIndex = version.indexOf('.'); + if (dotIndex >= 0) { + version = version.substring(0, dotIndex); + } + return Integer.parseInt(version); + } + private static void setField(Class type, Object target, String name, Object value) throws Exception { Field field = type.getDeclaredField(name); From 95a70a1b06a87f9c013b1d7fb5e5f93e3c32db99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 11:20:14 +0800 Subject: [PATCH 13/58] fix(java): harden static generated compatible serializers --- .../processing/ForyStructProcessor.java | 2 +- .../StaticSerializerSourceWriter.java | 29 ++++-- .../processing/ForyStructProcessorTest.java | 89 ++++++++++++++++++- .../builder/StaticCompatibleCodecBuilder.java | 26 ++++-- .../org/apache/fory/config/ForyBuilder.java | 15 ++++ .../java/org/apache/fory/meta/FieldInfo.java | 9 +- .../StaticGeneratedStructSerializer.java | 34 +++++++ .../serializer/converter/FieldConverters.java | 76 ++++++++++++++++ .../fory-core/native-image.properties | 2 +- .../org/apache/fory/ThreadSafeForyTest.java | 27 ++++++ 10 files changed, 292 insertions(+), 17 deletions(-) diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index c9b0628dbb..94b3ec3f8f 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -604,7 +604,7 @@ private String typeExtMetaExpression(TypeMirror type, String rawType, List tr return "meta(" + (typeId == null ? "Types.UNKNOWN" : typeId) + ", true, " - + booleanValue(ref, "enable", true) + + (ref != null && booleanValue(ref, "enable", true)) + ")"; } diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 3b62ad29c7..188988e8d7 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -444,11 +444,21 @@ private void writeCompatibleBeanDispatchGroup(int group) { SourceField field = struct.fields.get(i); builder.append(" case ").append(field.id).append(":\n"); builder - .append(" ") + .append(" if (canReadRemoteField(remoteField, fieldsById[") + .append(field.id) + .append("])) {\n"); + builder .append( - field.writeStatement( - "value", field.castExpression("readRemoteField(readContext, remoteField)"))) + " Object fieldValue = readCompatibleFieldValue(readContext, remoteField, fieldsById[") + .append(field.id) + .append("]);\n"); + builder + .append(" ") + .append(field.writeStatement("value", field.castExpression("fieldValue"))) .append("\n"); + builder.append(" } else {\n"); + builder.append(" skipField(readContext, remoteField);\n"); + builder.append(" }\n"); builder.append(" return;\n"); } builder.append(" default:\n"); @@ -468,9 +478,18 @@ private void writeCompatibleRecordDispatchGroup(int group) { SourceField field = struct.fields.get(i); builder.append(" case ").append(field.id).append(":\n"); builder - .append(" values[") + .append(" if (canReadRemoteField(remoteField, fieldsById[") + .append(field.id) + .append("])) {\n"); + builder + .append(" values[") + .append(field.id) + .append("] = readCompatibleFieldValue(readContext, remoteField, fieldsById[") .append(field.id) - .append("] = readRemoteField(readContext, remoteField);\n"); + .append("]);\n"); + builder.append(" } else {\n"); + builder.append(" skipField(readContext, remoteField);\n"); + builder.append(" }\n"); builder.append(" return;\n"); } builder.append(" default:\n"); diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index e5bcea15a3..770ce11e0d 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -38,6 +38,7 @@ import javax.tools.StandardJavaFileManager; import javax.tools.ToolProvider; import org.apache.fory.Fory; +import org.apache.fory.ThreadSafeFory; import org.apache.fory.builder.CodecUtils; import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; import org.apache.fory.context.MetaReadContext; @@ -254,6 +255,7 @@ public void testGeneratedDescriptorsCarryNestedTypeMetadata() throws Exception { Descriptor code = descriptor(serializer.getDescriptors(), "code"); Assert.assertTrue(code.getTypeRef().hasTypeExtMeta()); Assert.assertEquals(code.getTypeRef().getTypeExtMeta().typeId(), Types.UINT16); + Assert.assertFalse(code.getTypeRef().getTypeExtMeta().trackingRef()); } } @@ -305,7 +307,7 @@ public void testCompatibleReadUsesGeneratedSerializer() throws Exception { + "import org.apache.fory.annotation.ForyField;\n" + "import org.apache.fory.annotation.ForyStruct;\n" + "@ForyStruct public class EvolvingStruct {\n" - + " @ForyField(id = 1) public int id;\n" + + " @ForyField(id = 1, nullable = true) public String id;\n" + " @ForyField(id = 2, nullable = true) public String name;\n" + " public EvolvingStruct() {}\n" + "}\n"); @@ -334,7 +336,7 @@ public void testCompatibleReadUsesGeneratedSerializer() throws Exception { URLClassLoader readerLoader = readerResult.classLoader()) { Class writerType = writerLoader.loadClass("test.EvolvingStruct"); Object value = writerType.getConstructor().newInstance(); - setField(writerType, value, "id", 42); + setField(writerType, value, "id", "42"); setField(writerType, value, "name", "old"); Fory writer = Fory.builder() @@ -369,6 +371,17 @@ public void testCompatibleReadUsesGeneratedSerializer() throws Exception { } } + @Test + public void testStaticSerializerRoundTripsWithRuntimeSerializerSchemaConsistent() + throws Exception { + assertStaticRuntimeRoundTrip(false); + } + + @Test + public void testStaticSerializerRoundTripsWithRuntimeSerializerCompatible() throws Exception { + assertStaticRuntimeRoundTrip(true); + } + @Test public void testGraalvmStaticCompatibleSerializerReadsRuntimeRemoteTypeDef() throws Exception { if (AndroidSupport.IS_ANDROID) { @@ -463,6 +476,78 @@ public void testGraalvmStaticCompatibleSerializerReadsRuntimeRemoteTypeDef() thr } } + private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exception { + CompilationResult staticResult = + compile( + "test.RoundTripStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class RoundTripStruct {\n" + + " public int id;\n" + + " public String name;\n" + + " public RoundTripStruct() {}\n" + + "}\n"); + CompilationResult runtimeResult = + compile( + "test.RoundTripStruct", + "package test;\n" + + "public class RoundTripStruct {\n" + + " public int id;\n" + + " public String name;\n" + + " public RoundTripStruct() {}\n" + + "}\n"); + Assert.assertTrue(staticResult.success, staticResult.diagnostics()); + Assert.assertTrue(runtimeResult.success, runtimeResult.diagnostics()); + try (URLClassLoader staticLoader = staticResult.classLoader(); + URLClassLoader runtimeLoader = runtimeResult.classLoader()) { + Class staticType = staticLoader.loadClass("test.RoundTripStruct"); + Class runtimeType = runtimeLoader.loadClass("test.RoundTripStruct"); + ThreadSafeFory staticFory = threadSafeFory(staticLoader, false, compatible); + ThreadSafeFory runtimeFory = threadSafeFory(runtimeLoader, true, compatible); + staticFory.register(staticType, 9101); + runtimeFory.register(runtimeType, 9101); + + Object staticSerializer = + staticFory.execute( + fory -> fory.getTypeResolver().getTypeInfo(staticType).getSerializer()); + Object runtimeSerializer = + runtimeFory.execute( + fory -> fory.getTypeResolver().getTypeInfo(runtimeType).getSerializer()); + Assert.assertTrue(staticSerializer instanceof StaticGeneratedStructSerializer); + Assert.assertFalse(runtimeSerializer instanceof StaticGeneratedStructSerializer); + + Object staticValue = staticType.getConstructor().newInstance(); + setField(staticType, staticValue, "id", 101); + setField(staticType, staticValue, "name", compatible ? "compatible-static" : "static"); + Object runtimeRoundTrip = runtimeFory.deserialize(staticFory.serialize(staticValue)); + Assert.assertSame(runtimeRoundTrip.getClass(), runtimeType); + Assert.assertEquals(getField(runtimeType, runtimeRoundTrip, "id"), 101); + Assert.assertEquals( + getField(runtimeType, runtimeRoundTrip, "name"), + compatible ? "compatible-static" : "static"); + + Object runtimeValue = runtimeType.getConstructor().newInstance(); + setField(runtimeType, runtimeValue, "id", 202); + setField(runtimeType, runtimeValue, "name", compatible ? "compatible-runtime" : "runtime"); + Object staticRoundTrip = staticFory.deserialize(runtimeFory.serialize(runtimeValue)); + Assert.assertSame(staticRoundTrip.getClass(), staticType); + Assert.assertEquals(getField(staticType, staticRoundTrip, "id"), 202); + Assert.assertEquals( + getField(staticType, staticRoundTrip, "name"), + compatible ? "compatible-runtime" : "runtime"); + } + } + + private static ThreadSafeFory threadSafeFory( + ClassLoader classLoader, boolean codegen, boolean compatible) { + return Fory.builder() + .withClassLoader(classLoader) + .withCodegen(codegen) + .withCompatible(compatible) + .requireClassRegistration(true) + .buildThreadSafeForyPool(1); + } + private static CompilationResult compile(String typeName, String source) throws IOException { JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); Assert.assertNotNull(compiler, "Tests require a JDK compiler"); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 0be893ae3a..0622ff053d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -35,6 +35,7 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.MetaSharedSerializer; import org.apache.fory.type.Descriptor; import org.apache.fory.util.Preconditions; @@ -79,7 +80,7 @@ public String genCode() { ctx.setClassName(className); ctx.extendsClasses(ctx.type(parentSerializerClass)); ctx.reserveName(POJO_CLASS_TYPE_NAME); - ctx.addImports(List.class, TypeDef.class, Descriptor.class); + ctx.addImports(List.class, TypeDef.class, Descriptor.class, SerializationFieldInfo.class); String constructorCode = StringUtils.format( "" @@ -316,12 +317,22 @@ private String genObjectDispatchGroup(int group, TypeRef valueTypeRef) { code.append(" case ") .append(i) .append(": {\n") - .append(" Object _f_fieldValue = readRemoteField(") + .append(" if (hasFieldConverter(_f_remoteField)) {\n") + .append(" Object _f_fieldValue = readRemoteField(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField);\n") - .append(" if (hasFieldConverter(_f_remoteField)) {\n") .append(" setConvertedField(_f_value, _f_fieldValue, _f_remoteField);\n") .append(" } else {\n") + .append(" SerializationFieldInfo _f_localField = localFieldInfo(_f_matchedId);\n") + .append(" if (!canReadRemoteField(_f_remoteField, _f_localField)) {\n") + .append(" skipField(") + .append(READ_CONTEXT_NAME) + .append(", _f_remoteField);\n") + .append(" return;\n") + .append(" }\n") + .append(" Object _f_fieldValue = readCompatibleFieldValue(") + .append(READ_CONTEXT_NAME) + .append(", _f_remoteField, _f_localField);\n") .append(indent(genSetFieldCode(descriptor, valueTypeRef), 6)) .append('\n') .append(" }\n") @@ -353,10 +364,15 @@ private String genRecordDispatchGroup(int group) { .append(" readRemoteField(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField);\n") - .append(" } else {\n") + .append( + " } else if (canReadRemoteField(_f_remoteField, localFieldInfo(_f_matchedId))) {\n") .append(" _f_recordValues[") .append(componentIndex) - .append("] = readRemoteField(") + .append("] = readCompatibleFieldValue(") + .append(READ_CONTEXT_NAME) + .append(", _f_remoteField, localFieldInfo(_f_matchedId));\n") + .append(" } else {\n") + .append(" skipField(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField);\n") .append(" }\n") diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index 02702e12df..7ea308b688 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; import org.apache.fory.Fory; @@ -54,6 +55,7 @@ public final class ForyBuilder { private static final Logger LOG = LoggerFactory.getLogger(ForyBuilder.class); private static final int DEFAULT_THREAD_SAFE_POOL_SIZE = Math.max(1, Runtime.getRuntime().availableProcessors() * 4); + private static final AtomicInteger THREAD_SAFE_NAME_COUNTER = new AtomicInteger(); private static final boolean ENABLE_CLASS_REGISTRATION_FORCIBLY; @@ -104,6 +106,7 @@ public final class ForyBuilder { TypeChecker typeChecker; private List> actions = new ArrayList<>(); private boolean replayingActions = false; + private boolean nameExplicitlySet = false; public ForyBuilder() {} @@ -533,6 +536,7 @@ public ForyBuilder withScalaOptimizationEnabled(boolean enableScalaOptimization) /** Set name for Fory serialization. */ public ForyBuilder withName(String name) { this.name = name; + nameExplicitlySet = true; recordAction(b -> b.withName(name)); return this; } @@ -694,6 +698,7 @@ public Fory build() { * channel APIs keep a pooled {@link Fory} instance occupied for the whole blocking call. */ public ThreadSafeFory buildThreadSafeFory() { + assignGeneratedThreadSafeName(); finish(); ClassLoader loader = this.classLoader; this.classLoader = null; @@ -702,6 +707,7 @@ public ThreadSafeFory buildThreadSafeFory() { /** Build thread safe fory backed by {@link ThreadLocalFory}. */ public ThreadLocalFory buildThreadLocalFory() { + assignGeneratedThreadSafeName(); finish(); ClassLoader loader = this.classLoader; this.classLoader = null; @@ -724,12 +730,21 @@ public ThreadSafeFory buildThreadSafeForyPool(int poolSize) { String.format( "thread safe fory pool's size error, please check it, size:[%s]", poolSize)); } + assignGeneratedThreadSafeName(); finish(); ClassLoader loader = this.classLoader; this.classLoader = null; return new ThreadPoolFory(factory(loader), poolSize); } + private void assignGeneratedThreadSafeName() { + if (!nameExplicitlySet) { + String generatedName = "fory-thread-safe-" + THREAD_SAFE_NAME_COUNTER.incrementAndGet(); + name = generatedName; + recordAction(b -> b.withName(generatedName)); + } + } + private Function factory(ClassLoader loader) { List> actions = new ArrayList<>(this.actions); return builder -> { diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index 70494f2e79..59530e2f42 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -176,9 +176,12 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { builder.field(null); } } - FieldConverter converter = FieldConverters.getConverter(rawType, descriptor.getField()); - if (converter != null) { - builder.fieldConverter(converter); + if (descriptor.getField() != null) { + FieldConverter converter = + FieldConverters.getConverter(rawType, descriptor.getField()); + if (converter != null) { + builder.fieldConverter(converter); + } } } return builder.build(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 8827a2ffab..b23a6c4ae2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -34,6 +34,7 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; +import org.apache.fory.serializer.converter.FieldConverters; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.util.StringUtils; @@ -45,12 +46,14 @@ public abstract class StaticGeneratedStructSerializer extends AbstractObjectS protected final TypeDef typeDef; protected final List remoteFields; + private final SerializationFieldInfo[] localFieldsById; @SuppressWarnings("unchecked") public StaticGeneratedStructSerializer(TypeResolver typeResolver, Class type) { super(typeResolver, (Class) type); this.typeDef = null; this.remoteFields = Collections.emptyList(); + this.localFieldsById = new SerializationFieldInfo[0]; } @SuppressWarnings("unchecked") @@ -60,6 +63,7 @@ protected StaticGeneratedStructSerializer( this.typeDef = typeDef; this.remoteFields = typeDef == null ? Collections.emptyList() : buildRemoteFields(typeDef, descriptors); + this.localFieldsById = buildLocalFieldsById(descriptors); } @Override @@ -189,6 +193,25 @@ protected final int matchedId(RemoteFieldInfo remoteField) { return remoteField.matchedId; } + protected final SerializationFieldInfo localFieldInfo(int matchedId) { + return localFieldsById[matchedId]; + } + + protected final boolean canReadRemoteField( + RemoteFieldInfo remoteField, SerializationFieldInfo localFieldInfo) { + Class remoteType = remoteField.serializationFieldInfo.typeRef.getRawType(); + Class localType = localFieldInfo.typeRef.getRawType(); + return FieldConverters.canConvert(remoteType, localType); + } + + protected final Object readCompatibleFieldValue( + ReadContext readContext, RemoteFieldInfo remoteField, SerializationFieldInfo localFieldInfo) { + Object fieldValue = readRemoteField(readContext, remoteField); + Class remoteType = remoteField.serializationFieldInfo.typeRef.getRawType(); + Class localType = localFieldInfo.typeRef.getRawType(); + return FieldConverters.convertValue(remoteType, localType, fieldValue); + } + protected final boolean hasFieldConverter(RemoteFieldInfo remoteField) { return remoteField.serializationFieldInfo.fieldConverter != null; } @@ -254,6 +277,17 @@ private List buildRemoteFields( return Collections.unmodifiableList(remoteFields); } + private SerializationFieldInfo[] buildLocalFieldsById(List descriptors) { + FieldGroups fieldGroups = buildFieldGroups(descriptors); + SerializationFieldInfo[] allFields = fieldGroups.allFields; + int[] ids = localFieldIds(allFields, descriptors); + SerializationFieldInfo[] fieldsById = new SerializationFieldInfo[descriptors.size()]; + for (int i = 0; i < allFields.length; i++) { + fieldsById[ids[i]] = allFields[i]; + } + return fieldsById; + } + private int matchField( FieldInfo fieldInfo, Map fieldIds, Map fields) { Integer localId; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/converter/FieldConverters.java b/java/fory-core/src/main/java/org/apache/fory/serializer/converter/FieldConverters.java index fc602f512f..63ef581622 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/converter/FieldConverters.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/converter/FieldConverters.java @@ -126,6 +126,82 @@ public static FieldConverter getConverter(Class from, Field field) { return null; // No compatible converter found } + /** Returns whether a value of {@code from} can be assigned or converted to {@code to}. */ + public static boolean canConvert(Class from, Class to) { + if (isDirectlyAssignable(from, to)) { + return true; + } + Class wrappedFrom = TypeUtils.wrap(from); + if (to == int.class || to == Integer.class) { + return IntConverter.compatibleTypes.contains(wrappedFrom); + } else if (to == boolean.class || to == Boolean.class) { + return BooleanConverter.compatibleTypes.contains(wrappedFrom); + } else if (to == byte.class || to == Byte.class) { + return ByteConverter.compatibleTypes.contains(wrappedFrom); + } else if (to == short.class || to == Short.class) { + return ShortConverter.compatibleTypes.contains(wrappedFrom); + } else if (to == long.class || to == Long.class) { + return LongConverter.compatibleTypes.contains(wrappedFrom); + } else if (to == float.class || to == Float.class) { + return FloatConverter.compatibleTypes.contains(wrappedFrom); + } else if (to == double.class || to == Double.class) { + return DoubleConverter.compatibleTypes.contains(wrappedFrom); + } else if (to == String.class) { + return StringConverter.compatibleTypes.contains(wrappedFrom); + } + return false; + } + + /** + * Converts {@code value} from {@code from} to {@code to}, or returns it for direct assignment. + */ + public static Object convertValue(Class from, Class to, Object value) { + if (isDirectlyAssignable(from, to)) { + return value; + } else if (to == int.class) { + return IntConverter.convertFrom(value); + } else if (to == Integer.class) { + return BoxedIntConverter.convertFrom(value); + } else if (to == boolean.class) { + return BooleanConverter.convertFrom(value); + } else if (to == Boolean.class) { + return BoxedBooleanConverter.convertFrom(value); + } else if (to == byte.class) { + return ByteConverter.convertFrom(value); + } else if (to == Byte.class) { + return BoxedByteConverter.convertFrom(value); + } else if (to == short.class) { + return ShortConverter.convertFrom(value); + } else if (to == Short.class) { + return BoxedShortConverter.convertFrom(value); + } else if (to == long.class) { + return LongConverter.convertFrom(value); + } else if (to == Long.class) { + return BoxedLongConverter.convertFrom(value); + } else if (to == float.class) { + return FloatConverter.convertFrom(value); + } else if (to == Float.class) { + return BoxedFloatConverter.convertFrom(value); + } else if (to == double.class) { + return DoubleConverter.convertFrom(value); + } else if (to == Double.class) { + return BoxedDoubleConverter.convertFrom(value); + } else if (to == String.class) { + return StringConverter.convertFrom(value); + } + throw new UnsupportedOperationException("Incompatible type: " + from + " -> " + to); + } + + private static boolean isDirectlyAssignable(Class from, Class to) { + if (to.isAssignableFrom(from)) { + return true; + } + if (from.isPrimitive() && !to.isPrimitive()) { + return to.isAssignableFrom(TypeUtils.wrap(from)); + } + return false; + } + /** * Converter for primitive boolean fields. Converts compatible types to boolean values. Returns * false for null values and incompatible types. diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 99b763f971..402e2eee99 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -158,7 +158,7 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.builder.JITContext,\ org.apache.fory.builder.ObjectCodecBuilder,\ org.apache.fory.builder.MetaSharedCodecBuilder,\ - org.apache.fory.builder.CompatibleMetaSharedCodecBuilder,\ + org.apache.fory.builder.StaticCompatibleCodecBuilder,\ org.apache.fory.builder.Generated$GeneratedCompatibleMetaSharedSerializer,\ org.apache.fory.builder.CodecUtils,\ org.apache.fory.resolver.RefMode,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 480b1e4717..a24c54b2e6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -56,6 +56,33 @@ public void testBuildThreadSafeForyUsesThreadPoolFory() { assertTrue(fory instanceof ThreadPoolFory); } + @Test + public void testThreadSafeBuildersAssignGeneratedNames() { + ThreadSafeFory threadSafe = + Fory.builder().requireClassRegistration(false).buildThreadSafeFory(); + ThreadSafeFory threadLocal = + Fory.builder().requireClassRegistration(false).buildThreadLocalFory(); + ThreadSafeFory threadPool = + Fory.builder().requireClassRegistration(false).buildThreadSafeForyPool(1); + + String threadSafeName = threadSafe.execute(fory -> fory.getConfig().getName()); + String threadLocalName = threadLocal.execute(fory -> fory.getConfig().getName()); + String threadPoolName = threadPool.execute(fory -> fory.getConfig().getName()); + assertNotNull(threadSafeName); + assertNotNull(threadLocalName); + assertNotNull(threadPoolName); + Assert.assertNotEquals(threadSafeName, threadLocalName); + Assert.assertNotEquals(threadSafeName, threadPoolName); + Assert.assertNotEquals(threadLocalName, threadPoolName); + + ThreadSafeFory named = + Fory.builder() + .withName("explicit-thread-safe-name") + .requireClassRegistration(false) + .buildThreadSafeForyPool(1); + assertEquals(named.execute(fory -> fory.getConfig().getName()), "explicit-thread-safe-name"); + } + @Test public void testFunctionFactoryConstructorsUseBuilderProvidedClassLoader() { ClassLoader custom = new ClassLoader(ClassLoader.getSystemClassLoader()) {}; From 8a234cf93eca84b3578277723e599451e01c28eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 11:49:06 +0800 Subject: [PATCH 14/58] fix(java): cover static compatible read conversions --- .../processing/ForyStructProcessor.java | 36 ++-- .../processing/ForyStructProcessorTest.java | 198 ++++++++++++++++++ .../builder/StaticCompatibleCodecBuilder.java | 10 +- .../CompatibleCollectionArrayReader.java | 73 ++++++- .../StaticGeneratedStructSerializer.java | 20 +- 5 files changed, 309 insertions(+), 28 deletions(-) diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index 94b3ec3f8f..bf3f93c50f 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -258,11 +258,13 @@ private SourceField buildField( + "; use a record component or mark the field @Ignore/transient", field); } - SourceTypeNode typeNode = buildFieldTypeNode(field); + ForyFieldMeta foryField = foryField(field); + boolean nullable = + foryField.hasForyField ? foryField.nullable : !field.asType().getKind().isPrimitive(); + SourceTypeNode typeNode = buildFieldTypeNode(field, nullable); String erasedType = canonicalName(types.erasure(field.asType())); String declaringClass = elements.getBinaryName((TypeElement) field.getEnclosingElement()).toString(); - ForyFieldMeta foryField = foryField(field); SourceField.AccessKind readKind; SourceField.AccessKind writeKind; @@ -309,7 +311,7 @@ private SourceField buildField( writeAccess, foryField.hasForyField, foryField.id, - foryField.hasForyField ? foryField.nullable : !typeNode.primitive, + nullable, foryField.hasForyField && foryField.ref, foryField.dynamic); } @@ -521,8 +523,8 @@ private int reflectionModifiers(Set modifiers) { return value; } - private SourceTypeNode buildFieldTypeNode(VariableElement field) { - return buildTypeNode(field.asType(), typeTree(field)); + private SourceTypeNode buildFieldTypeNode(VariableElement field, boolean nullable) { + return buildTypeNode(field.asType(), typeTree(field), Boolean.toString(nullable)); } private Object typeTree(VariableElement field) { @@ -541,38 +543,41 @@ private Object typeTree(VariableElement field) { } private SourceTypeNode buildTypeNode(TypeMirror type) { - return buildTypeNode(type, null); + return buildTypeNode(type, null, "true"); } - private SourceTypeNode buildTypeNode(TypeMirror type, Object tree) { + private SourceTypeNode buildTypeNode(TypeMirror type, Object tree, String typeExtNullable) { TypeKind kind = type.getKind(); TypeTreeInfo treeInfo = typeTreeInfo(tree); if (kind == TypeKind.TYPEVAR) { TypeVariable typeVariable = (TypeVariable) type; - return buildTypeNode(typeVariable.getUpperBound()); + return buildTypeNode(typeVariable.getUpperBound(), null, typeExtNullable); } if (kind == TypeKind.WILDCARD) { WildcardType wildcard = (WildcardType) type; TypeMirror bound = wildcard.getExtendsBound(); return buildTypeNode( - bound == null ? elements.getTypeElement("java.lang.Object").asType() : bound); + bound == null ? elements.getTypeElement("java.lang.Object").asType() : bound, + null, + typeExtNullable); } List arguments = new ArrayList<>(); SourceTypeNode componentType = null; if (kind == TypeKind.ARRAY) { componentType = - buildTypeNode(((ArrayType) type).getComponentType(), treeInfo.arrayComponentTree()); + buildTypeNode( + ((ArrayType) type).getComponentType(), treeInfo.arrayComponentTree(), "true"); } else if (type instanceof DeclaredType) { List argumentTrees = treeInfo.typeArgumentTrees(); int index = 0; for (TypeMirror argument : ((DeclaredType) type).getTypeArguments()) { Object argumentTree = index < argumentTrees.size() ? argumentTrees.get(index) : null; - arguments.add(buildTypeNode(argument, argumentTree)); + arguments.add(buildTypeNode(argument, argumentTree, "true")); index++; } } String rawType = canonicalName(types.erasure(type)); - String extMeta = typeExtMetaExpression(type, rawType, treeInfo.annotations); + String extMeta = typeExtMetaExpression(type, rawType, treeInfo.annotations, typeExtNullable); boolean primitive = kind.isPrimitive(); boolean nestedStruct = isForyStructType(type); return new SourceTypeNode( @@ -595,7 +600,8 @@ private boolean isForyStructType(TypeMirror type) { return element instanceof TypeElement && hasAnnotation(element, FORY_STRUCT); } - private String typeExtMetaExpression(TypeMirror type, String rawType, List treeAnnotations) { + private String typeExtMetaExpression( + TypeMirror type, String rawType, List treeAnnotations, String nullable) { String typeId = scalarTypeId(type, rawType, treeAnnotations); TypeUseAnnotation ref = typeUseAnnotation(type, treeAnnotations, REF); if (typeId == null && ref == null) { @@ -603,7 +609,9 @@ private String typeExtMetaExpression(TypeMirror type, String rawType, List tr } return "meta(" + (typeId == null ? "Types.UNKNOWN" : typeId) - + ", true, " + + ", " + + nullable + + ", " + (ref != null && booleanValue(ref, "enable", true)) + ")"; } diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 770ce11e0d..6eaeb351e5 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -255,6 +255,7 @@ public void testGeneratedDescriptorsCarryNestedTypeMetadata() throws Exception { Descriptor code = descriptor(serializer.getDescriptors(), "code"); Assert.assertTrue(code.getTypeRef().hasTypeExtMeta()); Assert.assertEquals(code.getTypeRef().getTypeExtMeta().typeId(), Types.UINT16); + Assert.assertFalse(code.getTypeRef().getTypeExtMeta().nullable()); Assert.assertFalse(code.getTypeRef().getTypeExtMeta().trackingRef()); } } @@ -476,14 +477,97 @@ public void testGraalvmStaticCompatibleSerializerReadsRuntimeRemoteTypeDef() thr } } + @Test + public void testGraalvmStaticCompatibleRecordSerializerConvertsRemoteField() throws Exception { + assumeRecordSupport(); + if (AndroidSupport.IS_ANDROID) { + return; + } + CompilationResult writerResult = + compile( + "test.NativeRecordStruct", + "package test;\n" + + "public class NativeRecordStruct {\n" + + " public String id;\n" + + " public NativeRecordStruct() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.NativeRecordStruct", + "package test;\n" + "public record NativeRecordStruct(int id) {}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.NativeRecordStruct"); + Class readerType = readerLoader.loadClass("test.NativeRecordStruct"); + Fory writer = compatibleFory(writerLoader, false); + Fory reader = compatibleFory(readerLoader, true); + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "id", "73"); + + Object result = generatedCompatibleRead(writer, reader, writerType, readerType, writerValue); + Assert.assertSame(result.getClass(), readerType); + Assert.assertEquals(invoke(readerType, result, "id"), 73); + } + } + + @Test + public void testGraalvmStaticCompatibleSerializerUsesListArrayAction() throws Exception { + if (AndroidSupport.IS_ANDROID) { + return; + } + CompilationResult writerResult = + compile( + "test.NativeArrayShapeStruct", + "package test;\n" + + "import org.apache.fory.annotation.Int32Type;\n" + + "import org.apache.fory.collection.Int32List;\n" + + "import org.apache.fory.config.Int32Encoding;\n" + + "public class NativeArrayShapeStruct {\n" + + " @Int32Type(encoding = Int32Encoding.FIXED) public Int32List values;\n" + + " public NativeArrayShapeStruct() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.NativeArrayShapeStruct", + "package test;\n" + + "public class NativeArrayShapeStruct {\n" + + " public int[] values;\n" + + " public NativeArrayShapeStruct() {}\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.NativeArrayShapeStruct"); + Class readerType = readerLoader.loadClass("test.NativeArrayShapeStruct"); + Fory writer = xlangCompatibleFory(writerLoader, writerType, true, "NativeArrayShapeStruct"); + Fory reader = xlangCompatibleFory(readerLoader, readerType, true, "NativeArrayShapeStruct"); + Object writerValue = writerType.getConstructor().newInstance(); + setField( + writerType, + writerValue, + "values", + new org.apache.fory.collection.Int32List(new int[] {4, 5, 6})); + + Object result = generatedCompatibleRead(writer, reader, writerType, readerType, writerValue); + Assert.assertSame(result.getClass(), readerType); + Assert.assertTrue( + Arrays.equals((int[]) getField(readerType, result, "values"), new int[] {4, 5, 6})); + } + } + private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exception { CompilationResult staticResult = compile( "test.RoundTripStruct", "package test;\n" + + "import org.apache.fory.annotation.UInt16Type;\n" + "import org.apache.fory.annotation.ForyStruct;\n" + "@ForyStruct public class RoundTripStruct {\n" + " public int id;\n" + + " public @UInt16Type int code;\n" + " public String name;\n" + " public RoundTripStruct() {}\n" + "}\n"); @@ -491,8 +575,10 @@ private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exce compile( "test.RoundTripStruct", "package test;\n" + + "import org.apache.fory.annotation.UInt16Type;\n" + "public class RoundTripStruct {\n" + " public int id;\n" + + " public @UInt16Type int code;\n" + " public String name;\n" + " public RoundTripStruct() {}\n" + "}\n"); @@ -518,20 +604,24 @@ private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exce Object staticValue = staticType.getConstructor().newInstance(); setField(staticType, staticValue, "id", 101); + setField(staticType, staticValue, "code", 513); setField(staticType, staticValue, "name", compatible ? "compatible-static" : "static"); Object runtimeRoundTrip = runtimeFory.deserialize(staticFory.serialize(staticValue)); Assert.assertSame(runtimeRoundTrip.getClass(), runtimeType); Assert.assertEquals(getField(runtimeType, runtimeRoundTrip, "id"), 101); + Assert.assertEquals(getField(runtimeType, runtimeRoundTrip, "code"), 513); Assert.assertEquals( getField(runtimeType, runtimeRoundTrip, "name"), compatible ? "compatible-static" : "static"); Object runtimeValue = runtimeType.getConstructor().newInstance(); setField(runtimeType, runtimeValue, "id", 202); + setField(runtimeType, runtimeValue, "code", 1024); setField(runtimeType, runtimeValue, "name", compatible ? "compatible-runtime" : "runtime"); Object staticRoundTrip = staticFory.deserialize(runtimeFory.serialize(runtimeValue)); Assert.assertSame(staticRoundTrip.getClass(), staticType); Assert.assertEquals(getField(staticType, staticRoundTrip, "id"), 202); + Assert.assertEquals(getField(staticType, staticRoundTrip, "code"), 1024); Assert.assertEquals( getField(staticType, staticRoundTrip, "name"), compatible ? "compatible-runtime" : "runtime"); @@ -548,6 +638,114 @@ private static ThreadSafeFory threadSafeFory( .buildThreadSafeForyPool(1); } + @Test + public void testStaticCompatibleReadUsesListArrayAction() throws Exception { + CompilationResult writerResult = + compile( + "test.ArrayShapeStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "import org.apache.fory.annotation.Int32Type;\n" + + "import org.apache.fory.collection.Int32List;\n" + + "import org.apache.fory.config.Int32Encoding;\n" + + "@ForyStruct public class ArrayShapeStruct {\n" + + " @Int32Type(encoding = Int32Encoding.FIXED) public Int32List values;\n" + + " public ArrayShapeStruct() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.ArrayShapeStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class ArrayShapeStruct {\n" + + " public int[] values;\n" + + " public ArrayShapeStruct() {}\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.ArrayShapeStruct"); + Class readerType = readerLoader.loadClass("test.ArrayShapeStruct"); + Fory writer = xlangCompatibleFory(writerLoader, writerType, false); + Fory reader = xlangCompatibleFory(readerLoader, readerType, false); + Object writerValue = writerType.getConstructor().newInstance(); + setField( + writerType, + writerValue, + "values", + new org.apache.fory.collection.Int32List(new int[] {1, 2, 3})); + + Object result = reader.deserialize(writer.serialize(writerValue)); + Assert.assertSame(result.getClass(), readerType); + Assert.assertTrue( + Arrays.equals((int[]) getField(readerType, result, "values"), new int[] {1, 2, 3})); + } + } + + private static Fory xlangCompatibleFory(ClassLoader classLoader, Class type, boolean codegen) { + return xlangCompatibleFory(classLoader, type, codegen, "ArrayShapeStruct"); + } + + private static Fory xlangCompatibleFory( + ClassLoader classLoader, Class type, boolean codegen, String typeName) { + Fory fory = + Fory.builder() + .withClassLoader(classLoader) + .withXlang(true) + .withCompatible(true) + .withCodegen(codegen) + .requireClassRegistration(false) + .build(); + fory.register(type, "test", typeName); + return fory; + } + + private static Fory compatibleFory(ClassLoader classLoader, boolean codegen) { + return Fory.builder() + .withClassLoader(classLoader) + .withCodegen(codegen) + .withMetaShare(true) + .withScopedMetaShare(false) + .withCompatible(true) + .requireClassRegistration(false) + .build(); + } + + private static Object generatedCompatibleRead( + Fory writer, Fory reader, Class writerType, Class readerType, Object writerValue) + throws Exception { + TypeDef remoteTypeDef = TypeDef.buildTypeDef(writer.getTypeResolver(), writerType); + Class> serializerClass = + CodecUtils.loadOrGenCompatibleMetaSharedCodecClass( + reader.getTypeResolver(), (Class) readerType, remoteTypeDef); + Assert.assertTrue( + GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(serializerClass)); + Serializer serializer = + serializerClass + .getConstructor(org.apache.fory.resolver.TypeResolver.class, Class.class, TypeDef.class) + .newInstance(reader.getTypeResolver(), (Class) readerType, remoteTypeDef); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(128); + Serializer writerSerializer = + writer.getConfig().isXlang() + ? writer.getTypeResolver().getTypeInfo(writerType).getSerializer() + : new MetaSharedSerializer<>( + writer.getTypeResolver(), (Class) writerType, remoteTypeDef); + writer.getWriteContext().prepare(buffer, null); + try { + writerSerializer.write(writer.getWriteContext(), writerValue); + } finally { + writer.getWriteContext().reset(); + } + buffer.readerIndex(0); + reader.getReadContext().prepare(buffer, null, false); + try { + return serializer.read(reader.getReadContext()); + } finally { + reader.getReadContext().reset(); + } + } + private static CompilationResult compile(String typeName, String source) throws IOException { JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); Assert.assertNotNull(compiler, "Tests require a JDK compiler"); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 0622ff053d..9003a2a7b0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -360,17 +360,13 @@ private String genRecordDispatchGroup(int group) { code.append(" case ") .append(i) .append(": {\n") - .append(" if (hasFieldConverter(_f_remoteField)) {\n") - .append(" readRemoteField(") - .append(READ_CONTEXT_NAME) - .append(", _f_remoteField);\n") - .append( - " } else if (canReadRemoteField(_f_remoteField, localFieldInfo(_f_matchedId))) {\n") + .append(" SerializationFieldInfo _f_localField = localFieldInfo(_f_matchedId);\n") + .append(" if (canReadRemoteField(_f_remoteField, _f_localField)) {\n") .append(" _f_recordValues[") .append(componentIndex) .append("] = readCompatibleFieldValue(") .append(READ_CONTEXT_NAME) - .append(", _f_remoteField, localFieldInfo(_f_matchedId));\n") + .append(", _f_remoteField, _f_localField);\n") .append(" } else {\n") .append(" skipField(") .append(READ_CONTEXT_NAME) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index 991841d3d9..50a1a2c763 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -42,6 +42,7 @@ import org.apache.fory.exception.DeserializationException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.NativeByteOrder; +import org.apache.fory.meta.FieldInfo; import org.apache.fory.meta.FieldTypes; import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.reflect.TypeRef; @@ -53,6 +54,7 @@ import org.apache.fory.type.Descriptor; import org.apache.fory.type.Float16; import org.apache.fory.type.Float16Array; +import org.apache.fory.type.TypeAnnotationUtils; import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; @@ -104,6 +106,35 @@ static ReadAction readAction(TypeResolver resolver, Descriptor descriptor) { return null; } + static ReadAction readAction( + TypeResolver resolver, FieldInfo remoteFieldInfo, Descriptor localDescriptor) { + if (localDescriptor == null || !resolver.isCrossLanguage()) { + return null; + } + FieldTypes.FieldType remoteFieldType = remoteFieldInfo.getFieldType(); + TypeRef localType = localDescriptor.getTypeRef(); + int peerListElementTypeId = listElementTypeId(remoteFieldType); + if (peerListElementTypeId != Types.UNKNOWN) { + int localArrayTypeId = arrayTypeId(localType); + if (localArrayTypeId != Types.UNKNOWN + && localArrayTypeId == denseArrayTypeId(peerListElementTypeId)) { + return new ReadAction( + READ_LIST_TO_ARRAY, localArrayTypeId, peerListElementTypeId, localDescriptor.getType()); + } + return null; + } + int peerArrayTypeId = arrayTypeId(remoteFieldType); + if (peerArrayTypeId != Types.UNKNOWN) { + int localListElementTypeId = listElementTypeId(localType); + if (localListElementTypeId != Types.UNKNOWN + && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { + return new ReadAction( + READ_ARRAY_TO_LIST, peerArrayTypeId, localListElementTypeId, localDescriptor.getType()); + } + } + return null; + } + static Object read(ReadContext readContext, RefMode refMode, ReadAction action) { return read( readContext, @@ -187,11 +218,16 @@ private static int listElementTypeId(FieldTypes.FieldType fieldType) { private static int listElementTypeId(TypeRef typeRef) { TypeExtMeta extMeta = typeRef.getTypeExtMeta(); - if (extMeta == null || extMeta.typeId() != Types.LIST) { - return Types.UNKNOWN; + if (extMeta != null && extMeta.typeId() == Types.LIST) { + TypeExtMeta elementExtMeta = TypeUtils.getElementType(typeRef).getTypeExtMeta(); + return elementExtMeta == null ? Types.UNKNOWN : elementExtMeta.typeId(); + } + if (TypeUtils.isPrimitiveListClass(typeRef.getRawType())) { + return extMeta != null && Types.isPrimitiveType(extMeta.typeId()) + ? extMeta.typeId() + : TypeAnnotationUtils.getDefaultPrimitiveListElementTypeId(typeRef.getRawType()); } - TypeExtMeta elementExtMeta = TypeUtils.getElementType(typeRef).getTypeExtMeta(); - return elementExtMeta == null ? Types.UNKNOWN : elementExtMeta.typeId(); + return Types.UNKNOWN; } private static int arrayTypeId(FieldTypes.FieldType fieldType) { @@ -209,6 +245,35 @@ private static int arrayTypeId(TypeRef typeRef) { if (extMeta != null && Types.isPrimitiveArray(extMeta.typeId())) { return extMeta.typeId(); } + Class rawType = typeRef.getRawType(); + if (rawType.isArray() && rawType.getComponentType().isPrimitive()) { + return primitiveArrayTypeId(rawType.getComponentType()); + } + return Types.UNKNOWN; + } + + private static int primitiveArrayTypeId(Class componentType) { + if (componentType == boolean.class) { + return Types.BOOL_ARRAY; + } + if (componentType == byte.class) { + return Types.INT8_ARRAY; + } + if (componentType == short.class) { + return Types.INT16_ARRAY; + } + if (componentType == int.class) { + return Types.INT32_ARRAY; + } + if (componentType == long.class) { + return Types.INT64_ARRAY; + } + if (componentType == float.class) { + return Types.FLOAT32_ARRAY; + } + if (componentType == double.class) { + return Types.FLOAT64_ARRAY; + } return Types.UNKNOWN; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index b23a6c4ae2..e8a1258bad 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -199,6 +199,9 @@ protected final SerializationFieldInfo localFieldInfo(int matchedId) { protected final boolean canReadRemoteField( RemoteFieldInfo remoteField, SerializationFieldInfo localFieldInfo) { + if (remoteField.compatibleCollectionArrayReadAction != null) { + return true; + } Class remoteType = remoteField.serializationFieldInfo.typeRef.getRawType(); Class localType = localFieldInfo.typeRef.getRawType(); return FieldConverters.canConvert(remoteType, localType); @@ -207,6 +210,9 @@ protected final boolean canReadRemoteField( protected final Object readCompatibleFieldValue( ReadContext readContext, RemoteFieldInfo remoteField, SerializationFieldInfo localFieldInfo) { Object fieldValue = readRemoteField(readContext, remoteField); + if (remoteField.compatibleCollectionArrayReadAction != null) { + return fieldValue; + } Class remoteType = remoteField.serializationFieldInfo.typeRef.getRawType(); Class localType = localFieldInfo.typeRef.getRawType(); return FieldConverters.convertValue(remoteType, localType, fieldValue); @@ -270,9 +276,16 @@ private List buildRemoteFields( int matchedId = matchField(fieldInfo, fieldIds, fields); SerializationFieldInfo serializationFieldInfo = FieldGroups.buildFieldInfo(typeResolver, descriptor); + Descriptor localDescriptor = + matchedId == UNKNOWN_FIELD ? null : localDescriptors.get(matchedId); remoteFields.add( new RemoteFieldInfo( - typeResolver, matchedId, fieldInfo, descriptor, serializationFieldInfo)); + typeResolver, + matchedId, + fieldInfo, + descriptor, + serializationFieldInfo, + localDescriptor)); } return Collections.unmodifiableList(remoteFields); } @@ -325,13 +338,14 @@ private RemoteFieldInfo( int matchedId, FieldInfo fieldInfo, Descriptor descriptor, - SerializationFieldInfo serializationFieldInfo) { + SerializationFieldInfo serializationFieldInfo, + Descriptor localDescriptor) { this.matchedId = matchedId; this.fieldInfo = fieldInfo; this.descriptor = descriptor; this.serializationFieldInfo = serializationFieldInfo; this.compatibleCollectionArrayReadAction = - CompatibleCollectionArrayReader.readAction(typeResolver, descriptor); + CompatibleCollectionArrayReader.readAction(typeResolver, fieldInfo, localDescriptor); } } } From 00703ac8d9eaf6fae51f9ec62667714af4f1eacf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 13:28:32 +0800 Subject: [PATCH 15/58] fix(java): stabilize static generated xlang serializers --- .github/workflows/ci.yml | 7 + ci/run_ci.py | 126 +++++++++++++++++- .../processing/ForyStructProcessor.java | 4 +- .../java/org/apache/fory/meta/FieldTypes.java | 52 +++++++- .../java/org/apache/fory/reflect/TypeRef.java | 4 +- .../apache/fory/resolver/TypeResolver.java | 16 ++- .../CompatibleCollectionArrayReader.java | 10 +- .../apache/fory/serializer/FieldGroups.java | 37 +++-- .../StaticGeneratedStructSerializer.java | 73 ++++++++-- .../fory/serializer/struct/Fingerprint.java | 5 +- .../org/apache/fory/xlang/XlangTestBase.java | 56 ++++++++ 11 files changed, 350 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9792b0e5ab..51cefc1644 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1270,6 +1270,13 @@ jobs: mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true cd fory-core mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.GoXlangTest + - name: Run Android Go Xlang Tests With Static Generated Serializers + env: + FORY_ANDROID_ENABLED: "1" + FORY_GO_JAVA_CI: "1" + FORY_STATIC_PROCESSOR_CI: "1" + ENABLE_FORY_DEBUG_OUTPUT: "1" + run: python ./ci/run_ci.py android-go-xlang-static-processor android_serializer: name: Android Serializer JVM Round Trip Test diff --git a/ci/run_ci.py b/ci/run_ci.py index e3877549ce..de181549b4 100644 --- a/ci/run_ci.py +++ b/ci/run_ci.py @@ -18,11 +18,14 @@ import argparse import logging import os +import shlex import shutil import subprocess import sys +import tempfile +import xml.etree.ElementTree as ET -from tasks import cpp, java, javascript, kotlin, rust, python, go, format +from tasks import common, cpp, java, javascript, kotlin, rust, python, go, format from tasks.common import is_windows # Configure logging @@ -104,6 +107,119 @@ def run_shell_script(command, *args): sys.exit(subprocess.call(cmd)) +def _pom_tag(namespace, name): + return f"{{{namespace}}}{name}" if namespace else name + + +def _pom_namespace(root): + if root.tag.startswith("{"): + return root.tag[1:].split("}", 1)[0] + return "" + + +def _find_child(element, namespace, name): + return element.find(_pom_tag(namespace, name)) + + +def _find_or_add_child(element, namespace, name): + child = _find_child(element, namespace, name) + if child is None: + child = ET.SubElement(element, _pom_tag(namespace, name)) + return child + + +def _child_text(element, namespace, name): + child = _find_child(element, namespace, name) + return "" if child is None or child.text is None else child.text.strip() + + +def _find_or_add_compiler_plugin(root, namespace): + build = _find_or_add_child(root, namespace, "build") + plugins = _find_or_add_child(build, namespace, "plugins") + for plugin in plugins.findall(_pom_tag(namespace, "plugin")): + if _child_text(plugin, namespace, "artifactId") == "maven-compiler-plugin": + return plugin + plugin = ET.SubElement(plugins, _pom_tag(namespace, "plugin")) + group_id = ET.SubElement(plugin, _pom_tag(namespace, "groupId")) + group_id.text = "org.apache.maven.plugins" + artifact_id = ET.SubElement(plugin, _pom_tag(namespace, "artifactId")) + artifact_id.text = "maven-compiler-plugin" + return plugin + + +def _has_processor_path(annotation_processor_paths, namespace, artifact_id): + for path in annotation_processor_paths.findall(_pom_tag(namespace, "path")): + if _child_text(path, namespace, "artifactId") == artifact_id: + return True + return False + + +def _add_annotation_processor_path(pom_path): + ET.register_namespace("", "http://maven.apache.org/POM/4.0.0") + ET.register_namespace("xsi", "http://www.w3.org/2001/XMLSchema-instance") + tree = ET.parse(pom_path) + root = tree.getroot() + namespace = _pom_namespace(root) + compiler_plugin = _find_or_add_compiler_plugin(root, namespace) + configuration = _find_or_add_child(compiler_plugin, namespace, "configuration") + annotation_processor_paths = _find_or_add_child( + configuration, namespace, "annotationProcessorPaths" + ) + annotation_processor_paths.set("combine.children", "append") + if not _has_processor_path( + annotation_processor_paths, namespace, "fory-annotation-processor" + ): + path = ET.SubElement(annotation_processor_paths, _pom_tag(namespace, "path")) + group_id = ET.SubElement(path, _pom_tag(namespace, "groupId")) + group_id.text = "org.apache.fory" + artifact_id = ET.SubElement(path, _pom_tag(namespace, "artifactId")) + artifact_id.text = "fory-annotation-processor" + version = ET.SubElement(path, _pom_tag(namespace, "version")) + version.text = "${project.version}" + tree.write(pom_path, encoding="UTF-8", xml_declaration=True) + + +def _exec_cmd(cmd, cwd): + logging.info(f"running command in {cwd}: {cmd}") + subprocess.check_call(cmd, shell=True, cwd=cwd) + + +def run_android_go_xlang_static_processor(): + """Run Android-mode Go xlang tests with javac static serializer generation enabled.""" + root = common.PROJECT_ROOT_DIR + java_dir = os.path.join(root, "java") + core_dir = os.path.join(java_dir, "fory-core") + core_pom = os.path.join(core_dir, "pom.xml") + fd, temp_pom = tempfile.mkstemp( + prefix="pom-static-processor-", suffix=".xml", dir=core_dir + ) + os.close(fd) + try: + shutil.copyfile(core_pom, temp_pom) + _add_annotation_processor_path(temp_pom) + os.environ.setdefault("FORY_ANDROID_ENABLED", "1") + os.environ.setdefault("FORY_GO_JAVA_CI", "1") + os.environ.setdefault("FORY_STATIC_PROCESSOR_CI", "1") + os.environ.setdefault("ENABLE_FORY_DEBUG_OUTPUT", "1") + _exec_cmd( + "mvn -T16 --no-transfer-progress install -DskipTests " + "-Dmaven.javadoc.skip=true -Dmaven.source.skip=true " + "-pl fory-annotation-processor -am", + java_dir, + ) + _exec_cmd( + "mvn -T16 --no-transfer-progress " + f"-f {shlex.quote(temp_pom)} " + "clean test -Dtest=org.apache.fory.xlang.GoXlangTest", + core_dir, + ) + finally: + try: + os.remove(temp_pom) + except FileNotFoundError: + pass + + def parse_args(): """Parse command-line arguments and dispatch to the appropriate task module.""" parser = argparse.ArgumentParser( @@ -225,6 +341,14 @@ def parse_args(): ) go_parser.set_defaults(func=go.run) + android_go_static_parser = subparsers.add_parser( + "android-go-xlang-static-processor", + description="Run Android-mode Go xlang tests with @ForyStruct annotation processing enabled", + help="Run Android Go xlang tests with static generated serializers", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + android_go_static_parser.set_defaults(func=run_android_go_xlang_static_processor) + # Format subparser format_parser = subparsers.add_parser( "format", diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index bf3f93c50f..e8f9815461 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -566,13 +566,13 @@ private SourceTypeNode buildTypeNode(TypeMirror type, Object tree, String typeEx if (kind == TypeKind.ARRAY) { componentType = buildTypeNode( - ((ArrayType) type).getComponentType(), treeInfo.arrayComponentTree(), "true"); + ((ArrayType) type).getComponentType(), treeInfo.arrayComponentTree(), "false"); } else if (type instanceof DeclaredType) { List argumentTrees = treeInfo.typeArgumentTrees(); int index = 0; for (TypeMirror argument : ((DeclaredType) type).getTypeArguments()) { Object argumentTree = index < argumentTrees.size() ? argumentTrees.get(index) : null; - arguments.add(buildTypeNode(argument, argumentTree, "true")); + arguments.add(buildTypeNode(argument, argumentTree, "false")); index++; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 38f0a4b7fb..e3b24ce097 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -117,11 +117,19 @@ private static FieldType buildFieldType( && descriptor != null && !primitiveList && TypeAnnotationUtils.isBoxedListArrayType(descriptor); + TypeExtMeta typeExtMeta = genericType.getTypeRef().getTypeExtMeta(); + TypeExtMeta primitiveListArgumentMeta = + primitiveList ? primitiveListArgumentMeta(genericType.getTypeRef()) : null; + TypeExtMeta primitiveListInlineMeta = + primitiveList ? primitiveListInlineMeta(genericType.getTypeRef()) : null; + TypeExtMeta primitiveListElementMeta = + primitiveListArgumentMeta != null ? primitiveListArgumentMeta : primitiveListInlineMeta; int primitiveListElementTypeId = primitiveList - ? TypeAnnotationUtils.getPrimitiveListElementTypeId(typeAnnotation, rawType, isXlang) + ? primitiveListElementMeta != null + ? primitiveListElementMeta.typeId() + : TypeAnnotationUtils.getPrimitiveListElementTypeId(typeAnnotation, rawType, isXlang) : Types.UNKNOWN; - TypeExtMeta typeExtMeta = genericType.getTypeRef().getTypeExtMeta(); if (typeExtMeta != null && typeExtMeta.typeId() != Types.UNKNOWN) { typeId = typeExtMeta.typeId(); } else if (isXlang && primitiveList) { @@ -186,8 +194,11 @@ private static FieldType buildFieldType( } // For xlang: ref tracking is false by default (no shared ownership like Rust's Rc/Arc) // For native: use the type's default tracking behavior + boolean descriptorCarriesFieldOptions = descriptor != null && field == null; boolean trackingRef = - isXlang + descriptorCarriesFieldOptions + ? descriptor.isTrackingRef() + : isXlang ? typeExtMeta != null && typeExtMeta.trackingRef() : genericType.trackingRef(resolver); // For xlang: nullable is false by default for top-level fields. @@ -196,9 +207,15 @@ private static FieldType buildFieldType( // Optional types are nullable (like Rust's Option). // For native: non-primitive types are nullable by default. boolean nullable; - if (isXlang) { - boolean nestedType = field == null; - nullable = nestedType || isOptionalType(rawType); + if (descriptorCarriesFieldOptions) { + nullable = descriptor.isNullable(); + } else if (isXlang) { + if (typeExtMeta != null) { + nullable = typeExtMeta.nullable(); + } else { + boolean nestedType = descriptor == null; + nullable = nestedType || isOptionalType(rawType); + } } else { // Primitives are never nullable, non-primitives are nullable by default // This applies to both top-level fields and nested types (in arrays, collections, maps) @@ -221,11 +238,18 @@ private static FieldType buildFieldType( } if (isXlang && primitiveList && !primitiveListArray) { + boolean elementNullable = true; + boolean elementTrackingRef = false; + if (primitiveListArgumentMeta != null) { + elementNullable = primitiveListArgumentMeta.nullable(); + elementTrackingRef = primitiveListArgumentMeta.trackingRef(); + } return new CollectionFieldType( Types.LIST, nullable, trackingRef, - new RegisteredFieldType(true, false, primitiveListElementTypeId, -1)); + new RegisteredFieldType( + elementNullable, elementTrackingRef, primitiveListElementTypeId, -1)); } if (COLLECTION_TYPE.isSupertypeOf(genericType.getTypeRef())) { @@ -313,6 +337,20 @@ private static Tuple2, TypeRef> getMapKeyValueType(GenericType gen return TypeUtils.getMapKeyValueType(genericType.getTypeRef()); } + private static TypeExtMeta primitiveListInlineMeta(TypeRef typeRef) { + TypeExtMeta typeExtMeta = typeRef.getTypeExtMeta(); + return typeExtMeta != null && Types.isPrimitiveType(typeExtMeta.typeId()) ? typeExtMeta : null; + } + + private static TypeExtMeta primitiveListArgumentMeta(TypeRef typeRef) { + if (!typeRef.hasExplicitTypeArguments()) { + return null; + } + TypeRef elementType = TypeUtils.getElementType(typeRef); + TypeExtMeta elementMeta = elementType.getTypeExtMeta(); + return elementMeta != null && Types.isPrimitiveType(elementMeta.typeId()) ? elementMeta : null; + } + public abstract static class FieldType implements Serializable { private static final int KIND_OBJECT = 0; private static final int KIND_MAP = 1; diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/TypeRef.java b/java/fory-core/src/main/java/org/apache/fory/reflect/TypeRef.java index 8a6dc4e08a..007221ee02 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/TypeRef.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/TypeRef.java @@ -187,9 +187,9 @@ private static TypeRef ofAnnotatedType(AnnotatedType annotatedType, boolean i Ref ref = annotatedType.getAnnotation(Ref.class); if (typeAnnotation != null) { int typeId = TypeAnnotationUtils.getTypeId(typeAnnotation, TypeUtils.getRawType(type)); - meta = TypeExtMeta.of(typeId, true, ref != null && ref.enable()); + meta = TypeExtMeta.of(typeId, false, ref != null && ref.enable()); } else if (ref != null) { - meta = TypeExtMeta.of(Types.UNKNOWN, true, ref.enable()); + meta = TypeExtMeta.of(Types.UNKNOWN, false, ref.enable()); } } return new TypeRef<>(type, meta, typeArguments, componentType); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index ff74c43b29..b406fffae2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1473,10 +1473,18 @@ private DescriptorGrouper buildDescriptorGrouper( .sort(); } + @Internal + public final DescriptorGrouper groupDescriptors( + Collection descriptors, + boolean descriptorsGroupedOrdered, + Function descriptorUpdator) { + return buildDescriptorGrouper(descriptors, descriptorsGroupedOrdered, descriptorUpdator); + } + private List buildFieldDescriptors(Class clz, boolean searchParent) { List staticDescriptors = getStaticGeneratedStructDescriptors(clz); if (staticDescriptors != null) { - return buildFieldDescriptors(clz, searchParent, staticDescriptors); + return normalizeFieldDescriptors(clz, searchParent, staticDescriptors); } SortedMap allDescriptors = getAllDescriptorsMap(clz, searchParent); List result = new ArrayList<>(allDescriptors.size()); @@ -1491,6 +1499,12 @@ private List buildFieldDescriptors(Class clz, boolean searchParen return buildFieldDescriptors(clz, searchParent, descriptors); } + @Internal + public final List normalizeFieldDescriptors( + Class clz, boolean searchParent, List descriptors) { + return buildFieldDescriptors(clz, searchParent, descriptors); + } + private List buildFieldDescriptors( Class clz, boolean searchParent, List descriptors) { List result = new ArrayList<>(descriptors.size()); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index 50a1a2c763..1b1f2c5f72 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -119,7 +119,10 @@ static ReadAction readAction( if (localArrayTypeId != Types.UNKNOWN && localArrayTypeId == denseArrayTypeId(peerListElementTypeId)) { return new ReadAction( - READ_LIST_TO_ARRAY, localArrayTypeId, peerListElementTypeId, localDescriptor.getType()); + READ_LIST_TO_ARRAY, + localArrayTypeId, + peerListElementTypeId, + localDescriptor.getRawType()); } return null; } @@ -129,7 +132,10 @@ static ReadAction readAction( if (localListElementTypeId != Types.UNKNOWN && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { return new ReadAction( - READ_ARRAY_TO_LIST, peerArrayTypeId, localListElementTypeId, localDescriptor.getType()); + READ_ARRAY_TO_LIST, + peerArrayTypeId, + localListElementTypeId, + localDescriptor.getRawType()); } } return null; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index a662af9f9f..f9ed1625b6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -41,6 +41,7 @@ import org.apache.fory.type.GenericType; import org.apache.fory.type.TypeAnnotationUtils; import org.apache.fory.type.TypeUtils; +import org.apache.fory.type.Types; import org.apache.fory.util.StringUtils; public class FieldGroups { @@ -87,16 +88,7 @@ static DescriptorGrouper buildDescriptorGrouper( Collection descriptors, boolean descriptorsGroupedOrdered, Function descriptorUpdator) { - return DescriptorGrouper.createDescriptorGrouper( - typeResolver::usesPrimitiveFieldOrdering, - typeResolver::isBuildIn, - typeResolver::isCollectionDescriptor, - descriptors, - descriptorsGroupedOrdered, - descriptorUpdator, - typeResolver.getPrimitiveComparator(), - typeResolver.getDescriptorComparator()) - .sort(); + return typeResolver.groupDescriptors(descriptors, descriptorsGroupedOrdered, descriptorUpdator); } public static FieldGroups buildFieldInfos(TypeResolver typeResolver, DescriptorGrouper grouper) { @@ -235,9 +227,7 @@ public static final class SerializationFieldInfo { GenericType t; if (primitiveListCollection) { - TypeRef elementTypeRef = - TypeAnnotationUtils.getPrimitiveListElementTypeRef( - d.getTypeAnnotation(), typeRef.getRawType()); + TypeRef elementTypeRef = primitiveListElementTypeRef(d); t = new GenericType(typeRef, true, resolver.buildGenericType(elementTypeRef)); } else { t = resolver.buildGenericType(typeRef); @@ -320,4 +310,25 @@ public String toString() { + '}'; } } + + private static TypeRef primitiveListElementTypeRef(Descriptor descriptor) { + TypeRef typeRef = descriptor.getTypeRef(); + TypeExtMeta inlineMeta = typeRef.getTypeExtMeta(); + if (inlineMeta != null && Types.isPrimitiveType(inlineMeta.typeId())) { + Class elementClass = TypeAnnotationUtils.getPrimitiveListElementClass(typeRef.getRawType()); + if (elementClass != null) { + return TypeRef.of( + elementClass, TypeExtMeta.of(inlineMeta.typeId(), true, inlineMeta.trackingRef())); + } + } + if (typeRef.hasExplicitTypeArguments()) { + TypeRef elementTypeRef = TypeUtils.getElementType(typeRef); + TypeExtMeta elementMeta = elementTypeRef.getTypeExtMeta(); + if (elementMeta != null && Types.isPrimitiveType(elementMeta.typeId())) { + return elementTypeRef; + } + } + return TypeAnnotationUtils.getPrimitiveListElementTypeRef( + descriptor.getTypeAnnotation(), typeRef.getRawType()); + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index e8a1258bad..e7c4a21b33 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -60,10 +60,11 @@ public StaticGeneratedStructSerializer(TypeResolver typeResolver, Class type) protected StaticGeneratedStructSerializer( TypeResolver typeResolver, Class type, TypeDef typeDef, List descriptors) { super(typeResolver, (Class) type); + List runtimeDescriptors = runtimeDescriptors(descriptors); this.typeDef = typeDef; this.remoteFields = - typeDef == null ? Collections.emptyList() : buildRemoteFields(typeDef, descriptors); - this.localFieldsById = buildLocalFieldsById(descriptors); + typeDef == null ? Collections.emptyList() : buildRemoteFields(typeDef, runtimeDescriptors); + this.localFieldsById = buildLocalFieldsById(runtimeDescriptors); } @Override @@ -80,12 +81,17 @@ protected StaticGeneratedStructSerializer( public abstract T readCompatible(ReadContext readContext); protected final FieldGroups buildFieldGroups(List descriptors) { + descriptors = runtimeDescriptors(descriptors); DescriptorGrouper grouper = FieldGroups.buildDescriptorGrouper( typeResolver, descriptors, false, descriptor -> descriptor); return FieldGroups.buildFieldInfos(typeResolver, grouper); } + protected final List runtimeDescriptors(List descriptors) { + return typeResolver.normalizeFieldDescriptors(type, true, descriptors); + } + protected final int[] localFieldIds( SerializationFieldInfo[] fieldInfos, List descriptors) { Map localIds = new HashMap<>(); @@ -233,6 +239,7 @@ protected final Object copyFieldValue( } protected final int computeClassVersionHash(List descriptors) { + descriptors = runtimeDescriptors(descriptors); return ObjectSerializer.computeStructHash( typeResolver, FieldGroups.buildDescriptorGrouper( @@ -260,6 +267,12 @@ private List buildRemoteFields( List remoteFieldInfos = remoteTypeDef.getFieldsInfo(); List remoteDescriptors = remoteTypeDef.getDescriptors(typeResolver, type, localDescriptors); + Map remoteFieldInfosByKey = new HashMap<>(); + for (int i = 0; i < remoteFieldInfos.size(); i++) { + FieldInfo fieldInfo = remoteFieldInfos.get(i); + Descriptor descriptor = remoteDescriptors.get(i); + putRemoteFieldInfo(remoteFieldInfosByKey, fieldInfo, descriptor); + } Map fieldIds = new HashMap<>(); Map fields = new HashMap<>(); for (int i = 0; i < localDescriptors.size(); i++) { @@ -269,13 +282,38 @@ private List buildRemoteFields( } fields.put(fieldKey(descriptor), i); } + FieldGroups remoteFieldGroups = + FieldGroups.buildFieldInfos( + typeResolver, + FieldGroups.buildDescriptorGrouper( + typeResolver, remoteDescriptors, false, descriptor -> descriptor)); List remoteFields = new ArrayList<>(remoteFieldInfos.size()); - for (int i = 0; i < remoteFieldInfos.size(); i++) { - FieldInfo fieldInfo = remoteFieldInfos.get(i); - Descriptor descriptor = remoteDescriptors.get(i); + appendRemoteFields( + remoteFields, remoteFieldGroups.buildInFields, remoteFieldInfosByKey, fieldIds, fields, + localDescriptors); + appendRemoteFields( + remoteFields, remoteFieldGroups.containerFields, remoteFieldInfosByKey, fieldIds, fields, + localDescriptors); + appendRemoteFields( + remoteFields, remoteFieldGroups.userTypeFields, remoteFieldInfosByKey, fieldIds, fields, + localDescriptors); + return Collections.unmodifiableList(remoteFields); + } + + private void appendRemoteFields( + List remoteFields, + SerializationFieldInfo[] serializationFieldInfos, + Map remoteFieldInfosByKey, + Map fieldIds, + Map fields, + List localDescriptors) { + for (SerializationFieldInfo serializationFieldInfo : serializationFieldInfos) { + Descriptor descriptor = serializationFieldInfo.descriptor; + FieldInfo fieldInfo = remoteFieldInfosByKey.get(remoteFieldKey(descriptor)); + if (fieldInfo == null) { + throw new IllegalStateException("Missing remote field metadata for " + descriptor); + } int matchedId = matchField(fieldInfo, fieldIds, fields); - SerializationFieldInfo serializationFieldInfo = - FieldGroups.buildFieldInfo(typeResolver, descriptor); Descriptor localDescriptor = matchedId == UNKNOWN_FIELD ? null : localDescriptors.get(matchedId); remoteFields.add( @@ -287,7 +325,6 @@ private List buildRemoteFields( serializationFieldInfo, localDescriptor)); } - return Collections.unmodifiableList(remoteFields); } private SerializationFieldInfo[] buildLocalFieldsById(List descriptors) { @@ -324,6 +361,26 @@ private static String fieldKey(Descriptor descriptor) { return descriptor.getDeclaringClass() + "." + descriptor.getName(); } + private static void putRemoteFieldInfo( + Map remoteFieldInfosByKey, FieldInfo fieldInfo, Descriptor descriptor) { + remoteFieldInfosByKey.put(remoteFieldKey(descriptor), fieldInfo); + if (fieldInfo.hasFieldId()) { + remoteFieldInfosByKey.put(remoteFieldKey(fieldInfo), fieldInfo); + } + } + + private static String remoteFieldKey(FieldInfo fieldInfo) { + return fieldInfo.hasFieldId() + ? "id:" + fieldInfo.getFieldId() + : fieldInfo.getDefinedClass() + "." + fieldInfo.getFieldName(); + } + + private static String remoteFieldKey(Descriptor descriptor) { + return descriptor.hasForyFieldId() + ? "id:" + descriptor.getForyFieldId() + : fieldKey(descriptor); + } + /** Remote field metadata consumed by generated compatible read methods. */ @Internal protected static final class RemoteFieldInfo { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java index dbf35e8bfe..1ebc9989c2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java @@ -94,10 +94,7 @@ public static String computeStructFingerprint( List fieldInfos = new ArrayList<>(descriptors.size()); for (Descriptor descriptor : descriptors) { Class rawType = descriptor.getTypeRef().getRawType(); - FieldTypes.FieldType fieldType = - descriptor.getField() == null - ? null - : FieldTypes.buildFieldType(resolver, descriptor.getField()); + FieldTypes.FieldType fieldType = FieldTypes.buildFieldType(resolver, descriptor); int typeId = fieldType != null ? fingerprintTypeId(fieldType) : getTypeId(resolver, descriptor); diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index 517f4e6a3a..ec74a5d53a 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -65,7 +65,9 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.DeferedLazySerializer; import org.apache.fory.serializer.Serializer; +import org.apache.fory.serializer.StaticGeneratedStructSerializer; import org.apache.fory.test.TestUtils; import org.apache.fory.type.BFloat16; import org.apache.fory.type.Float16; @@ -191,6 +193,7 @@ public void ensurePeerReadyForTests() { protected static final String FIXED_OVERRIDE_STRUCT_TYPE_NAME = "evolving_off"; @Data + @ForyStruct(evolving = Evolution.ENABLED) protected static class EvolvingOverrideStruct { String f1; } @@ -297,6 +300,19 @@ protected static void assertStructEvolvingOverride(Fory fory) { Assert.assertNotNull(fixedInfo); Assert.assertEquals(evolvingInfo.getTypeId(), Types.NAMED_COMPATIBLE_STRUCT); Assert.assertEquals(fixedInfo.getTypeId(), Types.NAMED_STRUCT); + if ("1".equals(System.getenv("FORY_STATIC_PROCESSOR_CI"))) { + Serializer fixedSerializer = + fory.getTypeResolver().getSerializer(FixedOverrideStruct.class); + if (fixedSerializer instanceof DeferedLazySerializer) { + fixedSerializer = ((DeferedLazySerializer) fixedSerializer).resolveSerializer(); + } + Assert.assertTrue( + fixedSerializer instanceof StaticGeneratedStructSerializer, + fixedSerializer.getClass().getName()); + Assert.assertEquals( + fixedSerializer.getClass().getName(), + FixedOverrideStruct.class.getName() + "__ForyStaticSerializer__"); + } EvolvingOverrideStruct evolving = newEvolvingOverrideStruct(); FixedOverrideStruct fixed = newFixedOverrideStruct(); @@ -605,11 +621,13 @@ public void testCrossLanguageSerializer() throws Exception { } @Data + @ForyStruct static class Item { String name; } @Data + @ForyStruct static class SimpleStruct { HashMap f1; int f2; @@ -787,6 +805,7 @@ public void testMap(boolean enableCodegen) throws java.io.IOException { assertEqualsNullTolerant(fory.deserialize(buffer2), itemMap); } + @ForyStruct static class Item1 { int f1; int f2; @@ -951,6 +970,7 @@ public void testColor(boolean enableCodegen) throws java.io.IOException { // AbstractObjectSerializer.readFinalObjectFieldValue and readOtherFieldValue // have special handling for Union types to skip reading type_id. @Data + @ForyStruct static class StructWithUnion2 { org.apache.fory.type.union.Union2 union; } @@ -988,6 +1008,7 @@ public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { } @Data + @ForyStruct static class StructWithList { List items; } @@ -1021,11 +1042,13 @@ public void testStructWithList(boolean enableCodegen) throws java.io.IOException } @Data + @ForyStruct static class StructWithMap { Map data; } @Data + @ForyStruct static class SignedNestedAnnotatedContainerSchemaConsistent { Map< @Int32Type(encoding = Int32Encoding.FIXED) Integer, @@ -1060,6 +1083,7 @@ protected void testSignedNestedAnnotatedContainerSchemaConsistent(boolean enable } @Data + @ForyStruct static class SignedNestedAnnotatedContainerCompatible { Map< @Int32Type(encoding = Int32Encoding.FIXED) Integer, @@ -1098,6 +1122,7 @@ protected void testSignedNestedAnnotatedContainerCompatible(boolean enableCodege } @Data + @ForyStruct static class NestedAnnotatedContainerSchemaConsistent { Map< @UInt32Type(encoding = Int32Encoding.FIXED) Long, @@ -1131,6 +1156,7 @@ protected void testNestedAnnotatedContainerSchemaConsistent(boolean enableCodege } @Data + @ForyStruct static class NestedAnnotatedContainerCompatible { Map< @UInt32Type(encoding = Int32Encoding.FIXED) Long, @@ -1139,6 +1165,7 @@ static class NestedAnnotatedContainerCompatible { } @Data + @ForyStruct static class ManualSchemaKindStruct { public List orderedValues; @@ -1257,6 +1284,7 @@ public void testStructWithMap(boolean enableCodegen) throws java.io.IOException } @Data + @ForyStruct static class MyStruct { int id; @@ -1303,6 +1331,7 @@ public MyExt read(ReadContext readContext) { } @Data + @ForyStruct static class MyWrapper { Color color; MyExt myExt; @@ -1310,6 +1339,7 @@ static class MyWrapper { } @Data + @ForyStruct static class EmptyWrapper {} private void _testSkipCustom(Fory fory1, Fory fory2, String caseName) throws IOException { @@ -1410,6 +1440,7 @@ public void testConsistentNamed(boolean enableCodegen) throws java.io.IOExceptio } @Data + @ForyStruct static class VersionCheckStruct { int f1; @@ -1459,6 +1490,7 @@ public interface Animal { } @Data + @ForyStruct public static class Dog implements Animal { int age; @@ -1477,6 +1509,7 @@ public String speak() { } @Data + @ForyStruct public static class Cat implements Animal { int age; int lives; @@ -1493,11 +1526,13 @@ public String speak() { } @Data + @ForyStruct static class AnimalListHolder { List animals; } @Data + @ForyStruct static class AnimalMapHolder { // Using snake_case field name to test fallback lookup in TypeDef.getDescriptors() Map animal_map; @@ -1667,21 +1702,25 @@ private static void assertManualIntegralListEquals( // ============================================================================ @Data + @ForyStruct static class EmptyStruct {} @Data + @ForyStruct static class OneStringFieldStruct { @ForyField(nullable = true) String f1; } @Data + @ForyStruct static class TwoStringFieldStruct { String f1; String f2; } @Data + @ForyStruct static class ReducedPrecisionFloatStruct { Float16 float16Value; BFloat16 bfloat16Value; @@ -1719,6 +1758,7 @@ protected static void assertReducedPrecisionFloatStruct(ReducedPrecisionFloatStr } @Data + @ForyStruct static class XlangCompatibleInt32ListField { @ForyField(id = 1) @Int32Type(encoding = Int32Encoding.FIXED) @@ -1726,12 +1766,14 @@ static class XlangCompatibleInt32ListField { } @Data + @ForyStruct static class XlangCompatibleNullableInt32ListField { @ForyField(id = 1) List values; } @Data + @ForyStruct static class XlangCompatibleInt32ArrayField { @ForyField(id = 1) int[] values; @@ -2046,11 +2088,13 @@ enum TestEnum { } @Data + @ForyStruct static class OneEnumFieldStruct { TestEnum f1; } @Data + @ForyStruct static class TwoEnumFieldStruct { TestEnum f1; TestEnum f2; @@ -2202,6 +2246,7 @@ public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java * */ @Data + @ForyStruct static class NullableComprehensiveSchemaConsistent { // Base non-nullable primitive fields byte byteField; @@ -2372,6 +2417,7 @@ public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) *

In other languages, group 1 fields should be nullable, group 2 fields should be not-null. */ @Data + @ForyStruct static class NullableComprehensiveCompatible { // Base non-nullable primitive fields byte byteField; @@ -2666,6 +2712,7 @@ protected void assertEqualsNullTolerant(Object actual, Object expected) { * simple struct with id and name fields. */ @Data + @ForyStruct static class RefInnerSchemaConsistent { int id; String name; @@ -2676,6 +2723,7 @@ static class RefInnerSchemaConsistent { * can point to the same RefInnerSchemaConsistent instance. Both fields have ref tracking enabled. */ @Data + @ForyStruct static class RefOuterSchemaConsistent { @ForyField(ref = true, nullable = true, dynamic = ForyField.Dynamic.FALSE) RefInnerSchemaConsistent inner1; @@ -2743,6 +2791,7 @@ public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOExce * with id and name fields. */ @Data + @ForyStruct static class RefInnerCompatible { int id; String name; @@ -2753,6 +2802,7 @@ static class RefInnerCompatible { * point to the same RefInnerCompatible instance. Both fields have ref tracking enabled. */ @Data + @ForyStruct static class RefOuterCompatible { @ForyField(ref = true, nullable = true) RefInnerCompatible inner1; @@ -2821,12 +2871,14 @@ public void testRefCompatible(boolean enableCodegen) throws java.io.IOException // ============================================================================ @Data + @ForyStruct static class RefOverrideElement { int id; String name; } @Data + @ForyStruct static class RefOverrideContainer { List<@Ref(enable = false) RefOverrideElement> listField; Set<@Ref(enable = false) RefOverrideElement> setField; @@ -2976,6 +3028,7 @@ public void testCollectionElementRefRemoteTracking(boolean enableCodegen) * 'selfRef' instead of 'self' because 'self' is a reserved keyword in Rust. */ @Data + @ForyStruct static class CircularRefStruct { String name; @@ -3115,6 +3168,7 @@ private Object normalizeNulls(Object obj) { * with different encoding options. */ @Data + @ForyStruct static class UnsignedSchemaConsistent { // Primitive unsigned fields (use Field suffix to avoid reserved keywords in Rust/Go) @UInt8Type int u8Field; @@ -3166,6 +3220,7 @@ static class UnsignedSchemaConsistent { } @Data + @ForyStruct static class UnsignedSchemaConsistentSimple { @UInt64Type(encoding = Int64Encoding.TAGGED) long u64Tagged; @@ -3259,6 +3314,7 @@ private static String bytesToHex(byte[] bytes) { * nullability: Group 1 is Optional, Group 2 is non-Optional. */ @Data + @ForyStruct static class UnsignedSchemaCompatible { // Group 1: Primitive unsigned fields (non-nullable in Java, Optional in other languages) @UInt8Type int u8Field1; From 7abd0256aae475e1e63da30f15ae56730e48e047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 13:36:57 +0800 Subject: [PATCH 16/58] fix name --- .../processing/ForyStructProcessorTest.java | 10 ++++---- .../org/apache/fory/builder/CodecUtils.java | 24 +++++++++---------- .../apache/fory/resolver/ClassResolver.java | 2 +- .../apache/fory/resolver/TypeResolver.java | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 6eaeb351e5..4139510b7a 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -434,12 +434,12 @@ public void testGraalvmStaticCompatibleSerializerReadsRuntimeRemoteTypeDef() thr Assert.assertNotEquals( remoteTypeDef.getId(), TypeDef.buildTypeDef(reader.getTypeResolver(), readerType).getId()); - Class> serializerClass = - CodecUtils.loadOrGenCompatibleMetaSharedCodecClass( + Class serializerClass = + CodecUtils.loadOrGenStaticCompatibleCodecClass( reader.getTypeResolver(), (Class) readerType, remoteTypeDef); Assert.assertTrue( GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(serializerClass)); - Constructor> constructor = + Constructor constructor = serializerClass.getConstructor( org.apache.fory.resolver.TypeResolver.class, Class.class, TypeDef.class); Serializer serializer = @@ -716,8 +716,8 @@ private static Object generatedCompatibleRead( Fory writer, Fory reader, Class writerType, Class readerType, Object writerValue) throws Exception { TypeDef remoteTypeDef = TypeDef.buildTypeDef(writer.getTypeResolver(), writerType); - Class> serializerClass = - CodecUtils.loadOrGenCompatibleMetaSharedCodecClass( + Class serializerClass = + CodecUtils.loadOrGenStaticCompatibleCodecClass( reader.getTypeResolver(), (Class) readerType, remoteTypeDef); Assert.assertTrue( GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(serializerClass)); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index 2bb4665f40..cf0319c419 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -44,7 +44,7 @@ public class CodecUtils { // TODO(chaokunyang) how to uninstall org.apache.fory.codegen/builder classes for graalvm build // time // maybe use a temporal URLClassLoader - public static Class> loadOrGenObjectCodecClass( + public static Class loadOrGenObjectCodecClass( Class cls, Fory fory) { Preconditions.checkNotNull(fory); return loadSerializer( @@ -54,7 +54,7 @@ public static Class> loadOrGenObjectCodecClass( () -> loadOrGenCodecClass(cls, fory, new ObjectCodecBuilder(cls, fory))); } - public static Class> loadOrGenMetaSharedCodecClass( + public static Class loadOrGenMetaSharedCodecClass( Fory fory, Class cls, TypeDef typeDef) { Preconditions.checkNotNull(fory); return loadSerializer( @@ -66,18 +66,18 @@ public static Class> loadOrGenMetaSharedCodecClass( cls, fory, new MetaSharedCodecBuilder(TypeRef.of(cls), fory, typeDef))); } - public static Class> loadOrGenMetaSharedCodecClass( + public static Class loadOrGenMetaSharedCodecClass( TypeResolver typeResolver, Class cls, TypeDef typeDef) { return typeResolver .getJITContext() .asyncVisitFory(f -> loadOrGenMetaSharedCodecClass(f, cls, typeDef)); } - public static Class> loadOrGenCompatibleMetaSharedCodecClass( + public static Class loadOrGenStaticCompatibleCodecClass( Fory fory, Class cls, TypeDef typeDef) { Preconditions.checkNotNull(fory); return loadSerializer( - "loadOrGenCompatibleMetaSharedCodecClass", + "loadOrGenStaticCompatibleCodecClass", cls, fory, () -> @@ -85,11 +85,11 @@ public static Class> loadOrGenCompatibleMetaSharedCo cls, fory, new StaticCompatibleCodecBuilder(TypeRef.of(cls), fory, typeDef))); } - public static Class> loadOrGenCompatibleMetaSharedCodecClass( + public static Class loadOrGenStaticCompatibleCodecClass( TypeResolver typeResolver, Class cls, TypeDef typeDef) { return typeResolver .getJITContext() - .asyncVisitFory(f -> loadOrGenCompatibleMetaSharedCodecClass(f, cls, typeDef)); + .asyncVisitFory(f -> loadOrGenStaticCompatibleCodecClass(f, cls, typeDef)); } /** @@ -101,7 +101,7 @@ public static Class> loadOrGenCompatibleMetaSharedCo * @param layerMarkerClass the marker class for this layer * @return the generated serializer class */ - public static Class> loadOrGenMetaSharedLayerCodecClass( + public static Class loadOrGenMetaSharedLayerCodecClass( Class cls, Fory fory, TypeDef layerTypeDef, Class layerMarkerClass) { Preconditions.checkNotNull(fory); return loadSerializer( @@ -117,7 +117,7 @@ public static Class> loadOrGenMetaSharedLayerCodecCl } @SuppressWarnings("unchecked") - static Class> loadOrGenCodecClass( + static Class loadOrGenCodecClass( Class beanClass, Fory fory, BaseObjectCodecBuilder codecBuilder) { // use genCodeFunc to avoid gen code repeatedly CompileUnit compileUnit = @@ -140,7 +140,7 @@ static Class> loadOrGenCodecClass( Collections.singletonList(compileUnit), compileState -> compileState.lock.lock()); String className = codecBuilder.codecQualifiedClassName(beanClass); try { - return (Class>) classLoader.loadClass(className); + return (Class) classLoader.loadClass(className); } catch (ClassNotFoundException e) { throw new IllegalStateException("Impossible because we just compiled class", e); } @@ -170,8 +170,8 @@ private static CodeGenerator getCodeGenerator( return codeGenerator; } - private static Class> loadSerializer( - String name, Class cls, Fory fory, Callable>> func) { + private static Class loadSerializer( + String name, Class cls, Fory fory, Callable> func) { int configHash = fory.getConfig().getConfigHash(); if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { Tuple3, Integer> key = Tuple3.of(name, cls, configHash); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 9bcb250058..4af60b556b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1794,7 +1794,7 @@ private void registerGraalvmSerializerClass(Class cls) { typeDef.getId(), getMetaSharedDeserializerClassForGraalvmBuild(cls, typeDef)); getGraalvmClassRegistry() .putCompatibleDeserializerClass( - cls, CodecUtils.loadOrGenCompatibleMetaSharedCodecClass(this, cls, typeDef)); + cls, CodecUtils.loadOrGenStaticCompatibleCodecClass(this, cls, typeDef)); } typeInfoCache = NIL_TYPE_INFO; if (RecordUtils.isRecord(cls)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index b406fffae2..50ecb2a93d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1082,7 +1082,7 @@ private Class loadGraalvmMetaSharedDeserializerClass( if (typeDef.getId() == TypeDef.buildTypeDef(this, cls).getId()) { return CodecUtils.loadOrGenMetaSharedCodecClass(this, cls, typeDef); } - return CodecUtils.loadOrGenCompatibleMetaSharedCodecClass(this, cls, typeDef); + return CodecUtils.loadOrGenStaticCompatibleCodecClass(this, cls, typeDef); } protected int buildUnregisteredTypeId(Class cls, Serializer serializer) { From 7325b9cf5d865624f03a71e26ab9678b79db59e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 15:23:01 +0800 Subject: [PATCH 17/58] fix inconsistent field processing --- .github/workflows/ci.yml | 6 ++- ci/run_ci.py | 51 ++++++++++++++----- compiler/fory_compiler/generators/java.py | 14 +++-- .../tests/test_generated_code.py | 5 ++ integration_tests/idl_tests/run_java_tests.sh | 3 +- .../processing/ForyStructProcessor.java | 10 ++-- .../processing/ForyStructProcessorTest.java | 8 +++ .../java/org/apache/fory/reflect/TypeRef.java | 4 +- .../apache/fory/resolver/XtypeResolver.java | 14 ++--- .../CompatibleCollectionArrayReader.java | 40 ++++++++++++++- .../StaticGeneratedStructSerializer.java | 5 ++ .../java/org/apache/fory/type/TypeUtils.java | 2 +- .../org/apache/fory/reflect/TypeRefTest.java | 28 ++++++++++ 13 files changed, 155 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51cefc1644..062b5dc8cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1260,7 +1260,7 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - name: Run Android Go Xlang Tests + - name: Run Android Go Xlang And IDL Tests env: FORY_ANDROID_ENABLED: "1" FORY_GO_JAVA_CI: "1" @@ -1270,7 +1270,9 @@ jobs: mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true cd fory-core mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.GoXlangTest - - name: Run Android Go Xlang Tests With Static Generated Serializers + cd ../.. + ./integration_tests/idl_tests/run_go_tests.sh + - name: Run Android Go Xlang And IDL Tests With Static Generated Serializers env: FORY_ANDROID_ENABLED: "1" FORY_GO_JAVA_CI: "1" diff --git a/ci/run_ci.py b/ci/run_ci.py index de181549b4..334ad7c77c 100644 --- a/ci/run_ci.py +++ b/ci/run_ci.py @@ -179,6 +179,25 @@ def _add_annotation_processor_path(pom_path): tree.write(pom_path, encoding="UTF-8", xml_declaration=True) +def _copy_pom_with_annotation_processor(pom_path): + pom_dir = os.path.dirname(pom_path) + fd, temp_pom = tempfile.mkstemp( + prefix="pom-static-processor-", suffix=".xml", dir=pom_dir + ) + os.close(fd) + shutil.copyfile(pom_path, temp_pom) + _add_annotation_processor_path(temp_pom) + return temp_pom + + +def _remove_file(path): + if path: + try: + os.remove(path) + except FileNotFoundError: + pass + + def _exec_cmd(cmd, cwd): logging.info(f"running command in {cwd}: {cmd}") subprocess.check_call(cmd, shell=True, cwd=cwd) @@ -190,34 +209,40 @@ def run_android_go_xlang_static_processor(): java_dir = os.path.join(root, "java") core_dir = os.path.join(java_dir, "fory-core") core_pom = os.path.join(core_dir, "pom.xml") - fd, temp_pom = tempfile.mkstemp( - prefix="pom-static-processor-", suffix=".xml", dir=core_dir - ) - os.close(fd) + idl_java_dir = os.path.join(root, "integration_tests", "idl_tests", "java") + idl_java_pom = os.path.join(idl_java_dir, "pom.xml") + temp_core_pom = None + temp_idl_java_pom = None try: - shutil.copyfile(core_pom, temp_pom) - _add_annotation_processor_path(temp_pom) + temp_core_pom = _copy_pom_with_annotation_processor(core_pom) + temp_idl_java_pom = _copy_pom_with_annotation_processor(idl_java_pom) os.environ.setdefault("FORY_ANDROID_ENABLED", "1") os.environ.setdefault("FORY_GO_JAVA_CI", "1") os.environ.setdefault("FORY_STATIC_PROCESSOR_CI", "1") os.environ.setdefault("ENABLE_FORY_DEBUG_OUTPUT", "1") _exec_cmd( - "mvn -T16 --no-transfer-progress install -DskipTests " + "mvn -T16 --no-transfer-progress clean install -DskipTests " "-Dmaven.javadoc.skip=true -Dmaven.source.skip=true " - "-pl fory-annotation-processor -am", + "-pl fory-core,fory-annotation-processor -am", java_dir, ) _exec_cmd( "mvn -T16 --no-transfer-progress " - f"-f {shlex.quote(temp_pom)} " + f"-f {shlex.quote(temp_core_pom)} " "clean test -Dtest=org.apache.fory.xlang.GoXlangTest", core_dir, ) + env = os.environ.copy() + env["IDL_JAVA_POM"] = temp_idl_java_pom + logging.info( + f"running command in {root}: ./integration_tests/idl_tests/run_go_tests.sh" + ) + subprocess.check_call( + ["./integration_tests/idl_tests/run_go_tests.sh"], cwd=root, env=env + ) finally: - try: - os.remove(temp_pom) - except FileNotFoundError: - pass + _remove_file(temp_core_pom) + _remove_file(temp_idl_java_pom) def parse_args(): diff --git a/compiler/fory_compiler/generators/java.py b/compiler/fory_compiler/generators/java.py index e46aba3d05..67b64eb7cc 100644 --- a/compiler/fory_compiler/generators/java.py +++ b/compiler/fory_compiler/generators/java.py @@ -406,6 +406,12 @@ def generate_union_file(self, union: Union) -> GeneratedFile: return GeneratedFile(path=path, content="\n".join(lines)) + def get_struct_annotation(self, message: Message) -> str: + """Return the ForyStruct annotation for a generated message.""" + if self.get_effective_evolving(message): + return "@ForyStruct" + return "@ForyStruct(evolving = Evolution.DISABLED)" + # Generates a Java class file from a message schema definition. def generate_message_file(self, message: Message) -> GeneratedFile: """Generate a Java class file for a message.""" @@ -435,8 +441,7 @@ def generate_message_file(self, message: Message) -> GeneratedFile: comment = self.format_type_id_comment(message, "//") if comment: lines.append(comment) - if not self.get_effective_evolving(message): - lines.append("@ForyStruct(evolving = Evolution.DISABLED)") + lines.append(self.get_struct_annotation(message)) lines.append(f"public class {message.name} {{") # Generate nested enums as static inner classes @@ -588,8 +593,8 @@ def collect_message_imports(self, message: Message, imports: Set[str]): for field in message.fields: self.collect_field_imports(field, imports) + imports.add("org.apache.fory.annotation.ForyStruct") if not self.get_effective_evolving(message): - imports.add("org.apache.fory.annotation.ForyStruct") imports.add("org.apache.fory.annotation.ForyStruct.Evolution") # Add imports for equals/hashCode @@ -1042,8 +1047,7 @@ def generate_nested_message( comment = self.format_type_id_comment(message, " " * indent + "//") if comment: lines.append(comment) - if not self.get_effective_evolving(message): - lines.append("@ForyStruct(evolving = Evolution.DISABLED)") + lines.append(self.get_struct_annotation(message)) lines.append(f"public static class {message.name} {{") # Generate nested enums diff --git a/compiler/fory_compiler/tests/test_generated_code.py b/compiler/fory_compiler/tests/test_generated_code.py index 79a201f5ff..bd76ae3ab8 100644 --- a/compiler/fory_compiler/tests/test_generated_code.py +++ b/compiler/fory_compiler/tests/test_generated_code.py @@ -616,6 +616,10 @@ def test_java_evolving_false_generation_uses_struct_evolution_enum(): """ package gen; + message DefaultEvolving { + string value = 1; + } + message Stable [evolving=false] { string name = 1; @@ -630,6 +634,7 @@ def test_java_evolving_false_generation_uses_struct_evolution_enum(): assert "import org.apache.fory.annotation.ForyStruct;" in java_output assert "import org.apache.fory.annotation.ForyStruct.Evolution;" in java_output assert java_output.count("@ForyStruct(evolving = Evolution.DISABLED)") == 2 + assert java_output.count("@ForyStruct") == 3 assert "@ForyStruct(evolving = false)" not in java_output diff --git a/integration_tests/idl_tests/run_java_tests.sh b/integration_tests/idl_tests/run_java_tests.sh index f945a5d5d5..eb6fcd6843 100755 --- a/integration_tests/idl_tests/run_java_tests.sh +++ b/integration_tests/idl_tests/run_java_tests.sh @@ -22,6 +22,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" JAVA_TEST_PATTERN="${IDL_JAVA_TEST_PATTERN:-}" +JAVA_POM="${IDL_JAVA_POM:-${ROOT_DIR}/integration_tests/idl_tests/java/pom.xml}" python "${SCRIPT_DIR}/generate_idl.py" --lang java @@ -30,5 +31,5 @@ MAVEN_ARGS=(-T16 --no-transfer-progress) if [[ -n "${JAVA_TEST_PATTERN}" ]]; then MAVEN_ARGS+=("-Dtest=${JAVA_TEST_PATTERN}") fi -MAVEN_ARGS+=(test -f "${ROOT_DIR}/integration_tests/idl_tests/java/pom.xml") +MAVEN_ARGS+=(test -f "${JAVA_POM}") ENABLE_FORY_DEBUG_OUTPUT=1 mvn "${MAVEN_ARGS[@]}" diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index e8f9815461..c2c78c0779 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -564,15 +564,15 @@ private SourceTypeNode buildTypeNode(TypeMirror type, Object tree, String typeEx List arguments = new ArrayList<>(); SourceTypeNode componentType = null; if (kind == TypeKind.ARRAY) { + TypeMirror componentMirror = ((ArrayType) type).getComponentType(); componentType = - buildTypeNode( - ((ArrayType) type).getComponentType(), treeInfo.arrayComponentTree(), "false"); + buildTypeNode(componentMirror, treeInfo.arrayComponentTree(), nestedNullable(componentMirror)); } else if (type instanceof DeclaredType) { List argumentTrees = treeInfo.typeArgumentTrees(); int index = 0; for (TypeMirror argument : ((DeclaredType) type).getTypeArguments()) { Object argumentTree = index < argumentTrees.size() ? argumentTrees.get(index) : null; - arguments.add(buildTypeNode(argument, argumentTree, "false")); + arguments.add(buildTypeNode(argument, argumentTree, nestedNullable(argument))); index++; } } @@ -616,6 +616,10 @@ private String typeExtMetaExpression( + ")"; } + private String nestedNullable(TypeMirror type) { + return Boolean.toString(!type.getKind().isPrimitive()); + } + private String scalarTypeId(TypeMirror type, String rawType, List treeAnnotations) { if (hasTypeAnnotation(type, treeAnnotations, INT8_TYPE)) { return rawType.equals("byte[]") ? "Types.INT8_ARRAY" : "Types.INT8"; diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 4139510b7a..cea0168783 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -226,9 +226,12 @@ public void testGeneratedDescriptorsCarryNestedTypeMetadata() throws Exception { + "import java.util.List;\n" + "import org.apache.fory.annotation.ForyStruct;\n" + "import org.apache.fory.annotation.Ref;\n" + + "import org.apache.fory.annotation.Int32Type;\n" + "import org.apache.fory.annotation.UInt16Type;\n" + + "import org.apache.fory.config.Int32Encoding;\n" + "@ForyStruct public class MetadataStruct {\n" + " public List<@Ref String> names;\n" + + " public List<@Int32Type(encoding = Int32Encoding.FIXED) Integer> codes;\n" + " public @UInt16Type int code;\n" + " public MetadataStruct() {}\n" + "}\n"); @@ -252,6 +255,11 @@ public void testGeneratedDescriptorsCarryNestedTypeMetadata() throws Exception { Assert.assertTrue(names.getTypeRef().getTypeArguments().get(0).hasTypeExtMeta()); Assert.assertTrue( names.getTypeRef().getTypeArguments().get(0).getTypeExtMeta().trackingRef()); + Assert.assertTrue(names.getTypeRef().getTypeArguments().get(0).getTypeExtMeta().nullable()); + Descriptor codes = descriptor(serializer.getDescriptors(), "codes"); + Assert.assertEquals( + codes.getTypeRef().getTypeArguments().get(0).getTypeExtMeta().typeId(), Types.INT32); + Assert.assertTrue(codes.getTypeRef().getTypeArguments().get(0).getTypeExtMeta().nullable()); Descriptor code = descriptor(serializer.getDescriptors(), "code"); Assert.assertTrue(code.getTypeRef().hasTypeExtMeta()); Assert.assertEquals(code.getTypeRef().getTypeExtMeta().typeId(), Types.UINT16); diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/TypeRef.java b/java/fory-core/src/main/java/org/apache/fory/reflect/TypeRef.java index 007221ee02..8a6dc4e08a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/TypeRef.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/TypeRef.java @@ -187,9 +187,9 @@ private static TypeRef ofAnnotatedType(AnnotatedType annotatedType, boolean i Ref ref = annotatedType.getAnnotation(Ref.class); if (typeAnnotation != null) { int typeId = TypeAnnotationUtils.getTypeId(typeAnnotation, TypeUtils.getRawType(type)); - meta = TypeExtMeta.of(typeId, false, ref != null && ref.enable()); + meta = TypeExtMeta.of(typeId, true, ref != null && ref.enable()); } else if (ref != null) { - meta = TypeExtMeta.of(Types.UNKNOWN, false, ref.enable()); + meta = TypeExtMeta.of(Types.UNKNOWN, true, ref.enable()); } } return new TypeRef<>(type, meta, typeArguments, componentType); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index de82a7446e..0220aa0655 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -1292,13 +1292,6 @@ public boolean usesPrimitiveFieldOrdering(Descriptor descriptor) { } private byte getInternalTypeId(Descriptor descriptor) { - TypeExtMeta extMeta = descriptor.getTypeRef().getTypeExtMeta(); - if (extMeta != null && extMeta.typeId() != Types.UNKNOWN) { - return (byte) extMeta.typeId(); - } - if (TypeAnnotationUtils.isBoxedListArrayType(descriptor.getField())) { - return (byte) TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor.getField()); - } Class cls = descriptor.getRawType(); if (TypeUtils.isPrimitiveListClass(cls) && TypeAnnotationUtils.isArrayType(descriptor)) { return (byte) TypeAnnotationUtils.getPrimitiveListArrayTypeId(cls); @@ -1309,6 +1302,13 @@ private byte getInternalTypeId(Descriptor descriptor) { descriptor.getTypeAnnotation(), cls)) { return Types.LIST; } + TypeExtMeta extMeta = descriptor.getTypeRef().getTypeExtMeta(); + if (extMeta != null && extMeta.typeId() != Types.UNKNOWN) { + return (byte) extMeta.typeId(); + } + if (TypeAnnotationUtils.isBoxedListArrayType(descriptor.getField())) { + return (byte) TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor.getField()); + } if (cls.isArray() && cls.getComponentType().isPrimitive()) { return (byte) Types.getDescriptorTypeId(this, descriptor); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index 1b1f2c5f72..185ec4ab6b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -47,6 +47,7 @@ import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.RefMode; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.collection.CollectionFlags; import org.apache.fory.type.BFloat16; @@ -61,6 +62,7 @@ final class CompatibleCollectionArrayReader { static final int READ_LIST_TO_ARRAY = 1; static final int READ_ARRAY_TO_LIST = 2; + static final int READ_LIST_TO_LIST = 3; static final class ReadAction { final int mode; @@ -92,6 +94,14 @@ static ReadAction readAction(TypeResolver resolver, Descriptor descriptor) { return new ReadAction( READ_LIST_TO_ARRAY, localArrayTypeId, peerListElementTypeId, field.getType()); } + int localListElementTypeId = listElementTypeId(localFieldType); + int peerArrayTypeId = denseArrayTypeId(peerListElementTypeId); + if (localListElementTypeId != Types.UNKNOWN + && peerArrayTypeId != Types.UNKNOWN + && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { + return new ReadAction( + READ_LIST_TO_LIST, peerArrayTypeId, peerListElementTypeId, field.getType()); + } return null; } int peerArrayTypeId = arrayTypeId(descriptor.getTypeRef()); @@ -124,6 +134,17 @@ static ReadAction readAction( peerListElementTypeId, localDescriptor.getRawType()); } + int localListElementTypeId = listElementTypeId(localType); + int peerArrayTypeId = denseArrayTypeId(peerListElementTypeId); + if (localListElementTypeId != Types.UNKNOWN + && peerArrayTypeId != Types.UNKNOWN + && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { + return new ReadAction( + READ_LIST_TO_LIST, + peerArrayTypeId, + peerListElementTypeId, + localDescriptor.getRawType()); + } return null; } int peerArrayTypeId = arrayTypeId(remoteFieldType); @@ -202,6 +223,13 @@ private static Object readNotNull( } return materializeTarget(array, arrayTypeId, targetType); } + if (readMode == READ_LIST_TO_LIST) { + Object array = readListPayloadAsPrimitiveArray(readContext, arrayTypeId, elementTypeId); + if (array == null) { + return null; + } + return materializeTarget(array, arrayTypeId, targetType); + } if (readMode == READ_ARRAY_TO_LIST) { Object array = readDenseArrayPayload(readContext, arrayTypeId); return materializeTarget(array, arrayTypeId, targetType); @@ -339,10 +367,20 @@ private static Object readListPayloadAsPrimitiveArray( throw new DeserializationException( "Cannot read nullable or ref-tracked peer list payload into local array field"); } - if (!sameType || !declared) { + if (!sameType) { throw new DeserializationException( "Cannot read peer list payload into local array field"); } + if (!declared) { + TypeInfo payloadElementTypeInfo = readContext.getTypeResolver().readTypeInfo(readContext); + if (payloadElementTypeInfo.getTypeId() != elementTypeId) { + throw new DeserializationException( + "Cannot read peer list element type id " + + payloadElementTypeInfo.getTypeId() + + " as local element type id " + + elementTypeId); + } + } } return readListPrimitiveElements(buffer, numElements, arrayTypeId, elementTypeId); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index e7c4a21b33..f355048652 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -235,6 +235,11 @@ protected final void setConvertedField( protected final Object copyFieldValue( CopyContext copyContext, Object fieldValue, SerializationFieldInfo fieldInfo) { + if (fieldInfo.containerSerializerOverride != null) { + @SuppressWarnings("unchecked") + Serializer serializer = (Serializer) fieldInfo.containerSerializerOverride; + return copyContext.copyObject(fieldValue, serializer); + } return copyContext.copyObject(fieldValue, fieldInfo.dispatchId); } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index a1e6a4a56c..20b7f414fa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -578,7 +578,7 @@ public static TypeRef getCollectionType(TypeRef typeRef) { /** Returns key/value type of map. */ public static Tuple2, TypeRef> getMapKeyValueType(TypeRef typeRef) { - if (typeRef.hasTypeExtMeta() && typeRef.hasExplicitTypeArguments()) { + if (typeRef.hasExplicitTypeArguments()) { List> typeArguments = typeRef.getTypeArguments(); if (typeArguments.size() == 2) { Class rawType = getRawType(typeRef); diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/TypeRefTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/TypeRefTest.java index f00716cf1e..e15fc953d6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/TypeRefTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/TypeRefTest.java @@ -21,6 +21,7 @@ import static org.testng.Assert.*; +import java.lang.reflect.Field; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.lang.reflect.WildcardType; @@ -32,7 +33,12 @@ import lombok.EqualsAndHashCode; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.annotation.Int32Type; +import org.apache.fory.annotation.Ref; import org.apache.fory.collection.Tuple2; +import org.apache.fory.config.Int32Encoding; +import org.apache.fory.meta.TypeExtMeta; +import org.apache.fory.type.Types; import org.apache.fory.type.TypeUtils; import org.testng.Assert; import org.testng.annotations.Test; @@ -89,6 +95,28 @@ public void testWildcardTypeSerialization(boolean enableCodegen) { serDeCheck(fory, new MyClass()); } + static class TypeUseMetadataStruct { + List<@Ref(enable = false) String> names; + List<@Int32Type(encoding = Int32Encoding.FIXED) Integer> codes; + } + + @Test + public void testNestedTypeUseMetadataKeepsNestedNullableDefault() throws Exception { + Field namesField = TypeUseMetadataStruct.class.getDeclaredField("names"); + TypeRef namesType = TypeRef.of(namesField.getAnnotatedType()); + TypeExtMeta namesElementMeta = namesType.getTypeArguments().get(0).getTypeExtMeta(); + Assert.assertEquals(namesElementMeta.typeId(), Types.UNKNOWN); + Assert.assertTrue(namesElementMeta.nullable()); + Assert.assertFalse(namesElementMeta.trackingRef()); + + Field codesField = TypeUseMetadataStruct.class.getDeclaredField("codes"); + TypeRef codesType = TypeRef.of(codesField.getAnnotatedType()); + TypeExtMeta codesElementMeta = codesType.getTypeArguments().get(0).getTypeExtMeta(); + Assert.assertEquals(codesElementMeta.typeId(), Types.INT32); + Assert.assertTrue(codesElementMeta.nullable()); + Assert.assertFalse(codesElementMeta.trackingRef()); + } + @Test public void testIsWildcard() { // Test with direct wildcard types extracted from parameterized types From 0d2a003f2791dbd691b4106d827e5fe629eda18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 16:07:53 +0800 Subject: [PATCH 18/58] fix(java): align static compatible field metadata --- .../processing/ForyStructProcessorTest.java | 228 ------------ java/fory-core/pom.xml | 5 + .../apache/fory/serializer/FieldGroups.java | 6 +- .../StaticCompatibleCodecBuilderTest.java | 344 ++++++++++++++++++ .../org.mockito.plugins.MockMaker | 1 + java/pom.xml | 6 + 6 files changed, 359 insertions(+), 231 deletions(-) create mode 100644 java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java create mode 100644 java/fory-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index cea0168783..45017aab07 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -20,7 +20,6 @@ package org.apache.fory.annotation.processing; import java.io.IOException; -import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.net.URL; import java.net.URLClassLoader; @@ -39,15 +38,8 @@ import javax.tools.ToolProvider; import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; -import org.apache.fory.builder.CodecUtils; -import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.TypeDef; -import org.apache.fory.platform.AndroidSupport; -import org.apache.fory.serializer.MetaSharedSerializer; -import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.StaticGeneratedStructSerializer; import org.apache.fory.type.Descriptor; import org.apache.fory.type.Types; @@ -391,181 +383,6 @@ public void testStaticSerializerRoundTripsWithRuntimeSerializerCompatible() thro assertStaticRuntimeRoundTrip(true); } - @Test - public void testGraalvmStaticCompatibleSerializerReadsRuntimeRemoteTypeDef() throws Exception { - if (AndroidSupport.IS_ANDROID) { - return; - } - CompilationResult writerResult = - compile( - "test.NativeImageStruct", - "package test;\n" - + "public class NativeImageStruct {\n" - + " public int id;\n" - + " public int legacy;\n" - + " public NativeImageStruct() {}\n" - + "}\n"); - CompilationResult readerResult = - compile( - "test.NativeImageStruct", - "package test;\n" - + "public class NativeImageStruct {\n" - + " public int id;\n" - + " public String added = \"default\";\n" - + " public NativeImageStruct() {}\n" - + "}\n"); - Assert.assertTrue(writerResult.success, writerResult.diagnostics()); - Assert.assertTrue(readerResult.success, readerResult.diagnostics()); - try (URLClassLoader writerLoader = writerResult.classLoader(); - URLClassLoader readerLoader = readerResult.classLoader()) { - Fory writer = - Fory.builder() - .withClassLoader(writerLoader) - .withCodegen(false) - .withMetaShare(true) - .withScopedMetaShare(false) - .withCompatible(true) - .requireClassRegistration(false) - .build(); - Fory reader = - Fory.builder() - .withClassLoader(readerLoader) - .withCodegen(true) - .withMetaShare(true) - .withScopedMetaShare(false) - .withCompatible(true) - .requireClassRegistration(false) - .build(); - Class writerType = writerLoader.loadClass("test.NativeImageStruct"); - Class readerType = readerLoader.loadClass("test.NativeImageStruct"); - TypeDef remoteTypeDef = TypeDef.buildTypeDef(writer.getTypeResolver(), writerType); - Assert.assertNotEquals( - remoteTypeDef.getId(), - TypeDef.buildTypeDef(reader.getTypeResolver(), readerType).getId()); - Class serializerClass = - CodecUtils.loadOrGenStaticCompatibleCodecClass( - reader.getTypeResolver(), (Class) readerType, remoteTypeDef); - Assert.assertTrue( - GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(serializerClass)); - Constructor constructor = - serializerClass.getConstructor( - org.apache.fory.resolver.TypeResolver.class, Class.class, TypeDef.class); - Serializer serializer = - constructor.newInstance( - reader.getTypeResolver(), (Class) readerType, remoteTypeDef); - Object writerValue = writerType.getConstructor().newInstance(); - setField(writerType, writerValue, "id", 42); - setField(writerType, writerValue, "legacy", 99); - MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(128); - MetaSharedSerializer writerSerializer = - new MetaSharedSerializer<>( - writer.getTypeResolver(), (Class) writerType, remoteTypeDef); - writer.getWriteContext().prepare(buffer, null); - try { - writerSerializer.write(writer.getWriteContext(), writerValue); - } finally { - writer.getWriteContext().reset(); - } - buffer.readerIndex(0); - reader.getReadContext().prepare(buffer, null, false); - Object result; - try { - result = serializer.read(reader.getReadContext()); - } finally { - reader.getReadContext().reset(); - } - Assert.assertSame(result.getClass(), readerType); - Assert.assertEquals(getField(readerType, result, "id"), 42); - Assert.assertEquals(getField(readerType, result, "added"), "default"); - Assert.assertThrows( - UnsupportedOperationException.class, - () -> - serializer.write( - reader.getWriteContext(), readerType.getConstructor().newInstance())); - } - } - - @Test - public void testGraalvmStaticCompatibleRecordSerializerConvertsRemoteField() throws Exception { - assumeRecordSupport(); - if (AndroidSupport.IS_ANDROID) { - return; - } - CompilationResult writerResult = - compile( - "test.NativeRecordStruct", - "package test;\n" - + "public class NativeRecordStruct {\n" - + " public String id;\n" - + " public NativeRecordStruct() {}\n" - + "}\n"); - CompilationResult readerResult = - compile( - "test.NativeRecordStruct", - "package test;\n" + "public record NativeRecordStruct(int id) {}\n"); - Assert.assertTrue(writerResult.success, writerResult.diagnostics()); - Assert.assertTrue(readerResult.success, readerResult.diagnostics()); - try (URLClassLoader writerLoader = writerResult.classLoader(); - URLClassLoader readerLoader = readerResult.classLoader()) { - Class writerType = writerLoader.loadClass("test.NativeRecordStruct"); - Class readerType = readerLoader.loadClass("test.NativeRecordStruct"); - Fory writer = compatibleFory(writerLoader, false); - Fory reader = compatibleFory(readerLoader, true); - Object writerValue = writerType.getConstructor().newInstance(); - setField(writerType, writerValue, "id", "73"); - - Object result = generatedCompatibleRead(writer, reader, writerType, readerType, writerValue); - Assert.assertSame(result.getClass(), readerType); - Assert.assertEquals(invoke(readerType, result, "id"), 73); - } - } - - @Test - public void testGraalvmStaticCompatibleSerializerUsesListArrayAction() throws Exception { - if (AndroidSupport.IS_ANDROID) { - return; - } - CompilationResult writerResult = - compile( - "test.NativeArrayShapeStruct", - "package test;\n" - + "import org.apache.fory.annotation.Int32Type;\n" - + "import org.apache.fory.collection.Int32List;\n" - + "import org.apache.fory.config.Int32Encoding;\n" - + "public class NativeArrayShapeStruct {\n" - + " @Int32Type(encoding = Int32Encoding.FIXED) public Int32List values;\n" - + " public NativeArrayShapeStruct() {}\n" - + "}\n"); - CompilationResult readerResult = - compile( - "test.NativeArrayShapeStruct", - "package test;\n" - + "public class NativeArrayShapeStruct {\n" - + " public int[] values;\n" - + " public NativeArrayShapeStruct() {}\n" - + "}\n"); - Assert.assertTrue(writerResult.success, writerResult.diagnostics()); - Assert.assertTrue(readerResult.success, readerResult.diagnostics()); - try (URLClassLoader writerLoader = writerResult.classLoader(); - URLClassLoader readerLoader = readerResult.classLoader()) { - Class writerType = writerLoader.loadClass("test.NativeArrayShapeStruct"); - Class readerType = readerLoader.loadClass("test.NativeArrayShapeStruct"); - Fory writer = xlangCompatibleFory(writerLoader, writerType, true, "NativeArrayShapeStruct"); - Fory reader = xlangCompatibleFory(readerLoader, readerType, true, "NativeArrayShapeStruct"); - Object writerValue = writerType.getConstructor().newInstance(); - setField( - writerType, - writerValue, - "values", - new org.apache.fory.collection.Int32List(new int[] {4, 5, 6})); - - Object result = generatedCompatibleRead(writer, reader, writerType, readerType, writerValue); - Assert.assertSame(result.getClass(), readerType); - Assert.assertTrue( - Arrays.equals((int[]) getField(readerType, result, "values"), new int[] {4, 5, 6})); - } - } - private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exception { CompilationResult staticResult = compile( @@ -709,51 +526,6 @@ private static Fory xlangCompatibleFory( return fory; } - private static Fory compatibleFory(ClassLoader classLoader, boolean codegen) { - return Fory.builder() - .withClassLoader(classLoader) - .withCodegen(codegen) - .withMetaShare(true) - .withScopedMetaShare(false) - .withCompatible(true) - .requireClassRegistration(false) - .build(); - } - - private static Object generatedCompatibleRead( - Fory writer, Fory reader, Class writerType, Class readerType, Object writerValue) - throws Exception { - TypeDef remoteTypeDef = TypeDef.buildTypeDef(writer.getTypeResolver(), writerType); - Class serializerClass = - CodecUtils.loadOrGenStaticCompatibleCodecClass( - reader.getTypeResolver(), (Class) readerType, remoteTypeDef); - Assert.assertTrue( - GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(serializerClass)); - Serializer serializer = - serializerClass - .getConstructor(org.apache.fory.resolver.TypeResolver.class, Class.class, TypeDef.class) - .newInstance(reader.getTypeResolver(), (Class) readerType, remoteTypeDef); - MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(128); - Serializer writerSerializer = - writer.getConfig().isXlang() - ? writer.getTypeResolver().getTypeInfo(writerType).getSerializer() - : new MetaSharedSerializer<>( - writer.getTypeResolver(), (Class) writerType, remoteTypeDef); - writer.getWriteContext().prepare(buffer, null); - try { - writerSerializer.write(writer.getWriteContext(), writerValue); - } finally { - writer.getWriteContext().reset(); - } - buffer.readerIndex(0); - reader.getReadContext().prepare(buffer, null, false); - try { - return serializer.read(reader.getReadContext()); - } finally { - reader.getReadContext().reset(); - } - } - private static CompilationResult compile(String typeName, String source) throws IOException { JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); Assert.assertNotNull(compiler, "Tests require a JDK compiler"); diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 940c17a93b..5231c1dcb9 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -62,6 +62,11 @@ ${project.version} test + + org.mockito + mockito-core + test + diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index f9ed1625b6..71d1e34a31 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -213,10 +213,10 @@ public static final class SerializationFieldInfo { // This determines how to write the value to the object (UnsafeOps.putInt vs putObject). isPrimitiveField = typeRef.getRawType().isPrimitive(); fieldConverter = d.getFieldConverter(); - // For xlang compatibility, check TypeExtMeta first (from remote peer's type meta) - // This ensures we read data correctly when remote's nullable differs from local + // Primitive-list carrier TypeExtMeta describes the element wire type, not the field + // nullability/ref mode. The field mode must stay aligned with Descriptor/TypeDef metadata. TypeExtMeta extMeta = typeRef.getTypeExtMeta(); - if (extMeta != null) { + if (extMeta != null && !primitiveListArray && !primitiveListCollection) { nullable = extMeta.nullable(); trackingRef = extMeta.trackingRef(); } else { diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java new file mode 100644 index 0000000000..78263664b5 --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.builder; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.atLeastOnce; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import javax.tools.Diagnostic; +import javax.tools.DiagnosticCollector; +import javax.tools.JavaCompiler; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; +import org.apache.fory.Fory; +import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.context.MetaReadContext; +import org.apache.fory.context.MetaWriteContext; +import org.apache.fory.meta.TypeDef; +import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.Serializer; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.SkipException; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class StaticCompatibleCodecBuilderTest { + @DataProvider + public static Object[][] xlangModes() { + return new Object[][] {{false}, {true}}; + } + + @Test(dataProvider = "xlangModes") + public void testStaticCompatibleSerializerReadsRuntimeRemoteTypeDef(boolean xlang) + throws Exception { + CompilationResult writerResult = + compile( + "test.StaticCompatiblePayload", + "package test;\n" + + "public class StaticCompatiblePayload {\n" + + " public int id;\n" + + " public int legacy;\n" + + " public String name;\n" + + " public StaticCompatiblePayload() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.StaticCompatiblePayload", + "package test;\n" + + "public class StaticCompatiblePayload {\n" + + " public int id;\n" + + " public String added = \"default\";\n" + + " public String name;\n" + + " public StaticCompatiblePayload() {}\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.StaticCompatiblePayload"); + Class readerType = readerLoader.loadClass("test.StaticCompatiblePayload"); + Fory writer = compatibleFory(writerLoader, writerType, xlang, "writer"); + Fory reader = compatibleFory(readerLoader, readerType, xlang, "reader"); + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "id", 42); + setField(writerType, writerValue, "legacy", 99); + setField(writerType, writerValue, "name", xlang ? "xlang" : "native"); + + Object result = + roundTripThroughStaticCompatibleSerializer( + writer, reader, writerType, readerType, writerValue); + Assert.assertSame(result.getClass(), readerType); + Assert.assertEquals(getField(readerType, result, "id"), 42); + Assert.assertEquals(getField(readerType, result, "added"), "default"); + Assert.assertEquals(getField(readerType, result, "name"), xlang ? "xlang" : "native"); + } + } + + @Test + public void testStaticCompatibleRecordSerializerConvertsRemoteField() throws Exception { + assumeRecordSupport(); + CompilationResult writerResult = + compile( + "test.StaticCompatibleRecordPayload", + "package test;\n" + + "public class StaticCompatibleRecordPayload {\n" + + " public String id;\n" + + " public StaticCompatibleRecordPayload() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.StaticCompatibleRecordPayload", + "package test;\n" + "public record StaticCompatibleRecordPayload(int id) {}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.StaticCompatibleRecordPayload"); + Class readerType = readerLoader.loadClass("test.StaticCompatibleRecordPayload"); + Fory writer = compatibleFory(writerLoader, writerType, false, "record-writer"); + Fory reader = compatibleFory(readerLoader, readerType, false, "record-reader"); + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "id", "73"); + + Object result = + roundTripThroughStaticCompatibleSerializer( + writer, reader, writerType, readerType, writerValue); + Assert.assertSame(result.getClass(), readerType); + Assert.assertEquals(invoke(readerType, result, "id"), 73); + } + } + + @Test + public void testStaticCompatibleSerializerUsesListArrayAction() throws Exception { + CompilationResult writerResult = + compile( + "test.StaticCompatibleArrayPayload", + "package test;\n" + + "import org.apache.fory.annotation.Int32Type;\n" + + "import org.apache.fory.collection.Int32List;\n" + + "import org.apache.fory.config.Int32Encoding;\n" + + "public class StaticCompatibleArrayPayload {\n" + + " @Int32Type(encoding = Int32Encoding.FIXED) public Int32List values;\n" + + " public StaticCompatibleArrayPayload() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.StaticCompatibleArrayPayload", + "package test;\n" + + "public class StaticCompatibleArrayPayload {\n" + + " public int[] values;\n" + + " public StaticCompatibleArrayPayload() {}\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.StaticCompatibleArrayPayload"); + Class readerType = readerLoader.loadClass("test.StaticCompatibleArrayPayload"); + Fory writer = compatibleFory(writerLoader, writerType, true, "array-writer"); + Fory reader = compatibleFory(readerLoader, readerType, true, "array-reader"); + Object writerValue = writerType.getConstructor().newInstance(); + setField( + writerType, + writerValue, + "values", + new org.apache.fory.collection.Int32List(new int[] {4, 5, 6})); + + Object result = + roundTripThroughStaticCompatibleSerializer( + writer, reader, writerType, readerType, writerValue); + Assert.assertSame(result.getClass(), readerType); + Assert.assertTrue( + Arrays.equals((int[]) getField(readerType, result, "values"), new int[] {4, 5, 6})); + } + } + + private static Object roundTripThroughStaticCompatibleSerializer( + Fory writer, Fory reader, Class writerType, Class readerType, Object writerValue) + throws Exception { + TypeDef remoteTypeDef = TypeDef.buildTypeDef(writer.getTypeResolver(), writerType); + Assert.assertNotEquals( + remoteTypeDef.getId(), TypeDef.buildTypeDef(reader.getTypeResolver(), readerType).getId()); + Class readerClass = cast(readerType); + Class compatibleSerializerClass = + CodecUtils.loadOrGenStaticCompatibleCodecClass( + reader.getTypeResolver(), readerClass, remoteTypeDef); + Assert.assertTrue( + GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(compatibleSerializerClass)); + Serializer compatibleSerializer = + compatibleSerializerClass + .getConstructor(TypeResolver.class, Class.class, TypeDef.class) + .newInstance(reader.getTypeResolver(), readerClass, remoteTypeDef); + Assert.assertThrows( + UnsupportedOperationException.class, + () -> compatibleSerializer.write(reader.getWriteContext(), null)); + + writer.setMetaWriteContext(new MetaWriteContext()); + byte[] bytes = writer.serialize(writerValue); + reader.setMetaReadContext(new MetaReadContext()); + try (MockedStatic codecUtils = + Mockito.mockStatic(CodecUtils.class, Mockito.CALLS_REAL_METHODS)) { + codecUtils + .when( + () -> + CodecUtils.loadOrGenMetaSharedCodecClass( + same(reader.getTypeResolver()), eq(readerClass), any(TypeDef.class))) + .thenReturn(compatibleSerializerClass); + Object result = reader.deserialize(bytes); + codecUtils.verify( + () -> + CodecUtils.loadOrGenMetaSharedCodecClass( + same(reader.getTypeResolver()), eq(readerClass), any(TypeDef.class)), + atLeastOnce()); + return result; + } + } + + private static Fory compatibleFory( + ClassLoader classLoader, Class type, boolean xlang, String role) { + Fory fory = + Fory.builder() + .withName("static-compatible-" + role + "-" + (xlang ? "xlang" : "native")) + .withClassLoader(classLoader) + .withXlang(xlang) + .withCodegen(true) + .withMetaShare(true) + .withScopedMetaShare(false) + .withCompatible(true) + .requireClassRegistration(false) + .build(); + if (xlang) { + fory.register(type, "test", type.getSimpleName()); + } + return fory; + } + + @SuppressWarnings("unchecked") + private static Class cast(Class type) { + return (Class) type; + } + + private static CompilationResult compile(String typeName, String source) throws IOException { + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + Assert.assertNotNull(compiler, "Tests require a JDK compiler"); + Path root = Files.createTempDirectory("fory-static-compatible-test"); + Path sourceRoot = root.resolve("src"); + Path classRoot = root.resolve("classes"); + Files.createDirectories(sourceRoot); + Files.createDirectories(classRoot); + Path sourceFile = sourceRoot.resolve(typeName.replace('.', '/') + ".java"); + Files.createDirectories(sourceFile.getParent()); + Files.write(sourceFile, source.getBytes(StandardCharsets.UTF_8)); + DiagnosticCollector diagnostics = new DiagnosticCollector<>(); + try (StandardJavaFileManager fileManager = + compiler.getStandardFileManager(diagnostics, null, StandardCharsets.UTF_8)) { + Iterable units = + fileManager.getJavaFileObjectsFromFiles(Collections.singletonList(sourceFile.toFile())); + List options = + Arrays.asList( + "-classpath", + System.getProperty("java.class.path"), + "-d", + classRoot.toString()); + JavaCompiler.CompilationTask task = + compiler.getTask(null, fileManager, diagnostics, options, null, units); + return new CompilationResult(classRoot, task.call(), diagnostics.getDiagnostics()); + } + } + + private static void assumeRecordSupport() { + if (javaSpecificationVersion() < 16) { + throw new SkipException("Record source tests require JDK 16 or newer"); + } + } + + private static int javaSpecificationVersion() { + String version = System.getProperty("java.specification.version"); + if (version.startsWith("1.")) { + version = version.substring(2); + } + int dotIndex = version.indexOf('.'); + if (dotIndex >= 0) { + version = version.substring(0, dotIndex); + } + return Integer.parseInt(version); + } + + private static void setField(Class type, Object target, String name, Object value) + throws Exception { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } + + private static Object getField(Class type, Object target, String name) throws Exception { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } + + private static Object invoke(Class type, Object target, String name) throws Exception { + java.lang.reflect.Method method = type.getDeclaredMethod(name); + method.setAccessible(true); + return method.invoke(target); + } + + private static final class CompilationResult { + final Path classRoot; + final boolean success; + final List> diagnostics; + + CompilationResult( + Path classRoot, boolean success, List> diagnostics) { + this.classRoot = classRoot; + this.success = success; + this.diagnostics = new ArrayList<>(diagnostics); + } + + URLClassLoader classLoader() throws IOException { + URL[] urls = {classRoot.toUri().toURL()}; + return new URLClassLoader(urls, StaticCompatibleCodecBuilderTest.class.getClassLoader()); + } + + String diagnostics() { + StringBuilder builder = new StringBuilder(); + for (Diagnostic diagnostic : diagnostics) { + builder.append(diagnostic).append('\n'); + } + return builder.toString(); + } + } +} diff --git a/java/fory-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/java/fory-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..1f0955d450 --- /dev/null +++ b/java/fory-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/java/pom.xml b/java/pom.xml index b3000ffa10..3fabe04bc3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -76,6 +76,7 @@ ${basedir} 3.3.0 1.18.38 + 4.11.0 @@ -157,6 +158,11 @@ lombok ${lombok.version} + + org.mockito + mockito-core + ${mockito.version} + From 3ee8866667b46ecb3ba3d24f1b4a5c1116e0f5c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 16:09:50 +0800 Subject: [PATCH 19/58] docs(java): clarify primitive-list field metadata --- .../src/main/java/org/apache/fory/serializer/FieldGroups.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 71d1e34a31..2b947a62c1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -214,7 +214,8 @@ public static final class SerializationFieldInfo { isPrimitiveField = typeRef.getRawType().isPrimitive(); fieldConverter = d.getFieldConverter(); // Primitive-list carrier TypeExtMeta describes the element wire type, not the field - // nullability/ref mode. The field mode must stay aligned with Descriptor/TypeDef metadata. + // nullability/ref mode. Treating it as field metadata writes an extra null marker that + // remote TypeDef payload readers then consume as list length. TypeExtMeta extMeta = typeRef.getTypeExtMeta(); if (extMeta != null && !primitiveListArray && !primitiveListCollection) { nullable = extMeta.nullable(); From c12da074406fae8d033ca648fb7a2325bc2dcd5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 17:35:22 +0800 Subject: [PATCH 20/58] add more tests and fix field meta and order --- .../processing/ForyStructProcessor.java | 24 +- .../annotation/processing/SourceStruct.java | 2 +- .../annotation/processing/SourceTypeNode.java | 14 +- .../StaticSerializerSourceWriter.java | 7 +- .../org/apache/fory/builder/CodecUtils.java | 3 +- .../builder/StaticCompatibleCodecBuilder.java | 4 +- .../java/org/apache/fory/meta/FieldInfo.java | 10 +- .../java/org/apache/fory/meta/FieldTypes.java | 7 +- .../java/org/apache/fory/meta/TypeDef.java | 3 +- .../apache/fory/resolver/ClassResolver.java | 20 +- .../apache/fory/resolver/TypeResolver.java | 13 +- .../apache/fory/resolver/XtypeResolver.java | 6 +- .../apache/fory/serializer/FieldGroups.java | 3 +- .../apache/fory/serializer/FieldSkipper.java | 7 + .../StaticGeneratedStructSerializer.java | 67 ++- .../StaticCompatibleCodecBuilderTest.java | 5 +- .../org/apache/fory/reflect/TypeRefTest.java | 2 +- java/fory-latest-jdk-tests/pom.xml | 29 ++ .../integration_tests/ExampleMessage.java | 453 ++++++++++++++++ .../ExampleRecordMessage.java | 207 ++++++++ .../ExampleStaticGeneratedSerializerTest.java | 490 ++++++++++++++++++ .../org.mockito.plugins.MockMaker | 1 + 22 files changed, 1318 insertions(+), 59 deletions(-) create mode 100644 java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleMessage.java create mode 100644 java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleRecordMessage.java create mode 100644 java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java create mode 100644 java/fory-latest-jdk-tests/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index c2c78c0779..15eb83a594 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -566,7 +566,8 @@ private SourceTypeNode buildTypeNode(TypeMirror type, Object tree, String typeEx if (kind == TypeKind.ARRAY) { TypeMirror componentMirror = ((ArrayType) type).getComponentType(); componentType = - buildTypeNode(componentMirror, treeInfo.arrayComponentTree(), nestedNullable(componentMirror)); + buildTypeNode( + componentMirror, treeInfo.arrayComponentTree(), nestedNullable(componentMirror)); } else if (type instanceof DeclaredType) { List argumentTrees = treeInfo.typeArgumentTrees(); int index = 0; @@ -579,7 +580,7 @@ private SourceTypeNode buildTypeNode(TypeMirror type, Object tree, String typeEx String rawType = canonicalName(types.erasure(type)); String extMeta = typeExtMetaExpression(type, rawType, treeInfo.annotations, typeExtNullable); boolean primitive = kind.isPrimitive(); - boolean nestedStruct = isForyStructType(type); + boolean nestedStruct = isCompatibleForyStructType(type); return new SourceTypeNode( rawType, typeName(type), extMeta, arguments, componentType, primitive, nestedStruct); } @@ -594,10 +595,25 @@ private TypeTreeInfo typeTreeInfo(Object tree) { return new TypeTreeInfo(annotations, current); } - private boolean isForyStructType(TypeMirror type) { + private boolean isCompatibleForyStructType(TypeMirror type) { TypeMirror erased = types.erasure(type); Element element = types.asElement(erased); - return element instanceof TypeElement && hasAnnotation(element, FORY_STRUCT); + if (!(element instanceof TypeElement)) { + return false; + } + AnnotationMirror mirror = annotationMirror(element, FORY_STRUCT); + if (mirror == null) { + return false; + } + Map values = + elements.getElementValuesWithDefaults(mirror); + for (Map.Entry entry : + values.entrySet()) { + if (entry.getKey().getSimpleName().contentEquals("evolving")) { + return !"DISABLED".equals(enumConstant(String.valueOf(entry.getValue().getValue()))); + } + } + return true; } private String typeExtMetaExpression( diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java index 925ba55172..e07eed36fd 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java @@ -48,7 +48,7 @@ final class SourceStruct { Collections.unmodifiableList(new ArrayList<>(recordConstructorFields)); boolean hasNestedStruct = false; for (SourceField field : fields) { - hasNestedStruct |= field.typeNode.hasNestedStruct(); + hasNestedStruct |= field.typeNode.hasNestedCompatibleStruct(); } this.hasNestedCompatibleStructFields = hasNestedStruct; } diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceTypeNode.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceTypeNode.java index 80804eedab..e67dd60f98 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceTypeNode.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceTypeNode.java @@ -30,7 +30,7 @@ final class SourceTypeNode { final List typeArguments; final SourceTypeNode componentType; final boolean primitive; - final boolean nestedStruct; + final boolean nestedCompatibleStruct; SourceTypeNode( String rawType, @@ -39,7 +39,7 @@ final class SourceTypeNode { List typeArguments, SourceTypeNode componentType, boolean primitive, - boolean nestedStruct) { + boolean nestedCompatibleStruct) { this.rawType = rawType; this.typeName = typeName; this.typeExtMeta = typeExtMeta; @@ -49,7 +49,7 @@ final class SourceTypeNode { : Collections.unmodifiableList(new ArrayList<>(typeArguments)); this.componentType = componentType; this.primitive = primitive; - this.nestedStruct = nestedStruct; + this.nestedCompatibleStruct = nestedCompatibleStruct; } String typeRefExpression() { @@ -82,15 +82,15 @@ private String typeArgumentsExpression() { return builder.toString(); } - boolean hasNestedStruct() { - if (nestedStruct) { + boolean hasNestedCompatibleStruct() { + if (nestedCompatibleStruct) { return true; } - if (componentType != null && componentType.hasNestedStruct()) { + if (componentType != null && componentType.hasNestedCompatibleStruct()) { return true; } for (SourceTypeNode typeArgument : typeArguments) { - if (typeArgument.hasNestedStruct()) { + if (typeArgument.hasNestedCompatibleStruct()) { return true; } } diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 188988e8d7..2aaa465b2c 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -139,7 +139,7 @@ private void writeConstructors() { .append(struct.serializerName) .append("(TypeResolver typeResolver, Class type) {\n"); builder.append(" super(typeResolver, type);\n"); - writeConstructorBody("false"); + writeConstructorBody("buildFieldGroups(DESCRIPTORS)", "false"); builder.append(" }\n\n"); builder .append(" public ") @@ -147,12 +147,13 @@ private void writeConstructors() { .append("(TypeResolver typeResolver, Class type, TypeDef typeDef) {\n"); builder.append(" super(typeResolver, type, typeDef, DESCRIPTORS);\n"); writeConstructorBody( + "buildLocalFieldGroups(DESCRIPTORS)", "typeDef != null && !HAS_NESTED_COMPATIBLE_STRUCT_FIELDS && typeDef.getId() == TypeDef.buildTypeDef(typeResolver, type).getId()"); builder.append(" }\n\n"); } - private void writeConstructorBody(String sameSchemaExpression) { - builder.append(" FieldGroups fieldGroups = buildFieldGroups(DESCRIPTORS);\n"); + private void writeConstructorBody(String fieldGroupsExpression, String sameSchemaExpression) { + builder.append(" FieldGroups fieldGroups = ").append(fieldGroupsExpression).append(";\n"); builder.append(" this.buildInFields = fieldGroups.buildInFields;\n"); builder.append(" this.buildInFieldIds = localFieldIds(buildInFields, DESCRIPTORS);\n"); builder.append(" this.containerFields = fieldGroups.containerFields;\n"); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index cf0319c419..5856e58460 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -44,8 +44,7 @@ public class CodecUtils { // TODO(chaokunyang) how to uninstall org.apache.fory.codegen/builder classes for graalvm build // time // maybe use a temporal URLClassLoader - public static Class loadOrGenObjectCodecClass( - Class cls, Fory fory) { + public static Class loadOrGenObjectCodecClass(Class cls, Fory fory) { Preconditions.checkNotNull(fory); return loadSerializer( "loadOrGenObjectCodecClass", diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 9003a2a7b0..c9c14147f7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -49,8 +49,8 @@ * *

The generated class is keyed by the local Java class, not by a fixed remote schema. Its * constructor receives the runtime remote {@link TypeDef}; {@link - * org.apache.fory.serializer.StaticGeneratedStructSerializer} derives the remote field list from - * that definition and the generated local descriptors. + * org.apache.fory.serializer.StaticGeneratedStructSerializer} rebuilds remote read order through + * the same descriptor-grouper owner used by {@link MetaSharedSerializer}. * * @see ForyBuilder#withMetaShare * @see MetaSharedSerializer diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index 59530e2f42..d6ec155256 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -23,6 +23,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.Objects; +import org.apache.fory.annotation.ForyField; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.converter.FieldConverter; @@ -186,7 +187,9 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { } return builder.build(); } - // This field doesn't exist in peer class, so any legal modifier will be OK. + // This field doesn't exist in peer class, so any legal modifier will be OK. Preserve the + // remote tag id in the synthetic descriptor because descriptor grouping uses it as the sort key + // for compatible payload order. // Use constant instead of reflection to avoid GraalVM native image issues. int stubModifiers = Modifier.PRIVATE | Modifier.FINAL; return new Descriptor( @@ -195,8 +198,11 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { fieldName, stubModifiers, definedClass, + hasFieldId(), + fieldId, + remoteNullable, remoteTrackingRef, - remoteNullable); + ForyField.Dynamic.AUTO); } private boolean isTopLevelListArrayCompatibleReadPair( diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index e3b24ce097..c967f99b24 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -128,7 +128,8 @@ private static FieldType buildFieldType( primitiveList ? primitiveListElementMeta != null ? primitiveListElementMeta.typeId() - : TypeAnnotationUtils.getPrimitiveListElementTypeId(typeAnnotation, rawType, isXlang) + : TypeAnnotationUtils.getPrimitiveListElementTypeId( + typeAnnotation, rawType, isXlang) : Types.UNKNOWN; if (typeExtMeta != null && typeExtMeta.typeId() != Types.UNKNOWN) { typeId = typeExtMeta.typeId(); @@ -199,8 +200,8 @@ private static FieldType buildFieldType( descriptorCarriesFieldOptions ? descriptor.isTrackingRef() : isXlang - ? typeExtMeta != null && typeExtMeta.trackingRef() - : genericType.trackingRef(resolver); + ? typeExtMeta != null && typeExtMeta.trackingRef() + : genericType.trackingRef(resolver); // For xlang: nullable is false by default for top-level fields. // Nested element types are nullable by default to align with cross-language collection // semantics. diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java index 1602ffe42c..90caf3aef9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java @@ -452,7 +452,8 @@ public TypeDef replaceRootClassTo(TypeResolver resolver, Class targetCls) { .map( fieldInfo -> { if (fieldInfo.definedClass.equals(classSpec.entireClassName)) { - return new FieldInfo(name, fieldInfo.fieldName, fieldInfo.fieldType); + return new FieldInfo( + name, fieldInfo.fieldName, fieldInfo.fieldType, fieldInfo.fieldId); } else { return fieldInfo; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 4af60b556b..272a57dfa8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1725,7 +1725,7 @@ private Serializer createSerializer(Class cls) { } Class serializerClass = getSerializerClass(cls); - Serializer serializer = Serializers.newSerializer(this, cls, serializerClass); + Serializer serializer = newSerializer(cls, serializerClass); if (ForyCopyable.class.isAssignableFrom(cls)) { serializer = new ForyCopyableSerializer<>(config, cls, serializer); } @@ -2093,19 +2093,17 @@ public boolean isPrimitive(int classId) { /** * Normalize type name for consistent ordering between serialization and deserialization. - * Collection subtypes (List, Set, etc.) are normalized to "java.util.Collection". Map subtypes - * are normalized to "java.util.Map". This ensures fields have the same order regardless of - * whether the peer has the field locally. + * Collection descriptors are normalized to "java.util.Collection". Map subtypes are normalized to + * "java.util.Map". This ensures fields have the same order regardless of whether the peer has the + * field locally. */ private String getNormalizedTypeName(Descriptor d) { + if (isCollectionDescriptor(d)) { + return "java.util.Collection"; + } Class rawType = d.getRawType(); - if (rawType != null) { - if (isCollection(rawType)) { - return "java.util.Collection"; - } - if (isMap(rawType)) { - return "java.util.Map"; - } + if (rawType != null && isMap(rawType)) { + return "java.util.Map"; } return d.getTypeName(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 50ecb2a93d..18a4b31662 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -97,7 +97,6 @@ import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.GenericType; import org.apache.fory.type.ScalaTypes; -import org.apache.fory.type.TypeAnnotationUtils; import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; @@ -426,9 +425,7 @@ public boolean usesPrimitiveFieldOrdering(Descriptor descriptor) { } public boolean isCollectionDescriptor(Descriptor descriptor) { - return isCollection(descriptor.getRawType()) - || TypeAnnotationUtils.usesCollectionProtocolForPrimitiveList( - descriptor.getTypeAnnotation(), descriptor.getRawType()); + return isCollection(descriptor.getRawType()); } public abstract boolean isMonomorphic(Descriptor descriptor); @@ -1618,6 +1615,14 @@ protected final StaticGeneratedStructSerializer newStaticGeneratedStructSeria } } + protected final Serializer newSerializer( + Class cls, Class serializerClass) { + if (isShareMeta() && StaticGeneratedStructSerializer.class.isAssignableFrom(serializerClass)) { + return newStaticGeneratedStructSerializer(serializerClass, cls, getTypeDef(cls, true)); + } + return Serializers.newSerializer(this, cls, serializerClass); + } + private List getStaticGeneratedStructDescriptors(Class cls) { Class serializerClass = getStaticGeneratedStructSerializerClass(cls); if (serializerClass == null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 0220aa0655..197d8400be 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -322,8 +322,8 @@ private void register( type, shareMeta, config.isCodeGenEnabled(), - sc -> ref.set(Serializers.newSerializer(this, type, sc))); - ref.set(Serializers.newSerializer(this, type, c)); + sc -> ref.set(newSerializer(type, sc))); + ref.set(newSerializer(type, c)); if (!config.isAsyncCompilationEnabled()) { updated.set(true); } @@ -452,7 +452,7 @@ private TypeInfo newTypeInfo( public void registerSerializer(Class type, Class serializerClass) { checkRegisterAllowed(); - registerSerializer(type, Serializers.newSerializer(this, type, serializerClass)); + registerSerializer(type, newSerializer(type, serializerClass)); } public void registerSerializer(Class type, Serializer serializer) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 2b947a62c1..be201be9ad 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -316,7 +316,8 @@ private static TypeRef primitiveListElementTypeRef(Descriptor descriptor) { TypeRef typeRef = descriptor.getTypeRef(); TypeExtMeta inlineMeta = typeRef.getTypeExtMeta(); if (inlineMeta != null && Types.isPrimitiveType(inlineMeta.typeId())) { - Class elementClass = TypeAnnotationUtils.getPrimitiveListElementClass(typeRef.getRawType()); + Class elementClass = + TypeAnnotationUtils.getPrimitiveListElementClass(typeRef.getRawType()); if (elementClass != null) { return TypeRef.of( elementClass, TypeExtMeta.of(inlineMeta.typeId(), true, inlineMeta.trackingRef())); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java index 3a43ea0f6e..72b873a897 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java @@ -53,6 +53,13 @@ static void skipField( int dispatchId = fieldInfo.dispatchId; RefMode refMode = fieldInfo.refMode; + if (typeResolver.isCollectionDescriptor(fieldInfo.descriptor) + || typeResolver.isMap(fieldInfo.type)) { + AbstractObjectSerializer.readContainerFieldValue( + readContext, typeResolver, refReader, readContext.getGenerics(), fieldInfo, buffer); + return; + } + // For non-basic types, fall back to binding.readField if (!DispatchId.isBasicType(dispatchId)) { AbstractObjectSerializer.readField(readContext, typeResolver, refReader, fieldInfo, buffer); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index f355048652..1d1cdcce6b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -29,6 +29,7 @@ import org.apache.fory.context.ReadContext; import org.apache.fory.context.RefReader; import org.apache.fory.context.WriteContext; +import org.apache.fory.exception.DeserializationException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.FieldInfo; import org.apache.fory.meta.TypeDef; @@ -88,6 +89,18 @@ protected final FieldGroups buildFieldGroups(List descriptors) { return FieldGroups.buildFieldInfos(typeResolver, grouper); } + protected final FieldGroups buildLocalFieldGroups(List descriptors) { + if (!typeResolver.isShareMeta()) { + return buildFieldGroups(descriptors); + } + // Meta-share writers use the local TypeDef-reified descriptor grouping, matching + // ObjectSerializer. The constructor TypeDef may be a remote schema for compatible reads, so it + // must not own local field access ordering. + DescriptorGrouper grouper = + typeResolver.createDescriptorGrouper(typeResolver.getTypeDef(type, true), type); + return FieldGroups.buildFieldInfos(typeResolver, grouper); + } + protected final List runtimeDescriptors(List descriptors) { return typeResolver.normalizeFieldDescriptors(type, true, descriptors); } @@ -192,7 +205,12 @@ protected final void skipField( RefReader refReader, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { - FieldSkipper.skipField(readContext, typeResolver, refReader, fieldInfo, buffer); + try { + FieldSkipper.skipField(readContext, typeResolver, refReader, fieldInfo, buffer); + } catch (RuntimeException e) { + throw new DeserializationException( + "Failed to skip remote field " + fieldInfo.descriptor.getName(), e); + } } protected final int matchedId(RemoteFieldInfo remoteField) { @@ -269,9 +287,10 @@ private Object readField(ReadContext readContext, SerializationFieldInfo fieldIn private List buildRemoteFields( TypeDef remoteTypeDef, List localDescriptors) { + Class remoteDescriptorClass = remoteDescriptorClass(remoteTypeDef); List remoteFieldInfos = remoteTypeDef.getFieldsInfo(); List remoteDescriptors = - remoteTypeDef.getDescriptors(typeResolver, type, localDescriptors); + remoteTypeDef.getDescriptors(typeResolver, remoteDescriptorClass); Map remoteFieldInfosByKey = new HashMap<>(); for (int i = 0; i < remoteFieldInfos.size(); i++) { FieldInfo fieldInfo = remoteFieldInfos.get(i); @@ -287,24 +306,54 @@ private List buildRemoteFields( } fields.put(fieldKey(descriptor), i); } + // Keep compatible-read descriptor ordering owned by TypeResolver, matching MetaSharedSerializer + // and MetaSharedCodecBuilder. TypeDef itself is metadata; it is not the read-order owner. FieldGroups remoteFieldGroups = FieldGroups.buildFieldInfos( typeResolver, - FieldGroups.buildDescriptorGrouper( - typeResolver, remoteDescriptors, false, descriptor -> descriptor)); + typeResolver.createDescriptorGrouper(remoteTypeDef, remoteDescriptorClass)); List remoteFields = new ArrayList<>(remoteFieldInfos.size()); appendRemoteFields( - remoteFields, remoteFieldGroups.buildInFields, remoteFieldInfosByKey, fieldIds, fields, + remoteFields, + remoteFieldGroups.buildInFields, + remoteFieldInfosByKey, + fieldIds, + fields, localDescriptors); appendRemoteFields( - remoteFields, remoteFieldGroups.containerFields, remoteFieldInfosByKey, fieldIds, fields, + remoteFields, + remoteFieldGroups.containerFields, + remoteFieldInfosByKey, + fieldIds, + fields, localDescriptors); appendRemoteFields( - remoteFields, remoteFieldGroups.userTypeFields, remoteFieldInfosByKey, fieldIds, fields, + remoteFields, + remoteFieldGroups.userTypeFields, + remoteFieldInfosByKey, + fieldIds, + fields, localDescriptors); return Collections.unmodifiableList(remoteFields); } + private Class remoteDescriptorClass(TypeDef remoteTypeDef) { + String className = remoteTypeDef.getClassName(); + if (className.equals(type.getName())) { + return type; + } + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + try { + return Class.forName(className, false, contextClassLoader); + } catch (ClassNotFoundException | LinkageError e) { + try { + return Class.forName(className, false, type.getClassLoader()); + } catch (ClassNotFoundException | LinkageError ignored) { + return type; + } + } + } + private void appendRemoteFields( List remoteFields, SerializationFieldInfo[] serializationFieldInfos, @@ -381,9 +430,7 @@ private static String remoteFieldKey(FieldInfo fieldInfo) { } private static String remoteFieldKey(Descriptor descriptor) { - return descriptor.hasForyFieldId() - ? "id:" + descriptor.getForyFieldId() - : fieldKey(descriptor); + return descriptor.hasForyFieldId() ? "id:" + descriptor.getForyFieldId() : fieldKey(descriptor); } /** Remote field metadata consumed by generated compatible read methods. */ diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java index 78263664b5..d345b3f029 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -269,10 +269,7 @@ private static CompilationResult compile(String typeName, String source) throws fileManager.getJavaFileObjectsFromFiles(Collections.singletonList(sourceFile.toFile())); List options = Arrays.asList( - "-classpath", - System.getProperty("java.class.path"), - "-d", - classRoot.toString()); + "-classpath", System.getProperty("java.class.path"), "-d", classRoot.toString()); JavaCompiler.CompilationTask task = compiler.getTask(null, fileManager, diagnostics, options, null, units); return new CompilationResult(classRoot, task.call(), diagnostics.getDiagnostics()); diff --git a/java/fory-core/src/test/java/org/apache/fory/reflect/TypeRefTest.java b/java/fory-core/src/test/java/org/apache/fory/reflect/TypeRefTest.java index e15fc953d6..3930f84f1d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/reflect/TypeRefTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/reflect/TypeRefTest.java @@ -38,8 +38,8 @@ import org.apache.fory.collection.Tuple2; import org.apache.fory.config.Int32Encoding; import org.apache.fory.meta.TypeExtMeta; -import org.apache.fory.type.Types; import org.apache.fory.type.TypeUtils; +import org.apache.fory.type.Types; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/java/fory-latest-jdk-tests/pom.xml b/java/fory-latest-jdk-tests/pom.xml index 67826287ba..af70803df8 100644 --- a/java/fory-latest-jdk-tests/pom.xml +++ b/java/fory-latest-jdk-tests/pom.xml @@ -53,6 +53,17 @@ fory-test-core ${project.version} + + org.apache.fory + fory-annotation-processor + ${project.version} + test + + + org.mockito + mockito-core + test + @@ -64,6 +75,24 @@ true + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.projectlombok + lombok + ${lombok.version} + + + org.apache.fory + fory-annotation-processor + ${project.version} + + + + diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleMessage.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleMessage.java new file mode 100644 index 0000000000..b8b49d86f8 --- /dev/null +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleMessage.java @@ -0,0 +1,453 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests; + +import java.util.List; +import java.util.Map; +import org.apache.fory.annotation.ForyField; +import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.ForyStruct.Evolution; +import org.apache.fory.annotation.Int32Type; +import org.apache.fory.annotation.Int64Type; +import org.apache.fory.annotation.Int8Type; +import org.apache.fory.annotation.Ref; +import org.apache.fory.annotation.UInt16Type; +import org.apache.fory.annotation.UInt32Type; +import org.apache.fory.annotation.UInt64Type; +import org.apache.fory.annotation.UInt8Type; +import org.apache.fory.collection.BFloat16List; +import org.apache.fory.collection.BoolList; +import org.apache.fory.collection.Float16List; +import org.apache.fory.collection.Float32List; +import org.apache.fory.collection.Float64List; +import org.apache.fory.collection.Int16List; +import org.apache.fory.collection.Int32List; +import org.apache.fory.collection.Int64List; +import org.apache.fory.collection.Int8List; +import org.apache.fory.collection.UInt16List; +import org.apache.fory.collection.UInt32List; +import org.apache.fory.collection.UInt64List; +import org.apache.fory.collection.UInt8List; +import org.apache.fory.config.Int32Encoding; +import org.apache.fory.config.Int64Encoding; +import org.apache.fory.type.BFloat16; +import org.apache.fory.type.BFloat16Array; +import org.apache.fory.type.Float16; +import org.apache.fory.type.Float16Array; + +@ForyStruct +public class ExampleMessage { + @ForyField(id = 1) + public boolean boolValue; + + @ForyField(id = 2) + public byte int8Value; + + @ForyField(id = 3) + public short int16Value; + + @ForyField(id = 4) + public @Int32Type(encoding = Int32Encoding.FIXED) int fixedI32Value; + + @ForyField(id = 5) + public int varintI32Value; + + @ForyField(id = 6) + public @Int64Type(encoding = Int64Encoding.FIXED) long fixedI64Value; + + @ForyField(id = 7) + public long varintI64Value; + + @ForyField(id = 8) + public @Int64Type(encoding = Int64Encoding.TAGGED) long taggedI64Value; + + @ForyField(id = 9) + public @UInt8Type int uint8Value; + + @ForyField(id = 10) + public @UInt16Type int uint16Value; + + @ForyField(id = 11) + public @UInt32Type(encoding = Int32Encoding.FIXED) long fixedU32Value; + + @ForyField(id = 12) + public @UInt32Type long varintU32Value; + + @ForyField(id = 13) + public @UInt64Type(encoding = Int64Encoding.FIXED) long fixedU64Value; + + @ForyField(id = 14) + public @UInt64Type long varintU64Value; + + @ForyField(id = 15) + public @UInt64Type(encoding = Int64Encoding.TAGGED) long taggedU64Value; + + @ForyField(id = 16) + public Float16 float16Value; + + @ForyField(id = 17) + public BFloat16 bfloat16Value; + + @ForyField(id = 18) + public float float32Value; + + @ForyField(id = 19) + public double float64Value; + + @ForyField(id = 20) + public String stringValue; + + @ForyField(id = 21) + public byte[] bytesValue; + + @ForyField(id = 22) + public java.time.LocalDate dateValue; + + @ForyField(id = 23) + public java.time.Instant timestampValue; + + @ForyField(id = 24) + public java.time.Duration durationValue; + + @ForyField(id = 25) + public java.math.BigDecimal decimalValue; + + @ForyField(id = 26) + public State enumValue; + + @ForyField(id = 27, nullable = true) + public Leaf messageValue; + + @ForyField(id = 101) + public BoolList boolList; + + @ForyField(id = 102) + public Int8List int8List; + + @ForyField(id = 103) + public Int16List int16List; + + @ForyField(id = 104) + public @Int32Type(encoding = Int32Encoding.FIXED) Int32List fixedI32List; + + @ForyField(id = 105) + public Int32List varintI32List; + + @ForyField(id = 106) + public @Int64Type(encoding = Int64Encoding.FIXED) Int64List fixedI64List; + + @ForyField(id = 107) + public Int64List varintI64List; + + @ForyField(id = 108) + public @Int64Type(encoding = Int64Encoding.TAGGED) Int64List taggedI64List; + + @ForyField(id = 109) + public UInt8List uint8List; + + @ForyField(id = 110) + public UInt16List uint16List; + + @ForyField(id = 111) + public @UInt32Type(encoding = Int32Encoding.FIXED) UInt32List fixedU32List; + + @ForyField(id = 112) + public UInt32List varintU32List; + + @ForyField(id = 113) + public @UInt64Type(encoding = Int64Encoding.FIXED) UInt64List fixedU64List; + + @ForyField(id = 114) + public UInt64List varintU64List; + + @ForyField(id = 115) + public @UInt64Type(encoding = Int64Encoding.TAGGED) UInt64List taggedU64List; + + @ForyField(id = 116) + public Float16List float16List; + + @ForyField(id = 117) + public BFloat16List bfloat16List; + + @ForyField(id = 118) + public List maybeFloat16List; + + @ForyField(id = 119) + public List maybeBfloat16List; + + @ForyField(id = 120) + public Float32List float32List; + + @ForyField(id = 121) + public Float64List float64List; + + @ForyField(id = 122) + public List stringList; + + @ForyField(id = 123) + public List bytesList; + + @ForyField(id = 124) + public List dateList; + + @ForyField(id = 125) + public List timestampList; + + @ForyField(id = 126) + public List durationList; + + @ForyField(id = 127) + public List decimalList; + + @ForyField(id = 128) + public List enumList; + + @ForyField(id = 129) + public List<@Ref(enable = false) Leaf> messageList; + + @ForyField(id = 131) + public List<@Int32Type(encoding = Int32Encoding.FIXED) Integer> maybeFixedI32List; + + @ForyField(id = 132) + public List<@UInt64Type Long> maybeUint64List; + + @ForyField(id = 301) + public boolean[] boolArray; + + @ForyField(id = 302) + public @Int8Type byte[] int8Array; + + @ForyField(id = 303) + public short[] int16Array; + + @ForyField(id = 304) + public int[] int32Array; + + @ForyField(id = 305) + public long[] int64Array; + + @ForyField(id = 306) + public @UInt8Type byte[] uint8Array; + + @ForyField(id = 307) + public @UInt16Type short[] uint16Array; + + @ForyField(id = 308) + public @UInt32Type int[] uint32Array; + + @ForyField(id = 309) + public @UInt64Type long[] uint64Array; + + @ForyField(id = 310) + public Float16Array float16Array; + + @ForyField(id = 311) + public BFloat16Array bfloat16Array; + + @ForyField(id = 312) + public float[] float32Array; + + @ForyField(id = 313) + public double[] float64Array; + + @ForyField(id = 314) + public List int32ArrayList; + + @ForyField(id = 315) + public List<@UInt8Type byte[]> uint8ArrayList; + + @ForyField(id = 201) + public Map stringValuesByBool; + + @ForyField(id = 202) + public Map stringValuesByInt8; + + @ForyField(id = 203) + public Map stringValuesByInt16; + + @ForyField(id = 204) + public Map<@Int32Type(encoding = Int32Encoding.FIXED) Integer, String> stringValuesByFixedI32; + + @ForyField(id = 205) + public Map stringValuesByVarintI32; + + @ForyField(id = 206) + public Map<@Int64Type(encoding = Int64Encoding.FIXED) Long, String> stringValuesByFixedI64; + + @ForyField(id = 207) + public Map stringValuesByVarintI64; + + @ForyField(id = 208) + public Map<@Int64Type(encoding = Int64Encoding.TAGGED) Long, String> stringValuesByTaggedI64; + + @ForyField(id = 209) + public Map<@UInt8Type Integer, String> stringValuesByUint8; + + @ForyField(id = 210) + public Map<@UInt16Type Integer, String> stringValuesByUint16; + + @ForyField(id = 211) + public Map<@UInt32Type(encoding = Int32Encoding.FIXED) Long, String> stringValuesByFixedU32; + + @ForyField(id = 212) + public Map<@UInt32Type Long, String> stringValuesByVarintU32; + + @ForyField(id = 213) + public Map<@UInt64Type(encoding = Int64Encoding.FIXED) Long, String> stringValuesByFixedU64; + + @ForyField(id = 214) + public Map<@UInt64Type Long, String> stringValuesByVarintU64; + + @ForyField(id = 215) + public Map<@UInt64Type(encoding = Int64Encoding.TAGGED) Long, String> stringValuesByTaggedU64; + + @ForyField(id = 218) + public Map stringValuesByString; + + @ForyField(id = 219) + public Map stringValuesByTimestamp; + + @ForyField(id = 220) + public Map stringValuesByDuration; + + @ForyField(id = 221) + public Map stringValuesByEnum; + + @ForyField(id = 222) + public Map float16ValuesByName; + + @ForyField(id = 223) + public Map maybeFloat16ValuesByName; + + @ForyField(id = 224) + public Map bfloat16ValuesByName; + + @ForyField(id = 225) + public Map maybeBfloat16ValuesByName; + + @ForyField(id = 226) + public Map bytesValuesByName; + + @ForyField(id = 227) + public Map dateValuesByName; + + @ForyField(id = 228) + public Map decimalValuesByName; + + @ForyField(id = 229) + public Map messageValuesByName; + + @ForyField(id = 231) + public Map uint8ArrayValuesByName; + + @ForyField(id = 232) + public Map float32ArrayValuesByName; + + @ForyField(id = 233) + public Map int32ArrayValuesByName; + + @ForyField(id = 234) + public Map stringValuesByDate; + + @ForyField(id = 236) + public Map boolValuesByName; + + @ForyField(id = 237) + public Map int8ValuesByName; + + @ForyField(id = 238) + public Map int16ValuesByName; + + @ForyField(id = 239) + public Map fixedI32ValuesByName; + + @ForyField(id = 240) + public Map varintI32ValuesByName; + + @ForyField(id = 241) + public Map fixedI64ValuesByName; + + @ForyField(id = 242) + public Map varintI64ValuesByName; + + @ForyField(id = 243) + public Map taggedI64ValuesByName; + + @ForyField(id = 244) + public Map uint8ValuesByName; + + @ForyField(id = 245) + public Map uint16ValuesByName; + + @ForyField(id = 246) + public Map fixedU32ValuesByName; + + @ForyField(id = 247) + public Map varintU32ValuesByName; + + @ForyField(id = 248) + public Map fixedU64ValuesByName; + + @ForyField(id = 249) + public Map varintU64ValuesByName; + + @ForyField(id = 250) + public Map taggedU64ValuesByName; + + @ForyField(id = 251) + public Map float32ValuesByName; + + @ForyField(id = 252) + public Map float64ValuesByName; + + @ForyField(id = 253) + public Map timestampValuesByName; + + @ForyField(id = 254) + public Map durationValuesByName; + + @ForyField(id = 255) + public Map enumValuesByName; + + public ExampleMessage() {} + + @ForyStruct(evolving = Evolution.DISABLED) + public static class Leaf { + @ForyField(id = 1) + public String label; + + @ForyField(id = 2) + public int count; + + public Leaf() {} + } + + public enum State { + UNKNOWN(0), + READY(1), + FAILED(2); + + @org.apache.fory.annotation.ForyEnumId public final int id; + + State(int id) { + this.id = id; + } + } +} diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleRecordMessage.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleRecordMessage.java new file mode 100644 index 0000000000..375e3e8dbc --- /dev/null +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleRecordMessage.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests; + +import java.util.List; +import java.util.Map; +import org.apache.fory.annotation.ForyField; +import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.ForyStruct.Evolution; +import org.apache.fory.annotation.Int32Type; +import org.apache.fory.annotation.Int64Type; +import org.apache.fory.annotation.Int8Type; +import org.apache.fory.annotation.Ref; +import org.apache.fory.annotation.UInt16Type; +import org.apache.fory.annotation.UInt32Type; +import org.apache.fory.annotation.UInt64Type; +import org.apache.fory.annotation.UInt8Type; +import org.apache.fory.collection.BFloat16List; +import org.apache.fory.collection.BoolList; +import org.apache.fory.collection.Float16List; +import org.apache.fory.collection.Float32List; +import org.apache.fory.collection.Float64List; +import org.apache.fory.collection.Int16List; +import org.apache.fory.collection.Int32List; +import org.apache.fory.collection.Int64List; +import org.apache.fory.collection.Int8List; +import org.apache.fory.collection.UInt16List; +import org.apache.fory.collection.UInt32List; +import org.apache.fory.collection.UInt64List; +import org.apache.fory.collection.UInt8List; +import org.apache.fory.config.Int32Encoding; +import org.apache.fory.config.Int64Encoding; +import org.apache.fory.type.BFloat16; +import org.apache.fory.type.BFloat16Array; +import org.apache.fory.type.Float16; +import org.apache.fory.type.Float16Array; + +@ForyStruct +public record ExampleRecordMessage( + @ForyField(id = 1) boolean boolValue, + @ForyField(id = 2) byte int8Value, + @ForyField(id = 3) short int16Value, + @ForyField(id = 4) @Int32Type(encoding = Int32Encoding.FIXED) int fixedI32Value, + @ForyField(id = 5) int varintI32Value, + @ForyField(id = 6) @Int64Type(encoding = Int64Encoding.FIXED) long fixedI64Value, + @ForyField(id = 7) long varintI64Value, + @ForyField(id = 8) @Int64Type(encoding = Int64Encoding.TAGGED) long taggedI64Value, + @ForyField(id = 9) @UInt8Type int uint8Value, + @ForyField(id = 10) @UInt16Type int uint16Value, + @ForyField(id = 11) @UInt32Type(encoding = Int32Encoding.FIXED) long fixedU32Value, + @ForyField(id = 12) @UInt32Type long varintU32Value, + @ForyField(id = 13) @UInt64Type(encoding = Int64Encoding.FIXED) long fixedU64Value, + @ForyField(id = 14) @UInt64Type long varintU64Value, + @ForyField(id = 15) @UInt64Type(encoding = Int64Encoding.TAGGED) long taggedU64Value, + @ForyField(id = 16) Float16 float16Value, + @ForyField(id = 17) BFloat16 bfloat16Value, + @ForyField(id = 18) float float32Value, + @ForyField(id = 19) double float64Value, + @ForyField(id = 20) String stringValue, + @ForyField(id = 21) byte[] bytesValue, + @ForyField(id = 22) java.time.LocalDate dateValue, + @ForyField(id = 23) java.time.Instant timestampValue, + @ForyField(id = 24) java.time.Duration durationValue, + @ForyField(id = 25) java.math.BigDecimal decimalValue, + @ForyField(id = 26) State enumValue, + @ForyField(id = 27, nullable = true) Leaf messageValue, + @ForyField(id = 101) BoolList boolList, + @ForyField(id = 102) Int8List int8List, + @ForyField(id = 103) Int16List int16List, + @ForyField(id = 104) @Int32Type(encoding = Int32Encoding.FIXED) Int32List fixedI32List, + @ForyField(id = 105) Int32List varintI32List, + @ForyField(id = 106) @Int64Type(encoding = Int64Encoding.FIXED) Int64List fixedI64List, + @ForyField(id = 107) Int64List varintI64List, + @ForyField(id = 108) @Int64Type(encoding = Int64Encoding.TAGGED) Int64List taggedI64List, + @ForyField(id = 109) UInt8List uint8List, + @ForyField(id = 110) UInt16List uint16List, + @ForyField(id = 111) @UInt32Type(encoding = Int32Encoding.FIXED) UInt32List fixedU32List, + @ForyField(id = 112) UInt32List varintU32List, + @ForyField(id = 113) @UInt64Type(encoding = Int64Encoding.FIXED) UInt64List fixedU64List, + @ForyField(id = 114) UInt64List varintU64List, + @ForyField(id = 115) @UInt64Type(encoding = Int64Encoding.TAGGED) UInt64List taggedU64List, + @ForyField(id = 116) Float16List float16List, + @ForyField(id = 117) BFloat16List bfloat16List, + @ForyField(id = 118) List maybeFloat16List, + @ForyField(id = 119) List maybeBfloat16List, + @ForyField(id = 120) Float32List float32List, + @ForyField(id = 121) Float64List float64List, + @ForyField(id = 122) List stringList, + @ForyField(id = 123) List bytesList, + @ForyField(id = 124) List dateList, + @ForyField(id = 125) List timestampList, + @ForyField(id = 126) List durationList, + @ForyField(id = 127) List decimalList, + @ForyField(id = 128) List enumList, + @ForyField(id = 129) List<@Ref(enable = false) Leaf> messageList, + @ForyField(id = 131) List<@Int32Type(encoding = Int32Encoding.FIXED) Integer> maybeFixedI32List, + @ForyField(id = 132) List<@UInt64Type Long> maybeUint64List, + @ForyField(id = 301) boolean[] boolArray, + @ForyField(id = 302) @Int8Type byte[] int8Array, + @ForyField(id = 303) short[] int16Array, + @ForyField(id = 304) int[] int32Array, + @ForyField(id = 305) long[] int64Array, + @ForyField(id = 306) @UInt8Type byte[] uint8Array, + @ForyField(id = 307) @UInt16Type short[] uint16Array, + @ForyField(id = 308) @UInt32Type int[] uint32Array, + @ForyField(id = 309) @UInt64Type long[] uint64Array, + @ForyField(id = 310) Float16Array float16Array, + @ForyField(id = 311) BFloat16Array bfloat16Array, + @ForyField(id = 312) float[] float32Array, + @ForyField(id = 313) double[] float64Array, + @ForyField(id = 314) List int32ArrayList, + @ForyField(id = 315) List<@UInt8Type byte[]> uint8ArrayList, + @ForyField(id = 201) Map stringValuesByBool, + @ForyField(id = 202) Map stringValuesByInt8, + @ForyField(id = 203) Map stringValuesByInt16, + @ForyField(id = 204) + Map<@Int32Type(encoding = Int32Encoding.FIXED) Integer, String> stringValuesByFixedI32, + @ForyField(id = 205) Map stringValuesByVarintI32, + @ForyField(id = 206) + Map<@Int64Type(encoding = Int64Encoding.FIXED) Long, String> stringValuesByFixedI64, + @ForyField(id = 207) Map stringValuesByVarintI64, + @ForyField(id = 208) + Map<@Int64Type(encoding = Int64Encoding.TAGGED) Long, String> stringValuesByTaggedI64, + @ForyField(id = 209) Map<@UInt8Type Integer, String> stringValuesByUint8, + @ForyField(id = 210) Map<@UInt16Type Integer, String> stringValuesByUint16, + @ForyField(id = 211) + Map<@UInt32Type(encoding = Int32Encoding.FIXED) Long, String> stringValuesByFixedU32, + @ForyField(id = 212) Map<@UInt32Type Long, String> stringValuesByVarintU32, + @ForyField(id = 213) + Map<@UInt64Type(encoding = Int64Encoding.FIXED) Long, String> stringValuesByFixedU64, + @ForyField(id = 214) Map<@UInt64Type Long, String> stringValuesByVarintU64, + @ForyField(id = 215) + Map<@UInt64Type(encoding = Int64Encoding.TAGGED) Long, String> stringValuesByTaggedU64, + @ForyField(id = 218) Map stringValuesByString, + @ForyField(id = 219) Map stringValuesByTimestamp, + @ForyField(id = 220) Map stringValuesByDuration, + @ForyField(id = 221) Map stringValuesByEnum, + @ForyField(id = 222) Map float16ValuesByName, + @ForyField(id = 223) Map maybeFloat16ValuesByName, + @ForyField(id = 224) Map bfloat16ValuesByName, + @ForyField(id = 225) Map maybeBfloat16ValuesByName, + @ForyField(id = 226) Map bytesValuesByName, + @ForyField(id = 227) Map dateValuesByName, + @ForyField(id = 228) Map decimalValuesByName, + @ForyField(id = 229) Map messageValuesByName, + @ForyField(id = 231) Map uint8ArrayValuesByName, + @ForyField(id = 232) Map float32ArrayValuesByName, + @ForyField(id = 233) Map int32ArrayValuesByName, + @ForyField(id = 234) Map stringValuesByDate, + @ForyField(id = 236) Map boolValuesByName, + @ForyField(id = 237) Map int8ValuesByName, + @ForyField(id = 238) Map int16ValuesByName, + @ForyField(id = 239) + Map fixedI32ValuesByName, + @ForyField(id = 240) Map varintI32ValuesByName, + @ForyField(id = 241) + Map fixedI64ValuesByName, + @ForyField(id = 242) Map varintI64ValuesByName, + @ForyField(id = 243) + Map taggedI64ValuesByName, + @ForyField(id = 244) Map uint8ValuesByName, + @ForyField(id = 245) Map uint16ValuesByName, + @ForyField(id = 246) + Map fixedU32ValuesByName, + @ForyField(id = 247) Map varintU32ValuesByName, + @ForyField(id = 248) + Map fixedU64ValuesByName, + @ForyField(id = 249) Map varintU64ValuesByName, + @ForyField(id = 250) + Map taggedU64ValuesByName, + @ForyField(id = 251) Map float32ValuesByName, + @ForyField(id = 252) Map float64ValuesByName, + @ForyField(id = 253) Map timestampValuesByName, + @ForyField(id = 254) Map durationValuesByName, + @ForyField(id = 255) Map enumValuesByName) { + @ForyStruct(evolving = Evolution.DISABLED) + public record Leaf(@ForyField(id = 1) String label, @ForyField(id = 2) int count) {} + + public enum State { + UNKNOWN(0), + READY(1), + FAILED(2); + + @org.apache.fory.annotation.ForyEnumId public final int id; + + State(int id) { + this.id = id; + } + } +} diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java new file mode 100644 index 0000000000..0c897495ea --- /dev/null +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java @@ -0,0 +1,490 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.atLeastOnce; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.RecordComponent; +import java.lang.reflect.Type; +import java.math.BigDecimal; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.fory.Fory; +import org.apache.fory.ThreadSafeFory; +import org.apache.fory.annotation.ForyField; +import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.Int32Type; +import org.apache.fory.builder.CodecUtils; +import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.collection.BFloat16List; +import org.apache.fory.collection.BoolList; +import org.apache.fory.collection.Float16List; +import org.apache.fory.collection.Float32List; +import org.apache.fory.collection.Float64List; +import org.apache.fory.collection.Int16List; +import org.apache.fory.collection.Int32List; +import org.apache.fory.collection.Int64List; +import org.apache.fory.collection.Int8List; +import org.apache.fory.collection.UInt16List; +import org.apache.fory.collection.UInt32List; +import org.apache.fory.collection.UInt64List; +import org.apache.fory.collection.UInt8List; +import org.apache.fory.config.Int32Encoding; +import org.apache.fory.context.MetaReadContext; +import org.apache.fory.context.MetaWriteContext; +import org.apache.fory.meta.TypeDef; +import org.apache.fory.serializer.DeferedLazySerializer; +import org.apache.fory.serializer.Serializer; +import org.apache.fory.serializer.StaticGeneratedStructSerializer; +import org.apache.fory.type.BFloat16; +import org.apache.fory.type.BFloat16Array; +import org.apache.fory.type.Float16; +import org.apache.fory.type.Float16Array; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class ExampleStaticGeneratedSerializerTest { + private static final AtomicInteger FORY_ID = new AtomicInteger(); + + @DataProvider + public static Object[][] modes() { + return new Object[][] {{false, false}, {true, false}, {false, true}, {true, true}}; + } + + @DataProvider + public static Object[][] xlangModes() { + return new Object[][] {{false}, {true}}; + } + + @Test(dataProvider = "modes") + public void testStaticClass(boolean xlang, boolean compatible) throws Exception { + ExampleMessage value = newStruct(ExampleMessage.class); + ThreadSafeFory fory = threadSafeFory(ExampleMessage.class, xlang, compatible, false); + assertStaticSerializer(fory, ExampleMessage.class); + + ExampleMessage roundTrip = deserialize(fory, serialize(fory, value), ExampleMessage.class); + assertStructEquals(value, roundTrip, ExampleMessage.class); + } + + @Test(dataProvider = "modes") + public void testStaticRecord(boolean xlang, boolean compatible) throws Exception { + ExampleRecordMessage value = newRecord(ExampleRecordMessage.class); + ThreadSafeFory fory = threadSafeFory(ExampleRecordMessage.class, xlang, compatible, false); + assertStaticSerializer(fory, ExampleRecordMessage.class); + + ExampleRecordMessage roundTrip = + deserialize(fory, serialize(fory, value), ExampleRecordMessage.class); + assertRecordEquals(value, roundTrip); + } + + @Test(dataProvider = "xlangModes") + public void testCompatibleClassEvolution(boolean xlang) throws Exception { + ExampleMessage value = newStruct(ExampleMessage.class); + ThreadSafeFory writer = threadSafeFory(ExampleMessage.class, xlang, true, false); + byte[] bytes = serialize(writer, value); + + ThreadSafeFory emptyReader = threadSafeFory(EmptyMessage.class, xlang, true, false); + Assert.assertNotNull(deserialize(emptyReader, bytes, EmptyMessage.class)); + + ThreadSafeFory reader = threadSafeFory(InconsistentMessage.class, xlang, true, false); + InconsistentMessage result = deserialize(reader, bytes, InconsistentMessage.class); + Assert.assertEquals(result.fixedI32Value, value.fixedI32Value); + Assert.assertEquals(result.stringValue, value.stringValue); + Assert.assertEquals(result.added, "default"); + } + + @Test(dataProvider = "xlangModes") + public void testCompatibleRecordEvolution(boolean xlang) throws Exception { + ExampleRecordMessage value = newRecord(ExampleRecordMessage.class); + ThreadSafeFory writer = threadSafeFory(ExampleRecordMessage.class, xlang, true, false); + byte[] bytes = serialize(writer, value); + + ThreadSafeFory emptyReader = threadSafeFory(EmptyRecordMessage.class, xlang, true, false); + Assert.assertNotNull(deserialize(emptyReader, bytes, EmptyRecordMessage.class)); + + ThreadSafeFory reader = threadSafeFory(InconsistentRecordMessage.class, xlang, true, false); + InconsistentRecordMessage result = deserialize(reader, bytes, InconsistentRecordMessage.class); + Assert.assertEquals(result.fixedI32Value(), value.fixedI32Value()); + Assert.assertEquals(result.stringValue(), value.stringValue()); + Assert.assertNull(result.added()); + } + + @Test(dataProvider = "xlangModes") + public void testStaticCompatibleBuilder(boolean xlang) throws Exception { + ExampleMessage value = newStruct(ExampleMessage.class); + Fory writer = fory(ExampleMessage.class, xlang, true, true); + Fory reader = fory(InconsistentMessage.class, xlang, true, true); + TypeDef remoteTypeDef = TypeDef.buildTypeDef(writer.getTypeResolver(), ExampleMessage.class); + Assert.assertNotEquals( + remoteTypeDef.getId(), + TypeDef.buildTypeDef(reader.getTypeResolver(), InconsistentMessage.class).getId()); + + Class compatibleSerializerClass = + CodecUtils.loadOrGenStaticCompatibleCodecClass( + reader.getTypeResolver(), InconsistentMessage.class, remoteTypeDef); + Assert.assertTrue( + GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(compatibleSerializerClass)); + writer.setMetaWriteContext(new MetaWriteContext()); + byte[] bytes = writer.serialize(value); + + try (MockedStatic codecUtils = + Mockito.mockStatic(CodecUtils.class, Mockito.CALLS_REAL_METHODS)) { + codecUtils + .when( + () -> + CodecUtils.loadOrGenMetaSharedCodecClass( + same(reader.getTypeResolver()), + eq(InconsistentMessage.class), + any(TypeDef.class))) + .thenReturn(compatibleSerializerClass); + reader.setMetaReadContext(new MetaReadContext()); + InconsistentMessage result = (InconsistentMessage) reader.deserialize(bytes); + codecUtils.verify( + () -> + CodecUtils.loadOrGenMetaSharedCodecClass( + same(reader.getTypeResolver()), + eq(InconsistentMessage.class), + any(TypeDef.class)), + atLeastOnce()); + Assert.assertEquals(result.fixedI32Value, value.fixedI32Value); + Assert.assertEquals(result.stringValue, value.stringValue); + Assert.assertEquals(result.added, "default"); + } + } + + @ForyStruct + public static class EmptyMessage { + public EmptyMessage() {} + } + + @ForyStruct + public static class InconsistentMessage { + @ForyField(id = 4) + public @Int32Type(encoding = Int32Encoding.FIXED) int fixedI32Value; + + @ForyField(id = 20) + public String stringValue; + + @ForyField(id = 900, nullable = true) + public String added = "default"; + + public InconsistentMessage() {} + } + + @ForyStruct + public record EmptyRecordMessage() {} + + @ForyStruct + public record InconsistentRecordMessage( + @ForyField(id = 4) @Int32Type(encoding = Int32Encoding.FIXED) int fixedI32Value, + @ForyField(id = 20) String stringValue, + @ForyField(id = 900, nullable = true) String added) {} + + private static ThreadSafeFory threadSafeFory( + Class type, boolean xlang, boolean compatible, boolean codegen) { + ThreadSafeFory fory = + Fory.builder() + .withName("latest-static-" + FORY_ID.incrementAndGet()) + .withXlang(xlang) + .withCodegen(codegen) + .withMetaShare(compatible) + .withScopedMetaShare(false) + .withCompatible(compatible) + .requireClassRegistration(false) + .buildThreadSafeForyPool(1); + registerType(fory, type, xlang); + return fory; + } + + private static Fory fory(Class type, boolean xlang, boolean compatible, boolean codegen) { + Fory fory = + Fory.builder() + .withName("latest-static-" + FORY_ID.incrementAndGet()) + .withXlang(xlang) + .withCodegen(codegen) + .withMetaShare(compatible) + .withScopedMetaShare(false) + .withCompatible(compatible) + .requireClassRegistration(false) + .build(); + registerType(fory, type, xlang); + return fory; + } + + private static void registerType(org.apache.fory.BaseFory fory, Class type, boolean xlang) { + if (type == ExampleRecordMessage.class + || type == EmptyRecordMessage.class + || type == InconsistentRecordMessage.class) { + register(fory, type, xlang, 1510, "ExampleRecordMessage"); + register(fory, ExampleRecordMessage.Leaf.class, xlang, 1512, "Leaf"); + register(fory, ExampleRecordMessage.State.class, xlang, 1513, "State"); + } else { + register(fory, type, xlang, 1500, "ExampleMessage"); + register(fory, ExampleMessage.Leaf.class, xlang, 1502, "Leaf"); + register(fory, ExampleMessage.State.class, xlang, 1503, "State"); + } + } + + private static void register( + org.apache.fory.BaseFory fory, Class type, boolean xlang, int nativeId, String typeName) { + if (xlang) { + fory.register(type, "example", typeName); + } else { + fory.register(type, nativeId); + } + } + + private static void assertStaticSerializer(ThreadSafeFory fory, Class type) { + Serializer serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); + if (serializer instanceof DeferedLazySerializer) { + serializer = ((DeferedLazySerializer) serializer).resolveSerializer(); + } + Assert.assertTrue( + serializer instanceof StaticGeneratedStructSerializer, serializer.getClass().getName()); + } + + private static byte[] serialize(ThreadSafeFory fory, Object value) { + return fory.execute( + f -> { + f.setMetaWriteContext(new MetaWriteContext()); + return f.serialize(value); + }); + } + + private static T deserialize(ThreadSafeFory fory, byte[] bytes, Class type) { + return fory.execute( + f -> { + f.setMetaReadContext(new MetaReadContext()); + return f.deserialize(bytes, type); + }); + } + + private static T newStruct(Class type) throws Exception { + T value = type.getConstructor().newInstance(); + for (Field field : type.getFields()) { + if (!java.lang.reflect.Modifier.isStatic(field.getModifiers())) { + field.set(value, valueFor(field.getGenericType(), field.getType(), field.getName())); + } + } + return value; + } + + private static T newRecord(Class type) throws Exception { + RecordComponent[] components = type.getRecordComponents(); + Object[] args = new Object[components.length]; + Class[] parameterTypes = new Class[components.length]; + for (int i = 0; i < components.length; i++) { + args[i] = + valueFor( + components[i].getGenericType(), components[i].getType(), components[i].getName()); + parameterTypes[i] = components[i].getType(); + } + return type.getDeclaredConstructor(parameterTypes).newInstance(args); + } + + private static Object valueFor(Type genericType, Class type, String name) throws Exception { + if (type == boolean.class || type == Boolean.class) { + return true; + } else if (type == byte.class || type == Byte.class) { + return (byte) 7; + } else if (type == short.class || type == Short.class) { + return (short) 8; + } else if (type == int.class || type == Integer.class) { + return 42; + } else if (type == long.class || type == Long.class) { + return 43L; + } else if (type == float.class || type == Float.class) { + return 44.5f; + } else if (type == double.class || type == Double.class) { + return 45.5d; + } else if (type == String.class) { + return name + "-value"; + } else if (type == byte[].class) { + return new byte[] {1, 2, 3}; + } else if (type == boolean[].class) { + return new boolean[] {true, false}; + } else if (type == short[].class) { + return new short[] {1, 2}; + } else if (type == int[].class) { + return new int[] {1, 2}; + } else if (type == long[].class) { + return new long[] {1L, 2L}; + } else if (type == float[].class) { + return new float[] {1.5f, 2.5f}; + } else if (type == double[].class) { + return new double[] {1.5d, 2.5d}; + } else if (type == LocalDate.class) { + return LocalDate.of(2026, 5, 12); + } else if (type == Instant.class) { + return Instant.parse("2026-05-12T00:00:00Z"); + } else if (type == Duration.class) { + return Duration.ofSeconds(123); + } else if (type == BigDecimal.class) { + return new BigDecimal("123.45"); + } else if (type == Float16.class) { + return Float16.valueOf(1.5f); + } else if (type == BFloat16.class) { + return BFloat16.valueOf(2.5f); + } else if (type == Float16Array.class) { + return Float16Array.of(1.5f, 2.5f); + } else if (type == BFloat16Array.class) { + return BFloat16Array.of(1.5f, 2.5f); + } else if (type == BoolList.class) { + return new BoolList(new boolean[] {true, false}); + } else if (type == Int8List.class) { + return new Int8List(new byte[] {1, 2}); + } else if (type == Int16List.class) { + return new Int16List(new short[] {1, 2}); + } else if (type == Int32List.class) { + return new Int32List(new int[] {1, 2}); + } else if (type == Int64List.class) { + return new Int64List(new long[] {1L, 2L}); + } else if (type == UInt8List.class) { + return new UInt8List(new byte[] {1, 2}); + } else if (type == UInt16List.class) { + return new UInt16List(new short[] {1, 2}); + } else if (type == UInt32List.class) { + return new UInt32List(new int[] {1, 2}); + } else if (type == UInt64List.class) { + return new UInt64List(new long[] {1L, 2L}); + } else if (type == Float16List.class) { + return new Float16List(new short[] {Float16.toBits(1.5f), Float16.toBits(2.5f)}); + } else if (type == BFloat16List.class) { + return new BFloat16List(new short[] {BFloat16.toBits(1.5f), BFloat16.toBits(2.5f)}); + } else if (type == Float32List.class) { + return new Float32List(new float[] {1.5f, 2.5f}); + } else if (type == Float64List.class) { + return new Float64List(new double[] {1.5d, 2.5d}); + } else if (type == ExampleMessage.Leaf.class || type == ExampleRecordMessage.Leaf.class) { + return type.isRecord() ? newRecord(type) : newStruct(type); + } else if (type.isEnum()) { + return type.getEnumConstants()[1]; + } else if (List.class.isAssignableFrom(type)) { + ArrayList list = new ArrayList<>(); + list.add(valueFor(typeArgument(genericType, 0), typeArgumentClass(genericType, 0), name)); + return list; + } else if (Map.class.isAssignableFrom(type)) { + LinkedHashMap map = new LinkedHashMap<>(); + map.put( + valueFor(typeArgument(genericType, 0), typeArgumentClass(genericType, 0), name + "Key"), + valueFor( + typeArgument(genericType, 1), typeArgumentClass(genericType, 1), name + "Value")); + return map; + } + throw new IllegalArgumentException(type.getName()); + } + + private static Type typeArgument(Type type, int index) { + return ((ParameterizedType) type).getActualTypeArguments()[index]; + } + + private static Class typeArgumentClass(Type type, int index) { + Type argument = typeArgument(type, index); + if (argument instanceof Class) { + return (Class) argument; + } + if (argument instanceof ParameterizedType) { + return (Class) ((ParameterizedType) argument).getRawType(); + } + return Object.class; + } + + private static void assertStructEquals(Object expected, Object actual, Class type) + throws Exception { + Assert.assertSame(actual.getClass(), type); + for (Field field : type.getFields()) { + if (!java.lang.reflect.Modifier.isStatic(field.getModifiers())) { + assertDeepEquals(field.get(expected), field.get(actual), field.getName()); + } + } + } + + private static void assertRecordEquals(Object expected, Object actual) throws Exception { + Assert.assertSame(actual.getClass(), expected.getClass()); + for (RecordComponent component : expected.getClass().getRecordComponents()) { + assertDeepEquals( + component.getAccessor().invoke(expected), + component.getAccessor().invoke(actual), + component.getName()); + } + } + + private static void assertDeepEquals(Object expected, Object actual, String name) { + if (expected != null && expected.getClass().isArray()) { + Assert.assertTrue(actual != null && actual.getClass().isArray(), name); + Assert.assertEquals(Array.getLength(actual), Array.getLength(expected), name); + for (int i = 0; i < Array.getLength(expected); i++) { + assertDeepEquals(Array.get(expected, i), Array.get(actual, i), name + "[" + i + "]"); + } + } else if (expected instanceof List) { + List expectedList = (List) expected; + List actualList = (List) actual; + Assert.assertEquals(actualList.size(), expectedList.size(), name); + for (int i = 0; i < expectedList.size(); i++) { + assertDeepEquals(expectedList.get(i), actualList.get(i), name + "[" + i + "]"); + } + } else if (expected instanceof Map) { + Map expectedMap = (Map) expected; + Map actualMap = (Map) actual; + Assert.assertEquals(actualMap.size(), expectedMap.size(), name); + for (Map.Entry entry : expectedMap.entrySet()) { + Assert.assertTrue(actualMap.containsKey(entry.getKey()), name); + assertDeepEquals(entry.getValue(), actualMap.get(entry.getKey()), name); + } + } else if (isExampleStruct(expected)) { + try { + assertStructEquals(expected, actual, expected.getClass()); + } catch (Exception e) { + throw new AssertionError(name, e); + } + } else if (isExampleRecord(expected)) { + try { + assertRecordEquals(expected, actual); + } catch (Exception e) { + throw new AssertionError(name, e); + } + } else { + Assert.assertEquals(actual, expected, name); + } + } + + private static boolean isExampleStruct(Object value) { + return value instanceof ExampleMessage.Leaf; + } + + private static boolean isExampleRecord(Object value) { + return value instanceof ExampleRecordMessage.Leaf; + } +} diff --git a/java/fory-latest-jdk-tests/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/java/fory-latest-jdk-tests/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..1f0955d450 --- /dev/null +++ b/java/fory-latest-jdk-tests/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline From 064018fbd00743e77086cd3fe115041f9a272e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Tue, 12 May 2026 23:29:43 +0800 Subject: [PATCH 21/58] fix more type meta err --- .../fory/builder/BaseObjectCodecBuilder.java | 8 +-- .../java/org/apache/fory/meta/FieldTypes.java | 30 ++++++---- .../java/org/apache/fory/meta/TypeDef.java | 10 ++-- .../apache/fory/resolver/ClassResolver.java | 11 ++++ .../apache/fory/serializer/FieldGroups.java | 25 ++------ .../StaticGeneratedStructSerializer.java | 59 +++++++++++-------- .../apache/fory/type/TypeAnnotationUtils.java | 20 +++++++ .../java/org/apache/fory/type/TypeUtils.java | 2 +- .../main/java/org/apache/fory/type/Types.java | 14 ++--- 9 files changed, 104 insertions(+), 75 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index b70a2bbf8d..639b23907a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -515,9 +515,7 @@ private Expression serializeForNotNullForField( TypeRef elementType = null; if (usesPrimitiveListCollectionProtocol(descriptor)) { serializer = getPrimitiveListCollectionSerializer(clz); - elementType = - TypeAnnotationUtils.getPrimitiveListElementTypeRef( - descriptor.getTypeAnnotation(), clz); + elementType = TypeAnnotationUtils.getPrimitiveListElementTypeRef(descriptor); } action = serializeForCollection(buffer, inputObject, typeRef, serializer, false, elementType); @@ -2268,9 +2266,7 @@ private Expression deserializeForNotNullForField( TypeRef elementType = null; if (usesPrimitiveListCollectionProtocol(descriptor)) { serializer = getPrimitiveListCollectionSerializer(cls); - elementType = - TypeAnnotationUtils.getPrimitiveListElementTypeRef( - descriptor.getTypeAnnotation(), cls); + elementType = TypeAnnotationUtils.getPrimitiveListElementTypeRef(descriptor); } obj = deserializeForCollection(buffer, typeRef, serializer, null, elementType); } else if (useMapSerialization(typeRef)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index c967f99b24..2e1d8a43c7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -129,18 +129,19 @@ private static FieldType buildFieldType( ? primitiveListElementMeta != null ? primitiveListElementMeta.typeId() : TypeAnnotationUtils.getPrimitiveListElementTypeId( - typeAnnotation, rawType, isXlang) + typeAnnotation, rawType, true) : Types.UNKNOWN; - if (typeExtMeta != null && typeExtMeta.typeId() != Types.UNKNOWN) { + // Primitive-list TypeExtMeta describes the list element wire type. Only @ArrayType changes the + // top-level schema to a dense primitive array. + if (primitiveListArray) { + typeId = TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType); + } else if (primitiveList) { + typeId = Types.LIST; + } else if (!primitiveList && typeExtMeta != null && typeExtMeta.typeId() != Types.UNKNOWN) { typeId = typeExtMeta.typeId(); - } else if (isXlang && primitiveList) { - typeId = - primitiveListArray - ? TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType) - : Types.LIST; } else if (boxedListArray) { typeId = TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor); - } else if (primitiveListElementTypeId != Types.UNKNOWN) { + } else if (isXlang && primitiveListElementTypeId != Types.UNKNOWN) { typeId = TypeAnnotationUtils.getPrimitiveListTypeId(typeAnnotation, rawType); } else if (TypeUtils.unwrap(rawType).isPrimitive()) { if (field != null) { @@ -238,7 +239,7 @@ private static FieldType buildFieldType( return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } - if (isXlang && primitiveList && !primitiveListArray) { + if (primitiveList && !primitiveListArray) { boolean elementNullable = true; boolean elementTrackingRef = false; if (primitiveListArgumentMeta != null) { @@ -451,13 +452,16 @@ public void write(MemoryBuffer buffer, boolean writeHeader) { buffer.writeVarUInt32Small7(arrayFieldType.getDimensions()); (arrayFieldType).getComponentType().write(buffer); } else if (this instanceof CollectionFieldType) { + CollectionFieldType collectionFieldType = (CollectionFieldType) this; buffer.writeUInt8(kindHeader); + buffer.writeVarUInt32Small7(collectionFieldType.typeId); // TODO remove it when new collection deserialization jit finished. - ((CollectionFieldType) this).getElementType().write(buffer); + collectionFieldType.getElementType().write(buffer); } else if (this instanceof MapFieldType) { buffer.writeUInt8(kindHeader); // TODO remove it when new map deserialization jit finished. MapFieldType mapFieldType = (MapFieldType) this; + buffer.writeVarUInt32Small7(mapFieldType.typeId); mapFieldType.getKeyType().write(buffer); mapFieldType.getValueType().write(buffer); } else { @@ -492,10 +496,12 @@ public static FieldType read( if (kind == 0) { return new ObjectFieldType(Types.UNKNOWN, nullable, trackingRef); } else if (kind == 1) { + int typeId = buffer.readVarUInt32Small7(); return new MapFieldType( - -1, nullable, trackingRef, read(buffer, resolver), read(buffer, resolver)); + typeId, nullable, trackingRef, read(buffer, resolver), read(buffer, resolver)); } else if (kind == 2) { - return new CollectionFieldType(-1, nullable, trackingRef, read(buffer, resolver)); + int typeId = buffer.readVarUInt32Small7(); + return new CollectionFieldType(typeId, nullable, trackingRef, read(buffer, resolver)); } else if (kind == 3) { int dims = buffer.readVarUInt32Small7(); return new ArrayFieldType(-1, nullable, trackingRef, read(buffer, resolver), dims); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java index 90caf3aef9..04a36ebfe6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java @@ -459,11 +459,9 @@ public TypeDef replaceRootClassTo(TypeResolver resolver, Class targetCls) { } }) .collect(Collectors.toList()); - if (resolver.isCrossLanguage()) { - return TypeDefEncoder.buildTypeDefWithFieldInfos( - (XtypeResolver) resolver, targetCls, fieldInfos); - } - return NativeTypeDefEncoder.buildTypeDefWithFieldInfos( - (ClassResolver) resolver, targetCls, fieldInfos); + // Keep the remote class spec/id/encoded metadata. Compatible readers still need the remote + // schema identity to rebuild descriptor order; only root field ownership is rewritten so local + // field matching by declaring class keeps the existing target-class semantics. + return new TypeDef(classSpec, fieldInfos, id, encoded); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 272a57dfa8..3a53d8fb4a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1029,6 +1029,17 @@ public boolean isBuildIn(Descriptor descriptor) { return isMonomorphic(descriptor); } + @Override + public boolean isCollectionDescriptor(Descriptor descriptor) { + Class rawType = descriptor.getRawType(); + if (TypeUtils.isPrimitiveListClass(rawType)) { + return !org.apache.fory.type.TypeAnnotationUtils.isArrayType(descriptor) + && org.apache.fory.type.TypeAnnotationUtils.usesCollectionProtocolForPrimitiveList( + descriptor.getTypeAnnotation(), rawType); + } + return super.isCollectionDescriptor(descriptor); + } + public boolean isInternalRegistered(int classId) { if (Types.isUserDefinedType((byte) classId)) { return false; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index be201be9ad..a1c14409b8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -217,6 +217,7 @@ public static final class SerializationFieldInfo { // nullability/ref mode. Treating it as field metadata writes an extra null marker that // remote TypeDef payload readers then consume as list length. TypeExtMeta extMeta = typeRef.getTypeExtMeta(); + int schemaTypeId = extMeta == null ? Types.UNKNOWN : extMeta.typeId(); if (extMeta != null && !primitiveListArray && !primitiveListCollection) { nullable = extMeta.nullable(); trackingRef = extMeta.trackingRef(); @@ -265,6 +266,10 @@ public static final class SerializationFieldInfo { resolver, TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor.getField()), qualifiedFieldName); + } else if (!resolver.isCrossLanguage() && schemaTypeId == Types.LIST) { + containerSerializerOverride = + new org.apache.fory.serializer.collection.CollectionSerializers.ArrayListSerializer( + resolver); } else { containerSerializerOverride = null; } @@ -313,24 +318,6 @@ public String toString() { } private static TypeRef primitiveListElementTypeRef(Descriptor descriptor) { - TypeRef typeRef = descriptor.getTypeRef(); - TypeExtMeta inlineMeta = typeRef.getTypeExtMeta(); - if (inlineMeta != null && Types.isPrimitiveType(inlineMeta.typeId())) { - Class elementClass = - TypeAnnotationUtils.getPrimitiveListElementClass(typeRef.getRawType()); - if (elementClass != null) { - return TypeRef.of( - elementClass, TypeExtMeta.of(inlineMeta.typeId(), true, inlineMeta.trackingRef())); - } - } - if (typeRef.hasExplicitTypeArguments()) { - TypeRef elementTypeRef = TypeUtils.getElementType(typeRef); - TypeExtMeta elementMeta = elementTypeRef.getTypeExtMeta(); - if (elementMeta != null && Types.isPrimitiveType(elementMeta.typeId())) { - return elementTypeRef; - } - } - return TypeAnnotationUtils.getPrimitiveListElementTypeRef( - descriptor.getTypeAnnotation(), typeRef.getRawType()); + return TypeAnnotationUtils.getPrimitiveListElementTypeRef(descriptor); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 1d1cdcce6b..6adce9c1e7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -206,7 +206,30 @@ protected final void skipField( SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { try { + int startIndex = buffer.readerIndex(); + System.err.println( + "STATIC_SKIP_START name=" + + fieldInfo.descriptor.getName() + + " index=" + + startIndex + + " next=" + + (buffer.getByte(startIndex) & 0xFF) + + " ref=" + + fieldInfo.refMode + + " typeRef=" + + fieldInfo.typeRef + + " schemaMeta=" + + fieldInfo.typeRef.getTypeExtMeta() + + " override=" + + fieldInfo.containerSerializerOverride); FieldSkipper.skipField(readContext, typeResolver, refReader, fieldInfo, buffer); + System.err.println( + "STATIC_SKIP_END name=" + + fieldInfo.descriptor.getName() + + " start=" + + startIndex + + " end=" + + buffer.readerIndex()); } catch (RuntimeException e) { throw new DeserializationException( "Failed to skip remote field " + fieldInfo.descriptor.getName(), e); @@ -306,30 +329,17 @@ private List buildRemoteFields( } fields.put(fieldKey(descriptor), i); } - // Keep compatible-read descriptor ordering owned by TypeResolver, matching MetaSharedSerializer - // and MetaSharedCodecBuilder. TypeDef itself is metadata; it is not the read-order owner. - FieldGroups remoteFieldGroups = - FieldGroups.buildFieldInfos( - typeResolver, - typeResolver.createDescriptorGrouper(remoteTypeDef, remoteDescriptorClass)); + // Keep compatible-read descriptor ordering owned by TypeResolver, matching the sorted + // DescriptorGrouper order used by ObjectCodecBuilder and MetaSharedCodecBuilder. FieldGroups + // may regroup descriptors for helper ownership, so it must not drive remote payload order. + List remoteDescriptorsInWireOrder = + typeResolver + .createDescriptorGrouper(remoteTypeDef, remoteDescriptorClass) + .getSortedDescriptors(); List remoteFields = new ArrayList<>(remoteFieldInfos.size()); appendRemoteFields( remoteFields, - remoteFieldGroups.buildInFields, - remoteFieldInfosByKey, - fieldIds, - fields, - localDescriptors); - appendRemoteFields( - remoteFields, - remoteFieldGroups.containerFields, - remoteFieldInfosByKey, - fieldIds, - fields, - localDescriptors); - appendRemoteFields( - remoteFields, - remoteFieldGroups.userTypeFields, + remoteDescriptorsInWireOrder, remoteFieldInfosByKey, fieldIds, fields, @@ -356,13 +366,14 @@ private Class remoteDescriptorClass(TypeDef remoteTypeDef) { private void appendRemoteFields( List remoteFields, - SerializationFieldInfo[] serializationFieldInfos, + List remoteDescriptorsInWireOrder, Map remoteFieldInfosByKey, Map fieldIds, Map fields, List localDescriptors) { - for (SerializationFieldInfo serializationFieldInfo : serializationFieldInfos) { - Descriptor descriptor = serializationFieldInfo.descriptor; + for (Descriptor descriptor : remoteDescriptorsInWireOrder) { + SerializationFieldInfo serializationFieldInfo = + FieldGroups.buildFieldInfo(typeResolver, descriptor); FieldInfo fieldInfo = remoteFieldInfosByKey.get(remoteFieldKey(descriptor)); if (fieldInfo == null) { throw new IllegalStateException("Missing remote field metadata for " + descriptor); diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java index 79ed581942..397d19f4d9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java @@ -547,6 +547,26 @@ public static TypeRef getPrimitiveListElementTypeRef( return TypeRef.of(elementClass, TypeExtMeta.of(elementTypeId, true, false)); } + public static TypeRef getPrimitiveListElementTypeRef(Descriptor descriptor) { + TypeRef typeRef = descriptor.getTypeRef(); + TypeExtMeta inlineMeta = typeRef.getTypeExtMeta(); + if (inlineMeta != null && Types.isPrimitiveType(inlineMeta.typeId())) { + Class elementClass = getPrimitiveListElementClass(typeRef.getRawType()); + if (elementClass != null) { + return TypeRef.of( + elementClass, TypeExtMeta.of(inlineMeta.typeId(), true, inlineMeta.trackingRef())); + } + } + if (typeRef.hasExplicitTypeArguments()) { + TypeRef elementTypeRef = TypeUtils.getElementType(typeRef); + TypeExtMeta elementMeta = elementTypeRef.getTypeExtMeta(); + if (elementMeta != null && Types.isPrimitiveType(elementMeta.typeId())) { + return elementTypeRef; + } + } + return getPrimitiveListElementTypeRef(descriptor.getTypeAnnotation(), typeRef.getRawType()); + } + public static boolean isArrayType(Descriptor descriptor) { if (descriptor.isArrayType()) { return true; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index 20b7f414fa..45abe10073 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -546,7 +546,7 @@ public static TypeRef getElementType(TypeRef typeRef) { List> typeArguments = typeRef.getTypeArguments(); if (typeArguments.size() == 1) { Class rawType = getRawType(typeRef); - if (Iterable.class.isAssignableFrom(rawType) && rawType.getTypeParameters().length == 1) { + if (Iterable.class.isAssignableFrom(rawType)) { return typeArguments.get(0); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Types.java b/java/fory-core/src/main/java/org/apache/fory/type/Types.java index f669603eb9..23910a439c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Types.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Types.java @@ -414,11 +414,17 @@ public static int getDescriptorTypeId(TypeResolver resolver, Descriptor d) { return TypeAnnotationUtils.getBoxedListArrayTypeId(d); } TypeRef typeRef = d.getTypeRef(); + Class rawType = typeRef.getRawType(); + if (TypeUtils.isPrimitiveListClass(rawType)) { + if (TypeAnnotationUtils.isArrayType(d)) { + return TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType); + } + return TypeAnnotationUtils.getPrimitiveListTypeId(d.getTypeAnnotation(), rawType); + } TypeExtMeta extMeta = typeRef.getTypeExtMeta(); if (extMeta != null) { return extMeta.typeId(); } else { - Class rawType = typeRef.getRawType(); TypeRef componentType = typeRef.getComponentType(); if (rawType.isArray() && componentType != null) { TypeExtMeta componentMeta = componentType.getTypeExtMeta(); @@ -431,12 +437,6 @@ public static int getDescriptorTypeId(TypeResolver resolver, Descriptor d) { } } Annotation typeAnnotation = d.getTypeAnnotation(); - if (TypeUtils.isPrimitiveListClass(rawType)) { - if (TypeAnnotationUtils.isArrayType(d)) { - return TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType); - } - return TypeAnnotationUtils.getPrimitiveListTypeId(typeAnnotation, rawType); - } if (typeAnnotation != null) { int primitiveListTypeId = TypeAnnotationUtils.getPrimitiveListTypeId(typeAnnotation, rawType); From 4f509cca8b7e866c7f22e8cd9a796a364b107bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 00:11:09 +0800 Subject: [PATCH 22/58] fix meta error --- .../java/org/apache/fory/meta/FieldTypes.java | 20 ++-- .../apache/fory/resolver/ClassResolver.java | 11 --- .../apache/fory/serializer/FieldGroups.java | 6 -- .../StaticGeneratedStructSerializer.java | 91 ++++++++++++------- .../main/java/org/apache/fory/type/Types.java | 2 +- 5 files changed, 67 insertions(+), 63 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 2e1d8a43c7..2670eb85b0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -111,7 +111,8 @@ private static FieldType buildFieldType( int typeId; Annotation typeAnnotation = descriptor == null ? null : descriptor.getTypeAnnotation(); boolean primitiveList = TypeUtils.isPrimitiveListClass(rawType); - boolean primitiveListArray = descriptor != null && TypeAnnotationUtils.isArrayType(descriptor); + boolean primitiveListArray = + primitiveList && descriptor != null && TypeAnnotationUtils.isArrayType(descriptor); boolean boxedListArray = isXlang && descriptor != null @@ -133,9 +134,9 @@ private static FieldType buildFieldType( : Types.UNKNOWN; // Primitive-list TypeExtMeta describes the list element wire type. Only @ArrayType changes the // top-level schema to a dense primitive array. - if (primitiveListArray) { + if (isXlang && primitiveListArray) { typeId = TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType); - } else if (primitiveList) { + } else if (isXlang && primitiveList) { typeId = Types.LIST; } else if (!primitiveList && typeExtMeta != null && typeExtMeta.typeId() != Types.UNKNOWN) { typeId = typeExtMeta.typeId(); @@ -239,7 +240,7 @@ private static FieldType buildFieldType( return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } - if (primitiveList && !primitiveListArray) { + if (isXlang && primitiveList && !primitiveListArray) { boolean elementNullable = true; boolean elementTrackingRef = false; if (primitiveListArgumentMeta != null) { @@ -452,16 +453,13 @@ public void write(MemoryBuffer buffer, boolean writeHeader) { buffer.writeVarUInt32Small7(arrayFieldType.getDimensions()); (arrayFieldType).getComponentType().write(buffer); } else if (this instanceof CollectionFieldType) { - CollectionFieldType collectionFieldType = (CollectionFieldType) this; buffer.writeUInt8(kindHeader); - buffer.writeVarUInt32Small7(collectionFieldType.typeId); // TODO remove it when new collection deserialization jit finished. - collectionFieldType.getElementType().write(buffer); + ((CollectionFieldType) this).getElementType().write(buffer); } else if (this instanceof MapFieldType) { buffer.writeUInt8(kindHeader); // TODO remove it when new map deserialization jit finished. MapFieldType mapFieldType = (MapFieldType) this; - buffer.writeVarUInt32Small7(mapFieldType.typeId); mapFieldType.getKeyType().write(buffer); mapFieldType.getValueType().write(buffer); } else { @@ -496,12 +494,10 @@ public static FieldType read( if (kind == 0) { return new ObjectFieldType(Types.UNKNOWN, nullable, trackingRef); } else if (kind == 1) { - int typeId = buffer.readVarUInt32Small7(); return new MapFieldType( - typeId, nullable, trackingRef, read(buffer, resolver), read(buffer, resolver)); + -1, nullable, trackingRef, read(buffer, resolver), read(buffer, resolver)); } else if (kind == 2) { - int typeId = buffer.readVarUInt32Small7(); - return new CollectionFieldType(typeId, nullable, trackingRef, read(buffer, resolver)); + return new CollectionFieldType(-1, nullable, trackingRef, read(buffer, resolver)); } else if (kind == 3) { int dims = buffer.readVarUInt32Small7(); return new ArrayFieldType(-1, nullable, trackingRef, read(buffer, resolver), dims); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 3a53d8fb4a..272a57dfa8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1029,17 +1029,6 @@ public boolean isBuildIn(Descriptor descriptor) { return isMonomorphic(descriptor); } - @Override - public boolean isCollectionDescriptor(Descriptor descriptor) { - Class rawType = descriptor.getRawType(); - if (TypeUtils.isPrimitiveListClass(rawType)) { - return !org.apache.fory.type.TypeAnnotationUtils.isArrayType(descriptor) - && org.apache.fory.type.TypeAnnotationUtils.usesCollectionProtocolForPrimitiveList( - descriptor.getTypeAnnotation(), rawType); - } - return super.isCollectionDescriptor(descriptor); - } - public boolean isInternalRegistered(int classId) { if (Types.isUserDefinedType((byte) classId)) { return false; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index a1c14409b8..26b4153e06 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -41,7 +41,6 @@ import org.apache.fory.type.GenericType; import org.apache.fory.type.TypeAnnotationUtils; import org.apache.fory.type.TypeUtils; -import org.apache.fory.type.Types; import org.apache.fory.util.StringUtils; public class FieldGroups { @@ -217,7 +216,6 @@ public static final class SerializationFieldInfo { // nullability/ref mode. Treating it as field metadata writes an extra null marker that // remote TypeDef payload readers then consume as list length. TypeExtMeta extMeta = typeRef.getTypeExtMeta(); - int schemaTypeId = extMeta == null ? Types.UNKNOWN : extMeta.typeId(); if (extMeta != null && !primitiveListArray && !primitiveListCollection) { nullable = extMeta.nullable(); trackingRef = extMeta.trackingRef(); @@ -266,10 +264,6 @@ public static final class SerializationFieldInfo { resolver, TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor.getField()), qualifiedFieldName); - } else if (!resolver.isCrossLanguage() && schemaTypeId == Types.LIST) { - containerSerializerOverride = - new org.apache.fory.serializer.collection.CollectionSerializers.ArrayListSerializer( - resolver); } else { containerSerializerOverride = null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 6adce9c1e7..fcd26ac155 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -68,6 +68,23 @@ protected StaticGeneratedStructSerializer( this.localFieldsById = buildLocalFieldsById(runtimeDescriptors); } + @SuppressWarnings("unchecked") + protected StaticGeneratedStructSerializer( + TypeResolver typeResolver, + Class type, + TypeDef typeDef, + List localDescriptors, + List remoteDescriptors) { + super(typeResolver, (Class) type); + List runtimeDescriptors = runtimeDescriptors(localDescriptors); + this.typeDef = typeDef; + this.remoteFields = + typeDef == null + ? Collections.emptyList() + : buildRemoteFields(typeDef, runtimeDescriptors, remoteDescriptors); + this.localFieldsById = buildLocalFieldsById(runtimeDescriptors); + } + @Override public abstract void write(WriteContext writeContext, T value); @@ -206,30 +223,7 @@ protected final void skipField( SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { try { - int startIndex = buffer.readerIndex(); - System.err.println( - "STATIC_SKIP_START name=" - + fieldInfo.descriptor.getName() - + " index=" - + startIndex - + " next=" - + (buffer.getByte(startIndex) & 0xFF) - + " ref=" - + fieldInfo.refMode - + " typeRef=" - + fieldInfo.typeRef - + " schemaMeta=" - + fieldInfo.typeRef.getTypeExtMeta() - + " override=" - + fieldInfo.containerSerializerOverride); FieldSkipper.skipField(readContext, typeResolver, refReader, fieldInfo, buffer); - System.err.println( - "STATIC_SKIP_END name=" - + fieldInfo.descriptor.getName() - + " start=" - + startIndex - + " end=" - + buffer.readerIndex()); } catch (RuntimeException e) { throw new DeserializationException( "Failed to skip remote field " + fieldInfo.descriptor.getName(), e); @@ -310,15 +304,32 @@ private Object readField(ReadContext readContext, SerializationFieldInfo fieldIn private List buildRemoteFields( TypeDef remoteTypeDef, List localDescriptors) { + return buildRemoteFields(remoteTypeDef, localDescriptors, null); + } + + private List buildRemoteFields( + TypeDef remoteTypeDef, + List localDescriptors, + List generatedRemoteDescriptors) { Class remoteDescriptorClass = remoteDescriptorClass(remoteTypeDef); List remoteFieldInfos = remoteTypeDef.getFieldsInfo(); - List remoteDescriptors = - remoteTypeDef.getDescriptors(typeResolver, remoteDescriptorClass); + List remoteDescriptors; + if (generatedRemoteDescriptors == null) { + remoteDescriptors = remoteTypeDef.getDescriptors(typeResolver, remoteDescriptorClass); + } else { + remoteDescriptors = generatedRemoteDescriptors; + } Map remoteFieldInfosByKey = new HashMap<>(); - for (int i = 0; i < remoteFieldInfos.size(); i++) { - FieldInfo fieldInfo = remoteFieldInfos.get(i); - Descriptor descriptor = remoteDescriptors.get(i); - putRemoteFieldInfo(remoteFieldInfosByKey, fieldInfo, descriptor); + if (generatedRemoteDescriptors == null) { + for (int i = 0; i < remoteFieldInfos.size(); i++) { + FieldInfo fieldInfo = remoteFieldInfos.get(i); + Descriptor descriptor = remoteDescriptors.get(i); + putRemoteFieldInfo(remoteFieldInfosByKey, fieldInfo, descriptor); + } + } else { + for (FieldInfo fieldInfo : remoteFieldInfos) { + putRemoteFieldInfo(remoteFieldInfosByKey, fieldInfo); + } } Map fieldIds = new HashMap<>(); Map fields = new HashMap<>(); @@ -332,10 +343,18 @@ private List buildRemoteFields( // Keep compatible-read descriptor ordering owned by TypeResolver, matching the sorted // DescriptorGrouper order used by ObjectCodecBuilder and MetaSharedCodecBuilder. FieldGroups // may regroup descriptors for helper ownership, so it must not drive remote payload order. - List remoteDescriptorsInWireOrder = - typeResolver - .createDescriptorGrouper(remoteTypeDef, remoteDescriptorClass) - .getSortedDescriptors(); + List remoteDescriptorsInWireOrder; + if (generatedRemoteDescriptors == null) { + remoteDescriptorsInWireOrder = + typeResolver + .createDescriptorGrouper(remoteTypeDef, remoteDescriptorClass) + .getSortedDescriptors(); + } else { + remoteDescriptorsInWireOrder = + typeResolver + .groupDescriptors(generatedRemoteDescriptors, false, descriptor -> descriptor) + .getSortedDescriptors(); + } List remoteFields = new ArrayList<>(remoteFieldInfos.size()); appendRemoteFields( remoteFields, @@ -434,6 +453,12 @@ private static void putRemoteFieldInfo( } } + private static void putRemoteFieldInfo( + Map remoteFieldInfosByKey, FieldInfo fieldInfo) { + remoteFieldInfosByKey.put(remoteFieldKey(fieldInfo), fieldInfo); + remoteFieldInfosByKey.put("name:" + fieldInfo.getFieldName(), fieldInfo); + } + private static String remoteFieldKey(FieldInfo fieldInfo) { return fieldInfo.hasFieldId() ? "id:" + fieldInfo.getFieldId() diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Types.java b/java/fory-core/src/main/java/org/apache/fory/type/Types.java index 23910a439c..8760f0607a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Types.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Types.java @@ -415,7 +415,7 @@ public static int getDescriptorTypeId(TypeResolver resolver, Descriptor d) { } TypeRef typeRef = d.getTypeRef(); Class rawType = typeRef.getRawType(); - if (TypeUtils.isPrimitiveListClass(rawType)) { + if (resolver.isCrossLanguage() && TypeUtils.isPrimitiveListClass(rawType)) { if (TypeAnnotationUtils.isArrayType(d)) { return TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType); } From 30db8eecce7d06ff0dcd60b4692b9d732af139fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 02:48:10 +0800 Subject: [PATCH 23/58] fix(java): split static serializers by protocol mode --- .../processing/ForyStructProcessor.java | 73 +++++++- .../annotation/processing/SourceStruct.java | 3 + .../StaticSerializerSourceWriter.java | 68 +++++++ .../processing/ForyStructProcessorTest.java | 16 +- .../apache/fory/annotation/ForyStruct.java | 8 + .../org/apache/fory/builder/Generated.java | 9 + .../builder/StaticCompatibleCodecBuilder.java | 87 ++++++++- .../java/org/apache/fory/meta/FieldTypes.java | 19 +- .../java/org/apache/fory/meta/TypeDef.java | 120 +++++++++++- .../apache/fory/resolver/ClassResolver.java | 175 ++++++++++++++++-- .../apache/fory/resolver/TypeResolver.java | 91 +++++++-- .../apache/fory/serializer/FieldGroups.java | 12 +- .../fory/serializer/PrimitiveSerializers.java | 32 ++++ .../StaticGeneratedStructSerializer.java | 157 ++++++++++------ .../main/java/org/apache/fory/type/Types.java | 2 +- .../org/apache/fory/xlang/XlangTestBase.java | 2 +- .../ExampleStaticGeneratedSerializerTest.java | 47 +++++ 17 files changed, 804 insertions(+), 117 deletions(-) diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index 15eb83a594..08b738aa1f 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -86,6 +86,17 @@ public final class ForyStructProcessor extends AbstractProcessor { // javac's public tree API reflectively while keeping this processor targetable to Java 8. private Object trees; + private enum SerializerMode { + XLANG("__ForySerializer__"), + NATIVE("__ForyNativeSerializer__"); + + final String serializerSuffix; + + SerializerMode(String serializerSuffix) { + this.serializerSuffix = serializerSuffix; + } + } + @Override public synchronized void init(ProcessingEnvironment processingEnv) { super.init(processingEnv); @@ -128,8 +139,7 @@ public boolean process(Set annotations, RoundEnvironment continue; } try { - SourceStruct struct = buildStruct(type); - if (struct != null) { + for (SourceStruct struct : buildStructs(type)) { emit(struct, type); } } catch (InvalidStructException e) { @@ -144,7 +154,14 @@ public boolean process(Set annotations, RoundEnvironment return true; } - private SourceStruct buildStruct(TypeElement type) { + private List buildStructs(TypeElement type) { + List structs = new ArrayList<>(2); + structs.add(buildStruct(type, SerializerMode.XLANG)); + structs.add(buildStruct(type, SerializerMode.NATIVE)); + return structs; + } + + private SourceStruct buildStruct(TypeElement type, SerializerMode mode) { if (type.getModifiers().contains(Modifier.PRIVATE)) { throw new InvalidStructException("@ForyStruct classes must not be private", type); } @@ -163,7 +180,7 @@ private SourceStruct buildStruct(TypeElement type) { String binaryName = elements.getBinaryName(type).toString(); String serializerName = binaryName.substring(packageName.isEmpty() ? 0 : packageName.length() + 1) - + "__ForyStaticSerializer__"; + + mode.serializerSuffix; String qualifiedSerializerName = packageName.isEmpty() ? serializerName : packageName + "." + serializerName; TypeElement existing = elements.getTypeElement(qualifiedSerializerName); @@ -193,7 +210,7 @@ boolean record = isRecord(type); for (VariableElement field : fields) { boolean serialized = isSerializableRecordField(field, type); int id = serialized ? serializedId++ : -1; - SourceField sourceField = buildField(id, type, packageName, field, true, serialized); + SourceField sourceField = buildField(id, type, packageName, field, true, serialized, mode); recordConstructorFields.add(sourceField); if (serialized) { validateForyFieldId(binaryName, fieldIds, field); @@ -204,7 +221,7 @@ boolean record = isRecord(type); for (int i = 0; i < fields.size(); i++) { VariableElement field = fields.get(i); validateForyFieldId(binaryName, fieldIds, field); - SourceField sourceField = buildField(i, type, packageName, field, false, true); + SourceField sourceField = buildField(i, type, packageName, field, false, true, mode); sourceFields.add(sourceField); recordConstructorFields.add(sourceField); } @@ -214,10 +231,27 @@ boolean record = isRecord(type); canonicalName(type.asType()), serializerName, record, + foryStructDebug(type), sourceFields, recordConstructorFields); } + private boolean foryStructDebug(TypeElement type) { + AnnotationMirror mirror = annotationMirror(type, FORY_STRUCT); + if (mirror == null) { + return false; + } + Map values = + elements.getElementValuesWithDefaults(mirror); + for (Map.Entry entry : + values.entrySet()) { + if (entry.getKey().getSimpleName().contentEquals("debug")) { + return Boolean.TRUE.equals(entry.getValue().getValue()); + } + } + return false; + } + private void validateForyFieldId( String binaryName, Map fieldIds, VariableElement field) { ForyFieldMeta foryField = foryField(field); @@ -249,7 +283,8 @@ private SourceField buildField( String generatedPackage, VariableElement field, boolean record, - boolean serialized) { + boolean serialized, + SerializerMode mode) { Set modifiers = field.getModifiers(); if (!record && modifiers.contains(Modifier.FINAL)) { throw new InvalidStructException( @@ -259,8 +294,7 @@ private SourceField buildField( field); } ForyFieldMeta foryField = foryField(field); - boolean nullable = - foryField.hasForyField ? foryField.nullable : !field.asType().getKind().isPrimitive(); + boolean nullable = fieldNullable(field.asType(), foryField, mode); SourceTypeNode typeNode = buildFieldTypeNode(field, nullable); String erasedType = canonicalName(types.erasure(field.asType())); String declaringClass = @@ -316,6 +350,27 @@ private SourceField buildField( foryField.dynamic); } + private boolean fieldNullable(TypeMirror type, ForyFieldMeta foryField, SerializerMode mode) { + if (type.getKind().isPrimitive()) { + return false; + } + if (mode == SerializerMode.NATIVE) { + return true; + } + if (foryField.hasForyField) { + return foryField.nullable; + } + return isOptionalType(type); + } + + private boolean isOptionalType(TypeMirror type) { + String erasedType = canonicalName(types.erasure(type)); + return erasedType.equals("java.util.Optional") + || erasedType.equals("java.util.OptionalInt") + || erasedType.equals("java.util.OptionalLong") + || erasedType.equals("java.util.OptionalDouble"); + } + private List serializableFields(TypeElement type) { List hierarchy = hierarchy(type); List fields = new ArrayList<>(); diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java index e07eed36fd..c94e61632e 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/SourceStruct.java @@ -28,6 +28,7 @@ final class SourceStruct { final String typeName; final String serializerName; final boolean record; + final boolean debug; final boolean hasNestedCompatibleStructFields; final List fields; final List recordConstructorFields; @@ -37,12 +38,14 @@ final class SourceStruct { String typeName, String serializerName, boolean record, + boolean debug, List fields, List recordConstructorFields) { this.packageName = packageName; this.typeName = typeName; this.serializerName = serializerName; this.record = record; + this.debug = debug; this.fields = Collections.unmodifiableList(new ArrayList<>(fields)); this.recordConstructorFields = Collections.unmodifiableList(new ArrayList<>(recordConstructorFields)); diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 2aaa465b2c..6aa36b27b6 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -81,6 +81,9 @@ private void writeClassStart() { .append(" private static final boolean HAS_NESTED_COMPATIBLE_STRUCT_FIELDS = ") .append(struct.hasNestedCompatibleStructFields) .append(";\n"); + if (struct.debug) { + builder.append(" private static final boolean FORY_STRUCT_DEBUG = true;\n"); + } builder.append(" private static final List DESCRIPTORS = buildDescriptors();\n\n"); builder.append(" private final SerializationFieldInfo[] buildInFields;\n"); builder.append(" private final int[] buildInFieldIds;\n"); @@ -264,12 +267,14 @@ private void writeWriteGroup( builder.append(" switch (").append(idsName).append("[i]) {\n"); for (SourceField field : struct.fields) { builder.append(" case ").append(field.id).append(":\n"); + appendDebugWrite("before", "fieldInfo", 10); builder .append(" ") .append(helperName) .append("(writeContext, fieldInfo, ") .append(field.readExpression("value")) .append(");\n"); + appendDebugWrite("after", "fieldInfo", 10); builder.append(" break;\n"); } builder.append(" default:\n"); @@ -306,10 +311,12 @@ private void writeReadBeanGroup( .append(" value) {\n"); builder.append(" for (int i = 0; i < ").append(fieldsName).append(".length; i++) {\n"); builder.append(" SerializationFieldInfo fieldInfo = ").append(fieldsName).append("[i];\n"); + appendDebugRead("before", "fieldInfo", 6); builder .append(" Object fieldValue = ") .append(helperName) .append("(readContext, fieldInfo);\n"); + appendDebugRead("after", "fieldInfo", 6); builder.append(" switch (").append(idsName).append("[i]) {\n"); for (SourceField field : struct.fields) { builder.append(" case ").append(field.id).append(":\n"); @@ -337,12 +344,14 @@ private void writeReadRecordGroup( .append("RecordFields(ReadContext readContext, Object[] values) {\n"); builder.append(" for (int i = 0; i < ").append(fieldsName).append(".length; i++) {\n"); builder.append(" SerializationFieldInfo fieldInfo = ").append(fieldsName).append("[i];\n"); + appendDebugRead("before", "fieldInfo", 6); builder .append(" values[") .append(idsName) .append("[i]] = ") .append(helperName) .append("(readContext, fieldInfo);\n"); + appendDebugRead("after", "fieldInfo", 6); builder.append(" }\n"); builder.append(" }\n\n"); } @@ -430,7 +439,9 @@ private void writeCompatibleDispatchRouter(String methodName, boolean record, in builder.append(" return;\n"); builder.append(" }\n"); } + appendDebugRemoteRead("before skip", "remoteField", 4); builder.append(" skipField(readContext, remoteField);\n"); + appendDebugRemoteRead("after skip", "remoteField", 4); builder.append(" }\n\n"); } @@ -444,6 +455,7 @@ private void writeCompatibleBeanDispatchGroup(int group) { for (int i = start; i < end; i++) { SourceField field = struct.fields.get(i); builder.append(" case ").append(field.id).append(":\n"); + appendDebugRemoteRead("before read", "remoteField", 8); builder .append(" if (canReadRemoteField(remoteField, fieldsById[") .append(field.id) @@ -453,17 +465,22 @@ private void writeCompatibleBeanDispatchGroup(int group) { " Object fieldValue = readCompatibleFieldValue(readContext, remoteField, fieldsById[") .append(field.id) .append("]);\n"); + appendDebugRemoteRead("after read", "remoteField", 10); builder .append(" ") .append(field.writeStatement("value", field.castExpression("fieldValue"))) .append("\n"); builder.append(" } else {\n"); + appendDebugRemoteRead("before skip", "remoteField", 10); builder.append(" skipField(readContext, remoteField);\n"); + appendDebugRemoteRead("after skip", "remoteField", 10); builder.append(" }\n"); builder.append(" return;\n"); } builder.append(" default:\n"); + appendDebugRemoteRead("before skip", "remoteField", 8); builder.append(" skipField(readContext, remoteField);\n"); + appendDebugRemoteRead("after skip", "remoteField", 8); builder.append(" }\n"); builder.append(" }\n\n"); } @@ -478,6 +495,7 @@ private void writeCompatibleRecordDispatchGroup(int group) { for (int i = start; i < end; i++) { SourceField field = struct.fields.get(i); builder.append(" case ").append(field.id).append(":\n"); + appendDebugRemoteRead("before read", "remoteField", 8); builder .append(" if (canReadRemoteField(remoteField, fieldsById[") .append(field.id) @@ -488,13 +506,18 @@ private void writeCompatibleRecordDispatchGroup(int group) { .append("] = readCompatibleFieldValue(readContext, remoteField, fieldsById[") .append(field.id) .append("]);\n"); + appendDebugRemoteRead("after read", "remoteField", 10); builder.append(" } else {\n"); + appendDebugRemoteRead("before skip", "remoteField", 10); builder.append(" skipField(readContext, remoteField);\n"); + appendDebugRemoteRead("after skip", "remoteField", 10); builder.append(" }\n"); builder.append(" return;\n"); } builder.append(" default:\n"); + appendDebugRemoteRead("before skip", "remoteField", 8); builder.append(" skipField(readContext, remoteField);\n"); + appendDebugRemoteRead("after skip", "remoteField", 8); builder.append(" }\n"); builder.append(" }\n\n"); } @@ -519,6 +542,51 @@ private void appendCompatibleDispatchArguments(boolean record) { builder.append("remoteField, matchedId"); } + private void appendDebugWrite(String stage, String fieldInfoName, int indent) { + if (!struct.debug) { + return; + } + appendIndent(indent); + builder + .append("if (FORY_STRUCT_DEBUG) { debugWriteField(\"") + .append(stage) + .append("\", ") + .append(fieldInfoName) + .append(", writeContext); }\n"); + } + + private void appendDebugRead(String stage, String fieldInfoName, int indent) { + if (!struct.debug) { + return; + } + appendIndent(indent); + builder + .append("if (FORY_STRUCT_DEBUG) { debugReadField(\"") + .append(stage) + .append("\", ") + .append(fieldInfoName) + .append(", readContext); }\n"); + } + + private void appendDebugRemoteRead(String stage, String remoteFieldName, int indent) { + if (!struct.debug) { + return; + } + appendIndent(indent); + builder + .append("if (FORY_STRUCT_DEBUG) { debugRemoteReadField(\"") + .append(stage) + .append("\", ") + .append(remoteFieldName) + .append(", readContext); }\n"); + } + + private void appendIndent(int spaces) { + for (int i = 0; i < spaces; i++) { + builder.append(' '); + } + } + private void writeCopy() { builder.append(" @Override\n"); builder diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 45017aab07..ded30e40c6 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -63,7 +63,7 @@ public void testStaticSerializerSelectedWithCodegenDisabled() throws Exception { Assert.assertTrue(result.success, result.diagnostics()); try (URLClassLoader loader = result.classLoader()) { Class type = loader.loadClass("test.SimpleStruct"); - Class serializerType = loader.loadClass("test.SimpleStruct__ForyStaticSerializer__"); + Class serializerType = loader.loadClass("test.SimpleStruct__ForyNativeSerializer__"); Assert.assertTrue(StaticGeneratedStructSerializer.class.isAssignableFrom(serializerType)); Object value = type.getConstructor().newInstance(); @@ -186,8 +186,10 @@ public void testInnerTypeGeneratedAsTopLevelBinaryTail() throws Exception { + "}\n"); Assert.assertTrue(result.success, result.diagnostics()); try (URLClassLoader loader = result.classLoader()) { - Class serializer = loader.loadClass("test.Outer$Inner__ForyStaticSerializer__"); - Assert.assertTrue(StaticGeneratedStructSerializer.class.isAssignableFrom(serializer)); + Class xlangSerializer = loader.loadClass("test.Outer$Inner__ForySerializer__"); + Class nativeSerializer = loader.loadClass("test.Outer$Inner__ForyNativeSerializer__"); + Assert.assertTrue(StaticGeneratedStructSerializer.class.isAssignableFrom(xlangSerializer)); + Assert.assertTrue(StaticGeneratedStructSerializer.class.isAssignableFrom(nativeSerializer)); } } @@ -204,7 +206,7 @@ public void testGeneratedNameCollisionFailsCompilation() throws Exception { + " public Inner() {}\n" + " }\n" + "}\n" - + "class Outer$Inner__ForyStaticSerializer__ {}\n"); + + "class Outer$Inner__ForySerializer__ {}\n"); Assert.assertFalse(result.success); Assert.assertTrue(result.diagnostics().contains("collides"), result.diagnostics()); } @@ -230,7 +232,7 @@ public void testGeneratedDescriptorsCarryNestedTypeMetadata() throws Exception { Assert.assertTrue(result.success, result.diagnostics()); try (URLClassLoader loader = result.classLoader()) { Class type = loader.loadClass("test.MetadataStruct"); - Class serializerType = loader.loadClass("test.MetadataStruct__ForyStaticSerializer__"); + Class serializerType = loader.loadClass("test.MetadataStruct__ForySerializer__"); Fory fory = Fory.builder() .withClassLoader(loader) @@ -274,7 +276,7 @@ public void testRecordReadAndCopyUseCanonicalConstructor() throws Exception { + "}\n"); Assert.assertTrue(result.success, result.diagnostics()); String generatedSource = - result.generatedSource("test/RecordStruct__ForyStaticSerializer__.java"); + result.generatedSource("test/RecordStruct__ForyNativeSerializer__.java"); Assert.assertTrue(generatedSource.contains("private void readCompatibleRecordField0(")); Assert.assertTrue(generatedSource.contains("switch (matchedId)")); try (URLClassLoader loader = result.classLoader()) { @@ -327,7 +329,7 @@ public void testCompatibleReadUsesGeneratedSerializer() throws Exception { Assert.assertTrue(writerResult.success, writerResult.diagnostics()); Assert.assertTrue(readerResult.success, readerResult.diagnostics()); String generatedSource = - readerResult.generatedSource("test/EvolvingStruct__ForyStaticSerializer__.java"); + readerResult.generatedSource("test/EvolvingStruct__ForyNativeSerializer__.java"); Assert.assertTrue( generatedSource.contains( "readCompatibleField(readContext, value, remoteField, matchedId(remoteField))")); diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java index 00f75732de..e6d471ad2f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java +++ b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java @@ -50,4 +50,12 @@ enum Evolution { * metadata is otherwise enabled. */ Evolution evolving() default Evolution.INHERIT; + + /** + * Emit generated serializer field-level debug tracing. + * + *

The generated code still prints only when {@code ENABLE_FORY_DEBUG_OUTPUT=1}; this option + * controls whether field tracing code is emitted for this struct. + */ + boolean debug() default false; } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java index e63bc95fbb..17291e16df 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java @@ -130,6 +130,15 @@ public GeneratedCompatibleMetaSharedSerializer( super(typeResolver, cls, typeDef, descriptors); } + public GeneratedCompatibleMetaSharedSerializer( + TypeResolver typeResolver, + Class cls, + TypeDef typeDef, + List descriptors, + Class remoteDescriptorClass) { + super(typeResolver, cls, typeDef, descriptors, remoteDescriptorClass); + } + @Override public final Object read(ReadContext readContext) { return readCompatible(readContext); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index c9c14147f7..5ed288982a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.List; import org.apache.fory.Fory; +import org.apache.fory.annotation.ForyStruct; import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; import org.apache.fory.codegen.Code; import org.apache.fory.codegen.CodeGenerator; @@ -59,6 +60,8 @@ public final class StaticCompatibleCodecBuilder extends ObjectCodecBuilder { private static final int DISPATCH_GROUP_SIZE = 8; private final List localDescriptors; + private final Class remoteDescriptorClass; + private final boolean debug; public StaticCompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { super(beanType, fory, GeneratedCompatibleMetaSharedSerializer.class); @@ -66,6 +69,9 @@ public StaticCompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef type !fory.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); localDescriptors = Collections.unmodifiableList(Descriptor.getDescriptors(beanClass)); + remoteDescriptorClass = resolveRemoteDescriptorClass(typeDef); + ForyStruct foryStruct = beanClass.getAnnotation(ForyStruct.class); + debug = foryStruct != null && foryStruct.debug(); } @Override @@ -84,7 +90,8 @@ public String genCode() { String constructorCode = StringUtils.format( "" - + "super(${typeResolver}, ${cls}, ${typeDef}, Descriptor.getDescriptors(${cls}));\n" + + "super(${typeResolver}, ${cls}, ${typeDef}, Descriptor.getDescriptors(${cls})," + + " ${remoteDescriptorClass});\n" + "this.${generatedTypeResolver} = (${generatedTypeResolverType}) ${typeResolver};\n", "typeResolver", CONSTRUCTOR_TYPE_RESOLVER_NAME, @@ -92,6 +99,8 @@ public String genCode() { POJO_CLASS_TYPE_NAME, "typeDef", "_f_typeDef", + "remoteDescriptorClass", + remoteDescriptorClassLiteral(), "generatedTypeResolver", TYPE_RESOLVER_NAME, "generatedTypeResolverType", @@ -121,6 +130,39 @@ protected void addCommonImports() { ctx.addImport(GeneratedCompatibleMetaSharedSerializer.class); } + private Class resolveRemoteDescriptorClass(TypeDef typeDef) { + String className = typeDef.getClassName(); + if (className.equals(beanClass.getName())) { + return null; + } + ClassLoader beanClassLoader = beanClass.getClassLoader(); + try { + return Class.forName(className, false, beanClassLoader); + } catch (ClassNotFoundException | LinkageError e) { + try { + return Class.forName(className, false, StaticCompatibleCodecBuilder.class.getClassLoader()); + } catch (ClassNotFoundException | LinkageError ignored) { + return null; + } + } + } + + private String remoteDescriptorClassLiteral() { + if (remoteDescriptorClass == null || !canReferenceRemoteDescriptorClass()) { + return "null"; + } + return ctx.type(remoteDescriptorClass) + ".class"; + } + + private boolean canReferenceRemoteDescriptorClass() { + if (!ctx.sourcePkgLevelAccessible(remoteDescriptorClass)) { + return false; + } + return ctx.sourcePublicAccessible(remoteDescriptorClass) + || CodeGenerator.getPackage(beanClass) + .equals(CodeGenerator.getPackage(remoteDescriptorClass)); + } + @Override public Expression buildEncodeExpression() { throw new IllegalStateException("unreachable"); @@ -304,7 +346,9 @@ private String genDispatchRouter(String methodPrefix, int groupCount) { code.append(isRecord ? "_f_recordValues" : "_f_value"); code.append(", _f_remoteField, _f_matchedId);\n").append(" return;\n").append("}\n"); } - code.append("skipField(").append(READ_CONTEXT_NAME).append(", _f_remoteField);"); + appendDebugRemoteRead(code, "before skip", "_f_remoteField", 0); + code.append("skipField(").append(READ_CONTEXT_NAME).append(", _f_remoteField);\n"); + appendDebugRemoteRead(code, "after skip", "_f_remoteField", 0); return code.toString(); } @@ -317,22 +361,27 @@ private String genObjectDispatchGroup(int group, TypeRef valueTypeRef) { code.append(" case ") .append(i) .append(": {\n") + .append(debugRemoteReadCode("before read", "_f_remoteField", 4)) .append(" if (hasFieldConverter(_f_remoteField)) {\n") .append(" Object _f_fieldValue = readRemoteField(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField);\n") + .append(debugRemoteReadCode("after read", "_f_remoteField", 6)) .append(" setConvertedField(_f_value, _f_fieldValue, _f_remoteField);\n") .append(" } else {\n") .append(" SerializationFieldInfo _f_localField = localFieldInfo(_f_matchedId);\n") .append(" if (!canReadRemoteField(_f_remoteField, _f_localField)) {\n") + .append(debugRemoteReadCode("before skip", "_f_remoteField", 8)) .append(" skipField(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField);\n") + .append(debugRemoteReadCode("after skip", "_f_remoteField", 8)) .append(" return;\n") .append(" }\n") .append(" Object _f_fieldValue = readCompatibleFieldValue(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField, _f_localField);\n") + .append(debugRemoteReadCode("after read", "_f_remoteField", 6)) .append(indent(genSetFieldCode(descriptor, valueTypeRef), 6)) .append('\n') .append(" }\n") @@ -340,9 +389,11 @@ private String genObjectDispatchGroup(int group, TypeRef valueTypeRef) { .append(" }\n"); } code.append(" default:\n") + .append(debugRemoteReadCode("before skip", "_f_remoteField", 4)) .append(" skipField(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField);\n") + .append(debugRemoteReadCode("after skip", "_f_remoteField", 4)) .append("}\n"); return code.toString(); } @@ -360,6 +411,7 @@ private String genRecordDispatchGroup(int group) { code.append(" case ") .append(i) .append(": {\n") + .append(debugRemoteReadCode("before read", "_f_remoteField", 4)) .append(" SerializationFieldInfo _f_localField = localFieldInfo(_f_matchedId);\n") .append(" if (canReadRemoteField(_f_remoteField, _f_localField)) {\n") .append(" _f_recordValues[") @@ -367,22 +419,53 @@ private String genRecordDispatchGroup(int group) { .append("] = readCompatibleFieldValue(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField, _f_localField);\n") + .append(debugRemoteReadCode("after read", "_f_remoteField", 6)) .append(" } else {\n") + .append(debugRemoteReadCode("before skip", "_f_remoteField", 6)) .append(" skipField(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField);\n") + .append(debugRemoteReadCode("after skip", "_f_remoteField", 6)) .append(" }\n") .append(" return;\n") .append(" }\n"); } code.append(" default:\n") + .append(debugRemoteReadCode("before skip", "_f_remoteField", 4)) .append(" skipField(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField);\n") + .append(debugRemoteReadCode("after skip", "_f_remoteField", 4)) .append("}\n"); return code.toString(); } + private void appendDebugRemoteRead( + StringBuilder code, String stage, String remoteField, int indent) { + if (!debug) { + return; + } + code.append(debugRemoteReadCode(stage, remoteField, indent)); + } + + private String debugRemoteReadCode(String stage, String remoteField, int indent) { + if (!debug) { + return ""; + } + StringBuilder code = new StringBuilder(); + for (int i = 0; i < indent; i++) { + code.append(' '); + } + code.append("if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { debugRemoteReadField(\"") + .append(stage) + .append("\", ") + .append(remoteField) + .append(", ") + .append(READ_CONTEXT_NAME) + .append("); }\n"); + return code.toString(); + } + private String genSetFieldCode(Descriptor descriptor, TypeRef valueTypeRef) { ctx.clearExprState(); Reference value = new Reference("_f_value", valueTypeRef, false); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 2670eb85b0..7e5adaeb6e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -129,20 +129,19 @@ private static FieldType buildFieldType( primitiveList ? primitiveListElementMeta != null ? primitiveListElementMeta.typeId() - : TypeAnnotationUtils.getPrimitiveListElementTypeId( - typeAnnotation, rawType, true) + : TypeAnnotationUtils.getPrimitiveListElementTypeId(typeAnnotation, rawType, true) : Types.UNKNOWN; // Primitive-list TypeExtMeta describes the list element wire type. Only @ArrayType changes the // top-level schema to a dense primitive array. - if (isXlang && primitiveListArray) { + if (primitiveListArray) { typeId = TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType); - } else if (isXlang && primitiveList) { + } else if (primitiveList) { typeId = Types.LIST; } else if (!primitiveList && typeExtMeta != null && typeExtMeta.typeId() != Types.UNKNOWN) { typeId = typeExtMeta.typeId(); } else if (boxedListArray) { typeId = TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor); - } else if (isXlang && primitiveListElementTypeId != Types.UNKNOWN) { + } else if (primitiveListElementTypeId != Types.UNKNOWN) { typeId = TypeAnnotationUtils.getPrimitiveListTypeId(typeAnnotation, rawType); } else if (TypeUtils.unwrap(rawType).isPrimitive()) { if (field != null) { @@ -225,9 +224,13 @@ private static FieldType buildFieldType( nullable = !genericType.getCls().isPrimitive(); } - // Apply @ForyField annotation if present + // Apply @ForyField annotation where the protocol uses it as field-wrapper metadata. Native + // reflected value fields stay nullable by default; generated/remote descriptors without a + // backing Field already carry the schema-owned value. if (descriptor != null && descriptor.hasForyField()) { - nullable = descriptor.isNullable(); + if (isXlang || descriptorCarriesFieldOptions) { + nullable = descriptor.isNullable(); + } trackingRef = descriptor.isTrackingRef(); } @@ -240,7 +243,7 @@ private static FieldType buildFieldType( return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } - if (isXlang && primitiveList && !primitiveListArray) { + if (primitiveList && !primitiveListArray) { boolean elementNullable = true; boolean elementTrackingRef = false; if (primitiveListArgumentMeta != null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java index 04a36ebfe6..a3eef88ad5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java @@ -45,7 +45,9 @@ import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; import org.apache.fory.serializer.MetaSharedSerializer; +import org.apache.fory.serializer.UnknownClass; import org.apache.fory.type.Descriptor; +import org.apache.fory.type.DescriptorBuilder; import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; @@ -407,6 +409,11 @@ private List buildDescriptors( } List descriptors = new ArrayList<>(fieldsInfo.size()); boolean isXlang = resolver.isCrossLanguage(); + Collection remoteDescriptors = null; + Map remoteDescriptorsMap = null; + Map remoteFieldIdToDescriptorMap = null; + Map> remoteDefinedClassDescriptors = new HashMap<>(); + Map> remoteDefinedClassFieldIds = new HashMap<>(); for (FieldInfo fieldInfo : fieldsInfo) { Descriptor descriptor; if (fieldInfo.hasFieldId()) { @@ -421,11 +428,122 @@ private List buildDescriptors( definedClass + "." + StringUtils.lowerCamelToLowerUnderscore(fieldName)); } } - descriptors.add(fieldInfo.toDescriptor(resolver, descriptor)); + boolean remoteOnly = false; + if (descriptor == null) { + if (remoteDescriptors == null) { + remoteDescriptors = tryLoadRemoteDescriptors(resolver, cls); + if (remoteDescriptors != null) { + remoteDescriptorsMap = new HashMap<>(); + remoteFieldIdToDescriptorMap = new HashMap<>(); + populateDescriptorMaps( + remoteDescriptors, remoteDescriptorsMap, remoteFieldIdToDescriptorMap); + } + } + if (remoteDescriptors != null) { + if (fieldInfo.hasFieldId()) { + descriptor = remoteFieldIdToDescriptorMap.get(fieldInfo.getFieldId()); + } else { + descriptor = + remoteDescriptorsMap.get( + fieldInfo.getDefinedClass() + "." + fieldInfo.getFieldName()); + } + remoteOnly = descriptor != null; + } + if (descriptor == null) { + String definedClass = fieldInfo.getDefinedClass(); + Map definedClassDescriptors = + remoteDefinedClassDescriptors.get(definedClass); + Map definedClassFieldIds = + remoteDefinedClassFieldIds.get(definedClass); + if (definedClassDescriptors == null) { + Collection descriptorsForDefinedClass = + tryLoadDescriptorsForClassName(resolver, definedClass, cls); + if (descriptorsForDefinedClass != null) { + definedClassDescriptors = new HashMap<>(); + definedClassFieldIds = new HashMap<>(); + populateDescriptorMaps( + descriptorsForDefinedClass, definedClassDescriptors, definedClassFieldIds); + remoteDefinedClassDescriptors.put(definedClass, definedClassDescriptors); + remoteDefinedClassFieldIds.put(definedClass, definedClassFieldIds); + } + } + if (definedClassDescriptors != null) { + if (fieldInfo.hasFieldId()) { + descriptor = definedClassFieldIds.get(fieldInfo.getFieldId()); + } else { + descriptor = + definedClassDescriptors.get( + fieldInfo.getDefinedClass() + "." + fieldInfo.getFieldName()); + } + remoteOnly = descriptor != null; + } + } + } + Descriptor resolved = fieldInfo.toDescriptor(resolver, descriptor); + if (remoteOnly) { + resolved = + new DescriptorBuilder(resolved) + .field(null) + .readMethod(null) + .writeMethod(null) + .fieldConverter(null) + .build(); + } + descriptors.add(resolved); } return descriptors; } + private static void populateDescriptorMaps( + Collection descriptors, + Map descriptorsMap, + Map fieldIdToDescriptorMap) { + for (Descriptor descriptor : descriptors) { + descriptorsMap.put(descriptor.getDeclaringClass() + "." + descriptor.getName(), descriptor); + if (descriptor.hasForyFieldId()) { + fieldIdToDescriptorMap.put((short) descriptor.getForyFieldId(), descriptor); + } + } + } + + private Collection tryLoadRemoteDescriptors( + TypeResolver resolver, Class localCls) { + if (resolver.isCrossLanguage() || !(resolver instanceof ClassResolver)) { + return null; + } + try { + Class remoteCls = + ((ClassResolver) resolver) + .loadClassForMeta(classSpec.entireClassName, classSpec.isEnum, classSpec.dimension); + if (remoteCls == localCls || UnknownClass.class.isAssignableFrom(remoteCls)) { + return null; + } + // Native TypeDef collection/map metadata intentionally does not carry Java implementation + // classes. When the writer class is locally loadable, use its descriptors only to recover + // descriptor-owned payload details such as primitive-list carriers; accessors are cleared + // before the descriptor is returned so compatible readers still skip remote-only fields. + return resolver.getFieldDescriptors(remoteCls, true); + } catch (RuntimeException e) { + return null; + } + } + + private Collection tryLoadDescriptorsForClassName( + TypeResolver resolver, String className, Class localCls) { + if (resolver.isCrossLanguage() || !(resolver instanceof ClassResolver)) { + return null; + } + try { + Class remoteCls = ((ClassResolver) resolver).loadClassForMeta(className, false, -1); + if (remoteCls == localCls || UnknownClass.class.isAssignableFrom(remoteCls)) { + return null; + } + return resolver.getFieldDescriptors(remoteCls, true); + } catch (RuntimeException e) { + return null; + } + } + public static TypeDef buildTypeDef(TypeResolver resolver, Class cls) { return buildTypeDef(resolver, cls, true); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 272a57dfa8..9d59822d2f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -35,7 +35,9 @@ import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.charset.Charset; +import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; @@ -103,6 +105,7 @@ import org.apache.fory.meta.Encoders; import org.apache.fory.meta.NativeTypeDefEncoder; import org.apache.fory.meta.TypeDef; +import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.platform.AndroidSupport; import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.reflect.ObjectCreators; @@ -159,9 +162,11 @@ import org.apache.fory.type.BFloat16; import org.apache.fory.type.BFloat16Array; import org.apache.fory.type.Descriptor; +import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Float16; import org.apache.fory.type.Float16Array; import org.apache.fory.type.GenericType; +import org.apache.fory.type.TypeAnnotationUtils; import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; import org.apache.fory.type.union.Union; @@ -1023,10 +1028,33 @@ public boolean isMonomorphic(Class clz) { public boolean isBuildIn(Descriptor descriptor) { Class rawType = descriptor.getRawType(); if (TypeUtils.isPrimitiveListClass(rawType)) { - return !org.apache.fory.type.TypeAnnotationUtils.usesCollectionProtocolForPrimitiveList( + return !TypeAnnotationUtils.usesCollectionProtocolForPrimitiveList( descriptor.getTypeAnnotation(), rawType); } - return isMonomorphic(descriptor); + int typeId = getDescriptorSortTypeId(descriptor); + return typeId != Types.UNKNOWN + && !Types.isUserDefinedType(typeId) + && typeId != Types.LIST + && typeId != Types.SET + && typeId != Types.MAP; + } + + @Override + public boolean usesPrimitiveFieldOrdering(Descriptor descriptor) { + if (super.usesPrimitiveFieldOrdering(descriptor)) { + return true; + } + int typeId = Types.getDescriptorTypeId(this, descriptor); + return typeId == Types.FLOAT16 || typeId == Types.BFLOAT16; + } + + @Override + public boolean isCollectionDescriptor(Descriptor descriptor) { + Class rawType = descriptor.getRawType(); + if (TypeUtils.isPrimitiveListClass(rawType)) { + return !TypeAnnotationUtils.isArrayType(descriptor); + } + return super.isCollectionDescriptor(descriptor); } public boolean isInternalRegistered(int classId) { @@ -2091,11 +2119,131 @@ public boolean isPrimitive(int classId) { return classId >= PRIMITIVE_VOID_ID && classId <= PRIMITIVE_FLOAT64_ID; } + private int getDescriptorSortTypeId(Descriptor d) { + int sortTypeId = getLogicalDescriptorSortTypeId(d); + if (sortTypeId != Types.UNKNOWN) { + return sortTypeId; + } + if (isCollectionDescriptor(d)) { + return Types.LIST; + } + Class rawType = d.getRawType(); + if (rawType != null && (rawType.isEnum() || rawType == UnknownClass.UnknownEnum.class)) { + return Types.ENUM; + } + if (rawType != null && isMap(rawType)) { + return Types.MAP; + } + try { + return Integer.parseInt(d.getTypeName()); + } catch (NumberFormatException ignored) { + return Types.getDescriptorTypeId(this, d); + } + } + + private int getLogicalDescriptorSortTypeId(Descriptor descriptor) { + Class rawType = descriptor.getRawType(); + if (rawType == null) { + return Types.UNKNOWN; + } + if (isCollectionDescriptor(descriptor)) { + return Types.LIST; + } + if (rawType.isArray() && !rawType.getComponentType().isPrimitive()) { + return Types.LIST; + } + if (isMap(rawType)) { + return Types.MAP; + } + if (rawType.isEnum() || rawType == UnknownClass.UnknownEnum.class) { + return Types.ENUM; + } + if (TypeUtils.isPrimitiveListClass(rawType)) { + if (TypeAnnotationUtils.isArrayType(descriptor)) { + return TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType); + } + return Types.LIST; + } + TypeExtMeta extMeta = descriptor.getTypeRef().getTypeExtMeta(); + if (extMeta != null && extMeta.typeId() != Types.UNKNOWN) { + int typeId = extMeta.typeId(); + if (typeId < Types.BOUND) { + return typeId; + } + } + if (TypeAnnotationUtils.isBoxedListArrayType(descriptor)) { + return TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor); + } + if (rawType.isArray() && rawType.getComponentType().isPrimitive()) { + int annotatedTypeId = Types.getDescriptorTypeId(this, descriptor); + if (annotatedTypeId > Types.UNKNOWN && annotatedTypeId < Types.BOUND) { + return annotatedTypeId; + } + return getPrimitiveArraySortTypeId(rawType); + } + return getBuiltinSortTypeId(rawType); + } + + private int getPrimitiveArraySortTypeId(Class rawType) { + if (rawType == boolean[].class) { + return Types.BOOL_ARRAY; + } + if (rawType == byte[].class) { + return Types.BINARY; + } + if (rawType == short[].class) { + return Types.INT16_ARRAY; + } + if (rawType == int[].class) { + return Types.INT32_ARRAY; + } + if (rawType == long[].class) { + return Types.INT64_ARRAY; + } + if (rawType == float[].class) { + return Types.FLOAT32_ARRAY; + } + if (rawType == double[].class) { + return Types.FLOAT64_ARRAY; + } + return Types.UNKNOWN; + } + + private int getBuiltinSortTypeId(Class rawType) { + if (rawType == Float16Array.class) { + return Types.FLOAT16_ARRAY; + } + if (rawType == BFloat16Array.class) { + return Types.BFLOAT16_ARRAY; + } + if (rawType == byte[].class || ByteBuffer.class.isAssignableFrom(rawType)) { + return Types.BINARY; + } + if (rawType == Duration.class) { + return Types.DURATION; + } + if (rawType == Instant.class || rawType == LocalDateTime.class || rawType == Date.class) { + return Types.TIMESTAMP; + } + if (rawType == LocalDate.class) { + return Types.DATE; + } + if (rawType == BigDecimal.class || rawType == BigInteger.class) { + return Types.DECIMAL; + } + TypeInfo typeInfo = getTypeInfo(rawType, false); + int typeId = typeInfo == null ? Types.UNKNOWN : typeInfo.getTypeId(); + return typeId > Types.UNKNOWN && typeId < Types.BOUND ? typeId : Types.UNKNOWN; + } + + @Override + protected DescriptorGrouper configureDescriptorGrouper(DescriptorGrouper descriptorGrouper) { + return descriptorGrouper.setOtherDescriptorComparator(TypeResolver::compareFieldSortKey); + } + /** - * Normalize type name for consistent ordering between serialization and deserialization. - * Collection descriptors are normalized to "java.util.Collection". Map subtypes are normalized to - * "java.util.Map". This ensures fields have the same order regardless of whether the peer has the - * field locally. + * Normalize type name for deterministic fallback ordering between serialization and + * deserialization. */ private String getNormalizedTypeName(Descriptor d) { if (isCollectionDescriptor(d)) { @@ -2109,17 +2257,20 @@ private String getNormalizedTypeName(Descriptor d) { } /** - * Creates a comparator for sorting descriptors by normalized type name and field name/id. This - * comparator normalizes Collection/Map types to ensure consistent field ordering between - * serialization and deserialization, even when peers have different Collection/Map subtypes. + * Creates a comparator for sorting descriptors by internal type id and field name/id. TypeDef + * descriptors preserve native internal ids as decimal type names; compare them numerically so ids + * such as 101 do not sort before 17. */ public Comparator createTypeAndNameComparator() { return (d1, d2) -> { // sort by type so that we can hit class info cache more possibly. // sort by field id/name to fix order if type is same. - // Use normalized type name so that Collection/Map subtypes have consistent order - // between processes even if the field doesn't exist in peer (e.g., List vs Collection). - int c = getNormalizedTypeName(d1).compareTo(getNormalizedTypeName(d2)); + int c = Integer.compare(getDescriptorSortTypeId(d1), getDescriptorSortTypeId(d2)); + if (c == 0) { + // Use normalized type name so that Collection/Map subtypes have consistent order + // between processes even if the field doesn't exist in peer (e.g., List vs Collection). + c = getNormalizedTypeName(d1).compareTo(getNormalizedTypeName(d2)); + } // noinspection Duplicates if (c == 0) { c = compareFieldSortKey(d1, d2); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 18a4b31662..c28fa6231c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -84,6 +84,7 @@ import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; import org.apache.fory.serializer.MetaSharedSerializer; import org.apache.fory.serializer.ObjectSerializer; +import org.apache.fory.serializer.PrimitiveSerializers; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.SerializerFactory; import org.apache.fory.serializer.Serializers; @@ -1191,22 +1192,75 @@ final Class loadClass( public abstract Serializer getSerializer(Class cls); public final Serializer getSerializer(TypeRef typeRef) { - if (!isCrossLanguage()) { - return getSerializer(typeRef.getRawType()); - } + Class rawType = typeRef.getRawType(); TypeExtMeta typeExtMeta = typeRef.getTypeExtMeta(); if (typeExtMeta != null && typeExtMeta.typeId() != Types.UNKNOWN - && !Types.isUserDefinedType((byte) typeExtMeta.typeId())) { - TypeInfo typeInfo = getInternalTypeInfoByTypeId(typeExtMeta.typeId()); - if (typeInfo != null && typeInfo.getSerializer() != null) { - return typeInfo.getSerializer(); + && !Types.isUserDefinedType(typeExtMeta.typeId())) { + if (isCrossLanguage()) { + TypeInfo typeInfo = getInternalTypeInfoByTypeId(typeExtMeta.typeId()); + if (typeInfo != null && typeInfo.getSerializer() != null) { + return typeInfo.getSerializer(); + } + } else { + Serializer serializer = getNativeTypedValueSerializer(typeExtMeta.typeId(), rawType); + if (serializer != null) { + return serializer; + } } } - Class rawType = typeRef.getRawType(); return getSerializer(rawType); } + private Serializer getNativeTypedValueSerializer(int typeId, Class rawType) { + // Native TypeExtMeta on a field wrapper is schema metadata, but on primitive-list elements it + // is the element wire type. These type ids are shared by native and xlang modes; wider built-in + // ids such as DECIMAL/BINARY are intentionally left to the native declared/raw type serializer. + switch (typeId) { + case Types.BOOL: + case Types.STRING: + return getSerializer(rawType); + case Types.INT8: + return new PrimitiveSerializers.ByteSerializer(config, rawType); + case Types.UINT8: + return new PrimitiveSerializers.UInt8Serializer(config); + case Types.INT16: + return new PrimitiveSerializers.ShortSerializer(config, rawType); + case Types.UINT16: + return new PrimitiveSerializers.UInt16Serializer(config); + case Types.INT32: + return new PrimitiveSerializers.FixedInt32Serializer(config); + case Types.VARINT32: + return new PrimitiveSerializers.VarInt32Serializer(config); + case Types.UINT32: + return new PrimitiveSerializers.FixedUInt32Serializer(config); + case Types.VAR_UINT32: + return new PrimitiveSerializers.VarUInt32Serializer(config); + case Types.INT64: + return new PrimitiveSerializers.FixedInt64Serializer(config); + case Types.VARINT64: + return new PrimitiveSerializers.VarInt64Serializer(config); + case Types.TAGGED_INT64: + return new PrimitiveSerializers.TaggedInt64Serializer(config); + case Types.UINT64: + return new PrimitiveSerializers.FixedUInt64Serializer(config); + case Types.VAR_UINT64: + return new PrimitiveSerializers.VarUInt64Serializer(config); + case Types.TAGGED_UINT64: + return new PrimitiveSerializers.TaggedUInt64Serializer(config); + case Types.FLOAT16: + return new PrimitiveSerializers.Float16Serializer(config, rawType); + case Types.BFLOAT16: + return new PrimitiveSerializers.BFloat16Serializer(config, rawType); + case Types.FLOAT32: + return new PrimitiveSerializers.FloatSerializer(config, rawType); + case Types.FLOAT64: + return new PrimitiveSerializers.DoubleSerializer(config, rawType); + default: + return null; + } + } + public abstract Serializer getRawSerializer(Class cls); public abstract void setSerializer(Class cls, Serializer serializer); @@ -1555,7 +1609,8 @@ protected final Class getStaticGeneratedStructSerializerCl if (!cls.isAnnotationPresent(ForyStruct.class)) { return null; } - String generatedName = cls.getName() + "__ForyStaticSerializer__"; + String generatedName = + cls.getName() + (isCrossLanguage() ? "__ForySerializer__" : "__ForyNativeSerializer__"); Class serializerClass = loadStaticGeneratedStructSerializerClass(cls, generatedName); if (serializerClass == null) { return null; @@ -1716,7 +1771,7 @@ public Comparator getPrimitiveComparator() { if ((t1Compress && t2Compress) || (!t1Compress && !t2Compress)) { int c = getPrimitiveFieldSize(d2) - getPrimitiveFieldSize(d1); if (c == 0) { - c = isCrossLanguage() ? typeId1 - typeId2 : typeId2 - typeId1; + c = typeId1 - typeId2; // noinspection Duplicates if (c == 0) { c = compareFieldSortKey(d1, d2); @@ -1764,11 +1819,13 @@ private int getPrimitiveFieldSize(Descriptor descriptor) { *

  • Otherwise: return true only for Optional types, false for all other non-primitives * * - *

    For native mode: use descriptor's nullable which defaults to true for non-primitives. + *

    For native mode: reflected value fields are nullable by default. Descriptors without a + * backing field already carry schema-owned nullability, for example TypeDef descriptors and + * annotation-processor generated native descriptors. * - *

    Important: This ensures the serialization format matches what the TypeDef metadata says. The - * TypeDef uses xlang defaults (nullable=false except for Optional types), so the actual - * serialization must use the same defaults to ensure consistency across languages. + *

    Important: this must match the TypeDef metadata for the same descriptor source. Xlang local + * descriptors use xlang defaults, native reflected descriptors use native nullable-by-default + * semantics, and descriptors rebuilt from TypeDef preserve the remote schema bit. */ private boolean isFieldNullable(Descriptor descriptor) { Class rawType = descriptor.getTypeRef().getRawType(); @@ -1785,8 +1842,10 @@ private boolean isFieldNullable(Descriptor descriptor) { // Default for xlang: false for all non-primitives, except Optional types return TypeUtils.isOptionalType(rawType); } - // For native mode: use descriptor's nullable (true for non-primitives by default) - return descriptor.isNullable(); + if (descriptor.getField() == null) { + return descriptor.isNullable(); + } + return true; } // thread safe diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 26b4153e06..9c3f9ec797 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -212,11 +212,15 @@ public static final class SerializationFieldInfo { // This determines how to write the value to the object (UnsafeOps.putInt vs putObject). isPrimitiveField = typeRef.getRawType().isPrimitive(); fieldConverter = d.getFieldConverter(); - // Primitive-list carrier TypeExtMeta describes the element wire type, not the field - // nullability/ref mode. Treating it as field metadata writes an extra null marker that - // remote TypeDef payload readers then consume as list length. + // TypeExtMeta is xlang field-wrapper metadata. Native local descriptors keep native + // nullable-by-default field semantics on the descriptor, while remote TypeDef descriptors + // already carry their schema nullability there. Primitive-list carrier TypeExtMeta is also + // element wire metadata, not field-wrapper metadata. TypeExtMeta extMeta = typeRef.getTypeExtMeta(); - if (extMeta != null && !primitiveListArray && !primitiveListCollection) { + if (resolver.isCrossLanguage() + && extMeta != null + && !primitiveListArray + && !primitiveListCollection) { nullable = extMeta.nullable(); trackingRef = extMeta.trackingRef(); } else { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java index bd6bb3e848..fbe00f1c71 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java @@ -185,6 +185,22 @@ public Integer read(ReadContext readContext) { } } + public static final class VarInt32Serializer extends Serializer implements Shareable { + public VarInt32Serializer(Config config) { + super(config, Integer.class); + } + + @Override + public void write(WriteContext writeContext, Integer value) { + writeContext.getBuffer().writeVarInt32(value); + } + + @Override + public Integer read(ReadContext readContext) { + return readContext.getBuffer().readVarInt32(); + } + } + public static final class FixedUInt32Serializer extends ImmutableSerializer implements Shareable { public FixedUInt32Serializer(Config config) { @@ -308,6 +324,22 @@ public Long read(ReadContext readContext) { } } + public static final class VarInt64Serializer extends Serializer implements Shareable { + public VarInt64Serializer(Config config) { + super(config, Long.class); + } + + @Override + public void write(WriteContext writeContext, Long value) { + writeContext.getBuffer().writeVarInt64(value); + } + + @Override + public Long read(ReadContext readContext) { + return readContext.getBuffer().readVarInt64(); + } + } + public static final class TaggedInt64Serializer extends Serializer implements Shareable { public TaggedInt64Serializer(Config config) { super(config, Long.class); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index fcd26ac155..9513f8fde8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -60,12 +60,7 @@ public StaticGeneratedStructSerializer(TypeResolver typeResolver, Class type) @SuppressWarnings("unchecked") protected StaticGeneratedStructSerializer( TypeResolver typeResolver, Class type, TypeDef typeDef, List descriptors) { - super(typeResolver, (Class) type); - List runtimeDescriptors = runtimeDescriptors(descriptors); - this.typeDef = typeDef; - this.remoteFields = - typeDef == null ? Collections.emptyList() : buildRemoteFields(typeDef, runtimeDescriptors); - this.localFieldsById = buildLocalFieldsById(runtimeDescriptors); + this(typeResolver, type, typeDef, descriptors, null); } @SuppressWarnings("unchecked") @@ -73,15 +68,15 @@ protected StaticGeneratedStructSerializer( TypeResolver typeResolver, Class type, TypeDef typeDef, - List localDescriptors, - List remoteDescriptors) { + List descriptors, + Class remoteDescriptorClass) { super(typeResolver, (Class) type); - List runtimeDescriptors = runtimeDescriptors(localDescriptors); + List runtimeDescriptors = runtimeDescriptors(descriptors); this.typeDef = typeDef; this.remoteFields = typeDef == null ? Collections.emptyList() - : buildRemoteFields(typeDef, runtimeDescriptors, remoteDescriptors); + : buildRemoteFields(typeDef, runtimeDescriptors, remoteDescriptorClass); this.localFieldsById = buildLocalFieldsById(runtimeDescriptors); } @@ -268,6 +263,75 @@ protected final void setConvertedField( remoteField.serializationFieldInfo.fieldConverter.set(targetObject, fieldValue); } + protected final void debugWriteField( + String stage, SerializationFieldInfo fieldInfo, WriteContext writeContext) { + if (!org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { + return; + } + MemoryBuffer buffer = writeContext.getBuffer(); + System.out.println( + "[ForyStruct] " + + stage + + " write " + + type.getName() + + "." + + fieldInfo.descriptor.getName() + + " localId=" + + localFieldId(fieldInfo.descriptor) + + " type=" + + fieldInfo.typeRef + + " writerIndex=" + + buffer.writerIndex()); + } + + protected final void debugReadField( + String stage, SerializationFieldInfo fieldInfo, ReadContext readContext) { + if (!org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { + return; + } + MemoryBuffer buffer = readContext.getBuffer(); + System.out.println( + "[ForyStruct] " + + stage + + " read " + + type.getName() + + "." + + fieldInfo.descriptor.getName() + + " localId=" + + localFieldId(fieldInfo.descriptor) + + " type=" + + fieldInfo.typeRef + + " readerIndex=" + + buffer.readerIndex()); + } + + protected final void debugRemoteReadField( + String stage, RemoteFieldInfo remoteField, ReadContext readContext) { + if (!org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { + return; + } + MemoryBuffer buffer = readContext.getBuffer(); + System.out.println( + "[ForyStruct] " + + stage + + " compatible read " + + type.getName() + + " remote=" + + remoteField.fieldInfo.getDefinedClass() + + "." + + remoteField.fieldInfo.getFieldName() + + " remoteFieldId=" + + remoteField.fieldInfo.getFieldId() + + " matchedId=" + + remoteField.matchedId + + " descriptor=" + + remoteField.descriptor.getName() + + " type=" + + remoteField.serializationFieldInfo.typeRef + + " readerIndex=" + + buffer.readerIndex()); + } + protected final Object copyFieldValue( CopyContext copyContext, Object fieldValue, SerializationFieldInfo fieldInfo) { if (fieldInfo.containerSerializerOverride != null) { @@ -303,33 +367,16 @@ private Object readField(ReadContext readContext, SerializationFieldInfo fieldIn } private List buildRemoteFields( - TypeDef remoteTypeDef, List localDescriptors) { - return buildRemoteFields(remoteTypeDef, localDescriptors, null); - } - - private List buildRemoteFields( - TypeDef remoteTypeDef, - List localDescriptors, - List generatedRemoteDescriptors) { - Class remoteDescriptorClass = remoteDescriptorClass(remoteTypeDef); + TypeDef remoteTypeDef, List localDescriptors, Class generatedRemoteClass) { + Class remoteDescriptorClass = remoteDescriptorClass(remoteTypeDef, generatedRemoteClass); List remoteFieldInfos = remoteTypeDef.getFieldsInfo(); - List remoteDescriptors; - if (generatedRemoteDescriptors == null) { - remoteDescriptors = remoteTypeDef.getDescriptors(typeResolver, remoteDescriptorClass); - } else { - remoteDescriptors = generatedRemoteDescriptors; - } + List remoteDescriptors = + remoteTypeDef.getDescriptors(typeResolver, remoteDescriptorClass); Map remoteFieldInfosByKey = new HashMap<>(); - if (generatedRemoteDescriptors == null) { - for (int i = 0; i < remoteFieldInfos.size(); i++) { - FieldInfo fieldInfo = remoteFieldInfos.get(i); - Descriptor descriptor = remoteDescriptors.get(i); - putRemoteFieldInfo(remoteFieldInfosByKey, fieldInfo, descriptor); - } - } else { - for (FieldInfo fieldInfo : remoteFieldInfos) { - putRemoteFieldInfo(remoteFieldInfosByKey, fieldInfo); - } + for (int i = 0; i < remoteFieldInfos.size(); i++) { + FieldInfo fieldInfo = remoteFieldInfos.get(i); + Descriptor descriptor = remoteDescriptors.get(i); + putRemoteFieldInfo(remoteFieldInfosByKey, fieldInfo, descriptor); } Map fieldIds = new HashMap<>(); Map fields = new HashMap<>(); @@ -343,18 +390,10 @@ private List buildRemoteFields( // Keep compatible-read descriptor ordering owned by TypeResolver, matching the sorted // DescriptorGrouper order used by ObjectCodecBuilder and MetaSharedCodecBuilder. FieldGroups // may regroup descriptors for helper ownership, so it must not drive remote payload order. - List remoteDescriptorsInWireOrder; - if (generatedRemoteDescriptors == null) { - remoteDescriptorsInWireOrder = - typeResolver - .createDescriptorGrouper(remoteTypeDef, remoteDescriptorClass) - .getSortedDescriptors(); - } else { - remoteDescriptorsInWireOrder = - typeResolver - .groupDescriptors(generatedRemoteDescriptors, false, descriptor -> descriptor) - .getSortedDescriptors(); - } + List remoteDescriptorsInWireOrder = + typeResolver + .createDescriptorGrouper(remoteTypeDef, remoteDescriptorClass) + .getSortedDescriptors(); List remoteFields = new ArrayList<>(remoteFieldInfos.size()); appendRemoteFields( remoteFields, @@ -366,7 +405,15 @@ private List buildRemoteFields( return Collections.unmodifiableList(remoteFields); } - private Class remoteDescriptorClass(TypeDef remoteTypeDef) { + private Class remoteDescriptorClass(TypeDef remoteTypeDef, Class generatedRemoteClass) { + if (generatedRemoteClass != null) { + // Native TypeDefs for registered classes carry the registered id, so a reader that binds the + // same id to an evolved class decodes the TypeDef as the local class. Static-compatible + // codegen may still know the writer-side class; use it to preserve descriptor-only details + // such as primitive-list carrier raw types while keeping wire order in + // createDescriptorGrouper. + return generatedRemoteClass; + } String className = remoteTypeDef.getClassName(); if (className.equals(type.getName())) { return type; @@ -391,12 +438,12 @@ private void appendRemoteFields( Map fields, List localDescriptors) { for (Descriptor descriptor : remoteDescriptorsInWireOrder) { - SerializationFieldInfo serializationFieldInfo = - FieldGroups.buildFieldInfo(typeResolver, descriptor); FieldInfo fieldInfo = remoteFieldInfosByKey.get(remoteFieldKey(descriptor)); if (fieldInfo == null) { throw new IllegalStateException("Missing remote field metadata for " + descriptor); } + SerializationFieldInfo serializationFieldInfo = + FieldGroups.buildFieldInfo(typeResolver, descriptor); int matchedId = matchField(fieldInfo, fieldIds, fields); Descriptor localDescriptor = matchedId == UNKNOWN_FIELD ? null : localDescriptors.get(matchedId); @@ -445,6 +492,10 @@ private static String fieldKey(Descriptor descriptor) { return descriptor.getDeclaringClass() + "." + descriptor.getName(); } + private static int localFieldId(Descriptor descriptor) { + return descriptor.hasForyFieldId() ? descriptor.getForyFieldId() : -1; + } + private static void putRemoteFieldInfo( Map remoteFieldInfosByKey, FieldInfo fieldInfo, Descriptor descriptor) { remoteFieldInfosByKey.put(remoteFieldKey(descriptor), fieldInfo); @@ -453,12 +504,6 @@ private static void putRemoteFieldInfo( } } - private static void putRemoteFieldInfo( - Map remoteFieldInfosByKey, FieldInfo fieldInfo) { - remoteFieldInfosByKey.put(remoteFieldKey(fieldInfo), fieldInfo); - remoteFieldInfosByKey.put("name:" + fieldInfo.getFieldName(), fieldInfo); - } - private static String remoteFieldKey(FieldInfo fieldInfo) { return fieldInfo.hasFieldId() ? "id:" + fieldInfo.getFieldId() diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Types.java b/java/fory-core/src/main/java/org/apache/fory/type/Types.java index 8760f0607a..23910a439c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Types.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Types.java @@ -415,7 +415,7 @@ public static int getDescriptorTypeId(TypeResolver resolver, Descriptor d) { } TypeRef typeRef = d.getTypeRef(); Class rawType = typeRef.getRawType(); - if (resolver.isCrossLanguage() && TypeUtils.isPrimitiveListClass(rawType)) { + if (TypeUtils.isPrimitiveListClass(rawType)) { if (TypeAnnotationUtils.isArrayType(d)) { return TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType); } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index ec74a5d53a..8d59f6b056 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -311,7 +311,7 @@ protected static void assertStructEvolvingOverride(Fory fory) { fixedSerializer.getClass().getName()); Assert.assertEquals( fixedSerializer.getClass().getName(), - FixedOverrideStruct.class.getName() + "__ForyStaticSerializer__"); + FixedOverrideStruct.class.getName() + "__ForySerializer__"); } EvolvingOverrideStruct evolving = newEvolvingOverrideStruct(); diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java index 0c897495ea..044040272c 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java @@ -88,6 +88,20 @@ public static Object[][] xlangModes() { return new Object[][] {{false}, {true}}; } + @DataProvider + public static Object[][] runtimeModes() { + return new Object[][] { + {false, false, false}, + {true, false, false}, + {false, false, true}, + {true, false, true}, + {false, true, false}, + {true, true, false}, + {false, true, true}, + {true, true, true} + }; + } + @Test(dataProvider = "modes") public void testStaticClass(boolean xlang, boolean compatible) throws Exception { ExampleMessage value = newStruct(ExampleMessage.class); @@ -125,6 +139,22 @@ public void testCompatibleClassEvolution(boolean xlang) throws Exception { Assert.assertEquals(result.added, "default"); } + @Test(dataProvider = "runtimeModes") + public void testRuntimeCompatibleEmptyReaderSkipsExampleMessage( + boolean xlang, boolean writerCodegen, boolean readerCodegen) throws Exception { + RuntimeExampleMessage value = newStruct(RuntimeExampleMessage.class); + Fory writer = fory(RuntimeExampleMessage.class, xlang, true, writerCodegen); + Fory emptyReader = fory(RuntimeEmptyMessage.class, xlang, true, readerCodegen); + + writer.setMetaWriteContext(new MetaWriteContext()); + byte[] bytes = writer.serialize(value); + assertNotStaticSerializer(writer, RuntimeExampleMessage.class); + + emptyReader.setMetaReadContext(new MetaReadContext()); + Assert.assertNotNull(emptyReader.deserialize(bytes, RuntimeEmptyMessage.class)); + assertNotStaticSerializer(emptyReader, RuntimeEmptyMessage.class); + } + @Test(dataProvider = "xlangModes") public void testCompatibleRecordEvolution(boolean xlang) throws Exception { ExampleRecordMessage value = newRecord(ExampleRecordMessage.class); @@ -189,6 +219,14 @@ public static class EmptyMessage { public EmptyMessage() {} } + public static class RuntimeEmptyMessage { + public RuntimeEmptyMessage() {} + } + + public static class RuntimeExampleMessage extends ExampleMessage { + public RuntimeExampleMessage() {} + } + @ForyStruct public static class InconsistentMessage { @ForyField(id = 4) @@ -275,6 +313,15 @@ private static void assertStaticSerializer(ThreadSafeFory fory, Class type) { serializer instanceof StaticGeneratedStructSerializer, serializer.getClass().getName()); } + private static void assertNotStaticSerializer(Fory fory, Class type) { + Serializer serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); + if (serializer instanceof DeferedLazySerializer) { + serializer = ((DeferedLazySerializer) serializer).resolveSerializer(); + } + Assert.assertFalse( + serializer instanceof StaticGeneratedStructSerializer, serializer.getClass().getName()); + } + private static byte[] serialize(ThreadSafeFory fory, Object value) { return fory.execute( f -> { From b150ed602dd5d7ee50775e8938bb35d5464c3677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 02:58:51 +0800 Subject: [PATCH 24/58] fix(java): classify compatible primitive array payloads --- .../CompatibleCollectionArrayReader.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index 185ec4ab6b..7143273a00 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -257,9 +257,19 @@ private static int listElementTypeId(TypeRef typeRef) { return elementExtMeta == null ? Types.UNKNOWN : elementExtMeta.typeId(); } if (TypeUtils.isPrimitiveListClass(typeRef.getRawType())) { - return extMeta != null && Types.isPrimitiveType(extMeta.typeId()) - ? extMeta.typeId() - : TypeAnnotationUtils.getDefaultPrimitiveListElementTypeId(typeRef.getRawType()); + if (extMeta != null) { + int typeId = extMeta.typeId(); + if (Types.isPrimitiveArray(typeId)) { + // A compatible descriptor can keep the local primitive-list raw carrier while the remote + // TypeDef says the peer wrote a dense array payload. Treat the TypeExtMeta as the remote + // wire shape here; otherwise array->list reads are misclassified as list->list reads. + return Types.UNKNOWN; + } + if (Types.isPrimitiveType(typeId)) { + return typeId; + } + } + return TypeAnnotationUtils.getDefaultPrimitiveListElementTypeId(typeRef.getRawType()); } return Types.UNKNOWN; } From 0b85f159361b2b69536ab9680ee4004b3ba18f9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 03:07:00 +0800 Subject: [PATCH 25/58] chore(java): satisfy descriptor helper checkstyle --- .../java/org/apache/fory/resolver/TypeResolver.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index c28fa6231c..c6205b0873 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1550,12 +1550,6 @@ private List buildFieldDescriptors(Class clz, boolean searchParen return buildFieldDescriptors(clz, searchParent, descriptors); } - @Internal - public final List normalizeFieldDescriptors( - Class clz, boolean searchParent, List descriptors) { - return buildFieldDescriptors(clz, searchParent, descriptors); - } - private List buildFieldDescriptors( Class clz, boolean searchParent, List descriptors) { List result = new ArrayList<>(descriptors.size()); @@ -1601,6 +1595,12 @@ private List buildFieldDescriptors( return result; } + @Internal + public final List normalizeFieldDescriptors( + Class clz, boolean searchParent, List descriptors) { + return buildFieldDescriptors(clz, searchParent, descriptors); + } + protected final Class getStaticGeneratedStructSerializerClass( Class cls) { if (GraalvmSupport.isGraalBuildTime() || GraalvmSupport.isGraalRuntime()) { From 28f6e81537882ea748b59e7171a814af95f0ca5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 03:55:11 +0800 Subject: [PATCH 26/58] fix(java): preserve compatible collection wire shape --- .../CompatibleCollectionArrayReader.java | 51 +++++++++++++- .../java/org/apache/fory/type/Descriptor.java | 4 +- .../fory/type/DescriptorGrouperTest.java | 68 +++++++++---------- 3 files changed, 84 insertions(+), 39 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index 7143273a00..d6f4b24e5b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -86,7 +86,7 @@ static ReadAction readAction(TypeResolver resolver, Descriptor descriptor) { return null; } FieldTypes.FieldType localFieldType = FieldTypes.buildFieldType(resolver, field); - int peerListElementTypeId = listElementTypeId(descriptor.getTypeRef()); + int peerListElementTypeId = listElementTypeId(descriptor); if (peerListElementTypeId != Types.UNKNOWN) { int localArrayTypeId = arrayTypeId(localFieldType); if (localArrayTypeId != Types.UNKNOWN @@ -104,7 +104,7 @@ static ReadAction readAction(TypeResolver resolver, Descriptor descriptor) { } return null; } - int peerArrayTypeId = arrayTypeId(descriptor.getTypeRef()); + int peerArrayTypeId = arrayTypeId(descriptor); if (peerArrayTypeId != Types.UNKNOWN) { int localListElementTypeId = listElementTypeId(localFieldType); if (localListElementTypeId != Types.UNKNOWN @@ -250,6 +250,42 @@ private static int listElementTypeId(FieldTypes.FieldType fieldType) { return Types.UNKNOWN; } + private static int listElementTypeId(Descriptor descriptor) { + Class rawType = descriptor.getRawType(); + if (TypeUtils.isPrimitiveListClass(rawType) && TypeAnnotationUtils.isArrayType(descriptor)) { + return Types.UNKNOWN; + } + TypeRef typeRef = descriptor.getTypeRef(); + TypeExtMeta extMeta = typeRef.getTypeExtMeta(); + if (TypeUtils.isPrimitiveListClass(rawType)) { + if (extMeta != null) { + int typeId = extMeta.typeId(); + if (Types.isPrimitiveArray(typeId)) { + // A compatible descriptor can keep the local primitive-list carrier while the remote + // TypeDef says the peer wrote a dense array payload. Treat the TypeExtMeta as the remote + // wire shape here; otherwise array->list reads are misclassified as list->list reads. + return Types.UNKNOWN; + } + if (Types.isPrimitiveType(typeId)) { + return typeId; + } + } + TypeRef elementTypeRef = TypeAnnotationUtils.getPrimitiveListElementTypeRef(descriptor); + if (elementTypeRef != null) { + TypeExtMeta elementExtMeta = elementTypeRef.getTypeExtMeta(); + if (elementExtMeta != null && Types.isPrimitiveType(elementExtMeta.typeId())) { + return elementExtMeta.typeId(); + } + } + return Types.UNKNOWN; + } + if (extMeta != null && extMeta.typeId() == Types.LIST) { + TypeExtMeta elementExtMeta = TypeUtils.getElementType(typeRef).getTypeExtMeta(); + return elementExtMeta == null ? Types.UNKNOWN : elementExtMeta.typeId(); + } + return Types.UNKNOWN; + } + private static int listElementTypeId(TypeRef typeRef) { TypeExtMeta extMeta = typeRef.getTypeExtMeta(); if (extMeta != null && extMeta.typeId() == Types.LIST) { @@ -274,6 +310,17 @@ private static int listElementTypeId(TypeRef typeRef) { return Types.UNKNOWN; } + private static int arrayTypeId(Descriptor descriptor) { + Class rawType = descriptor.getRawType(); + if (TypeUtils.isPrimitiveListClass(rawType) && TypeAnnotationUtils.isArrayType(descriptor)) { + return TypeAnnotationUtils.getPrimitiveListArrayTypeId(rawType); + } + if (TypeAnnotationUtils.isBoxedListArrayType(descriptor)) { + return TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor); + } + return arrayTypeId(descriptor.getTypeRef()); + } + private static int arrayTypeId(FieldTypes.FieldType fieldType) { if (fieldType instanceof FieldTypes.RegisteredFieldType) { int typeId = ((FieldTypes.RegisteredFieldType) fieldType).getTypeId(); diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java index a189578537..732ba4bac7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java @@ -219,7 +219,9 @@ public Descriptor( } else { this.nullable = nullable; } - this.trackingRef = hasForyField && trackingRef; + // Synthetic descriptors created from remote TypeDef fields must preserve schema-owned wrapper + // ref tracking even when the field has no tag id/@ForyField metadata. + this.trackingRef = trackingRef; } private Descriptor(Field field, Method readMethod) { diff --git a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java index 95fc8701d4..b1e6b05b7b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java @@ -83,25 +83,25 @@ public void testComparatorByTypeAndName() { List> expected = Arrays.asList( boolean.class, - byte.class, - char.class, - double.class, - float.class, - int.class, Boolean.class, + byte.class, Byte.class, - Character.class, - Double.class, - Float.class, + Short.class, + short.class, Integer.class, + int.class, Long.class, - Object.class, - Short.class, + long.class, + float.class, + Float.class, + double.class, + Double.class, String.class, Void.class, - long.class, - short.class, - void.class); + Character.class, + void.class, + char.class, + Object.class); assertEquals(classes, expected); } @@ -118,16 +118,15 @@ public void testPrimitiveComparator() { List> classes = descriptors.stream().map(Descriptor::getRawType).collect(Collectors.toList()); // With compression enabled (default): int/long are compressed and go to the end - // Non-compressed sorted by size (desc), then typeId (desc): char(25) > short(3), byte(2) > - // boolean(1) + // Non-compressed sorted by size (desc), then typeId (asc), matching xlang field order. List> expected = Arrays.asList( double.class, float.class, - char.class, short.class, - byte.class, + char.class, boolean.class, + byte.class, void.class, long.class, int.class); @@ -147,16 +146,15 @@ public void testPrimitiveCompressedComparator() { List> classes = descriptors.stream().map(Descriptor::getRawType).collect(Collectors.toList()); // With compression enabled (default): int/long are compressed and go to the end - // Non-compressed sorted by size (desc), then typeId (desc): char(25) > short(3), byte(2) > - // boolean(1) + // Non-compressed sorted by size (desc), then typeId (asc), matching xlang field order. List> expected = Arrays.asList( double.class, float.class, - char.class, short.class, - byte.class, + char.class, boolean.class, + byte.class, void.class, long.class, int.class); @@ -219,15 +217,15 @@ public void testGrouper() { grouper.getPrimitiveDescriptors().stream() .map(Descriptor::getRawType) .collect(Collectors.toList()); - // With compression enabled: int/long go to end, sorted by size then typeId (desc) + // With compression enabled: int/long go to end, sorted by size then typeId (asc) List> expected = Arrays.asList( double.class, float.class, - char.class, short.class, - byte.class, + char.class, boolean.class, + byte.class, void.class, long.class, int.class); @@ -238,15 +236,15 @@ public void testGrouper() { grouper.getBoxedDescriptors().stream() .map(Descriptor::getRawType) .collect(Collectors.toList()); - // With compression enabled: Integer/Long go to end, sorted by size then typeId (desc) + // With compression enabled: Integer/Long go to end, sorted by size then typeId (asc) List> expected = Arrays.asList( Double.class, Float.class, - Character.class, Short.class, - Byte.class, + Character.class, Boolean.class, + Byte.class, Void.class, Long.class, Integer.class); @@ -284,7 +282,7 @@ public void testGrouper() { grouper.getOtherDescriptors().stream() .map(Descriptor::getRawType) .collect(Collectors.toList()); - assertEquals(classes, Arrays.asList(Object.class, Object.class, Date.class)); + assertEquals(classes, Arrays.asList(Date.class, Object.class, Object.class)); } } @@ -305,16 +303,15 @@ public void testCompressedPrimitiveGrouper() { grouper.getPrimitiveDescriptors().stream() .map(Descriptor::getRawType) .collect(Collectors.toList()); - // With compression enabled: int/long go to end, sorted by size then typeId (desc) - // char has higher typeId (25) than short (3) + // With compression enabled: int/long go to end, sorted by size then typeId (asc) List> expected = Arrays.asList( double.class, float.class, - char.class, short.class, - byte.class, + char.class, boolean.class, + byte.class, void.class, long.class, int.class); @@ -325,16 +322,15 @@ public void testCompressedPrimitiveGrouper() { grouper.getBoxedDescriptors().stream() .map(Descriptor::getRawType) .collect(Collectors.toList()); - // With compression enabled: Integer/Long go to end, sorted by size then typeId (desc) - // Character has higher typeId than Short + // With compression enabled: Integer/Long go to end, sorted by size then typeId (asc) List> expected = Arrays.asList( Double.class, Float.class, - Character.class, Short.class, - Byte.class, + Character.class, Boolean.class, + Byte.class, Void.class, Long.class, Integer.class); From 7ca8bfd1eb7be56e8660d415852c0228af3adaef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 04:22:18 +0800 Subject: [PATCH 27/58] fix(java): preserve compatible list metadata --- .../java/org/apache/fory/meta/FieldTypes.java | 13 ++-- .../apache/fory/resolver/TypeResolver.java | 10 ++- .../CompatibleCollectionArrayReader.java | 66 ++++++++++++++++--- .../apache/fory/type/TypeAnnotationUtils.java | 5 +- 4 files changed, 73 insertions(+), 21 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 7e5adaeb6e..0fbe8a1943 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -224,13 +224,12 @@ private static FieldType buildFieldType( nullable = !genericType.getCls().isPrimitive(); } - // Apply @ForyField annotation where the protocol uses it as field-wrapper metadata. Native - // reflected value fields stay nullable by default; generated/remote descriptors without a - // backing Field already carry the schema-owned value. + // Native unannotated value fields are nullable by default, but explicit @ForyField options are + // still field-wrapper metadata in both native and xlang TypeDef. Compatible serializers are + // built from TypeDef descriptors, so dropping this bit makes the writer emit a different field + // payload shape from the schema it advertised. if (descriptor != null && descriptor.hasForyField()) { - if (isXlang || descriptorCarriesFieldOptions) { - nullable = descriptor.isNullable(); - } + nullable = descriptor.isNullable(); trackingRef = descriptor.isTrackingRef(); } @@ -244,7 +243,7 @@ private static FieldType buildFieldType( } if (primitiveList && !primitiveListArray) { - boolean elementNullable = true; + boolean elementNullable = false; boolean elementTrackingRef = false; if (primitiveListArgumentMeta != null) { elementNullable = primitiveListArgumentMeta.nullable(); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index c6205b0873..d2701f8f13 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1819,9 +1819,10 @@ private int getPrimitiveFieldSize(Descriptor descriptor) { *

  • Otherwise: return true only for Optional types, false for all other non-primitives * * - *

    For native mode: reflected value fields are nullable by default. Descriptors without a - * backing field already carry schema-owned nullability, for example TypeDef descriptors and - * annotation-processor generated native descriptors. + *

    For native mode: reflected value fields are nullable by default unless @ForyField gives an + * explicit field-wrapper nullability. Descriptors without a backing field already carry + * schema-owned nullability, for example TypeDef descriptors and annotation-processor generated + * native descriptors. * *

    Important: this must match the TypeDef metadata for the same descriptor source. Xlang local * descriptors use xlang defaults, native reflected descriptors use native nullable-by-default @@ -1842,6 +1843,9 @@ private boolean isFieldNullable(Descriptor descriptor) { // Default for xlang: false for all non-primitives, except Optional types return TypeUtils.isOptionalType(rawType); } + if (descriptor.hasForyField()) { + return descriptor.isNullable(); + } if (descriptor.getField() == null) { return descriptor.isNullable(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index d6f4b24e5b..11da62d732 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -94,9 +94,13 @@ static ReadAction readAction(TypeResolver resolver, Descriptor descriptor) { return new ReadAction( READ_LIST_TO_ARRAY, localArrayTypeId, peerListElementTypeId, field.getType()); } - int localListElementTypeId = listElementTypeId(localFieldType); + int localListElementTypeId = nonNullableListElementTypeId(localFieldType); int peerArrayTypeId = denseArrayTypeId(peerListElementTypeId); - if (localListElementTypeId != Types.UNKNOWN + // The list-to-list fast path materializes through a dense primitive array, so it cannot + // preserve nullable or ref-tracked peer elements. List-to-array still accepts nullable + // schemas above and rejects only actual null/ref payload flags while reading. + if (nonNullableListElementTypeId(descriptor) != Types.UNKNOWN + && localListElementTypeId != Types.UNKNOWN && peerArrayTypeId != Types.UNKNOWN && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { return new ReadAction( @@ -134,9 +138,13 @@ static ReadAction readAction( peerListElementTypeId, localDescriptor.getRawType()); } - int localListElementTypeId = listElementTypeId(localType); + int localListElementTypeId = nonNullableListElementTypeId(localType); int peerArrayTypeId = denseArrayTypeId(peerListElementTypeId); - if (localListElementTypeId != Types.UNKNOWN + // The list-to-list fast path materializes through a dense primitive array, so it cannot + // preserve nullable or ref-tracked peer elements. List-to-array still accepts nullable + // schemas above and rejects only actual null/ref payload flags while reading. + if (nonNullableListElementTypeId(remoteFieldType) != Types.UNKNOWN + && localListElementTypeId != Types.UNKNOWN && peerArrayTypeId != Types.UNKNOWN && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { return new ReadAction( @@ -238,6 +246,10 @@ private static Object readNotNull( } private static int listElementTypeId(FieldTypes.FieldType fieldType) { + return listElementTypeId(fieldType, false); + } + + private static int listElementTypeId(FieldTypes.FieldType fieldType, boolean requireNonNullable) { if (!(fieldType instanceof FieldTypes.CollectionFieldType) || fieldType.getTypeId() != Types.LIST) { return Types.UNKNOWN; @@ -245,12 +257,19 @@ private static int listElementTypeId(FieldTypes.FieldType fieldType) { FieldTypes.FieldType elementType = ((FieldTypes.CollectionFieldType) fieldType).getElementType(); if (elementType instanceof FieldTypes.RegisteredFieldType) { + if (requireNonNullable && (elementType.nullable() || elementType.trackingRef())) { + return Types.UNKNOWN; + } return ((FieldTypes.RegisteredFieldType) elementType).getTypeId(); } return Types.UNKNOWN; } private static int listElementTypeId(Descriptor descriptor) { + return listElementTypeId(descriptor, false); + } + + private static int listElementTypeId(Descriptor descriptor, boolean requireNonNullable) { Class rawType = descriptor.getRawType(); if (TypeUtils.isPrimitiveListClass(rawType) && TypeAnnotationUtils.isArrayType(descriptor)) { return Types.UNKNOWN; @@ -266,14 +285,15 @@ private static int listElementTypeId(Descriptor descriptor) { // wire shape here; otherwise array->list reads are misclassified as list->list reads. return Types.UNKNOWN; } - if (Types.isPrimitiveType(typeId)) { + if (Types.isPrimitiveType(typeId) + && (!requireNonNullable || (!extMeta.nullable() && !extMeta.trackingRef()))) { return typeId; } } TypeRef elementTypeRef = TypeAnnotationUtils.getPrimitiveListElementTypeRef(descriptor); if (elementTypeRef != null) { TypeExtMeta elementExtMeta = elementTypeRef.getTypeExtMeta(); - if (elementExtMeta != null && Types.isPrimitiveType(elementExtMeta.typeId())) { + if (isPrimitiveElement(elementExtMeta, requireNonNullable)) { return elementExtMeta.typeId(); } } @@ -281,16 +301,24 @@ private static int listElementTypeId(Descriptor descriptor) { } if (extMeta != null && extMeta.typeId() == Types.LIST) { TypeExtMeta elementExtMeta = TypeUtils.getElementType(typeRef).getTypeExtMeta(); - return elementExtMeta == null ? Types.UNKNOWN : elementExtMeta.typeId(); + return isPrimitiveElement(elementExtMeta, requireNonNullable) + ? elementExtMeta.typeId() + : Types.UNKNOWN; } return Types.UNKNOWN; } private static int listElementTypeId(TypeRef typeRef) { + return listElementTypeId(typeRef, false); + } + + private static int listElementTypeId(TypeRef typeRef, boolean requireNonNullable) { TypeExtMeta extMeta = typeRef.getTypeExtMeta(); if (extMeta != null && extMeta.typeId() == Types.LIST) { TypeExtMeta elementExtMeta = TypeUtils.getElementType(typeRef).getTypeExtMeta(); - return elementExtMeta == null ? Types.UNKNOWN : elementExtMeta.typeId(); + return isPrimitiveElement(elementExtMeta, requireNonNullable) + ? elementExtMeta.typeId() + : Types.UNKNOWN; } if (TypeUtils.isPrimitiveListClass(typeRef.getRawType())) { if (extMeta != null) { @@ -301,7 +329,8 @@ private static int listElementTypeId(TypeRef typeRef) { // wire shape here; otherwise array->list reads are misclassified as list->list reads. return Types.UNKNOWN; } - if (Types.isPrimitiveType(typeId)) { + if (Types.isPrimitiveType(typeId) + && (!requireNonNullable || (!extMeta.nullable() && !extMeta.trackingRef()))) { return typeId; } } @@ -310,6 +339,25 @@ private static int listElementTypeId(TypeRef typeRef) { return Types.UNKNOWN; } + private static int nonNullableListElementTypeId(FieldTypes.FieldType fieldType) { + return listElementTypeId(fieldType, true); + } + + private static int nonNullableListElementTypeId(Descriptor descriptor) { + return listElementTypeId(descriptor, true); + } + + private static int nonNullableListElementTypeId(TypeRef typeRef) { + return listElementTypeId(typeRef, true); + } + + private static boolean isPrimitiveElement( + TypeExtMeta elementExtMeta, boolean requireNonNullable) { + return elementExtMeta != null + && Types.isPrimitiveType(elementExtMeta.typeId()) + && (!requireNonNullable || (!elementExtMeta.nullable() && !elementExtMeta.trackingRef())); + } + private static int arrayTypeId(Descriptor descriptor) { Class rawType = descriptor.getRawType(); if (TypeUtils.isPrimitiveListClass(rawType) && TypeAnnotationUtils.isArrayType(descriptor)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java index 397d19f4d9..0fad594909 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java @@ -544,7 +544,7 @@ public static TypeRef getPrimitiveListElementTypeRef( if (elementTypeId == Types.UNKNOWN || elementClass == null) { return null; } - return TypeRef.of(elementClass, TypeExtMeta.of(elementTypeId, true, false)); + return TypeRef.of(elementClass, TypeExtMeta.of(elementTypeId, false, false)); } public static TypeRef getPrimitiveListElementTypeRef(Descriptor descriptor) { @@ -554,7 +554,8 @@ public static TypeRef getPrimitiveListElementTypeRef(Descriptor descriptor) { Class elementClass = getPrimitiveListElementClass(typeRef.getRawType()); if (elementClass != null) { return TypeRef.of( - elementClass, TypeExtMeta.of(inlineMeta.typeId(), true, inlineMeta.trackingRef())); + elementClass, + TypeExtMeta.of(inlineMeta.typeId(), inlineMeta.nullable(), inlineMeta.trackingRef())); } } if (typeRef.hasExplicitTypeArguments()) { From b59f8f736cc0d703e2609e3e86e7d18a7f634f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 04:35:33 +0800 Subject: [PATCH 28/58] fix(go): align list element typedef nullability --- go/fory/field_spec.go | 43 ++++++++++++++++++++++++++++++++++++++++++- go/fory/tag_test.go | 21 +++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/go/fory/field_spec.go b/go/fory/field_spec.go index 68f15a69b7..ee19687d1e 100644 --- a/go/fory/field_spec.go +++ b/go/fory/field_spec.go @@ -145,7 +145,10 @@ func (t *TypeSpec) Clone() *TypeSpec { } func (t *TypeSpec) declaredNullable() bool { - if t != nil && t.hasDeclNull { + if t == nil { + return false + } + if t.hasDeclNull { return t.declNullable } return true @@ -158,6 +161,41 @@ func (t *TypeSpec) declaredTrackRef() bool { return false } +func isNonNullableScalarElementType(typeID TypeId) bool { + switch typeID { + case BOOL, + INT8, + INT16, + INT32, + VARINT32, + INT64, + VARINT64, + TAGGED_INT64, + UINT8, + UINT16, + UINT32, + VAR_UINT32, + UINT64, + VAR_UINT64, + TAGGED_UINT64, + FLOAT8, + FLOAT16, + BFLOAT16, + FLOAT32, + FLOAT64: + return true + default: + return false + } +} + +// Primitive list carriers use LIST payloads but still declare scalar elements +// as non-null. Map key/value primitive schemas come from boxed Java types and +// keep the default nullable nested TypeSpec unless a tag overrides it. +func projectsAsNonNullableListElement(t *TypeSpec) bool { + return t != nil && !t.hasDeclNull && !t.Nullable && isNonNullableScalarElementType(t.TypeID) +} + func (t *TypeSpec) typeDefProjection(preserveRootFlags bool) *TypeSpec { return t.typeDefProjectionWithMode(true, preserveRootFlags) } @@ -187,6 +225,9 @@ func (t *TypeSpec) typeDefProjectionWithMode(isRoot bool, preserveRootFlags bool } if t.Element != nil { projected.Element = t.Element.typeDefProjectionWithMode(false, preserveRootFlags) + if t.TypeID == LIST && projectsAsNonNullableListElement(t.Element) { + projected.Element.Nullable = false + } projected.elementType = projected.Element } if t.Key != nil { diff --git a/go/fory/tag_test.go b/go/fory/tag_test.go index 9d50631f1a..a6f92cdcb5 100644 --- a/go/fory/tag_test.go +++ b/go/fory/tag_test.go @@ -122,6 +122,8 @@ func TestFieldSpecSerializerSelection(t *testing.T) { U8List []uint8 `fory:"id=4"` U8Dense []uint8 `fory:"id=5,type=array(element=uint8)"` Bytes []byte `fory:"id=6,type=bytes"` + Strings []string `fory:"id=7"` + StringByID map[int32]string } f := New(WithXlang(true), WithCompatible(false)) @@ -135,6 +137,8 @@ func TestFieldSpecSerializerSelection(t *testing.T) { require.EqualValues(t, VARINT32, defaultListPrimitive.elemTypeID) require.EqualValues(t, LIST, defaultListSpec.Type.TypeId()) require.EqualValues(t, VARINT32, defaultListSpec.Type.Element.TypeId()) + require.False(t, defaultListSpec.Type.Element.Nullable) + require.False(t, defaultListSpec.Type.typeDefProjection(false).Element.Nullable) denseSpec := mustParseFieldSpec(t, typ.Field(1)) denseSerializer, err := serializerForTypeSpec(f.typeResolver, typ.Field(1).Type, denseSpec.Type) @@ -149,12 +153,16 @@ func TestFieldSpecSerializerSelection(t *testing.T) { require.True(t, ok) require.EqualValues(t, INT32, explicitPrimitive.elemTypeID) require.EqualValues(t, LIST, explicitSpec.Type.TypeId()) + require.False(t, explicitSpec.Type.Element.Nullable) + require.False(t, explicitSpec.Type.typeDefProjection(false).Element.Nullable) ptrsSpec := mustParseFieldSpec(t, typ.Field(3)) ptrsSerializer, err := serializerForTypeSpec(f.typeResolver, typ.Field(3).Type, ptrsSpec.Type) require.NoError(t, err) require.IsType(t, &sliceSerializer{}, ptrsSerializer) require.EqualValues(t, LIST, ptrsSpec.Type.TypeId()) + require.True(t, ptrsSpec.Type.Element.Nullable) + require.True(t, ptrsSpec.Type.typeDefProjection(false).Element.Nullable) u8ListSpec := mustParseFieldSpec(t, typ.Field(4)) u8ListSerializer, err := serializerForTypeSpec(f.typeResolver, typ.Field(4).Type, u8ListSpec.Type) @@ -164,6 +172,8 @@ func TestFieldSpecSerializerSelection(t *testing.T) { require.EqualValues(t, UINT8, u8Primitive.elemTypeID) require.EqualValues(t, LIST, u8ListSpec.Type.TypeId()) require.EqualValues(t, UINT8, u8ListSpec.Type.Element.TypeId()) + require.False(t, u8ListSpec.Type.Element.Nullable) + require.False(t, u8ListSpec.Type.typeDefProjection(false).Element.Nullable) u8DenseSpec := mustParseFieldSpec(t, typ.Field(5)) u8DenseSerializer, err := serializerForTypeSpec(f.typeResolver, typ.Field(5).Type, u8DenseSpec.Type) @@ -176,6 +186,17 @@ func TestFieldSpecSerializerSelection(t *testing.T) { require.NoError(t, err) require.IsType(t, encodedByteSliceSerializer{}, bytesSerializer) require.EqualValues(t, BINARY, bytesSpec.Type.TypeId()) + + stringsSpec := mustParseFieldSpec(t, typ.Field(7)) + require.EqualValues(t, LIST, stringsSpec.Type.TypeId()) + require.False(t, stringsSpec.Type.Element.Nullable) + require.True(t, stringsSpec.Type.typeDefProjection(false).Element.Nullable) + + mapSpec := mustParseFieldSpec(t, typ.Field(8)) + mapProjection := mapSpec.Type.typeDefProjection(false) + require.EqualValues(t, MAP, mapProjection.TypeId()) + require.True(t, mapProjection.Key.Nullable) + require.True(t, mapProjection.Value.Nullable) } func TestGroupFieldsUsesFlatOrderForFullyTaggedStructs(t *testing.T) { From abee3aabfca9e8cab5f7d3def24a2d890ea233d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 04:58:40 +0800 Subject: [PATCH 29/58] fix(java): skip native primitive list fields from typedef --- .../java/org/apache/fory/meta/FieldInfo.java | 24 +++++++++++- .../java/org/apache/fory/meta/FieldTypes.java | 39 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index d6ec155256..059822d738 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -91,7 +91,7 @@ public FieldTypes.FieldType getFieldType() { * reflection should be used to get the descriptor. */ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { - TypeRef declared = descriptor != null ? descriptor.getTypeRef() : null; + TypeRef declared = descriptor != null ? descriptor.getTypeRef() : primitiveListCarrierType(); TypeRef typeRef = fieldType.toTypeToken(resolver, declared); String typeName = fieldType.getTypeName(resolver, typeRef); if (fieldType instanceof FieldTypes.RegisteredFieldType) { @@ -266,6 +266,28 @@ private static boolean isListField(FieldTypes.FieldType fieldType) { && fieldType.getTypeId() == Types.LIST; } + private TypeRef primitiveListCarrierType() { + if (!(fieldType instanceof FieldTypes.CollectionFieldType)) { + return null; + } + FieldTypes.CollectionFieldType collectionFieldType = (FieldTypes.CollectionFieldType) fieldType; + FieldTypes.FieldType elementType = collectionFieldType.getElementType(); + if (!(elementType instanceof FieldTypes.RegisteredFieldType) + || elementType.nullable() + || elementType.trackingRef()) { + return null; + } + int elementTypeId = ((FieldTypes.RegisteredFieldType) elementType).getTypeId(); + Class carrierClass = FieldTypes.getPrimitiveListClassForElementType(elementTypeId); + if (carrierClass == null) { + return null; + } + // Native registered TypeDefs may not carry the writer class name for a removed field. A + // LIST field with a non-null primitive element is the schema marker emitted by primitive-list + // carriers, whose payload omits the native collection class header. + return TypeRef.of(carrierClass); + } + private static int listElementTypeId(FieldTypes.FieldType fieldType) { if (!(fieldType instanceof FieldTypes.CollectionFieldType) || fieldType.getTypeId() != Types.LIST) { diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 0fbe8a1943..bea2cf5a3e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -768,6 +768,45 @@ private static Class getPrimitiveArrayClass(int typeId) { } } + static Class getPrimitiveListClassForElementType(int typeId) { + switch (typeId) { + case Types.BOOL: + return BoolList.class; + case Types.INT8: + return Int8List.class; + case Types.UINT8: + return UInt8List.class; + case Types.INT16: + return Int16List.class; + case Types.UINT16: + return UInt16List.class; + case Types.INT32: + case Types.VARINT32: + return Int32List.class; + case Types.UINT32: + case Types.VAR_UINT32: + return UInt32List.class; + case Types.INT64: + case Types.VARINT64: + case Types.TAGGED_INT64: + return Int64List.class; + case Types.UINT64: + case Types.VAR_UINT64: + case Types.TAGGED_UINT64: + return UInt64List.class; + case Types.FLOAT32: + return Float32List.class; + case Types.FLOAT16: + return Float16List.class; + case Types.BFLOAT16: + return BFloat16List.class; + case Types.FLOAT64: + return Float64List.class; + default: + return null; + } + } + private static Class getPrimitiveListClass(int typeId) { switch (typeId) { case Types.BOOL_ARRAY: From 816972497403747052df834960e022dd8b2d3c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 05:17:10 +0800 Subject: [PATCH 30/58] fix(java): honor native static field nullability --- .../processing/ForyStructProcessor.java | 6 +++--- .../processing/ForyStructProcessorTest.java | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index 08b738aa1f..f6982c6562 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -354,12 +354,12 @@ private boolean fieldNullable(TypeMirror type, ForyFieldMeta foryField, Serializ if (type.getKind().isPrimitive()) { return false; } - if (mode == SerializerMode.NATIVE) { - return true; - } if (foryField.hasForyField) { return foryField.nullable; } + if (mode == SerializerMode.NATIVE) { + return true; + } return isOptionalType(type); } diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index ded30e40c6..0fdae71cb1 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -390,23 +390,27 @@ private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exce compile( "test.RoundTripStruct", "package test;\n" + + "import org.apache.fory.annotation.ForyField;\n" + "import org.apache.fory.annotation.UInt16Type;\n" + "import org.apache.fory.annotation.ForyStruct;\n" + "@ForyStruct public class RoundTripStruct {\n" + " public int id;\n" + " public @UInt16Type int code;\n" + " public String name;\n" + + " @ForyField(id = 4) public String strictName;\n" + " public RoundTripStruct() {}\n" + "}\n"); CompilationResult runtimeResult = compile( "test.RoundTripStruct", "package test;\n" + + "import org.apache.fory.annotation.ForyField;\n" + "import org.apache.fory.annotation.UInt16Type;\n" + "public class RoundTripStruct {\n" + " public int id;\n" + " public @UInt16Type int code;\n" + " public String name;\n" + + " @ForyField(id = 4) public String strictName;\n" + " public RoundTripStruct() {}\n" + "}\n"); Assert.assertTrue(staticResult.success, staticResult.diagnostics()); @@ -433,6 +437,11 @@ private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exce setField(staticType, staticValue, "id", 101); setField(staticType, staticValue, "code", 513); setField(staticType, staticValue, "name", compatible ? "compatible-static" : "static"); + setField( + staticType, + staticValue, + "strictName", + compatible ? "compatible-strict-static" : "strict-static"); Object runtimeRoundTrip = runtimeFory.deserialize(staticFory.serialize(staticValue)); Assert.assertSame(runtimeRoundTrip.getClass(), runtimeType); Assert.assertEquals(getField(runtimeType, runtimeRoundTrip, "id"), 101); @@ -440,11 +449,19 @@ private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exce Assert.assertEquals( getField(runtimeType, runtimeRoundTrip, "name"), compatible ? "compatible-static" : "static"); + Assert.assertEquals( + getField(runtimeType, runtimeRoundTrip, "strictName"), + compatible ? "compatible-strict-static" : "strict-static"); Object runtimeValue = runtimeType.getConstructor().newInstance(); setField(runtimeType, runtimeValue, "id", 202); setField(runtimeType, runtimeValue, "code", 1024); setField(runtimeType, runtimeValue, "name", compatible ? "compatible-runtime" : "runtime"); + setField( + runtimeType, + runtimeValue, + "strictName", + compatible ? "compatible-strict-runtime" : "strict-runtime"); Object staticRoundTrip = staticFory.deserialize(runtimeFory.serialize(runtimeValue)); Assert.assertSame(staticRoundTrip.getClass(), staticType); Assert.assertEquals(getField(staticType, staticRoundTrip, "id"), 202); @@ -452,6 +469,9 @@ private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exce Assert.assertEquals( getField(staticType, staticRoundTrip, "name"), compatible ? "compatible-runtime" : "runtime"); + Assert.assertEquals( + getField(staticType, staticRoundTrip, "strictName"), + compatible ? "compatible-strict-runtime" : "strict-runtime"); } } From 17f7dcedbc40d9558703181e8972535e6518b41b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 05:29:56 +0800 Subject: [PATCH 31/58] fix(java): guard static serializer recursion --- docs/guide/java/schema-evolution.md | 5 +++ .../java/static-generated-serializers.md | 9 +++-- .../processing/ForyStructProcessorTest.java | 37 +++++++++++++++++++ .../StaticGeneratedStructSerializer.java | 12 ++++++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/docs/guide/java/schema-evolution.md b/docs/guide/java/schema-evolution.md index f79613fd2b..49518f945d 100644 --- a/docs/guide/java/schema-evolution.md +++ b/docs/guide/java/schema-evolution.md @@ -56,6 +56,11 @@ This compatible mode involves serializing class metadata into the serialized out - `Evolution.ENABLED`: require schema evolution metadata for this class. Registration or type resolution fails if the Fory instance cannot emit that metadata. - `Evolution.DISABLED`: force fixed-schema `STRUCT/NAMED_STRUCT` encoding even when compatible metadata is otherwise enabled. +Use `@ForyStruct(evolving = Evolution.DISABLED)` for fixed-schema structs. Older development +snapshots used a boolean form such as `@ForyStruct(evolving = false)`; the enum form is the +supported API because it distinguishes inherited behavior from explicitly required compatible +metadata. + If a class schema is stable and will not change, opt out of schema evolution on that class to avoid compatible metadata overhead: ```java diff --git a/docs/guide/java/static-generated-serializers.md b/docs/guide/java/static-generated-serializers.md index 9f8c5d80c2..18f9fff0fa 100644 --- a/docs/guide/java/static-generated-serializers.md +++ b/docs/guide/java/static-generated-serializers.md @@ -53,10 +53,11 @@ public class Order { } ``` -The processor emits a public top-level serializer in the same package. For `Order`, the generated -class is `Order__ForyStaticSerializer__`. For a static member type `Outer.Inner`, the generated -top-level class is `Outer$Inner__ForyStaticSerializer__`; the processor does not modify the -enclosing class and does not generate inner serializer classes. +The processor emits public top-level serializers in the same package. For `Order`, the generated +cross-language serializer is `Order__ForySerializer__` and the generated Java native serializer is +`Order__ForyNativeSerializer__`. For a static member type `Outer.Inner`, the generated top-level +classes are `Outer$Inner__ForySerializer__` and `Outer$Inner__ForyNativeSerializer__`; the processor +does not modify the enclosing class and does not generate inner serializer classes. ## Runtime Selection diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 0fdae71cb1..2fbc80ecb1 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -385,6 +385,43 @@ public void testStaticSerializerRoundTripsWithRuntimeSerializerCompatible() thro assertStaticRuntimeRoundTrip(true); } + @Test + public void testStaticSerializerHandlesMonomorphicRecursiveField() throws Exception { + CompilationResult result = + compile( + "test.RecursiveStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyField;\n" + + "import org.apache.fory.annotation.ForyField.Dynamic;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class RecursiveStruct {\n" + + " public int id;\n" + + " @ForyField(nullable = true, ref = true, dynamic = Dynamic.FALSE)\n" + + " public RecursiveStruct next;\n" + + " public RecursiveStruct() {}\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.RecursiveStruct"); + Object value = type.getConstructor().newInstance(); + setField(type, value, "id", 12); + setField(type, value, "next", value); + + Fory fory = + Fory.builder() + .withClassLoader(loader) + .withCodegen(false) + .withRefTracking(true) + .requireClassRegistration(false) + .build(); + Object serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); + Assert.assertTrue(serializer instanceof StaticGeneratedStructSerializer); + Object roundTrip = fory.deserialize(fory.serialize(value)); + Assert.assertEquals(getField(type, roundTrip, "id"), 12); + Assert.assertSame(getField(type, roundTrip, "next"), roundTrip); + } + } + private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exception { CompilationResult staticResult = compile( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 9513f8fde8..023a044c50 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -52,6 +52,7 @@ public abstract class StaticGeneratedStructSerializer extends AbstractObjectS @SuppressWarnings("unchecked") public StaticGeneratedStructSerializer(TypeResolver typeResolver, Class type) { super(typeResolver, (Class) type); + setSerializerIfAbsent(typeResolver, (Class) type); this.typeDef = null; this.remoteFields = Collections.emptyList(); this.localFieldsById = new SerializationFieldInfo[0]; @@ -71,6 +72,7 @@ protected StaticGeneratedStructSerializer( List descriptors, Class remoteDescriptorClass) { super(typeResolver, (Class) type); + setSerializerIfAbsent(typeResolver, (Class) type); List runtimeDescriptors = runtimeDescriptors(descriptors); this.typeDef = typeDef; this.remoteFields = @@ -80,6 +82,16 @@ protected StaticGeneratedStructSerializer( this.localFieldsById = buildLocalFieldsById(runtimeDescriptors); } + private void setSerializerIfAbsent(TypeResolver typeResolver, Class type) { + if (!typeResolver.isCrossLanguage() || typeResolver.getTypeInfo(type, false) != null) { + // Field-group construction resolves monomorphic field serializers. A generated serializer can + // therefore encounter its own type before the subclass constructor has finished, just like + // ObjectSerializer. Install this instance early so recursive fields reuse it instead of + // constructing another serializer for the same type. + typeResolver.setSerializerIfAbsent(type, this); + } + } + @Override public abstract void write(WriteContext writeContext, T value); From 985fdb04dfbf2e7b15831729692cd9403540af1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 05:56:46 +0800 Subject: [PATCH 32/58] fix: harden compatible generated serializers --- docs/guide/java/schema-evolution.md | 8 +- .../processing/ForyStructProcessor.java | 15 +- .../StaticSerializerSourceWriter.java | 3 + .../processing/ForyStructProcessorTest.java | 49 ++++++ .../apache/fory/annotation/ForyStruct.java | 18 ++- .../org/apache/fory/builder/CodecUtils.java | 3 +- .../builder/StaticCompatibleCodecBuilder.java | 10 +- .../apache/fory/resolver/TypeResolver.java | 30 +++- .../CompatibleCollectionArrayReader.java | 6 + .../apache/fory/serializer/FieldSkipper.java | 8 + .../StaticCompatibleCodecBuilderTest.java | 143 ++++++++++++++++++ .../apache/fory/resolver/TypeInfoTest.java | 17 ++- .../org/apache/fory/xlang/XlangTestBase.java | 4 +- .../integration_tests/ExampleMessage.java | 2 +- .../ExampleRecordMessage.java | 2 +- javascript/packages/core/lib/typeResolver.ts | 1 + javascript/test/typemeta.test.ts | 23 +++ 17 files changed, 316 insertions(+), 26 deletions(-) diff --git a/docs/guide/java/schema-evolution.md b/docs/guide/java/schema-evolution.md index 49518f945d..b99a3c5465 100644 --- a/docs/guide/java/schema-evolution.md +++ b/docs/guide/java/schema-evolution.md @@ -56,10 +56,8 @@ This compatible mode involves serializing class metadata into the serialized out - `Evolution.ENABLED`: require schema evolution metadata for this class. Registration or type resolution fails if the Fory instance cannot emit that metadata. - `Evolution.DISABLED`: force fixed-schema `STRUCT/NAMED_STRUCT` encoding even when compatible metadata is otherwise enabled. -Use `@ForyStruct(evolving = Evolution.DISABLED)` for fixed-schema structs. Older development -snapshots used a boolean form such as `@ForyStruct(evolving = false)`; the enum form is the -supported API because it distinguishes inherited behavior from explicitly required compatible -metadata. +Use `@ForyStruct(evolution = Evolution.DISABLED)` for fixed-schema structs. The legacy boolean +form `@ForyStruct(evolving = false)` is still supported as a fixed-schema opt-out. If a class schema is stable and will not change, opt out of schema evolution on that class to avoid compatible metadata overhead: @@ -67,7 +65,7 @@ If a class schema is stable and will not change, opt out of schema evolution on import org.apache.fory.annotation.ForyStruct; import org.apache.fory.annotation.ForyStruct.Evolution; -@ForyStruct(evolving = Evolution.DISABLED) +@ForyStruct(evolution = Evolution.DISABLED) public class StableMessage { public int id; public String name; diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index f6982c6562..75286fc77a 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -662,13 +662,22 @@ private boolean isCompatibleForyStructType(TypeMirror type) { } Map values = elements.getElementValuesWithDefaults(mirror); + boolean evolving = true; + String evolution = "INHERIT"; for (Map.Entry entry : values.entrySet()) { - if (entry.getKey().getSimpleName().contentEquals("evolving")) { - return !"DISABLED".equals(enumConstant(String.valueOf(entry.getValue().getValue()))); + String name = entry.getKey().getSimpleName().toString(); + Object value = entry.getValue().getValue(); + if (name.equals("evolving")) { + evolving = (Boolean) value; + } else if (name.equals("evolution")) { + evolution = enumConstant(String.valueOf(value)); } } - return true; + if ("DISABLED".equals(evolution)) { + return false; + } + return evolving || "ENABLED".equals(evolution); } private String typeExtMetaExpression( diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 6aa36b27b6..8666ebde2c 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -130,6 +130,9 @@ private void writeDescriptors() { } builder.append(" return Collections.unmodifiableList(descriptors);\n"); builder.append(" }\n\n"); + builder.append(" private static List getGeneratedDescriptors() {\n"); + builder.append(" return DESCRIPTORS;\n"); + builder.append(" }\n\n"); builder.append(" @Override\n"); builder.append(" public List getDescriptors() {\n"); builder.append(" return DESCRIPTORS;\n"); diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 2fbc80ecb1..56dd3af676 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -87,6 +87,24 @@ public void testStaticSerializerSelectedWithCodegenDisabled() throws Exception { } } + @Test + public void testLegacyBooleanEvolvingAnnotationCompiles() throws Exception { + CompilationResult result = + compile( + "test.LegacyFixedStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct(evolving = false) public class LegacyFixedStruct {\n" + + " public int id;\n" + + " public LegacyFixedStruct() {}\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + loader.loadClass("test.LegacyFixedStruct__ForySerializer__"); + loader.loadClass("test.LegacyFixedStruct__ForyNativeSerializer__"); + } + } + @Test public void testPrivateFieldUsesAccessibleAccessors() throws Exception { CompilationResult result = @@ -422,6 +440,37 @@ public void testStaticSerializerHandlesMonomorphicRecursiveField() throws Except } } + @Test + public void testGeneratedDescriptorDiscoveryDoesNotSelectStaticSerializerWhenCodegenEnabled() + throws Exception { + CompilationResult result = + compile( + "test.DescriptorStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class DescriptorStruct {\n" + + " public int id;\n" + + " public String name;\n" + + " public DescriptorStruct() {}\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.DescriptorStruct"); + Fory fory = + Fory.builder() + .withClassLoader(loader) + .withCodegen(true) + .requireClassRegistration(false) + .build(); + List descriptors = fory.getTypeResolver().getFieldDescriptors(type, true); + Assert.assertEquals(descriptors.size(), 2); + + Object serializer = fory.getTypeResolver().getTypeInfo(type).getSerializer(); + Assert.assertFalse( + serializer instanceof StaticGeneratedStructSerializer, serializer.getClass().getName()); + } + } + private static void assertStaticRuntimeRoundTrip(boolean compatible) throws Exception { CompilationResult staticResult = compile( diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java index e6d471ad2f..c92d41cc61 100644 --- a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java +++ b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyStruct.java @@ -41,15 +41,23 @@ enum Evolution { DISABLED } + /** + * Legacy per-struct schema evolution switch. + * + *

    Set this to {@code false} to force fixed-schema struct encoding. New code that needs to + * require schema evolution metadata should use {@link #evolution()}. + */ + boolean evolving() default true; + /** * Per-struct schema evolution policy. * - *

    {@link Evolution#INHERIT} follows the Fory instance's compatible/meta-share configuration. - * {@link Evolution#ENABLED} requires that configuration to emit schema evolution metadata for - * this struct. {@link Evolution#DISABLED} uses fixed-schema struct encoding even when compatible - * metadata is otherwise enabled. + *

    {@link Evolution#INHERIT} follows {@link #evolving()} and then the Fory instance's + * compatible/meta-share configuration. {@link Evolution#ENABLED} requires that configuration to + * emit schema evolution metadata for this struct. {@link Evolution#DISABLED} uses fixed-schema + * struct encoding even when compatible metadata is otherwise enabled. */ - Evolution evolving() default Evolution.INHERIT; + Evolution evolution() default Evolution.INHERIT; /** * Emit generated serializer field-level debug tracing. diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index 5856e58460..b1006fe858 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -76,7 +76,8 @@ public static Class loadOrGenStaticCompatibleCodecClas Fory fory, Class cls, TypeDef typeDef) { Preconditions.checkNotNull(fory); return loadSerializer( - "loadOrGenStaticCompatibleCodecClass", + "loadOrGenStaticCompatibleCodecClass_" + + StaticCompatibleCodecBuilder.remoteTypeDefSuffix(typeDef), cls, fory, () -> diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 5ed288982a..dc9e2a6144 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -61,6 +61,7 @@ public final class StaticCompatibleCodecBuilder extends ObjectCodecBuilder { private final List localDescriptors; private final Class remoteDescriptorClass; + private final String remoteTypeDefSuffix; private final boolean debug; public StaticCompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { @@ -70,13 +71,14 @@ public StaticCompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef type "Class version check should be disabled when compatible mode is enabled."); localDescriptors = Collections.unmodifiableList(Descriptor.getDescriptors(beanClass)); remoteDescriptorClass = resolveRemoteDescriptorClass(typeDef); + remoteTypeDefSuffix = remoteTypeDefSuffix(typeDef); ForyStruct foryStruct = beanClass.getAnnotation(ForyStruct.class); debug = foryStruct != null && foryStruct.debug(); } @Override protected String codecSuffix() { - return "CompatibleMetaShared"; + return "CompatibleMetaShared_" + remoteTypeDefSuffix; } @Override @@ -147,6 +149,12 @@ private Class resolveRemoteDescriptorClass(TypeDef typeDef) { } } + static String remoteTypeDefSuffix(TypeDef typeDef) { + return Long.toUnsignedString(typeDef.getId(), 16) + + "_" + + Integer.toHexString(typeDef.getClassName().hashCode()); + } + private String remoteDescriptorClassLiteral() { if (remoteDescriptorClass == null || !canReferenceRemoteDescriptorClass()) { return "null"; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index d2701f8f13..bec591e93a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -25,6 +25,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Member; +import java.lang.reflect.Method; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Arrays; @@ -1107,7 +1108,7 @@ protected boolean useStructEvolution(Class cls, boolean inheritedEvolutionEna if (evolution == Evolution.ENABLED) { throw new IllegalStateException( String.format( - "Class %s is annotated with @ForyStruct(evolving = ENABLED), but this Fory " + "Class %s is annotated with @ForyStruct(evolution = ENABLED), but this Fory " + "instance is not configured to write schema evolution metadata", cls.getName())); } @@ -1119,7 +1120,13 @@ private Evolution getStructEvolution(Class cls) { return Evolution.INHERIT; } ForyStruct annotation = cls.getAnnotation(ForyStruct.class); - return annotation == null ? Evolution.INHERIT : annotation.evolving(); + if (annotation == null) { + return Evolution.INHERIT; + } + if (annotation.evolution() != Evolution.INHERIT) { + return annotation.evolution(); + } + return annotation.evolving() ? Evolution.INHERIT : Evolution.DISABLED; } protected static boolean isStructSerializer(Serializer serializer) { @@ -1683,9 +1690,22 @@ private List getStaticGeneratedStructDescriptors(Class cls) { if (serializerClass == null) { return null; } - StaticGeneratedStructSerializer serializer = - (StaticGeneratedStructSerializer) Serializers.newSerializer(this, cls, serializerClass); - return serializer.getDescriptors(); + try { + Method descriptorsMethod = serializerClass.getDeclaredMethod("getGeneratedDescriptors"); + descriptorsMethod.setAccessible(true); + return (List) descriptorsMethod.invoke(null); + } catch (NoSuchMethodException e) { + // Descriptor discovery must be side-effect free; instantiating the generated serializer here + // can install it before normal serializer selection gets to choose the JIT path. + throw new ForyException( + "Generated static serializer " + + serializerClass.getName() + + " must define static getGeneratedDescriptors()", + e); + } catch (ReflectiveOperationException e) { + throw new ForyException( + "Failed to read generated static descriptors from " + serializerClass.getName(), e); + } } /** diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index 11da62d732..c9440880ed 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -336,6 +336,12 @@ private static int listElementTypeId(TypeRef typeRef, boolean requireNonNulla } return TypeAnnotationUtils.getDefaultPrimitiveListElementTypeId(typeRef.getRawType()); } + if (TypeUtils.isCollection(typeRef.getRawType())) { + TypeExtMeta elementExtMeta = TypeUtils.getElementType(typeRef).getTypeExtMeta(); + return isPrimitiveElement(elementExtMeta, requireNonNullable) + ? elementExtMeta.typeId() + : Types.UNKNOWN; + } return Types.UNKNOWN; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java index 72b873a897..e272dcf946 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java @@ -67,6 +67,14 @@ static void skipField( } // For nullable basic types, check null flag first + if (refMode == RefMode.TRACKING) { + // Tracking refs can be null, a new value, or a back-reference with no payload bytes. Delegate + // to the normal ref-aware field read path so skipping an unknown back-reference does not + // consume the next field's payload. + AbstractObjectSerializer.readBuildInFieldValue( + readContext, typeResolver, refReader, fieldInfo, buffer); + return; + } if (refMode != RefMode.NONE) { if (buffer.readByte() == Fory.NULL_FLAG) { return; // Field is null, nothing more to skip diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java index d345b3f029..ab907bca30 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -186,6 +186,149 @@ public void testStaticCompatibleSerializerUsesListArrayAction() throws Exception } } + @Test + public void testStaticCompatibleSkipsUnknownBackReferenceField() throws Exception { + CompilationResult writerResult = + compile( + "test.StaticCompatibleRefPayload", + "package test;\n" + + "import java.util.List;\n" + + "import org.apache.fory.annotation.ForyField;\n" + + "public class StaticCompatibleRefPayload {\n" + + " @ForyField(nullable = true, ref = true) public String name;\n" + + " @ForyField(nullable = true, ref = true) public String nameAlias;\n" + + " public List after;\n" + + " public StaticCompatibleRefPayload() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.StaticCompatibleRefPayload", + "package test;\n" + + "import java.util.List;\n" + + "import org.apache.fory.annotation.ForyField;\n" + + "public class StaticCompatibleRefPayload {\n" + + " @ForyField(nullable = true, ref = true) public String name;\n" + + " public List after;\n" + + " public StaticCompatibleRefPayload() {}\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.StaticCompatibleRefPayload"); + Class readerType = readerLoader.loadClass("test.StaticCompatibleRefPayload"); + Fory writer = compatibleFory(writerLoader, writerType, false, "ref-writer"); + Fory reader = compatibleFory(readerLoader, readerType, false, "ref-reader"); + Object writerValue = writerType.getConstructor().newInstance(); + String shared = new String("shared"); + setField(writerType, writerValue, "name", shared); + setField(writerType, writerValue, "nameAlias", shared); + setField(writerType, writerValue, "after", Arrays.asList("after")); + + Object result = + roundTripThroughStaticCompatibleSerializer( + writer, reader, writerType, readerType, writerValue); + Assert.assertSame(result.getClass(), readerType); + Assert.assertEquals(getField(readerType, result, "name"), "shared"); + Assert.assertEquals(getField(readerType, result, "after"), Arrays.asList("after")); + } + } + + @Test + public void testStaticCompatibleArrayPayloadReadsOrdinaryAnnotatedList() throws Exception { + CompilationResult writerResult = + compile( + "test.StaticCompatibleAnnotatedListPayload", + "package test;\n" + + "public class StaticCompatibleAnnotatedListPayload {\n" + + " public int[] values;\n" + + " public StaticCompatibleAnnotatedListPayload() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.StaticCompatibleAnnotatedListPayload", + "package test;\n" + + "import java.util.List;\n" + + "import org.apache.fory.annotation.Int32Type;\n" + + "import org.apache.fory.config.Int32Encoding;\n" + + "public class StaticCompatibleAnnotatedListPayload {\n" + + " public List<@Int32Type(encoding = Int32Encoding.FIXED) Integer> values;\n" + + " public StaticCompatibleAnnotatedListPayload() {}\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.StaticCompatibleAnnotatedListPayload"); + Class readerType = readerLoader.loadClass("test.StaticCompatibleAnnotatedListPayload"); + Fory writer = compatibleFory(writerLoader, writerType, true, "annotated-list-writer"); + Fory reader = compatibleFory(readerLoader, readerType, true, "annotated-list-reader"); + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "values", new int[] {7, 8, 9}); + + Object result = + roundTripThroughStaticCompatibleSerializer( + writer, reader, writerType, readerType, writerValue); + Assert.assertSame(result.getClass(), readerType); + Assert.assertEquals(getField(readerType, result, "values"), Arrays.asList(7, 8, 9)); + } + } + + @Test + public void testStaticCompatibleSerializerClassKeyIncludesRemoteTypeDef() throws Exception { + CompilationResult writerAResult = + compile( + "test.WriterPayloadA", + "package test;\n" + + "public class WriterPayloadA {\n" + + " public int id;\n" + + " public String oldName;\n" + + " public WriterPayloadA() {}\n" + + "}\n"); + CompilationResult writerBResult = + compile( + "test.WriterPayloadB", + "package test;\n" + + "public class WriterPayloadB {\n" + + " public int id;\n" + + " public long oldCount;\n" + + " public WriterPayloadB() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.ReaderPayload", + "package test;\n" + + "public class ReaderPayload {\n" + + " public int id;\n" + + " public ReaderPayload() {}\n" + + "}\n"); + Assert.assertTrue(writerAResult.success, writerAResult.diagnostics()); + Assert.assertTrue(writerBResult.success, writerBResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerALoader = writerAResult.classLoader(); + URLClassLoader writerBLoader = writerBResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerAType = writerALoader.loadClass("test.WriterPayloadA"); + Class writerBType = writerBLoader.loadClass("test.WriterPayloadB"); + Class readerType = readerLoader.loadClass("test.ReaderPayload"); + Fory writerA = compatibleFory(writerALoader, writerAType, false, "writer-a"); + Fory writerB = compatibleFory(writerBLoader, writerBType, false, "writer-b"); + Fory reader = compatibleFory(readerLoader, readerType, false, "reader"); + TypeDef typeDefA = TypeDef.buildTypeDef(writerA.getTypeResolver(), writerAType); + TypeDef typeDefB = TypeDef.buildTypeDef(writerB.getTypeResolver(), writerBType); + Assert.assertNotEquals(typeDefA.getId(), typeDefB.getId()); + + Class readerClass = cast(readerType); + Class serializerA = + CodecUtils.loadOrGenStaticCompatibleCodecClass( + reader.getTypeResolver(), readerClass, typeDefA); + Class serializerB = + CodecUtils.loadOrGenStaticCompatibleCodecClass( + reader.getTypeResolver(), readerClass, typeDefB); + Assert.assertNotEquals(serializerA, serializerB); + } + } + private static Object roundTripThroughStaticCompatibleSerializer( Fory writer, Fory reader, Class writerType, Class readerType, Object writerValue) throws Exception { diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java index 2e2c240e22..57c8335eae 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java @@ -34,16 +34,21 @@ public static class EvolvingStruct { public int id; } - @ForyStruct(evolving = Evolution.ENABLED) + @ForyStruct(evolution = Evolution.ENABLED) public static class ExplicitEvolvingStruct { public int id; } - @ForyStruct(evolving = Evolution.DISABLED) + @ForyStruct(evolution = Evolution.DISABLED) public static class FixedStruct { public int id; } + @ForyStruct(evolving = false) + public static class LegacyFixedStruct { + public int id; + } + @Test public void testEncodePackageNameAndTypeName() { Fory fory1 = Fory.builder().withXlang(false).requireClassRegistration(false).build(); @@ -58,17 +63,21 @@ public void testStructEvolvingOverride() { fory.register(EvolvingStruct.class, "test", "EvolvingStruct"); fory.register(ExplicitEvolvingStruct.class, "test", "ExplicitEvolvingStruct"); fory.register(FixedStruct.class, "test", "FixedStruct"); + fory.register(LegacyFixedStruct.class, "test", "LegacyFixedStruct"); TypeInfo evolvingInfo = fory.getTypeResolver().getTypeInfo(EvolvingStruct.class, false); TypeInfo explicitEvolvingInfo = fory.getTypeResolver().getTypeInfo(ExplicitEvolvingStruct.class, false); TypeInfo fixedInfo = fory.getTypeResolver().getTypeInfo(FixedStruct.class, false); + TypeInfo legacyFixedInfo = fory.getTypeResolver().getTypeInfo(LegacyFixedStruct.class, false); assertNotNull(evolvingInfo); assertNotNull(explicitEvolvingInfo); assertNotNull(fixedInfo); + assertNotNull(legacyFixedInfo); assertEquals(evolvingInfo.getTypeId(), Types.NAMED_COMPATIBLE_STRUCT); assertEquals(explicitEvolvingInfo.getTypeId(), Types.NAMED_COMPATIBLE_STRUCT); assertEquals(fixedInfo.getTypeId(), Types.NAMED_STRUCT); + assertEquals(legacyFixedInfo.getTypeId(), Types.NAMED_STRUCT); EvolvingStruct evolving = new EvolvingStruct(); evolving.id = 123; @@ -89,6 +98,7 @@ public void testStructEvolvingOverrideForRegisteredClasses() { fory.register(EvolvingStruct.class, 100); fory.register(ExplicitEvolvingStruct.class, 101); fory.register(FixedStruct.class, 102); + fory.register(LegacyFixedStruct.class, 103); assertEquals( fory.getTypeResolver().getTypeInfo(EvolvingStruct.class, false).getTypeId(), @@ -98,6 +108,9 @@ public void testStructEvolvingOverrideForRegisteredClasses() { Types.COMPATIBLE_STRUCT); assertEquals( fory.getTypeResolver().getTypeInfo(FixedStruct.class, false).getTypeId(), Types.STRUCT); + assertEquals( + fory.getTypeResolver().getTypeInfo(LegacyFixedStruct.class, false).getTypeId(), + Types.STRUCT); } @Test( diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index 8d59f6b056..b5c90437d8 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -193,13 +193,13 @@ public void ensurePeerReadyForTests() { protected static final String FIXED_OVERRIDE_STRUCT_TYPE_NAME = "evolving_off"; @Data - @ForyStruct(evolving = Evolution.ENABLED) + @ForyStruct(evolution = Evolution.ENABLED) protected static class EvolvingOverrideStruct { String f1; } @Data - @ForyStruct(evolving = Evolution.DISABLED) + @ForyStruct(evolution = Evolution.DISABLED) protected static class FixedOverrideStruct { String f1; } diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleMessage.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleMessage.java index b8b49d86f8..7f8af9e682 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleMessage.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleMessage.java @@ -428,7 +428,7 @@ public class ExampleMessage { public ExampleMessage() {} - @ForyStruct(evolving = Evolution.DISABLED) + @ForyStruct(evolution = Evolution.DISABLED) public static class Leaf { @ForyField(id = 1) public String label; diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleRecordMessage.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleRecordMessage.java index 375e3e8dbc..a8b5c59f57 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleRecordMessage.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleRecordMessage.java @@ -190,7 +190,7 @@ public record ExampleRecordMessage( @ForyField(id = 253) Map timestampValuesByName, @ForyField(id = 254) Map durationValuesByName, @ForyField(id = 255) Map enumValuesByName) { - @ForyStruct(evolving = Evolution.DISABLED) + @ForyStruct(evolution = Evolution.DISABLED) public record Leaf(@ForyField(id = 1) String label, @ForyField(id = 2) int count) {} public enum State { diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index c227a74f69..744b8aff92 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -292,6 +292,7 @@ export default class TypeResolver { regenerateReadSerializer(typeInfo: TypeInfo) { const serializer = this.generateReadSerializer(typeInfo); return this.registerSerializer(typeInfo, { + getTypeInfo: serializer.getTypeInfo, getHash: serializer.getHash, read: serializer.read, readNoRef: serializer.readNoRef, diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index 66fba3e69e..30a1fe80a5 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -241,6 +241,29 @@ describe("typemeta", () => { expect(reader.deserialize(localBytes)).toEqual({ value: 123 }); }); + test("keeps type info on regenerated compatible named serializers", () => { + const stringWriterFory = new Fory({ compatible: true }); + const numberWriterFory = new Fory({ compatible: true }); + const readerFory = new Fory({ compatible: true }); + + const stringWriterType = Type.struct("example.dynamicNamed", { + value: Type.string().setId(1), + }); + const numberWriterType = Type.struct("example.dynamicNamed", { + value: Type.int32().setId(1), + }); + + const stringBytes = stringWriterFory + .register(stringWriterType) + .serialize({ value: "hello" }); + const numberBytes = numberWriterFory + .register(numberWriterType) + .serialize({ value: 123 }); + + expect(readerFory.deserialize(stringBytes)).toEqual({ $tag1: "hello" }); + expect(readerFory.deserialize(numberBytes)).toEqual({ $tag1: 123 }); + }); + test("caches regenerated compatible readers for alternating nested schemas", () => { const stringWriterFory = new Fory({ compatible: true }); const boolWriterFory = new Fory({ compatible: true }); From 7d1a229d028dedb82614fb9339f7ac9d144e337b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 06:45:18 +0800 Subject: [PATCH 33/58] fix: harden list-array compatible reads --- compiler/fory_compiler/generators/java.py | 2 +- .../tests/test_generated_code.py | 2 +- cpp/fory/serialization/struct_serializer.h | 6 ++ go/fory/fory_compatible_test.go | 7 +- go/fory/struct_init.go | 7 +- .../processing/ForyStructProcessorTest.java | 67 +++++++++++++++++++ .../java/org/apache/fory/meta/FieldInfo.java | 26 ++++++- .../apache/fory/resolver/XtypeResolver.java | 4 +- .../CompatibleCollectionArrayReader.java | 29 ++++---- .../apache/fory/serializer/FieldGroups.java | 4 +- .../StaticGeneratedStructSerializer.java | 10 +++ .../apache/fory/type/TypeAnnotationUtils.java | 15 ++--- .../fory/xlang/MetaSharedXlangTest.java | 3 +- .../org/apache/fory/xlang/XlangTestBase.java | 10 +-- python/pyfory/meta/typedef.py | 7 +- rust/fory-core/src/serializer/collection.rs | 13 +++- 16 files changed, 168 insertions(+), 44 deletions(-) diff --git a/compiler/fory_compiler/generators/java.py b/compiler/fory_compiler/generators/java.py index 67b64eb7cc..235aa6cd39 100644 --- a/compiler/fory_compiler/generators/java.py +++ b/compiler/fory_compiler/generators/java.py @@ -410,7 +410,7 @@ def get_struct_annotation(self, message: Message) -> str: """Return the ForyStruct annotation for a generated message.""" if self.get_effective_evolving(message): return "@ForyStruct" - return "@ForyStruct(evolving = Evolution.DISABLED)" + return "@ForyStruct(evolution = Evolution.DISABLED)" # Generates a Java class file from a message schema definition. def generate_message_file(self, message: Message) -> GeneratedFile: diff --git a/compiler/fory_compiler/tests/test_generated_code.py b/compiler/fory_compiler/tests/test_generated_code.py index bd76ae3ab8..a85d07af00 100644 --- a/compiler/fory_compiler/tests/test_generated_code.py +++ b/compiler/fory_compiler/tests/test_generated_code.py @@ -633,7 +633,7 @@ def test_java_evolving_false_generation_uses_struct_evolution_enum(): java_output = render_files(generate_files(schema, JavaGenerator)) assert "import org.apache.fory.annotation.ForyStruct;" in java_output assert "import org.apache.fory.annotation.ForyStruct.Evolution;" in java_output - assert java_output.count("@ForyStruct(evolving = Evolution.DISABLED)") == 2 + assert java_output.count("@ForyStruct(evolution = Evolution.DISABLED)") == 2 assert java_output.count("@ForyStruct") == 3 assert "@ForyStruct(evolving = false)" not in java_output diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 178ee464ee..6664ad26a6 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -3292,6 +3292,12 @@ void read_single_field_by_index_compatible(T &obj, ReadContext &ctx, return; } const auto &remote_element_type = remote_field_type.generics[0]; + if (FORY_PREDICT_FALSE(remote_element_type.nullable || + remote_element_type.track_ref)) { + ctx.set_error(Error::invalid_data( + "compatible list to array field requires non-null elements")); + return; + } constexpr int8_t child = configured_node_child(); FieldType result = read_configured_list_data_as_array_field< FieldType, T, Index, 0, child>(ctx, remote_element_type.type_id); diff --git a/go/fory/fory_compatible_test.go b/go/fory/fory_compatible_test.go index 4a78cd5c32..25d4c6aa4e 100644 --- a/go/fory/fory_compatible_test.go +++ b/go/fory/fory_compatible_test.go @@ -603,17 +603,14 @@ func TestCompatibleSerializationScenarios(t *testing.T) { }, }, { - name: "NullableInt32ListWithoutNullsMatchesArray", + name: "NullableInt32ListWithoutNullsDoesNotMatchArray", tag: "Int32Sequence", writeType: NullableInt32ListPayloadDataClass{}, readType: Int32ArrayPayloadDataClass{}, input: NullableInt32ListPayloadDataClass{ Payload: []*int32{ptr(int32(1)), ptr(int32(2)), ptr(int32(3))}, }, - assertFunc: func(t *testing.T, input any, output any) { - out := output.(Int32ArrayPayloadDataClass) - assert.Equal(t, [3]int32{1, 2, 3}, out.Payload) - }, + unmarshalErrContains: "compatible list to array field requires non-null elements", }, { name: "NullableInt32ListPayloadDoesNotMatchArray", diff --git a/go/fory/struct_init.go b/go/fory/struct_init.go index b64ae7d6fe..a20b0d9154 100644 --- a/go/fory/struct_init.go +++ b/go/fory/struct_init.go @@ -533,7 +533,10 @@ func (s *structSerializer) initFieldsFromTypeDef(typeResolver *TypeResolver) err ) { shouldRead = true fieldType = localType - } else if defTypeId == LIST && localFieldSpec != nil && compatibleListFieldCanReadLocalArray(def.typeSpec, localFieldSpec.Type, localType) { + } else if defTypeId == LIST && localFieldSpec != nil && compatibleListFieldHasPrimitiveArrayShape(def.typeSpec, localFieldSpec.Type, localType) { + if def.typeSpec.Element.Nullable || def.typeSpec.Element.TrackRef { + return fmt.Errorf("field %s: compatible list to array field requires non-null elements", def.name) + } shouldRead = true usesCompatibleCollectionArrayReader = true fieldType = localType @@ -810,7 +813,7 @@ func listFieldCanReadLocalArray(remoteSpec *TypeSpec, remoteNullable bool, remot return fieldSpecEqualForDiff(remoteSpec, remoteNullable, remoteTrackRef, localSpec, localNullable, localTrackRef) } -func compatibleListFieldCanReadLocalArray(remoteSpec *TypeSpec, localSpec *TypeSpec, localType reflect.Type) bool { +func compatibleListFieldHasPrimitiveArrayShape(remoteSpec *TypeSpec, localSpec *TypeSpec, localType reflect.Type) bool { if remoteSpec == nil || localSpec == nil || localType == nil { return false } diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 56dd3af676..a152fd65d3 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -616,6 +616,73 @@ public void testStaticCompatibleReadUsesListArrayAction() throws Exception { } } + @Test + public void testStaticArrayTypeListWritesDenseArrayPayload() throws Exception { + CompilationResult writerResult = + compile( + "test.DenseListStruct", + "package test;\n" + + "import java.util.List;\n" + + "import org.apache.fory.annotation.ArrayType;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "import org.apache.fory.annotation.UInt32Type;\n" + + "import org.apache.fory.type.Float16;\n" + + "@ForyStruct public class DenseListStruct {\n" + + " @ArrayType public List values;\n" + + " @ArrayType public List<@UInt32Type Long> unsignedValues;\n" + + " @ArrayType public List float16Values;\n" + + " public DenseListStruct() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.DenseListStruct", + "package test;\n" + + "import org.apache.fory.annotation.Float16Type;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "import org.apache.fory.annotation.UInt32Type;\n" + + "@ForyStruct public class DenseListStruct {\n" + + " public int[] values;\n" + + " public @UInt32Type int[] unsignedValues;\n" + + " public @Float16Type short[] float16Values;\n" + + " public DenseListStruct() {}\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.DenseListStruct"); + Class readerType = readerLoader.loadClass("test.DenseListStruct"); + Fory writer = xlangCompatibleFory(writerLoader, writerType, false, "DenseListStruct"); + Fory reader = xlangCompatibleFory(readerLoader, readerType, false, "DenseListStruct"); + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "values", Arrays.asList(1, 2, 3)); + setField( + writerType, + writerValue, + "unsignedValues", + Arrays.asList(1L, 2L, Integer.toUnsignedLong(-1))); + setField( + writerType, + writerValue, + "float16Values", + Arrays.asList( + org.apache.fory.type.Float16.fromBits((short) 0x0000), + org.apache.fory.type.Float16.fromBits((short) 0x3C00))); + + Object result = reader.deserialize(writer.serialize(writerValue)); + Assert.assertSame(result.getClass(), readerType); + Assert.assertTrue( + Arrays.equals((int[]) getField(readerType, result, "values"), new int[] {1, 2, 3})); + Assert.assertTrue( + Arrays.equals( + (int[]) getField(readerType, result, "unsignedValues"), new int[] {1, 2, -1})); + Assert.assertTrue( + Arrays.equals( + (short[]) getField(readerType, result, "float16Values"), + new short[] {(short) 0x0000, (short) 0x3C00})); + } + } + private static Fory xlangCompatibleFory(ClassLoader classLoader, Class type, boolean codegen) { return xlangCompatibleFory(classLoader, type, codegen, "ArrayShapeStruct"); } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index 059822d738..6b30855a47 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -129,6 +129,18 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { ? null : FieldTypes.buildFieldType(resolver, descriptor.getField()); int peerArrayTypeId = arrayTypeId(fieldType); + // Static @ArrayType List descriptors are fieldless, but the generated accessor still + // reads and writes a List. Preserve the local descriptor so field metadata installs the + // boxed-list dense-array serializer instead of treating the accessor value as a primitive + // array. + if (peerArrayTypeId != Types.UNKNOWN + && TypeAnnotationUtils.isBoxedListArrayType(descriptor) + && peerArrayTypeId == TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor)) { + return new DescriptorBuilder(descriptor) + .trackingRef(remoteTrackingRef) + .nullable(remoteNullable) + .build(); + } if (peerArrayTypeId != Types.UNKNOWN && peerArrayTypeId == arrayTypeId(localFieldType) && TypeAnnotationUtils.isArrayType(descriptor)) { @@ -214,9 +226,10 @@ private boolean isTopLevelListArrayCompatibleReadPair( FieldTypes.FieldType localFieldType = FieldTypes.buildFieldType(resolver, localField); int peerListElementTypeId = listElementTypeId(fieldType); if (peerListElementTypeId != Types.UNKNOWN) { + int nonNullablePeerListElementTypeId = nonNullableListElementTypeId(fieldType); int localArrayTypeId = arrayTypeId(localFieldType); return localArrayTypeId != Types.UNKNOWN - && localArrayTypeId == denseArrayTypeId(peerListElementTypeId); + && localArrayTypeId == denseArrayTypeId(nonNullablePeerListElementTypeId); } int peerArrayTypeId = arrayTypeId(fieldType); if (peerArrayTypeId != Types.UNKNOWN) { @@ -289,6 +302,10 @@ private TypeRef primitiveListCarrierType() { } private static int listElementTypeId(FieldTypes.FieldType fieldType) { + return listElementTypeId(fieldType, false); + } + + private static int listElementTypeId(FieldTypes.FieldType fieldType, boolean requireNonNullable) { if (!(fieldType instanceof FieldTypes.CollectionFieldType) || fieldType.getTypeId() != Types.LIST) { return Types.UNKNOWN; @@ -296,11 +313,18 @@ private static int listElementTypeId(FieldTypes.FieldType fieldType) { FieldTypes.FieldType elementType = ((FieldTypes.CollectionFieldType) fieldType).getElementType(); if (elementType instanceof FieldTypes.RegisteredFieldType) { + if (requireNonNullable && (elementType.nullable() || elementType.trackingRef())) { + return Types.UNKNOWN; + } return ((FieldTypes.RegisteredFieldType) elementType).getTypeId(); } return Types.UNKNOWN; } + private static int nonNullableListElementTypeId(FieldTypes.FieldType fieldType) { + return listElementTypeId(fieldType, true); + } + private static int arrayTypeId(FieldTypes.FieldType fieldType) { if (fieldType instanceof FieldTypes.RegisteredFieldType) { int typeId = ((FieldTypes.RegisteredFieldType) fieldType).getTypeId(); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 197d8400be..4f22870cc3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -1306,8 +1306,8 @@ private byte getInternalTypeId(Descriptor descriptor) { if (extMeta != null && extMeta.typeId() != Types.UNKNOWN) { return (byte) extMeta.typeId(); } - if (TypeAnnotationUtils.isBoxedListArrayType(descriptor.getField())) { - return (byte) TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor.getField()); + if (TypeAnnotationUtils.isBoxedListArrayType(descriptor)) { + return (byte) TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor); } if (cls.isArray() && cls.getComponentType().isPrimitive()) { return (byte) Types.getDescriptorTypeId(this, descriptor); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index c9440880ed..779888885d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -88,18 +88,21 @@ static ReadAction readAction(TypeResolver resolver, Descriptor descriptor) { FieldTypes.FieldType localFieldType = FieldTypes.buildFieldType(resolver, field); int peerListElementTypeId = listElementTypeId(descriptor); if (peerListElementTypeId != Types.UNKNOWN) { + int nonNullablePeerListElementTypeId = nonNullableListElementTypeId(descriptor); int localArrayTypeId = arrayTypeId(localFieldType); if (localArrayTypeId != Types.UNKNOWN - && localArrayTypeId == denseArrayTypeId(peerListElementTypeId)) { + && localArrayTypeId == denseArrayTypeId(nonNullablePeerListElementTypeId)) { return new ReadAction( - READ_LIST_TO_ARRAY, localArrayTypeId, peerListElementTypeId, field.getType()); + READ_LIST_TO_ARRAY, + localArrayTypeId, + nonNullablePeerListElementTypeId, + field.getType()); } int localListElementTypeId = nonNullableListElementTypeId(localFieldType); int peerArrayTypeId = denseArrayTypeId(peerListElementTypeId); - // The list-to-list fast path materializes through a dense primitive array, so it cannot - // preserve nullable or ref-tracked peer elements. List-to-array still accepts nullable - // schemas above and rejects only actual null/ref payload flags while reading. - if (nonNullableListElementTypeId(descriptor) != Types.UNKNOWN + // List-to-array and list-to-list materialize through a dense primitive array, so they cannot + // preserve nullable or ref-tracked peer elements. + if (nonNullablePeerListElementTypeId != Types.UNKNOWN && localListElementTypeId != Types.UNKNOWN && peerArrayTypeId != Types.UNKNOWN && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { @@ -129,21 +132,21 @@ static ReadAction readAction( TypeRef localType = localDescriptor.getTypeRef(); int peerListElementTypeId = listElementTypeId(remoteFieldType); if (peerListElementTypeId != Types.UNKNOWN) { - int localArrayTypeId = arrayTypeId(localType); + int nonNullablePeerListElementTypeId = nonNullableListElementTypeId(remoteFieldType); + int localArrayTypeId = arrayTypeId(localDescriptor); if (localArrayTypeId != Types.UNKNOWN - && localArrayTypeId == denseArrayTypeId(peerListElementTypeId)) { + && localArrayTypeId == denseArrayTypeId(nonNullablePeerListElementTypeId)) { return new ReadAction( READ_LIST_TO_ARRAY, localArrayTypeId, - peerListElementTypeId, + nonNullablePeerListElementTypeId, localDescriptor.getRawType()); } int localListElementTypeId = nonNullableListElementTypeId(localType); int peerArrayTypeId = denseArrayTypeId(peerListElementTypeId); - // The list-to-list fast path materializes through a dense primitive array, so it cannot - // preserve nullable or ref-tracked peer elements. List-to-array still accepts nullable - // schemas above and rejects only actual null/ref payload flags while reading. - if (nonNullableListElementTypeId(remoteFieldType) != Types.UNKNOWN + // List-to-array and list-to-list materialize through a dense primitive array, so they cannot + // preserve nullable or ref-tracked peer elements. + if (nonNullablePeerListElementTypeId != Types.UNKNOWN && localListElementTypeId != Types.UNKNOWN && peerArrayTypeId != Types.UNKNOWN && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 9c3f9ec797..c79dd08969 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -261,12 +261,12 @@ public static final class SerializationFieldInfo { resolver, type); } else if (primitiveListCollection) { containerSerializerOverride = new CollectionSerializer(resolver, (Class) type); - } else if (TypeAnnotationUtils.isBoxedListArrayType(descriptor.getField())) { + } else if (TypeAnnotationUtils.isBoxedListArrayType(descriptor)) { containerSerializerOverride = new org.apache.fory.serializer.collection.PrimitiveListSerializers .BoxedArrayAsListSerializer( resolver, - TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor.getField()), + TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor), qualifiedFieldName); } else { containerSerializerOverride = null; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 023a044c50..941683661b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -151,6 +151,12 @@ protected final int[] localFieldIds( protected final void writeBuildInFieldValue( WriteContext writeContext, SerializationFieldInfo fieldInfo, Object fieldValue) { + // Some schema-built-in fields still use container-shaped Java accessors, such as + // @ArrayType List. The override owns the accessor-to-payload conversion. + if (fieldInfo.containerSerializerOverride != null) { + writeContainerFieldValue(writeContext, fieldInfo, fieldValue); + return; + } AbstractObjectSerializer.writeBuildInFieldValue( writeContext, typeResolver, @@ -185,6 +191,10 @@ protected final void writeOtherFieldValue( protected final Object readBuildInFieldValue( ReadContext readContext, SerializationFieldInfo fieldInfo) { + // See writeBuildInFieldValue: built-in schema groups can still need container conversion. + if (fieldInfo.containerSerializerOverride != null) { + return readContainerFieldValue(readContext, fieldInfo); + } return AbstractObjectSerializer.readBuildInFieldValue( readContext, typeResolver, readContext.getRefReader(), fieldInfo, readContext.getBuffer()); } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java index 0fad594909..eef3aa93de 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeAnnotationUtils.java @@ -309,16 +309,13 @@ public static boolean isBoxedListArrayType(Descriptor descriptor) { if (TypeUtils.isPrimitiveListClass(rawType)) { return false; } + if (rawType.isArray() && rawType.getComponentType().isPrimitive()) { + return false; + } + // Fieldless descriptors can be TypeDef-reified schema views. Validate real source annotations + // through the Field overload; schema-only non-list descriptors are not boxed-list arrays. if (!List.class.isAssignableFrom(rawType)) { - if (Collection.class.isAssignableFrom(rawType)) { - throw new IllegalArgumentException( - "@ArrayType can only be applied to Fory primitive-list carriers or ordered " - + "java.util.List fields, but got " - + rawType.getName()); - } - throw new IllegalArgumentException( - "@ArrayType can only be applied to Fory primitive-list carriers or ordered " - + "java.util.List fields; primitive arrays already use array schema"); + return false; } return true; } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index 1fd546c5b8..51f4890514 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -176,8 +176,7 @@ public void testNullableListPayloadRejectedForArrayCompatibleRead() { byte[] listBytes = listFory.serialize(listStruct); Fory arrayFory = compatibleFory(DirectArrayField.class, codegen); - DirectArrayField arrayStruct = (DirectArrayField) arrayFory.deserialize(listBytes); - assertTrue(Arrays.equals(arrayStruct.values, new int[] {1, 2, 3})); + assertThrows(DeserializationException.class, () -> arrayFory.deserialize(listBytes)); listStruct.values = Arrays.asList(1, null, 3); byte[] nullablePayload = listFory.serialize(listStruct); diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index b5c90437d8..77193acf52 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -1879,13 +1879,15 @@ protected void testListArrayCompatibleRead(boolean enableCodegen) throws java.io newCompatibleNullableInt32ListField(1, 2, 3); buffer = MemoryBuffer.newHeapBuffer(256); nullableListFory.serialize(buffer, nullableListWithoutNulls); + byte[] nullableListWithoutNullsPayload = buffer.getBytes(0, buffer.writerIndex()); + Assert.expectThrows( + DeserializationException.class, + () -> arrayFory.deserialize(MemoryUtils.wrap(nullableListWithoutNullsPayload))); ctx = prepareExecution( - "test_list_array_compatible_list_to_array", buffer.getBytes(0, buffer.writerIndex())); + "test_list_array_compatible_nullable_list_to_array_error", + nullableListWithoutNullsPayload); runPeer(ctx); - XlangCompatibleInt32ArrayField nullableListArrayResult = - (XlangCompatibleInt32ArrayField) arrayFory.deserialize(readBuffer(ctx.dataFile())); - assertIntArrayEquals(nullableListArrayResult.values, 1, 2, 3); XlangCompatibleNullableInt32ListField nullableListValue = newCompatibleNullableInt32ListField(1, null, 3); diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py index cb3057962f..b3b6297918 100644 --- a/python/pyfory/meta/typedef.py +++ b/python/pyfory/meta/typedef.py @@ -635,9 +635,14 @@ def _list_array_element_type_matches(list_field_type: FieldType, array_field_typ array_element_type_id = _ARRAY_ELEMENT_TYPE_IDS.get(array_field_type.type_id) if array_element_type_id is None: return False - return list_field_type.type_id == TypeId.LIST and _list_element_type_matches_array_element( + return ( + list_field_type.type_id == TypeId.LIST + and not list_field_type.element_type.is_nullable + and not list_field_type.element_type.is_tracking_ref + and _list_element_type_matches_array_element( list_field_type.element_type.type_id, array_element_type_id ) + ) def _list_element_type_matches_array_element(list_element_type_id: TypeId, array_element_type_id: TypeId) -> bool: diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs index 421a224d6a..9d455e77a4 100644 --- a/rust/fory-core/src/serializer/collection.rs +++ b/rust/fory-core/src/serializer/collection.rs @@ -529,6 +529,12 @@ fn read_primitive_array_data_bulk( } fn list_element_type_matches_array(list: &FieldType, array: &FieldType) -> bool { + list_element_type_matches_array_shape(list, array) + && !list.generics[0].nullable + && !list.generics[0].track_ref +} + +fn list_element_type_matches_array_shape(list: &FieldType, array: &FieldType) -> bool { primitive_array_element_type_id(array.type_id).is_some_and(|element_type_id| { list.type_id == type_id::LIST && list.generics.len() == 1 @@ -829,8 +835,13 @@ where { if remote_field_type.type_id == type_id::LIST && !remote_field_type.generics.is_empty() - && list_element_type_matches_array(remote_field_type, local_field_type) + && list_element_type_matches_array_shape(remote_field_type, local_field_type) { + if remote_field_type.generics[0].nullable || remote_field_type.generics[0].track_ref { + return Err(Error::type_error( + "compatible list to array field requires non-null elements", + )); + } if field_ref_mode(remote_field_type) != RefMode::None { let ref_flag = context.reader.read_i8()?; if ref_flag == RefFlag::Null as i8 { From dc0819d9f4bca3a23f2244c6e9a55c860d5a95e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 07:21:57 +0800 Subject: [PATCH 34/58] fix(java): harden static compatible schema checks --- .../processing/ForyStructProcessor.java | 126 ++++++++++++++++-- .../StaticSerializerSourceWriter.java | 2 +- .../processing/ForyStructProcessorTest.java | 94 +++++++++++++ .../apache/fory/platform/GraalvmSupport.java | 40 +++++- .../apache/fory/resolver/ClassResolver.java | 4 +- .../apache/fory/resolver/TypeResolver.java | 10 +- .../CompatibleCollectionArrayReader.java | 44 ++++++ .../StaticGeneratedStructSerializer.java | 13 ++ .../StaticCompatibleCodecBuilderTest.java | 9 ++ 9 files changed, 318 insertions(+), 24 deletions(-) diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java index 75286fc77a..d6827c5d31 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/ForyStructProcessor.java @@ -579,7 +579,7 @@ private int reflectionModifiers(Set modifiers) { } private SourceTypeNode buildFieldTypeNode(VariableElement field, boolean nullable) { - return buildTypeNode(field.asType(), typeTree(field), Boolean.toString(nullable)); + return buildTypeNode(field.asType(), typeTree(field), Boolean.toString(nullable), field, false); } private Object typeTree(VariableElement field) { @@ -598,15 +598,21 @@ private Object typeTree(VariableElement field) { } private SourceTypeNode buildTypeNode(TypeMirror type) { - return buildTypeNode(type, null, "true"); + return buildTypeNode(type, null, "true", null, false); } - private SourceTypeNode buildTypeNode(TypeMirror type, Object tree, String typeExtNullable) { + private SourceTypeNode buildTypeNode( + TypeMirror type, + Object tree, + String typeExtNullable, + Element errorElement, + boolean arrayComponent) { TypeKind kind = type.getKind(); TypeTreeInfo treeInfo = typeTreeInfo(tree); if (kind == TypeKind.TYPEVAR) { TypeVariable typeVariable = (TypeVariable) type; - return buildTypeNode(typeVariable.getUpperBound(), null, typeExtNullable); + return buildTypeNode( + typeVariable.getUpperBound(), null, typeExtNullable, errorElement, arrayComponent); } if (kind == TypeKind.WILDCARD) { WildcardType wildcard = (WildcardType) type; @@ -614,7 +620,9 @@ private SourceTypeNode buildTypeNode(TypeMirror type, Object tree, String typeEx return buildTypeNode( bound == null ? elements.getTypeElement("java.lang.Object").asType() : bound, null, - typeExtNullable); + typeExtNullable, + errorElement, + arrayComponent); } List arguments = new ArrayList<>(); SourceTypeNode componentType = null; @@ -622,18 +630,25 @@ private SourceTypeNode buildTypeNode(TypeMirror type, Object tree, String typeEx TypeMirror componentMirror = ((ArrayType) type).getComponentType(); componentType = buildTypeNode( - componentMirror, treeInfo.arrayComponentTree(), nestedNullable(componentMirror)); + componentMirror, + treeInfo.arrayComponentTree(), + nestedNullable(componentMirror), + errorElement, + true); } else if (type instanceof DeclaredType) { List argumentTrees = treeInfo.typeArgumentTrees(); int index = 0; for (TypeMirror argument : ((DeclaredType) type).getTypeArguments()) { Object argumentTree = index < argumentTrees.size() ? argumentTrees.get(index) : null; - arguments.add(buildTypeNode(argument, argumentTree, nestedNullable(argument))); + arguments.add( + buildTypeNode(argument, argumentTree, nestedNullable(argument), errorElement, false)); index++; } } String rawType = canonicalName(types.erasure(type)); - String extMeta = typeExtMetaExpression(type, rawType, treeInfo.annotations, typeExtNullable); + String extMeta = + typeExtMetaExpression( + type, rawType, treeInfo.annotations, typeExtNullable, errorElement, arrayComponent); boolean primitive = kind.isPrimitive(); boolean nestedStruct = isCompatibleForyStructType(type); return new SourceTypeNode( @@ -681,8 +696,13 @@ private boolean isCompatibleForyStructType(TypeMirror type) { } private String typeExtMetaExpression( - TypeMirror type, String rawType, List treeAnnotations, String nullable) { - String typeId = scalarTypeId(type, rawType, treeAnnotations); + TypeMirror type, + String rawType, + List treeAnnotations, + String nullable, + Element errorElement, + boolean arrayComponent) { + String typeId = scalarTypeId(type, rawType, treeAnnotations, errorElement, arrayComponent); TypeUseAnnotation ref = typeUseAnnotation(type, treeAnnotations, REF); if (typeId == null && ref == null) { return null; @@ -700,18 +720,58 @@ private String nestedNullable(TypeMirror type) { return Boolean.toString(!type.getKind().isPrimitive()); } - private String scalarTypeId(TypeMirror type, String rawType, List treeAnnotations) { + private String scalarTypeId( + TypeMirror type, + String rawType, + List treeAnnotations, + Element errorElement, + boolean arrayComponent) { if (hasTypeAnnotation(type, treeAnnotations, INT8_TYPE)) { + validateScalarCarrier( + "@Int8Type", + rawType, + errorElement, + "byte", + "java.lang.Byte", + "byte[]", + "org.apache.fory.collection.Int8List"); return rawType.equals("byte[]") ? "Types.INT8_ARRAY" : "Types.INT8"; } if (hasTypeAnnotation(type, treeAnnotations, UINT8_TYPE)) { + validateScalarCarrier( + "@UInt8Type", + rawType, + errorElement, + arrayComponent + ? new String[] {"byte"} + : new String[] { + "int", "java.lang.Integer", "byte[]", "org.apache.fory.collection.UInt8List" + }); return rawType.equals("byte[]") ? "Types.UINT8_ARRAY" : "Types.UINT8"; } if (hasTypeAnnotation(type, treeAnnotations, UINT16_TYPE)) { + validateScalarCarrier( + "@UInt16Type", + rawType, + errorElement, + arrayComponent + ? new String[] {"short"} + : new String[] { + "int", "java.lang.Integer", "short[]", "org.apache.fory.collection.UInt16List" + }); return rawType.equals("short[]") ? "Types.UINT16_ARRAY" : "Types.UINT16"; } TypeUseAnnotation uint32 = typeUseAnnotation(type, treeAnnotations, UINT32_TYPE); if (uint32 != null) { + validateScalarCarrier( + "@UInt32Type", + rawType, + errorElement, + arrayComponent + ? new String[] {"int"} + : new String[] { + "long", "java.lang.Long", "int[]", "org.apache.fory.collection.UInt32List" + }); String encoding = int32Encoding(uint32); if (rawType.equals("int[]")) { return "Types.UINT32_ARRAY"; @@ -720,6 +780,15 @@ private String scalarTypeId(TypeMirror type, String rawType, List treeAnnotat } TypeUseAnnotation uint64 = typeUseAnnotation(type, treeAnnotations, UINT64_TYPE); if (uint64 != null) { + validateScalarCarrier( + "@UInt64Type", + rawType, + errorElement, + arrayComponent + ? new String[] {"long"} + : new String[] { + "long", "java.lang.Long", "long[]", "org.apache.fory.collection.UInt64List" + }); String encoding = int64Encoding(uint64); if (rawType.equals("long[]")) { return "Types.UINT64_ARRAY"; @@ -731,11 +800,25 @@ private String scalarTypeId(TypeMirror type, String rawType, List treeAnnotat } TypeUseAnnotation int32 = typeUseAnnotation(type, treeAnnotations, INT32_TYPE); if (int32 != null) { + validateScalarCarrier( + "@Int32Type", + rawType, + errorElement, + "int", + "java.lang.Integer", + "org.apache.fory.collection.Int32List"); String encoding = int32Encoding(int32); return "FIXED".equals(encoding) ? "Types.INT32" : "Types.VARINT32"; } TypeUseAnnotation int64 = typeUseAnnotation(type, treeAnnotations, INT64_TYPE); if (int64 != null) { + validateScalarCarrier( + "@Int64Type", + rawType, + errorElement, + "long", + "java.lang.Long", + "org.apache.fory.collection.Int64List"); String encoding = int64Encoding(int64); if ("FIXED".equals(encoding)) { return "Types.INT64"; @@ -743,14 +826,35 @@ private String scalarTypeId(TypeMirror type, String rawType, List treeAnnotat return "TAGGED".equals(encoding) ? "Types.TAGGED_INT64" : "Types.VARINT64"; } if (hasTypeAnnotation(type, treeAnnotations, FLOAT16_TYPE)) { + validateScalarCarrier( + "@Float16Type", + rawType, + errorElement, + arrayComponent ? new String[] {"short"} : new String[] {"short[]"}); return "Types.FLOAT16_ARRAY"; } if (hasTypeAnnotation(type, treeAnnotations, BFLOAT16_TYPE)) { + validateScalarCarrier( + "@BFloat16Type", + rawType, + errorElement, + arrayComponent ? new String[] {"short"} : new String[] {"short[]"}); return "Types.BFLOAT16_ARRAY"; } return null; } + private void validateScalarCarrier( + String annotationName, String rawType, Element errorElement, String... allowedTypes) { + for (String allowedType : allowedTypes) { + if (rawType.equals(allowedType)) { + return; + } + } + throw new InvalidStructException( + annotationName + " is not compatible with field type " + rawType, errorElement); + } + private boolean hasTypeAnnotation( TypeMirror type, List treeAnnotations, String annotationName) { return typeUseAnnotation(type, treeAnnotations, annotationName) != null; diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 8666ebde2c..bd027d4a43 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -130,7 +130,7 @@ private void writeDescriptors() { } builder.append(" return Collections.unmodifiableList(descriptors);\n"); builder.append(" }\n\n"); - builder.append(" private static List getGeneratedDescriptors() {\n"); + builder.append(" public static List getGeneratedDescriptors() {\n"); builder.append(" return DESCRIPTORS;\n"); builder.append(" }\n\n"); builder.append(" @Override\n"); diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index a152fd65d3..edfa311e52 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -40,6 +40,7 @@ import org.apache.fory.ThreadSafeFory; import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; +import org.apache.fory.exception.DeserializationException; import org.apache.fory.serializer.StaticGeneratedStructSerializer; import org.apache.fory.type.Descriptor; import org.apache.fory.type.Types; @@ -189,6 +190,26 @@ public void testDuplicateForyFieldIdFailsCompilation() throws Exception { result.diagnostics().contains("Duplicate @ForyField id 1"), result.diagnostics()); } + @Test + public void testInvalidScalarAnnotationCarrierFailsCompilation() throws Exception { + CompilationResult result = + compile( + "test.BadScalarCarrierStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "import org.apache.fory.annotation.UInt32Type;\n" + + "@ForyStruct public class BadScalarCarrierStruct {\n" + + " @UInt32Type public String id;\n" + + " public BadScalarCarrierStruct() {}\n" + + "}\n"); + Assert.assertFalse(result.success); + Assert.assertTrue( + result + .diagnostics() + .contains("@UInt32Type is not compatible with field type java.lang.String"), + result.diagnostics()); + } + @Test public void testInnerTypeGeneratedAsTopLevelBinaryTail() throws Exception { CompilationResult result = @@ -683,6 +704,79 @@ public void testStaticArrayTypeListWritesDenseArrayPayload() throws Exception { } } + @Test + public void testStaticIncompatibleListArrayCompatibleReadFails() throws Exception { + CompilationResult nullableListWriter = + compile( + "test.ListArrayMismatchStruct", + "package test;\n" + + "import java.util.List;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class ListArrayMismatchStruct {\n" + + " public List values;\n" + + " public ListArrayMismatchStruct() {}\n" + + "}\n"); + CompilationResult arrayReader = + compile( + "test.ListArrayMismatchStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class ListArrayMismatchStruct {\n" + + " public int[] values;\n" + + " public ListArrayMismatchStruct() {}\n" + + "}\n"); + Assert.assertTrue(nullableListWriter.success, nullableListWriter.diagnostics()); + Assert.assertTrue(arrayReader.success, arrayReader.diagnostics()); + try (URLClassLoader writerLoader = nullableListWriter.classLoader(); + URLClassLoader readerLoader = arrayReader.classLoader()) { + Class writerType = writerLoader.loadClass("test.ListArrayMismatchStruct"); + Class readerType = readerLoader.loadClass("test.ListArrayMismatchStruct"); + Fory writer = xlangCompatibleFory(writerLoader, writerType, false, "ListArrayMismatchStruct"); + Fory reader = xlangCompatibleFory(readerLoader, readerType, false, "ListArrayMismatchStruct"); + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "values", Arrays.asList(1, 2, 3)); + byte[] payload = writer.serialize(writerValue); + Assert.expectThrows(DeserializationException.class, () -> reader.deserialize(payload)); + } + + CompilationResult nestedListWriter = + compile( + "test.NestedListArrayMismatchStruct", + "package test;\n" + + "import java.util.Arrays;\n" + + "import java.util.List;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class NestedListArrayMismatchStruct {\n" + + " public List> values;\n" + + " public NestedListArrayMismatchStruct() {}\n" + + "}\n"); + CompilationResult nestedArrayReader = + compile( + "test.NestedListArrayMismatchStruct", + "package test;\n" + + "import java.util.List;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "@ForyStruct public class NestedListArrayMismatchStruct {\n" + + " public List values;\n" + + " public NestedListArrayMismatchStruct() {}\n" + + "}\n"); + Assert.assertTrue(nestedListWriter.success, nestedListWriter.diagnostics()); + Assert.assertTrue(nestedArrayReader.success, nestedArrayReader.diagnostics()); + try (URLClassLoader writerLoader = nestedListWriter.classLoader(); + URLClassLoader readerLoader = nestedArrayReader.classLoader()) { + Class writerType = writerLoader.loadClass("test.NestedListArrayMismatchStruct"); + Class readerType = readerLoader.loadClass("test.NestedListArrayMismatchStruct"); + Fory writer = + xlangCompatibleFory(writerLoader, writerType, false, "NestedListArrayMismatchStruct"); + Fory reader = + xlangCompatibleFory(readerLoader, readerType, false, "NestedListArrayMismatchStruct"); + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "values", Arrays.asList(Arrays.asList(1, 2, 3))); + byte[] payload = writer.serialize(writerValue); + Assert.expectThrows(DeserializationException.class, () -> reader.deserialize(payload)); + } + } + private static Fory xlangCompatibleFory(ClassLoader classLoader, Class type, boolean codegen) { return xlangCompatibleFory(classLoader, type, codegen, "ArrayShapeStruct"); } diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java index 573b379f83..955ae97306 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java @@ -381,7 +381,8 @@ public static class GraalvmClassRegistry { private final Map, Class> serializerClassMap; private final Map, Class> objectSerializerClassMap; private final Map> deserializerClassMap; - private final Map, Class> compatibleDeserializerClassMap; + private final Map> + compatibleDeserializerClassMap; private final Map> layerSerializerClassMap; private GraalvmClassRegistry() { @@ -447,13 +448,15 @@ public Map> getDeserializerClasses() { return Collections.unmodifiableMap(deserializerClassMap); } - public Class getCompatibleDeserializerClass(Class cls) { - return getRegisteredClassValue(compatibleDeserializerClassMap, cls); + public Class getCompatibleDeserializerClass( + Class cls, long typeDefId) { + return compatibleDeserializerClassMap.get(new CompatibleDeserializerKey(cls, typeDefId)); } public void putCompatibleDeserializerClass( - Class cls, Class serializerClass) { - compatibleDeserializerClassMap.put(cls, serializerClass); + Class cls, long typeDefId, Class serializerClass) { + compatibleDeserializerClassMap.put( + new CompatibleDeserializerKey(cls, typeDefId), serializerClass); } public Class getLayerSerializerClass(long typeDefId) { @@ -496,5 +499,32 @@ private static T getRegisteredClassValue(Map, T> registryMap, Class } return null; } + + private static final class CompatibleDeserializerKey { + private final Class cls; + private final long typeDefId; + + private CompatibleDeserializerKey(Class cls, long typeDefId) { + this.cls = cls; + this.typeDefId = typeDefId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CompatibleDeserializerKey)) { + return false; + } + CompatibleDeserializerKey that = (CompatibleDeserializerKey) o; + return typeDefId == that.typeDefId && cls == that.cls; + } + + @Override + public int hashCode() { + return 31 * System.identityHashCode(cls) + Long.hashCode(typeDefId); + } + } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 9d59822d2f..01ffe403d9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1822,7 +1822,9 @@ private void registerGraalvmSerializerClass(Class cls) { typeDef.getId(), getMetaSharedDeserializerClassForGraalvmBuild(cls, typeDef)); getGraalvmClassRegistry() .putCompatibleDeserializerClass( - cls, CodecUtils.loadOrGenStaticCompatibleCodecClass(this, cls, typeDef)); + cls, + typeDef.getId(), + CodecUtils.loadOrGenStaticCompatibleCodecClass(this, cls, typeDef)); } typeInfoCache = NIL_TYPE_INFO; if (RecordUtils.isRecord(cls)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index bec591e93a..9d7cccf467 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1062,7 +1062,7 @@ private TypeInfo getMetaSharedTypeInfo(TypeDef typeDef, Class clz) { } if (GraalvmSupport.isGraalBuildTime() && GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(sc)) { - getGraalvmClassRegistry().putCompatibleDeserializerClass(cls, sc); + getGraalvmClassRegistry().putCompatibleDeserializerClass(cls, typeDef.getId(), sc); typeInfo.setSerializer(this, new MetaSharedSerializer(this, cls, typeDef)); return typeInfo; } @@ -1658,8 +1658,7 @@ protected final StaticGeneratedStructSerializer newStaticGeneratedStructSeria Class serializerClass, Class cls, TypeDef typeDef) { try { Constructor constructor = - serializerClass.getDeclaredConstructor(TypeResolver.class, Class.class, TypeDef.class); - constructor.setAccessible(true); + serializerClass.getConstructor(TypeResolver.class, Class.class, TypeDef.class); return (StaticGeneratedStructSerializer) constructor.newInstance(this, cls, typeDef); } catch (NoSuchMethodException e) { throw new ForyException( @@ -1691,8 +1690,7 @@ private List getStaticGeneratedStructDescriptors(Class cls) { return null; } try { - Method descriptorsMethod = serializerClass.getDeclaredMethod("getGeneratedDescriptors"); - descriptorsMethod.setAccessible(true); + Method descriptorsMethod = serializerClass.getMethod("getGeneratedDescriptors"); return (List) descriptorsMethod.invoke(null); } catch (NoSuchMethodException e) { // Descriptor discovery must be side-effect free; instantiating the generated serializer here @@ -2059,7 +2057,7 @@ protected final Class getMetaSharedDeserializerClassFromGr if (deserializerClass != null) { return deserializerClass; } - deserializerClass = registry.getCompatibleDeserializerClass(cls); + deserializerClass = registry.getCompatibleDeserializerClass(cls, typeDef.getId()); if (deserializerClass != null && (!GraalvmSupport.isGraalBuildTime() || typeDef.getId() != TypeDef.buildTypeDef(this, cls).getId())) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index 779888885d..d8624aa85d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -173,6 +173,50 @@ static ReadAction readAction( return null; } + static boolean incompatibleCollectionArrayMatch( + TypeResolver resolver, FieldInfo remoteFieldInfo, Descriptor localDescriptor) { + if (localDescriptor == null || !resolver.isCrossLanguage()) { + return false; + } + if (readAction(resolver, remoteFieldInfo, localDescriptor) != null) { + return false; + } + FieldTypes.FieldType remoteFieldType = remoteFieldInfo.getFieldType(); + FieldTypes.FieldType localFieldType = FieldTypes.buildFieldType(resolver, localDescriptor); + return incompatibleCollectionArrayMatch(remoteFieldType, localFieldType); + } + + private static boolean incompatibleCollectionArrayMatch( + FieldTypes.FieldType remoteFieldType, FieldTypes.FieldType localFieldType) { + int remoteListElementTypeId = listElementTypeId(remoteFieldType); + int localArrayTypeId = arrayTypeId(localFieldType); + if (remoteListElementTypeId != Types.UNKNOWN + && localArrayTypeId != Types.UNKNOWN + && localArrayTypeId == denseArrayTypeId(remoteListElementTypeId)) { + return true; + } + int remoteArrayTypeId = arrayTypeId(remoteFieldType); + int localListElementTypeId = listElementTypeId(localFieldType); + if (remoteArrayTypeId != Types.UNKNOWN + && localListElementTypeId != Types.UNKNOWN + && remoteArrayTypeId == denseArrayTypeId(localListElementTypeId)) { + return true; + } + if (remoteFieldType instanceof FieldTypes.CollectionFieldType + && localFieldType instanceof FieldTypes.CollectionFieldType) { + return incompatibleCollectionArrayMatch( + ((FieldTypes.CollectionFieldType) remoteFieldType).getElementType(), + ((FieldTypes.CollectionFieldType) localFieldType).getElementType()); + } + if (remoteFieldType instanceof FieldTypes.ArrayFieldType + && localFieldType instanceof FieldTypes.ArrayFieldType) { + return incompatibleCollectionArrayMatch( + ((FieldTypes.ArrayFieldType) remoteFieldType).getComponentType(), + ((FieldTypes.ArrayFieldType) localFieldType).getComponentType()); + } + return false; + } + static Object read(ReadContext readContext, RefMode refMode, ReadAction action) { return read( readContext, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 941683661b..e5d763fe47 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -257,6 +257,15 @@ protected final SerializationFieldInfo localFieldInfo(int matchedId) { protected final boolean canReadRemoteField( RemoteFieldInfo remoteField, SerializationFieldInfo localFieldInfo) { + if (remoteField.incompatibleCollectionArrayMatch) { + throw new DeserializationException( + "Cannot read remote field " + + remoteField.descriptor.getName() + + " as local field " + + localFieldInfo.descriptor.getName() + + ": compatible list/array adaptation requires a matching non-null primitive element" + + " schema and does not apply recursively"); + } if (remoteField.compatibleCollectionArrayReadAction != null) { return true; } @@ -544,6 +553,7 @@ protected static final class RemoteFieldInfo { private final Descriptor descriptor; private final SerializationFieldInfo serializationFieldInfo; private final CompatibleCollectionArrayReader.ReadAction compatibleCollectionArrayReadAction; + private final boolean incompatibleCollectionArrayMatch; private RemoteFieldInfo( TypeResolver typeResolver, @@ -558,6 +568,9 @@ private RemoteFieldInfo( this.serializationFieldInfo = serializationFieldInfo; this.compatibleCollectionArrayReadAction = CompatibleCollectionArrayReader.readAction(typeResolver, fieldInfo, localDescriptor); + this.incompatibleCollectionArrayMatch = + CompatibleCollectionArrayReader.incompatibleCollectionArrayMatch( + typeResolver, fieldInfo, localDescriptor); } } } diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java index ab907bca30..7db7354820 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -46,6 +46,7 @@ import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; import org.apache.fory.meta.TypeDef; +import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.mockito.MockedStatic; @@ -326,6 +327,14 @@ public void testStaticCompatibleSerializerClassKeyIncludesRemoteTypeDef() throws CodecUtils.loadOrGenStaticCompatibleCodecClass( reader.getTypeResolver(), readerClass, typeDefB); Assert.assertNotEquals(serializerA, serializerB); + + GraalvmSupport.GraalvmClassRegistry registry = GraalvmSupport.getClassRegistry(0); + registry.putCompatibleDeserializerClass(readerType, typeDefA.getId(), serializerA); + registry.putCompatibleDeserializerClass(readerType, typeDefB.getId(), serializerB); + Assert.assertSame( + registry.getCompatibleDeserializerClass(readerType, typeDefA.getId()), serializerA); + Assert.assertSame( + registry.getCompatibleDeserializerClass(readerType, typeDefB.getId()), serializerB); } } From 7c3d32cfb59a3499f1e8806b5bc49e420d535463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 07:52:35 +0800 Subject: [PATCH 35/58] fix: reject nullable list array compatibility --- .../serialization/struct_compatible_test.cc | 6 +- cpp/fory/serialization/type_resolver.cc | 6 -- cpp/fory/serialization/type_resolver.h | 16 ---- csharp/src/Fory/TypeMeta.cs | 42 ++++++++- csharp/tests/Fory.Tests/ForyRuntimeTests.cs | 5 +- .../serializer/collection_serializers.dart | 20 +++- ...calar_and_typed_array_serializer_test.dart | 6 +- .../apache/fory/platform/GraalvmSupport.java | 40 +------- .../apache/fory/resolver/ClassResolver.java | 38 +------- .../apache/fory/resolver/TypeResolver.java | 4 +- .../CompatibleCollectionArrayReader.java | 50 +++++++++- .../StaticCompatibleCodecBuilderTest.java | 10 +- .../fory/type/DescriptorGrouperTest.java | 93 +++++-------------- javascript/packages/core/lib/context.ts | 3 + javascript/packages/core/lib/typeResolver.ts | 1 - javascript/test/typemeta.test.ts | 10 +- python/pyfory/tests/test_typedef_encoding.py | 5 +- rust/fory-core/src/meta/type_meta.rs | 32 ------- rust/tests/tests/compatible/test_struct.rs | 10 +- swift/Sources/Fory/FieldCodecs.swift | 3 + swift/Sources/Fory/TypeMeta.swift | 68 ++++++++++++-- swift/Sources/Fory/TypeResolver.swift | 7 +- .../Tests/ForyTests/CompatibilityTests.swift | 5 +- 23 files changed, 237 insertions(+), 243 deletions(-) diff --git a/cpp/fory/serialization/struct_compatible_test.cc b/cpp/fory/serialization/struct_compatible_test.cc index b688ba69f8..0078342618 100644 --- a/cpp/fory/serialization/struct_compatible_test.cc +++ b/cpp/fory/serialization/struct_compatible_test.cc @@ -506,8 +506,10 @@ TEST(SchemaEvolutionTest, NullableListElementsCannotReadIntoArrayCarrier) { auto decoded = reader.deserialize(payload.data(), payload.size()); - ASSERT_TRUE(decoded.ok()) << decoded.error().to_string(); - EXPECT_EQ(decoded.value().values, (std::vector{1, 2})); + ASSERT_FALSE(decoded.ok()); + EXPECT_NE(decoded.error().to_string().find("non-null elements"), + std::string::npos) + << decoded.error().to_string(); bytes = writer.serialize(CompatibleNullableListField{{1, std::nullopt}}); ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index 65e9220601..e84a986574 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1361,12 +1361,6 @@ int32_t TypeMeta::compute_struct_version(const TypeMeta &meta) { // Use the low 64 bits and then keep low 32 bits as i32. uint64_t low = static_cast(hash_out[0]); uint32_t version = static_cast(low & 0xFFFF'FFFFu); -#if defined(FORY_DEBUG) || defined(ENABLE_FORY_DEBUG_OUTPUT) - // DEBUG: Print fingerprint for debugging version mismatch - std::cerr << "[xlang][debug] struct_version type_name=" << meta.type_name - << ", fingerprint=\"" << fingerprint - << "\" version=" << static_cast(version) << std::endl; -#endif return static_cast(version); } diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 48f73445cf..a244ed5b22 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -1323,14 +1323,6 @@ template struct FieldInfoBuilder { field_type.nullable = is_nullable; field_type.track_ref = track_ref; field_type.ref_mode = make_ref_mode(is_nullable, track_ref); -#ifdef FORY_DEBUG - // DEBUG: Print field info for debugging fingerprint mismatch - std::cerr << "[xlang][debug] FieldInfoBuilder T=" << typeid(T).name() - << " Index=" << Index << " field=" << field_name - << " type_id=" << field_type.type_id - << " is_nullable=" << is_nullable << " track_ref=" << track_ref - << std::endl; -#endif FieldInfo info(std::move(field_name), std::move(field_type)); info.field_id = field_id; return info; @@ -1371,14 +1363,6 @@ template struct FieldInfoBuilder { field_type.nullable = is_nullable; field_type.track_ref = track_ref; field_type.ref_mode = make_ref_mode(is_nullable, track_ref); -#ifdef FORY_DEBUG - // DEBUG: Print field info for debugging fingerprint mismatch - std::cerr << "[xlang][debug] FieldInfoBuilder T=" << typeid(T).name() - << " Index=" << Index << " field=" << field_name - << " type_id=" << field_type.type_id - << " is_nullable=" << is_nullable << " track_ref=" << track_ref - << std::endl; -#endif FieldInfo info(std::move(field_name), std::move(field_type)); info.field_id = field_id; return info; diff --git a/csharp/src/Fory/TypeMeta.cs b/csharp/src/Fory/TypeMeta.cs index 216d34dfcd..6bdb8fc6ce 100644 --- a/csharp/src/Fory/TypeMeta.cs +++ b/csharp/src/Fory/TypeMeta.cs @@ -782,11 +782,13 @@ public static void AssignFieldIds( } } - if (localIndex >= 0 && - localMatch is not null && - IsCompatibleFieldType(remoteField.FieldType, localMatch.FieldType, topLevel: true)) + if (localIndex >= 0 && localMatch is not null) { - remoteField.AssignedFieldId = localIndex; + ThrowIfUnsupportedListArrayMismatch(remoteField.FieldType, localMatch.FieldType, topLevel: true); + remoteField.AssignedFieldId = + IsCompatibleFieldType(remoteField.FieldType, localMatch.FieldType, topLevel: true) + ? localIndex + : -1; } else { @@ -795,6 +797,28 @@ localMatch is not null && } } + private static void ThrowIfUnsupportedListArrayMismatch( + TypeMetaFieldType remote, + TypeMetaFieldType local, + bool topLevel) + { + if (topLevel && IsListArrayShapePair(remote, local)) + { + if (IsCompatibleListArrayFieldPair(remote, local)) + { + return; + } + if (remote.TypeId == (uint)global::Apache.Fory.TypeId.List && + TryPackedArrayElementTypeId(local.TypeId).HasValue && + remote.Generics.Count == 1 && + (remote.Generics[0].Nullable || remote.Generics[0].TrackRef)) + { + throw new InvalidDataException("compatible list to array field requires non-null elements"); + } + throw new InvalidDataException("unsupported compatible list/array schema mismatch"); + } + } + private static bool IsCompatibleFieldType(TypeMetaFieldType remote, TypeMetaFieldType local, bool topLevel) { if (topLevel && IsCompatibleListArrayFieldPair(remote, local)) @@ -830,6 +854,8 @@ private static bool IsCompatibleListArrayFieldPair(TypeMetaFieldType remote, Typ bool remoteListLocalArray = remote.TypeId == (uint)global::Apache.Fory.TypeId.List && localArrayElementTypeId.HasValue && remote.Generics.Count == 1 && + !remote.Generics[0].Nullable && + !remote.Generics[0].TrackRef && CompatibleScalarTypeId(localArrayElementTypeId.Value) == CompatibleScalarTypeId(remote.Generics[0].TypeId); if (remoteListLocalArray) @@ -844,6 +870,14 @@ private static bool IsCompatibleListArrayFieldPair(TypeMetaFieldType remote, Typ CompatibleScalarTypeId(local.Generics[0].TypeId); } + private static bool IsListArrayShapePair(TypeMetaFieldType remote, TypeMetaFieldType local) + { + return remote.TypeId == (uint)global::Apache.Fory.TypeId.List && + TryPackedArrayElementTypeId(local.TypeId).HasValue || + local.TypeId == (uint)global::Apache.Fory.TypeId.List && + TryPackedArrayElementTypeId(remote.TypeId).HasValue; + } + private static uint? TryPackedArrayElementTypeId(uint typeId) { return typeId switch diff --git a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs index 511713cf37..038058e94f 100644 --- a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs +++ b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs @@ -1080,8 +1080,9 @@ public void CompatibleReadRejectsNullableListElementsIntoArrayCarrier() reader.Register(308); byte[] nonNullPayload = writer.Serialize(new CompatibleNullableListSchema { Values = [1, 2] }); - CompatibleArraySchema decoded = reader.Deserialize(nonNullPayload); - Assert.Equal([1, 2], decoded.Values); + InvalidDataException nonNullException = + Assert.Throws(() => reader.Deserialize(nonNullPayload)); + Assert.Contains("compatible list to array field requires non-null elements", nonNullException.Message); byte[] payload = writer.Serialize(new CompatibleNullableListSchema { Values = [1, null] }); InvalidDataException exception = diff --git a/dart/packages/fory/lib/src/serializer/collection_serializers.dart b/dart/packages/fory/lib/src/serializer/collection_serializers.dart index 125292cc7f..0e757bd916 100644 --- a/dart/packages/fory/lib/src/serializer/collection_serializers.dart +++ b/dart/packages/fory/lib/src/serializer/collection_serializers.dart @@ -466,11 +466,19 @@ bool isCompatibleCollectionArrayTypePair( ) { if (isCompatibleArrayType(localType.typeId) && remoteType.typeId == TypeIds.list) { - return _listElementMatchesArray(remoteType, localType.typeId); + return _listElementMatchesArray( + remoteType, + localType.typeId, + requireNonNullableElement: true, + ); } if (localType.typeId == TypeIds.list && isCompatibleArrayType(remoteType.typeId)) { - return _listElementMatchesArray(localType, remoteType.typeId); + return _listElementMatchesArray( + localType, + remoteType.typeId, + requireNonNullableElement: false, + ); } return false; } @@ -485,10 +493,16 @@ bool isCompatibleCollectionArrayRootTypePair( (isCompatibleArrayType(localTypeId) && remoteTypeId == TypeIds.list); } -bool _listElementMatchesArray(FieldType listType, int arrayTypeId) { +bool _listElementMatchesArray( + FieldType listType, + int arrayTypeId, { + required bool requireNonNullableElement, +}) { final elementType = listType.arguments.isEmpty ? null : listType.arguments.single; return elementType != null && + (!requireNonNullableElement || + (!elementType.nullable && !elementType.ref)) && _arrayElementTypeId(arrayTypeId) == _compatibleArrayElementTypeId(elementType.typeId); } diff --git a/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart b/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart index 990de58709..12245bf897 100644 --- a/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart +++ b/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart @@ -640,8 +640,10 @@ void main() { final nonNullBytes = writer.serialize( CompatibleNullableListEnvelope()..values = [1, 2, 3], ); - final decoded = reader.deserialize(nonNullBytes); - expect(decoded.values, orderedEquals([1, 2, 3])); + expect( + () => reader.deserialize(nonNullBytes), + throwsStateError, + ); final nullableBytes = writer.serialize( CompatibleNullableListEnvelope()..values = [1, null, 3], diff --git a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java index 955ae97306..573b379f83 100644 --- a/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java +++ b/java/fory-core/src/main/java/org/apache/fory/platform/GraalvmSupport.java @@ -381,8 +381,7 @@ public static class GraalvmClassRegistry { private final Map, Class> serializerClassMap; private final Map, Class> objectSerializerClassMap; private final Map> deserializerClassMap; - private final Map> - compatibleDeserializerClassMap; + private final Map, Class> compatibleDeserializerClassMap; private final Map> layerSerializerClassMap; private GraalvmClassRegistry() { @@ -448,15 +447,13 @@ public Map> getDeserializerClasses() { return Collections.unmodifiableMap(deserializerClassMap); } - public Class getCompatibleDeserializerClass( - Class cls, long typeDefId) { - return compatibleDeserializerClassMap.get(new CompatibleDeserializerKey(cls, typeDefId)); + public Class getCompatibleDeserializerClass(Class cls) { + return getRegisteredClassValue(compatibleDeserializerClassMap, cls); } public void putCompatibleDeserializerClass( - Class cls, long typeDefId, Class serializerClass) { - compatibleDeserializerClassMap.put( - new CompatibleDeserializerKey(cls, typeDefId), serializerClass); + Class cls, Class serializerClass) { + compatibleDeserializerClassMap.put(cls, serializerClass); } public Class getLayerSerializerClass(long typeDefId) { @@ -499,32 +496,5 @@ private static T getRegisteredClassValue(Map, T> registryMap, Class } return null; } - - private static final class CompatibleDeserializerKey { - private final Class cls; - private final long typeDefId; - - private CompatibleDeserializerKey(Class cls, long typeDefId) { - this.cls = cls; - this.typeDefId = typeDefId; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof CompatibleDeserializerKey)) { - return false; - } - CompatibleDeserializerKey that = (CompatibleDeserializerKey) o; - return typeDefId == that.typeDefId && cls == that.cls; - } - - @Override - public int hashCode() { - return 31 * System.identityHashCode(cls) + Long.hashCode(typeDefId); - } - } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 01ffe403d9..67797d8921 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1822,9 +1822,7 @@ private void registerGraalvmSerializerClass(Class cls) { typeDef.getId(), getMetaSharedDeserializerClassForGraalvmBuild(cls, typeDef)); getGraalvmClassRegistry() .putCompatibleDeserializerClass( - cls, - typeDef.getId(), - CodecUtils.loadOrGenStaticCompatibleCodecClass(this, cls, typeDef)); + cls, CodecUtils.loadOrGenStaticCompatibleCodecClass(this, cls, typeDef)); } typeInfoCache = NIL_TYPE_INFO; if (RecordUtils.isRecord(cls)) { @@ -2149,7 +2147,7 @@ private int getLogicalDescriptorSortTypeId(Descriptor descriptor) { return Types.UNKNOWN; } if (isCollectionDescriptor(descriptor)) { - return Types.LIST; + return isSet(rawType) ? Types.SET : Types.LIST; } if (rawType.isArray() && !rawType.getComponentType().isPrimitive()) { return Types.LIST; @@ -2244,44 +2242,18 @@ protected DescriptorGrouper configureDescriptorGrouper(DescriptorGrouper descrip } /** - * Normalize type name for deterministic fallback ordering between serialization and - * deserialization. - */ - private String getNormalizedTypeName(Descriptor d) { - if (isCollectionDescriptor(d)) { - return "java.util.Collection"; - } - Class rawType = d.getRawType(); - if (rawType != null && isMap(rawType)) { - return "java.util.Map"; - } - return d.getTypeName(); - } - - /** - * Creates a comparator for sorting descriptors by internal type id and field name/id. TypeDef - * descriptors preserve native internal ids as decimal type names; compare them numerically so ids - * such as 101 do not sort before 17. + * Creates a comparator for sorting descriptors by logical internal type id and field name/id. + * Native compatible mode intentionally follows the xlang/spec field order, so equal type-id + * groups must not reintroduce Java raw-type ordering. */ public Comparator createTypeAndNameComparator() { return (d1, d2) -> { - // sort by type so that we can hit class info cache more possibly. - // sort by field id/name to fix order if type is same. int c = Integer.compare(getDescriptorSortTypeId(d1), getDescriptorSortTypeId(d2)); - if (c == 0) { - // Use normalized type name so that Collection/Map subtypes have consistent order - // between processes even if the field doesn't exist in peer (e.g., List vs Collection). - c = getNormalizedTypeName(d1).compareTo(getNormalizedTypeName(d2)); - } - // noinspection Duplicates if (c == 0) { c = compareFieldSortKey(d1, d2); if (c == 0) { - // Field name duplicate in super/child classes. c = d1.getDeclaringClass().compareTo(d2.getDeclaringClass()); if (c == 0) { - // Final tie-breaker: use actual field name to distinguish fields with same tag ID. - // This ensures TreeSet never treats different fields as duplicates. c = d1.getName().compareTo(d2.getName()); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 9d7cccf467..94c889fb4b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1062,7 +1062,7 @@ private TypeInfo getMetaSharedTypeInfo(TypeDef typeDef, Class clz) { } if (GraalvmSupport.isGraalBuildTime() && GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(sc)) { - getGraalvmClassRegistry().putCompatibleDeserializerClass(cls, typeDef.getId(), sc); + getGraalvmClassRegistry().putCompatibleDeserializerClass(cls, sc); typeInfo.setSerializer(this, new MetaSharedSerializer(this, cls, typeDef)); return typeInfo; } @@ -2057,7 +2057,7 @@ protected final Class getMetaSharedDeserializerClassFromGr if (deserializerClass != null) { return deserializerClass; } - deserializerClass = registry.getCompatibleDeserializerClass(cls, typeDef.getId()); + deserializerClass = registry.getCompatibleDeserializerClass(cls); if (deserializerClass != null && (!GraalvmSupport.isGraalBuildTime() || typeDef.getId() != TypeDef.buildTypeDef(this, cls).getId())) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index d8624aa85d..5834207083 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -105,7 +105,8 @@ static ReadAction readAction(TypeResolver resolver, Descriptor descriptor) { if (nonNullablePeerListElementTypeId != Types.UNKNOWN && localListElementTypeId != Types.UNKNOWN && peerArrayTypeId != Types.UNKNOWN - && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { + && peerArrayTypeId == denseArrayTypeId(localListElementTypeId) + && canMaterializeListTarget(field.getType(), peerArrayTypeId)) { return new ReadAction( READ_LIST_TO_LIST, peerArrayTypeId, peerListElementTypeId, field.getType()); } @@ -115,7 +116,8 @@ static ReadAction readAction(TypeResolver resolver, Descriptor descriptor) { if (peerArrayTypeId != Types.UNKNOWN) { int localListElementTypeId = listElementTypeId(localFieldType); if (localListElementTypeId != Types.UNKNOWN - && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { + && peerArrayTypeId == denseArrayTypeId(localListElementTypeId) + && canMaterializeListTarget(field.getType(), peerArrayTypeId)) { return new ReadAction( READ_ARRAY_TO_LIST, peerArrayTypeId, localListElementTypeId, field.getType()); } @@ -149,7 +151,8 @@ static ReadAction readAction( if (nonNullablePeerListElementTypeId != Types.UNKNOWN && localListElementTypeId != Types.UNKNOWN && peerArrayTypeId != Types.UNKNOWN - && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { + && peerArrayTypeId == denseArrayTypeId(localListElementTypeId) + && canMaterializeListTarget(localDescriptor.getRawType(), peerArrayTypeId)) { return new ReadAction( READ_LIST_TO_LIST, peerArrayTypeId, @@ -162,7 +165,8 @@ static ReadAction readAction( if (peerArrayTypeId != Types.UNKNOWN) { int localListElementTypeId = listElementTypeId(localType); if (localListElementTypeId != Types.UNKNOWN - && peerArrayTypeId == denseArrayTypeId(localListElementTypeId)) { + && peerArrayTypeId == denseArrayTypeId(localListElementTypeId) + && canMaterializeListTarget(localDescriptor.getRawType(), peerArrayTypeId)) { return new ReadAction( READ_ARRAY_TO_LIST, peerArrayTypeId, @@ -811,6 +815,44 @@ private static Object materializePrimitiveList( } } + private static boolean canMaterializeListTarget(Class targetType, int arrayTypeId) { + return canMaterializePrimitiveListTarget(targetType, arrayTypeId) + || targetType.isAssignableFrom(ArrayList.class); + } + + private static boolean canMaterializePrimitiveListTarget(Class targetType, int arrayTypeId) { + switch (arrayTypeId) { + case Types.BOOL_ARRAY: + return targetType == BoolList.class; + case Types.INT8_ARRAY: + return targetType == Int8List.class; + case Types.UINT8_ARRAY: + return targetType == UInt8List.class; + case Types.INT16_ARRAY: + return targetType == Int16List.class; + case Types.UINT16_ARRAY: + return targetType == UInt16List.class; + case Types.INT32_ARRAY: + return targetType == Int32List.class; + case Types.UINT32_ARRAY: + return targetType == UInt32List.class; + case Types.INT64_ARRAY: + return targetType == Int64List.class; + case Types.UINT64_ARRAY: + return targetType == UInt64List.class; + case Types.FLOAT16_ARRAY: + return targetType == Float16List.class; + case Types.BFLOAT16_ARRAY: + return targetType == BFloat16List.class; + case Types.FLOAT32_ARRAY: + return targetType == Float32List.class; + case Types.FLOAT64_ARRAY: + return targetType == Float64List.class; + default: + throw new IllegalArgumentException("Unsupported dense array type id " + arrayTypeId); + } + } + private static List materializeBoxedList(Object array, int arrayTypeId) { int size = java.lang.reflect.Array.getLength(array); ArrayList list = new ArrayList<>(size); diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java index 7db7354820..ee24ab00b2 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -276,7 +276,7 @@ public void testStaticCompatibleArrayPayloadReadsOrdinaryAnnotatedList() throws } @Test - public void testStaticCompatibleSerializerClassKeyIncludesRemoteTypeDef() throws Exception { + public void testGraalCompatibleSerializerRegistryUsesLocalReaderClass() throws Exception { CompilationResult writerAResult = compile( "test.WriterPayloadA", @@ -329,12 +329,8 @@ public void testStaticCompatibleSerializerClassKeyIncludesRemoteTypeDef() throws Assert.assertNotEquals(serializerA, serializerB); GraalvmSupport.GraalvmClassRegistry registry = GraalvmSupport.getClassRegistry(0); - registry.putCompatibleDeserializerClass(readerType, typeDefA.getId(), serializerA); - registry.putCompatibleDeserializerClass(readerType, typeDefB.getId(), serializerB); - Assert.assertSame( - registry.getCompatibleDeserializerClass(readerType, typeDefA.getId()), serializerA); - Assert.assertSame( - registry.getCompatibleDeserializerClass(readerType, typeDefB.getId()), serializerB); + registry.putCompatibleDeserializerClass(readerType, serializerA); + Assert.assertSame(registry.getCompatibleDeserializerClass(readerType), serializerA); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java index b1e6b05b7b..a079f4ee9c 100644 --- a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java @@ -23,6 +23,7 @@ import com.google.common.primitives.Primitives; import java.time.Instant; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -76,33 +77,17 @@ private List createDescriptors() { @Test public void testComparatorByTypeAndName() { Fory fory = Fory.builder().build(); - List descriptors = createDescriptors(); + List descriptors = new ArrayList<>(); + descriptors.add( + createDescriptor(TypeRef.of(Date.class), "z_timestamp", -1, "TestClass", false)); + descriptors.add( + createDescriptor(TypeRef.of(Instant.class), "a_timestamp", -1, "TestClass", false)); + descriptors.add( + createDescriptor(TypeRef.of(LocalDateTime.class), "m_timestamp", -1, "TestClass", false)); descriptors.sort(((ClassResolver) fory.getTypeResolver()).createTypeAndNameComparator()); - List> classes = - descriptors.stream().map(Descriptor::getRawType).collect(Collectors.toList()); - List> expected = - Arrays.asList( - boolean.class, - Boolean.class, - byte.class, - Byte.class, - Short.class, - short.class, - Integer.class, - int.class, - Long.class, - long.class, - float.class, - Float.class, - double.class, - Double.class, - String.class, - Void.class, - Character.class, - void.class, - char.class, - Object.class); - assertEquals(classes, expected); + assertEquals( + descriptors.stream().map(Descriptor::getName).collect(Collectors.toList()), + Arrays.asList("a_timestamp", "m_timestamp", "z_timestamp")); } @Test @@ -338,11 +323,7 @@ public void testCompressedPrimitiveGrouper() { } } - /** - * Test that ClassResolver's comparator normalizes Collection/Map subtypes for consistent - * ordering. This ensures List/ArrayList/HashSet are all treated as Collection, and - * HashMap/TreeMap are all treated as Map. - */ + /** Test that collection-like descriptors use xlang/spec type-id groups before field names. */ @Test public void testNormalizedTypeNameComparator() { Fory fory = builder().build(); @@ -381,53 +362,21 @@ public void testNormalizedTypeNameComparator() { List fieldNames = descriptors.stream().map(Descriptor::getName).collect(Collectors.toList()); - // All Collection types should be grouped together (sorted by field name within the group) - // All Map types should be grouped together (sorted by field name within the group) - // Collection types come before Map types alphabetically ("java.util.Collection" < - // "java.util.Map") + // LIST-like fields are ordered by field name, then SET, then MAP. List expected = Arrays.asList( - "arrayListField", - "collField", - "listField", - "setField", // Collection types - "hashMapField", - "mapField" // Map types - ); + "arrayListField", "collField", "listField", "setField", "hashMapField", "mapField"); assertEquals(fieldNames, expected); } - /** - * Test that the DescriptorGrouper's static COMPARATOR_BY_TYPE_AND_NAME does NOT normalize - * Collection/Map types (it uses the raw type name). - */ @Test - public void testStaticComparatorDoesNotNormalize() { - // Create descriptors with different Collection/Map subtypes - List descriptors = new ArrayList<>(); - descriptors.add( - createDescriptor(new TypeRef>() {}, "listField", -1, "TestClass", false)); - descriptors.add( - createDescriptor( - new TypeRef>() {}, "collField", -1, "TestClass", false)); - descriptors.add( - createDescriptor( - new TypeRef>() {}, "arrayListField", -1, "TestClass", false)); + public void testComparatorUsesFieldIdentifierBeforeRawTypeName() { Fory fory = Fory.builder().build(); - // Sort with the static comparator - descriptors.sort(((ClassResolver) fory.getTypeResolver()).createTypeAndNameComparator()); - - // Get type names after sorting - List typeNames = - descriptors.stream().map(Descriptor::getTypeName).collect(Collectors.toList()); - - // The static comparator should sort by actual type name, not normalized - // ArrayList < Collection < List (alphabetically) - assertEquals( - typeNames, - Arrays.asList( - "java.util.ArrayList", - "java.util.Collection", - "java.util.List")); + Comparator comparator = + ((ClassResolver) fory.getTypeResolver()).createTypeAndNameComparator(); + Descriptor date = createDescriptor(TypeRef.of(Date.class), "z_date", -1, "TestClass", false); + Descriptor instant = + createDescriptor(TypeRef.of(Instant.class), "a_instant", -1, "TestClass", false); + assertTrue(comparator.compare(date, instant) > 0); } } diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 0f59afe694..c2906e49e9 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -872,6 +872,9 @@ export class ReadContext { if (compatibleArrayElementTypeId(remoteElement.typeId) !== localElement) { return undefined; } + if (remoteElement.nullable === true || remoteElement.trackingRef === true) { + return undefined; + } return compatibleListToArrayTypeInfo(remoteElement, localElement); } const remoteArrayElement = denseArrayElementTypeId(remote.typeId); diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index 5680e11b83..c9936be86d 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -292,7 +292,6 @@ export default class TypeResolver { regenerateReadSerializer(typeInfo: TypeInfo) { const serializer = this.generateReadSerializer(typeInfo); return this.registerSerializer(typeInfo, { - getTypeInfo: serializer.getTypeInfo, getHash: serializer.getHash, getTypeInfo: serializer.getTypeInfo, read: serializer.read, diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index 9ec14d03aa..fe94088a19 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -452,13 +452,17 @@ describe("typemeta", () => { const nonNullBytes = serializer.serialize({ values: [1, 2, 3], }); - const result = readerFory.register(readerType).deserialize(nonNullBytes); - expect(Array.from(result.values as Int32Array)).toEqual([1, 2, 3]); + const reader = readerFory.register(readerType); + expect(() => reader.deserialize(nonNullBytes)).toThrow( + "unsupported compatible list/array schema mismatch", + ); const nullableBytes = serializer.serialize({ values: [1, null, 3], }); - expect(() => readerFory.register(readerType).deserialize(nullableBytes)).toThrow(); + expect(() => reader.deserialize(nullableBytes)).toThrow( + "unsupported compatible list/array schema mismatch", + ); }); test("rejects incompatible immediate list and dense array element fields", () => { diff --git a/python/pyfory/tests/test_typedef_encoding.py b/python/pyfory/tests/test_typedef_encoding.py index e91a42893e..d4522a70fa 100644 --- a/python/pyfory/tests/test_typedef_encoding.py +++ b/python/pyfory/tests/test_typedef_encoding.py @@ -590,9 +590,8 @@ def test_compatible_nullable_int32_list_payload_rejects_array_read(): _register_int32_payload(writer, NullableInt32ListPayload) _register_int32_payload(reader, Int32ArrayPayload) - decoded = reader.deserialize(writer.serialize(NullableInt32ListPayload(payload=[1, 2, 3]))) - assert isinstance(decoded, Int32ArrayPayload) - assert list(decoded.payload) == [1, 2, 3] + with pytest.raises(TypeNotCompatibleError): + reader.deserialize(writer.serialize(NullableInt32ListPayload(payload=[1, 2, 3]))) with pytest.raises(TypeNotCompatibleError): reader.deserialize(writer.serialize(NullableInt32ListPayload(payload=[1, None, 3]))) diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs index 02b96ba045..193b9abb84 100644 --- a/rust/fory-core/src/meta/type_meta.rs +++ b/rust/fory-core/src/meta/type_meta.rs @@ -984,28 +984,8 @@ impl TypeMeta { } fn assign_field_ids(type_info_current: &TypeInfo, field_infos: &mut [FieldInfo]) { - if crate::util::ENABLE_FORY_DEBUG_OUTPUT { - eprintln!( - "[fory-debug] assign_field_ids called for type: {:?}", - type_info_current.get_type_name() - ); - for f in field_infos.iter() { - eprintln!( - "[fory-debug] remote field before assign: name={}, field_id={}, type={:?}", - f.field_name, f.field_id, f.field_type - ); - } - } let type_meta = type_info_current.get_type_meta(); let local_field_infos = type_meta.get_field_infos(); - if crate::util::ENABLE_FORY_DEBUG_OUTPUT { - for f in local_field_infos.iter() { - eprintln!( - "[fory-debug] local field: name={}, field_id={}, type={:?}", - f.field_name, f.field_id, f.field_type - ); - } - } // Build maps for both name-based and ID-based lookup. // The value is the SORTED INDEX (position in local_field_infos), not the field's ID attribute. @@ -1048,20 +1028,8 @@ impl TypeMeta { // codec inspects the remote FieldType and either consumes it or // asks the caller to skip the remote payload. field.field_id = sorted_index as i16; - if crate::util::ENABLE_FORY_DEBUG_OUTPUT { - eprintln!( - "[fory-debug] matched field: name={}, assigned_field_id={}, remote_type={:?}, local_type={:?}", - field.field_name, field.field_id, field.field_type, local_info.field_type - ); - } } None => { - if crate::util::ENABLE_FORY_DEBUG_OUTPUT { - eprintln!( - "[fory-debug] no local match for field: name={}", - field.field_name - ); - } field.field_id = -1; // No match, skip } } diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index 38a7f38560..6d02c68679 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -132,8 +132,14 @@ fn compatible_list_array_field_pairs() { payload: vec![Some(1), Some(2), Some(3)], }) .unwrap(); - let decoded: ArrayPayload = reader.deserialize(&bytes).unwrap(); - assert_eq!(decoded.payload, vec![1, 2, 3]); + let err = reader + .deserialize::(&bytes) + .expect_err("expected nullable list schema to fail compatible array read"); + assert!( + err.to_string() + .contains("compatible list to array field requires non-null elements"), + "{err}" + ); let bytes = writer .serialize(&NullableListPayload { diff --git a/swift/Sources/Fory/FieldCodecs.swift b/swift/Sources/Fory/FieldCodecs.swift index 9d48cd6e74..16cec20891 100644 --- a/swift/Sources/Fory/FieldCodecs.swift +++ b/swift/Sources/Fory/FieldCodecs.swift @@ -711,6 +711,9 @@ public enum ArrayFieldCodec: FieldCodec { let element = remoteFieldType.generics.first, let localArrayTypeID = packedArrayTypeID(for: ElementCodec.self), TypeId.listElementTypeID(element.typeID, matchesDenseArrayTypeID: localArrayTypeID.rawValue) { + if element.nullable || element.trackRef { + throw ForyError.invalidData("compatible list-to-array field cannot read nullable elements") + } return try readListPayloadAsArray( context, refMode: refMode, diff --git a/swift/Sources/Fory/TypeMeta.swift b/swift/Sources/Fory/TypeMeta.swift index ebb6d78503..d20be245e6 100644 --- a/swift/Sources/Fory/TypeMeta.swift +++ b/swift/Sources/Fory/TypeMeta.swift @@ -622,16 +622,28 @@ public final class TypeMeta: Equatable, @unchecked Sendable { var localMatch: (Int, FieldInfo)? if let fieldID = field.fieldID, fieldID >= 0 { - if let candidate = fieldIndexByID[fieldID], - Self.isCompatibleFieldType(field.fieldType, candidate.1.fieldType) { - localMatch = candidate + if let candidate = fieldIndexByID[fieldID] { + try Self.throwIfUnsupportedListArrayMismatch( + remoteType: field.fieldType, + localType: candidate.1.fieldType, + topLevel: true + ) + if Self.isCompatibleFieldType(field.fieldType, candidate.1.fieldType) { + localMatch = candidate + } } } if localMatch == nil { - if let candidate = fieldIndexByName[toSnakeCase(field.fieldName)], - Self.isCompatibleFieldType(field.fieldType, candidate.1.fieldType) { - localMatch = candidate + if let candidate = fieldIndexByName[toSnakeCase(field.fieldName)] { + try Self.throwIfUnsupportedListArrayMismatch( + remoteType: field.fieldType, + localType: candidate.1.fieldType, + topLevel: true + ) + if Self.isCompatibleFieldType(field.fieldType, candidate.1.fieldType) { + localMatch = candidate + } } } @@ -700,28 +712,66 @@ public final class TypeMeta: Equatable, @unchecked Sendable { return true } + private static func throwIfUnsupportedListArrayMismatch( + remoteType: FieldType, + localType: FieldType, + topLevel: Bool + ) throws { + if topLevel, isListArrayShapePair(remoteType, localType) { + if isCompatibleTopLevelListArrayFieldType(remoteType, localType) { + return + } + if remoteType.typeID == TypeId.list.rawValue, + TypeId(rawValue: localType.typeID)?.denseArrayElementTypeID != nil, + let elementType = remoteType.generics.first, + elementType.nullable || elementType.trackRef { + throw ForyError.invalidData("compatible list-to-array field cannot read nullable elements") + } + throw ForyError.invalidData("unsupported compatible list/array schema mismatch") + } + } + + private static func isListArrayShapePair(_ remoteType: FieldType, _ localType: FieldType) -> Bool { + (remoteType.typeID == TypeId.list.rawValue + && TypeId(rawValue: localType.typeID)?.denseArrayElementTypeID != nil) + || (localType.typeID == TypeId.list.rawValue + && TypeId(rawValue: remoteType.typeID)?.denseArrayElementTypeID != nil) + } + private static func isCompatibleTopLevelListArrayFieldType( _ remoteType: FieldType, _ localType: FieldType ) -> Bool { if remoteType.typeID == TypeId.list.rawValue { - return listFieldType(remoteType, matchesDenseArrayTypeID: localType.typeID) + return listFieldType( + remoteType, + matchesDenseArrayTypeID: localType.typeID, + requireNonNullableElement: true + ) } if localType.typeID == TypeId.list.rawValue { - return listFieldType(localType, matchesDenseArrayTypeID: remoteType.typeID) + return listFieldType( + localType, + matchesDenseArrayTypeID: remoteType.typeID, + requireNonNullableElement: false + ) } return false } private static func listFieldType( _ listType: FieldType, - matchesDenseArrayTypeID arrayTypeID: UInt32 + matchesDenseArrayTypeID arrayTypeID: UInt32, + requireNonNullableElement: Bool ) -> Bool { guard listType.typeID == TypeId.list.rawValue, let elementType = listType.generics.first else { return false } + if requireNonNullableElement, (elementType.nullable || elementType.trackRef) { + return false + } return TypeId.listElementTypeID(elementType.typeID, matchesDenseArrayTypeID: arrayTypeID) } diff --git a/swift/Sources/Fory/TypeResolver.swift b/swift/Sources/Fory/TypeResolver.swift index a1ac088171..02ee5bb274 100644 --- a/swift/Sources/Fory/TypeResolver.swift +++ b/swift/Sources/Fory/TypeResolver.swift @@ -559,9 +559,10 @@ final class TypeResolver { return localTypeInfo } let canonicalTypeMeta: TypeMeta - if let localTypeMeta = localTypeInfo.typeMeta, - let remapped = try? typeMeta.assigningFieldIDs(from: localTypeMeta) { - canonicalTypeMeta = remapped + if let localTypeMeta = localTypeInfo.typeMeta { + // Field remapping validates compatible schema shape. Propagate those errors so an unsupported + // matched field cannot degrade into an unknown-field skip. + canonicalTypeMeta = try typeMeta.assigningFieldIDs(from: localTypeMeta) } else { canonicalTypeMeta = typeMeta } diff --git a/swift/Tests/ForyTests/CompatibilityTests.swift b/swift/Tests/ForyTests/CompatibilityTests.swift index 2afa3f255c..1850d67863 100644 --- a/swift/Tests/ForyTests/CompatibilityTests.swift +++ b/swift/Tests/ForyTests/CompatibilityTests.swift @@ -450,8 +450,9 @@ func compatibleReadRejectsNullableListElementsForArrayField() throws { reader.register(CompatibleArrayFieldV2.self, id: 9923) let bytes = try writer.serialize(CompatibleNullableListFieldV1(values: [1, 2, 3], extra: 9)) - let decoded: CompatibleArrayFieldV2 = try reader.deserialize(bytes) - #expect(decoded.values == [1, 2, 3]) + #expect(throws: ForyError.invalidData("compatible list-to-array field cannot read nullable elements")) { + let _: CompatibleArrayFieldV2 = try reader.deserialize(bytes) + } let nullableBytes = try writer.serialize(CompatibleNullableListFieldV1(values: [1, nil, 3], extra: 9)) #expect(throws: ForyError.invalidData("compatible list-to-array field cannot read nullable elements")) { From 05ee49021b9861e55bc8f5252c78477883c9e4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 08:12:08 +0800 Subject: [PATCH 36/58] fix: align compatible schema bridges --- .../org/apache/fory/builder/CodecUtils.java | 3 +- .../org/apache/fory/builder/Generated.java | 9 ---- .../builder/StaticCompatibleCodecBuilder.java | 50 +------------------ .../StaticGeneratedStructSerializer.java | 28 ++--------- .../StaticCompatibleCodecBuilderTest.java | 2 +- python/pyfory/meta/typedef.py | 25 ++++++---- python/pyfory/tests/test_typedef_encoding.py | 12 +++++ rust/fory-core/src/serializer/collection.rs | 12 ++--- rust/tests/tests/compatible/test_struct.rs | 20 ++++++-- 9 files changed, 55 insertions(+), 106 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index b1006fe858..5856e58460 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -76,8 +76,7 @@ public static Class loadOrGenStaticCompatibleCodecClas Fory fory, Class cls, TypeDef typeDef) { Preconditions.checkNotNull(fory); return loadSerializer( - "loadOrGenStaticCompatibleCodecClass_" - + StaticCompatibleCodecBuilder.remoteTypeDefSuffix(typeDef), + "loadOrGenStaticCompatibleCodecClass", cls, fory, () -> diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java index 17291e16df..e63bc95fbb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java @@ -130,15 +130,6 @@ public GeneratedCompatibleMetaSharedSerializer( super(typeResolver, cls, typeDef, descriptors); } - public GeneratedCompatibleMetaSharedSerializer( - TypeResolver typeResolver, - Class cls, - TypeDef typeDef, - List descriptors, - Class remoteDescriptorClass) { - super(typeResolver, cls, typeDef, descriptors, remoteDescriptorClass); - } - @Override public final Object read(ReadContext readContext) { return readCompatible(readContext); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index dc9e2a6144..7b8d0d5833 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -60,8 +60,6 @@ public final class StaticCompatibleCodecBuilder extends ObjectCodecBuilder { private static final int DISPATCH_GROUP_SIZE = 8; private final List localDescriptors; - private final Class remoteDescriptorClass; - private final String remoteTypeDefSuffix; private final boolean debug; public StaticCompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { @@ -70,15 +68,13 @@ public StaticCompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef type !fory.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); localDescriptors = Collections.unmodifiableList(Descriptor.getDescriptors(beanClass)); - remoteDescriptorClass = resolveRemoteDescriptorClass(typeDef); - remoteTypeDefSuffix = remoteTypeDefSuffix(typeDef); ForyStruct foryStruct = beanClass.getAnnotation(ForyStruct.class); debug = foryStruct != null && foryStruct.debug(); } @Override protected String codecSuffix() { - return "CompatibleMetaShared_" + remoteTypeDefSuffix; + return "CompatibleMetaShared"; } @Override @@ -92,8 +88,7 @@ public String genCode() { String constructorCode = StringUtils.format( "" - + "super(${typeResolver}, ${cls}, ${typeDef}, Descriptor.getDescriptors(${cls})," - + " ${remoteDescriptorClass});\n" + + "super(${typeResolver}, ${cls}, ${typeDef}, Descriptor.getDescriptors(${cls}));\n" + "this.${generatedTypeResolver} = (${generatedTypeResolverType}) ${typeResolver};\n", "typeResolver", CONSTRUCTOR_TYPE_RESOLVER_NAME, @@ -101,8 +96,6 @@ public String genCode() { POJO_CLASS_TYPE_NAME, "typeDef", "_f_typeDef", - "remoteDescriptorClass", - remoteDescriptorClassLiteral(), "generatedTypeResolver", TYPE_RESOLVER_NAME, "generatedTypeResolverType", @@ -132,45 +125,6 @@ protected void addCommonImports() { ctx.addImport(GeneratedCompatibleMetaSharedSerializer.class); } - private Class resolveRemoteDescriptorClass(TypeDef typeDef) { - String className = typeDef.getClassName(); - if (className.equals(beanClass.getName())) { - return null; - } - ClassLoader beanClassLoader = beanClass.getClassLoader(); - try { - return Class.forName(className, false, beanClassLoader); - } catch (ClassNotFoundException | LinkageError e) { - try { - return Class.forName(className, false, StaticCompatibleCodecBuilder.class.getClassLoader()); - } catch (ClassNotFoundException | LinkageError ignored) { - return null; - } - } - } - - static String remoteTypeDefSuffix(TypeDef typeDef) { - return Long.toUnsignedString(typeDef.getId(), 16) - + "_" - + Integer.toHexString(typeDef.getClassName().hashCode()); - } - - private String remoteDescriptorClassLiteral() { - if (remoteDescriptorClass == null || !canReferenceRemoteDescriptorClass()) { - return "null"; - } - return ctx.type(remoteDescriptorClass) + ".class"; - } - - private boolean canReferenceRemoteDescriptorClass() { - if (!ctx.sourcePkgLevelAccessible(remoteDescriptorClass)) { - return false; - } - return ctx.sourcePublicAccessible(remoteDescriptorClass) - || CodeGenerator.getPackage(beanClass) - .equals(CodeGenerator.getPackage(remoteDescriptorClass)); - } - @Override public Expression buildEncodeExpression() { throw new IllegalStateException("unreachable"); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index e5d763fe47..8696417d1d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -61,24 +61,12 @@ public StaticGeneratedStructSerializer(TypeResolver typeResolver, Class type) @SuppressWarnings("unchecked") protected StaticGeneratedStructSerializer( TypeResolver typeResolver, Class type, TypeDef typeDef, List descriptors) { - this(typeResolver, type, typeDef, descriptors, null); - } - - @SuppressWarnings("unchecked") - protected StaticGeneratedStructSerializer( - TypeResolver typeResolver, - Class type, - TypeDef typeDef, - List descriptors, - Class remoteDescriptorClass) { super(typeResolver, (Class) type); setSerializerIfAbsent(typeResolver, (Class) type); List runtimeDescriptors = runtimeDescriptors(descriptors); this.typeDef = typeDef; this.remoteFields = - typeDef == null - ? Collections.emptyList() - : buildRemoteFields(typeDef, runtimeDescriptors, remoteDescriptorClass); + typeDef == null ? Collections.emptyList() : buildRemoteFields(typeDef, runtimeDescriptors); this.localFieldsById = buildLocalFieldsById(runtimeDescriptors); } @@ -398,8 +386,8 @@ private Object readField(ReadContext readContext, SerializationFieldInfo fieldIn } private List buildRemoteFields( - TypeDef remoteTypeDef, List localDescriptors, Class generatedRemoteClass) { - Class remoteDescriptorClass = remoteDescriptorClass(remoteTypeDef, generatedRemoteClass); + TypeDef remoteTypeDef, List localDescriptors) { + Class remoteDescriptorClass = remoteDescriptorClass(remoteTypeDef); List remoteFieldInfos = remoteTypeDef.getFieldsInfo(); List remoteDescriptors = remoteTypeDef.getDescriptors(typeResolver, remoteDescriptorClass); @@ -436,15 +424,7 @@ private List buildRemoteFields( return Collections.unmodifiableList(remoteFields); } - private Class remoteDescriptorClass(TypeDef remoteTypeDef, Class generatedRemoteClass) { - if (generatedRemoteClass != null) { - // Native TypeDefs for registered classes carry the registered id, so a reader that binds the - // same id to an evolved class decodes the TypeDef as the local class. Static-compatible - // codegen may still know the writer-side class; use it to preserve descriptor-only details - // such as primitive-list carrier raw types while keeping wire order in - // createDescriptorGrouper. - return generatedRemoteClass; - } + private Class remoteDescriptorClass(TypeDef remoteTypeDef) { String className = remoteTypeDef.getClassName(); if (className.equals(type.getName())) { return type; diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java index ee24ab00b2..64be764f96 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -326,7 +326,7 @@ public void testGraalCompatibleSerializerRegistryUsesLocalReaderClass() throws E Class serializerB = CodecUtils.loadOrGenStaticCompatibleCodecClass( reader.getTypeResolver(), readerClass, typeDefB); - Assert.assertNotEquals(serializerA, serializerB); + Assert.assertSame(serializerA, serializerB); GraalvmSupport.GraalvmClassRegistry registry = GraalvmSupport.getClassRegistry(0); registry.putCompatibleDeserializerClass(readerType, serializerA); diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py index b3b6297918..91a00bf5f8 100644 --- a/python/pyfory/meta/typedef.py +++ b/python/pyfory/meta/typedef.py @@ -631,18 +631,19 @@ def __repr__(self): } -def _list_array_element_type_matches(list_field_type: FieldType, array_field_type: FieldType) -> bool: +def _list_array_element_type_matches( + list_field_type: FieldType, array_field_type: FieldType, require_non_nullable_elements: bool +) -> bool: array_element_type_id = _ARRAY_ELEMENT_TYPE_IDS.get(array_field_type.type_id) - if array_element_type_id is None: + if list_field_type.type_id != TypeId.LIST or array_element_type_id is None: return False - return ( - list_field_type.type_id == TypeId.LIST - and not list_field_type.element_type.is_nullable - and not list_field_type.element_type.is_tracking_ref - and _list_element_type_matches_array_element( + if require_non_nullable_elements and ( + list_field_type.element_type.is_nullable or list_field_type.element_type.is_tracking_ref + ): + return False + return _list_element_type_matches_array_element( list_field_type.element_type.type_id, array_element_type_id ) - ) def _list_element_type_matches_array_element(list_element_type_id: TypeId, array_element_type_id: TypeId) -> bool: @@ -656,9 +657,11 @@ def _is_root_list_array_pair(remote_field_type: FieldType, local_field_type: Fie if local_field_type is None: return False if remote_field_type.type_id == TypeId.LIST and local_field_type.type_id in _ARRAY_TYPE_IDS: - return _list_array_element_type_matches(remote_field_type, local_field_type) + return _list_array_element_type_matches(remote_field_type, local_field_type, True) if local_field_type.type_id == TypeId.LIST and remote_field_type.type_id in _ARRAY_TYPE_IDS: - return _list_array_element_type_matches(local_field_type, remote_field_type) + # A dense remote array can feed a nullable local list; only list-to-array requires rejecting + # nullable/ref-tracked elements because the local dense array has no carrier for them. + return _list_array_element_type_matches(local_field_type, remote_field_type, False) return False @@ -677,7 +680,7 @@ def _remote_list_to_local_array_allowed(remote_field_type: FieldType, local_fiel return ( remote_field_type.type_id == TypeId.LIST and local_field_type.type_id in _ARRAY_TYPE_IDS - and _list_array_element_type_matches(remote_field_type, local_field_type) + and _list_array_element_type_matches(remote_field_type, local_field_type, True) ) diff --git a/python/pyfory/tests/test_typedef_encoding.py b/python/pyfory/tests/test_typedef_encoding.py index d4522a70fa..bd0ee8ca49 100644 --- a/python/pyfory/tests/test_typedef_encoding.py +++ b/python/pyfory/tests/test_typedef_encoding.py @@ -559,6 +559,18 @@ def test_compatible_int32_array_assigns_to_list(): assert decoded.payload == [1, 2, 3] +def test_compatible_int32_array_assigns_to_nullable_list(): + writer = Fory(xlang=True, compatible=True) + reader = Fory(xlang=True, compatible=True) + _register_int32_payload(writer, Int32ArrayPayload) + _register_int32_payload(reader, NullableInt32ListPayload) + + decoded = reader.deserialize(writer.serialize(Int32ArrayPayload(payload=pyfory.Int32Array([1, 2, 3])))) + + assert isinstance(decoded, NullableInt32ListPayload) + assert decoded.payload == [1, 2, 3] + + @pytest.mark.skipif(np is None, reason="Requires numpy") def test_compatible_int32_ndarray_assigns_to_list(): writer = Fory(xlang=True, compatible=True) diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs index 9d455e77a4..b3506e817b 100644 --- a/rust/fory-core/src/serializer/collection.rs +++ b/rust/fory-core/src/serializer/collection.rs @@ -528,12 +528,6 @@ fn read_primitive_array_data_bulk( } } -fn list_element_type_matches_array(list: &FieldType, array: &FieldType) -> bool { - list_element_type_matches_array_shape(list, array) - && !list.generics[0].nullable - && !list.generics[0].track_ref -} - fn list_element_type_matches_array_shape(list: &FieldType, array: &FieldType) -> bool { primitive_array_element_type_id(array.type_id).is_some_and(|element_type_id| { list.type_id == type_id::LIST @@ -792,8 +786,12 @@ where T: 'static, C: Codec, { + // A dense remote array can materialize into a nullable local list by wrapping every element as + // Some(_). The reverse direction is stricter and is checked in + // read_primitive_array_vec_compatible_mismatch because a remote list may carry null/ref markers + // that a dense local array cannot represent. if local_field_type.type_id == type_id::LIST - && list_element_type_matches_array(local_field_type, remote_field_type) + && list_element_type_matches_array_shape(local_field_type, remote_field_type) { return read_array_data_as_vec_bridge::(context, remote_field_type).map(Some); } diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index 6d02c68679..55d2972531 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -125,8 +125,20 @@ fn compatible_list_array_field_pairs() { let mut writer = Fory::builder().compatible(true).build(); let mut reader = Fory::builder().compatible(true).build(); - writer.register::(993).unwrap(); - reader.register::(993).unwrap(); + writer.register::(993).unwrap(); + reader.register::(993).unwrap(); + let bytes = writer + .serialize(&ArrayPayload { + payload: vec![1, 2, 3], + }) + .unwrap(); + let decoded: NullableListPayload = reader.deserialize(&bytes).unwrap(); + assert_eq!(decoded.payload, vec![Some(1), Some(2), Some(3)]); + + let mut writer = Fory::builder().compatible(true).build(); + let mut reader = Fory::builder().compatible(true).build(); + writer.register::(994).unwrap(); + reader.register::(994).unwrap(); let bytes = writer .serialize(&NullableListPayload { payload: vec![Some(1), Some(2), Some(3)], @@ -157,8 +169,8 @@ fn compatible_list_array_field_pairs() { let mut writer = Fory::builder().compatible(true).build(); let mut reader = Fory::builder().compatible(true).build(); - writer.register::(994).unwrap(); - reader.register::(994).unwrap(); + writer.register::(995).unwrap(); + reader.register::(995).unwrap(); let bytes = writer .serialize(&NestedListPayload { payload: vec![vec![1, 2], vec![3]], From 79320a21470bcbce8657376b61f12e35c6aaa5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 08:23:17 +0800 Subject: [PATCH 37/58] fix: skip nested list array schema mismatches --- .../lib/src/serializer/struct_serializer.dart | 35 ++++++++- ...calar_and_typed_array_serializer_test.dart | 8 +- .../java/org/apache/fory/meta/FieldInfo.java | 61 +++++++++++++-- .../CompatibleCollectionArrayReader.java | 74 +++++++++++++++---- .../StaticGeneratedStructSerializer.java | 7 ++ .../fory/xlang/MetaSharedXlangTest.java | 23 +++--- javascript/packages/core/lib/context.ts | 46 +++++++++++- javascript/packages/core/lib/gen/struct.ts | 8 +- javascript/packages/core/lib/typeInfo.ts | 1 + 9 files changed, 221 insertions(+), 42 deletions(-) diff --git a/dart/packages/fory/lib/src/serializer/struct_serializer.dart b/dart/packages/fory/lib/src/serializer/struct_serializer.dart index 0f62f3416d..9b42c2d06c 100644 --- a/dart/packages/fory/lib/src/serializer/struct_serializer.dart +++ b/dart/packages/fory/lib/src/serializer/struct_serializer.dart @@ -186,6 +186,14 @@ final class StructSerializer extends Serializer { 'Compatible field ${localField.name} has unsupported list/array schema mismatch.', ); } + if (_hasNestedListArrayMismatch( + localField.field.fieldType, + remoteField.fieldType, + )) { + fields.add(null); + topLevelListArrayPairs?.add(false); + continue; + } if (topLevelListArrayPair) { topLevelListArrayPairs ??= List.filled(fields.length, false, growable: true); @@ -216,14 +224,37 @@ bool _topLevelListArrayPair(FieldInfo localField, FieldInfo remoteField) { return isCompatibleCollectionArrayFieldPair(localField, remoteField); } +bool _hasNestedListArrayMismatch(FieldType localType, FieldType remoteType) { + if (localType.typeId != remoteType.typeId || + localType.arguments.length != remoteType.arguments.length) { + return false; + } + for (var index = 0; index < localType.arguments.length; index += 1) { + if (_hasListArrayMismatch( + localType.arguments[index], + remoteType.arguments[index], + )) { + return true; + } + } + return false; +} + +bool _hasListArrayMismatch(FieldType localType, FieldType remoteType) { + if (isCompatibleCollectionArrayRootTypePair(localType, remoteType)) { + return true; + } + return _hasNestedListArrayMismatch(localType, remoteType); +} + bool _hasUnsupportedListArrayMismatch( FieldType localType, FieldType remoteType, { required bool topLevel, }) { if (isCompatibleCollectionArrayRootTypePair(localType, remoteType)) { - return !(topLevel && - isCompatibleCollectionArrayTypePair(localType, remoteType)); + return topLevel && + !isCompatibleCollectionArrayTypePair(localType, remoteType); } if (localType.typeId != remoteType.typeId || localType.arguments.length != remoteType.arguments.length) { diff --git a/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart b/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart index 12245bf897..b755f6e252 100644 --- a/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart +++ b/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart @@ -681,7 +681,7 @@ void main() { ); }); - test('rejects nested compatible list and dense array field positions', () { + test('skips nested compatible list and dense array field positions', () { final writer = Fory(); final reader = Fory(); ScalarAndTypedArraySerializerTestFory.register( @@ -704,10 +704,8 @@ void main() { ], ); - expect( - () => reader.deserialize(bytes), - throwsStateError, - ); + final decoded = reader.deserialize(bytes); + expect(decoded.values, isEmpty); }); test('enforces maxBinarySize on write and read', () { diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index 6b30855a47..50197b029c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -149,16 +149,29 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { .nullable(remoteNullable) .build(); } - if (localFieldType != null && hasListArrayShapeMismatch(fieldType, localFieldType)) { + if (localFieldType != null && isListArrayRootPair(fieldType, localFieldType)) { throw new IllegalArgumentException( StringUtils.format( - "Unsupported nested list/array compatible field mismatch for field " - + "{}.{}: peer={}, local={}", + "Unsupported list/array compatible field mismatch for field {}.{}: peer={}, local={}", definedClass, fieldName, fieldType, localFieldType)); } + if (localFieldType != null && hasNestedListArrayShapeMismatch(fieldType, localFieldType)) { + // List/array bridging is only defined for the matched field itself. If the shape differs + // deeper in a container, keep the remote descriptor for skipping but do not assign it to + // the local field. + TypeRef remoteTypeRef = fieldType.toTypeToken(resolver, null); + return new DescriptorBuilder(descriptor) + .typeName(fieldType.getTypeName(resolver, remoteTypeRef)) + .trackingRef(remoteTrackingRef) + .nullable(remoteNullable) + .typeRef(remoteTypeRef) + .type(remoteTypeRef.getRawType()) + .field(null) + .build(); + } if (remoteNullable == descriptor.isNullable() && remoteTrackingRef == descriptor.isTrackingRef() && typeRef.equals(descriptor.getTypeRef())) { @@ -242,11 +255,7 @@ private boolean isTopLevelListArrayCompatibleReadPair( private static boolean hasListArrayShapeMismatch( FieldTypes.FieldType peerFieldType, FieldTypes.FieldType localFieldType) { - boolean peerList = isListField(peerFieldType); - boolean localList = isListField(localFieldType); - boolean peerArray = arrayTypeId(peerFieldType) != Types.UNKNOWN; - boolean localArray = arrayTypeId(localFieldType) != Types.UNKNOWN; - if ((peerList && localArray) || (peerArray && localList)) { + if (isListArrayRootPair(peerFieldType, localFieldType)) { return true; } if (peerFieldType.getTypeId() != localFieldType.getTypeId()) { @@ -274,6 +283,42 @@ private static boolean hasListArrayShapeMismatch( return false; } + private static boolean hasNestedListArrayShapeMismatch( + FieldTypes.FieldType peerFieldType, FieldTypes.FieldType localFieldType) { + if (peerFieldType.getTypeId() != localFieldType.getTypeId()) { + return false; + } + if (peerFieldType instanceof FieldTypes.CollectionFieldType + && localFieldType instanceof FieldTypes.CollectionFieldType) { + return hasListArrayShapeMismatch( + ((FieldTypes.CollectionFieldType) peerFieldType).getElementType(), + ((FieldTypes.CollectionFieldType) localFieldType).getElementType()); + } + if (peerFieldType instanceof FieldTypes.MapFieldType + && localFieldType instanceof FieldTypes.MapFieldType) { + FieldTypes.MapFieldType peerMap = (FieldTypes.MapFieldType) peerFieldType; + FieldTypes.MapFieldType localMap = (FieldTypes.MapFieldType) localFieldType; + return hasListArrayShapeMismatch(peerMap.getKeyType(), localMap.getKeyType()) + || hasListArrayShapeMismatch(peerMap.getValueType(), localMap.getValueType()); + } + if (peerFieldType instanceof FieldTypes.ArrayFieldType + && localFieldType instanceof FieldTypes.ArrayFieldType) { + return hasListArrayShapeMismatch( + ((FieldTypes.ArrayFieldType) peerFieldType).getComponentType(), + ((FieldTypes.ArrayFieldType) localFieldType).getComponentType()); + } + return false; + } + + private static boolean isListArrayRootPair( + FieldTypes.FieldType peerFieldType, FieldTypes.FieldType localFieldType) { + boolean peerList = isListField(peerFieldType); + boolean localList = isListField(localFieldType); + boolean peerArray = arrayTypeId(peerFieldType) != Types.UNKNOWN; + boolean localArray = arrayTypeId(localFieldType) != Types.UNKNOWN; + return (peerList && localArray) || (peerArray && localList); + } + private static boolean isListField(FieldTypes.FieldType fieldType) { return fieldType instanceof FieldTypes.CollectionFieldType && fieldType.getTypeId() == Types.LIST; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index 5834207083..ff5e32ae56 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -187,40 +187,84 @@ static boolean incompatibleCollectionArrayMatch( } FieldTypes.FieldType remoteFieldType = remoteFieldInfo.getFieldType(); FieldTypes.FieldType localFieldType = FieldTypes.buildFieldType(resolver, localDescriptor); - return incompatibleCollectionArrayMatch(remoteFieldType, localFieldType); + return isListArrayRootPair(remoteFieldType, localFieldType); } - private static boolean incompatibleCollectionArrayMatch( + static boolean nestedCollectionArrayMatch( + TypeResolver resolver, FieldInfo remoteFieldInfo, Descriptor localDescriptor) { + if (localDescriptor == null || !resolver.isCrossLanguage()) { + return false; + } + FieldTypes.FieldType remoteFieldType = remoteFieldInfo.getFieldType(); + FieldTypes.FieldType localFieldType = FieldTypes.buildFieldType(resolver, localDescriptor); + return hasNestedCollectionArrayMatch(remoteFieldType, localFieldType); + } + + private static boolean hasCollectionArrayMatch( FieldTypes.FieldType remoteFieldType, FieldTypes.FieldType localFieldType) { - int remoteListElementTypeId = listElementTypeId(remoteFieldType); - int localArrayTypeId = arrayTypeId(localFieldType); - if (remoteListElementTypeId != Types.UNKNOWN - && localArrayTypeId != Types.UNKNOWN - && localArrayTypeId == denseArrayTypeId(remoteListElementTypeId)) { + if (isListArrayRootPair(remoteFieldType, localFieldType)) { return true; } - int remoteArrayTypeId = arrayTypeId(remoteFieldType); - int localListElementTypeId = listElementTypeId(localFieldType); - if (remoteArrayTypeId != Types.UNKNOWN - && localListElementTypeId != Types.UNKNOWN - && remoteArrayTypeId == denseArrayTypeId(localListElementTypeId)) { - return true; + if (remoteFieldType instanceof FieldTypes.CollectionFieldType + && localFieldType instanceof FieldTypes.CollectionFieldType) { + return hasCollectionArrayMatch( + ((FieldTypes.CollectionFieldType) remoteFieldType).getElementType(), + ((FieldTypes.CollectionFieldType) localFieldType).getElementType()); + } + if (remoteFieldType instanceof FieldTypes.MapFieldType + && localFieldType instanceof FieldTypes.MapFieldType) { + FieldTypes.MapFieldType remoteMap = (FieldTypes.MapFieldType) remoteFieldType; + FieldTypes.MapFieldType localMap = (FieldTypes.MapFieldType) localFieldType; + return hasCollectionArrayMatch(remoteMap.getKeyType(), localMap.getKeyType()) + || hasCollectionArrayMatch(remoteMap.getValueType(), localMap.getValueType()); + } + if (remoteFieldType instanceof FieldTypes.ArrayFieldType + && localFieldType instanceof FieldTypes.ArrayFieldType) { + return hasCollectionArrayMatch( + ((FieldTypes.ArrayFieldType) remoteFieldType).getComponentType(), + ((FieldTypes.ArrayFieldType) localFieldType).getComponentType()); + } + return false; + } + + private static boolean hasNestedCollectionArrayMatch( + FieldTypes.FieldType remoteFieldType, FieldTypes.FieldType localFieldType) { + if (remoteFieldType.getTypeId() != localFieldType.getTypeId()) { + return false; } if (remoteFieldType instanceof FieldTypes.CollectionFieldType && localFieldType instanceof FieldTypes.CollectionFieldType) { - return incompatibleCollectionArrayMatch( + return hasCollectionArrayMatch( ((FieldTypes.CollectionFieldType) remoteFieldType).getElementType(), ((FieldTypes.CollectionFieldType) localFieldType).getElementType()); } + if (remoteFieldType instanceof FieldTypes.MapFieldType + && localFieldType instanceof FieldTypes.MapFieldType) { + FieldTypes.MapFieldType remoteMap = (FieldTypes.MapFieldType) remoteFieldType; + FieldTypes.MapFieldType localMap = (FieldTypes.MapFieldType) localFieldType; + return hasCollectionArrayMatch(remoteMap.getKeyType(), localMap.getKeyType()) + || hasCollectionArrayMatch(remoteMap.getValueType(), localMap.getValueType()); + } if (remoteFieldType instanceof FieldTypes.ArrayFieldType && localFieldType instanceof FieldTypes.ArrayFieldType) { - return incompatibleCollectionArrayMatch( + return hasCollectionArrayMatch( ((FieldTypes.ArrayFieldType) remoteFieldType).getComponentType(), ((FieldTypes.ArrayFieldType) localFieldType).getComponentType()); } return false; } + private static boolean isListArrayRootPair( + FieldTypes.FieldType remoteFieldType, FieldTypes.FieldType localFieldType) { + return (isListField(remoteFieldType) && arrayTypeId(localFieldType) != Types.UNKNOWN) + || (arrayTypeId(remoteFieldType) != Types.UNKNOWN && isListField(localFieldType)); + } + + private static boolean isListField(FieldTypes.FieldType fieldType) { + return fieldType instanceof FieldTypes.CollectionFieldType + && fieldType.getTypeId() == Types.LIST; + } + static Object read(ReadContext readContext, RefMode refMode, ReadAction action) { return read( readContext, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 8696417d1d..21b410aec3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -254,6 +254,9 @@ protected final boolean canReadRemoteField( + ": compatible list/array adaptation requires a matching non-null primitive element" + " schema and does not apply recursively"); } + if (remoteField.nestedCollectionArrayMatch) { + return false; + } if (remoteField.compatibleCollectionArrayReadAction != null) { return true; } @@ -534,6 +537,7 @@ protected static final class RemoteFieldInfo { private final SerializationFieldInfo serializationFieldInfo; private final CompatibleCollectionArrayReader.ReadAction compatibleCollectionArrayReadAction; private final boolean incompatibleCollectionArrayMatch; + private final boolean nestedCollectionArrayMatch; private RemoteFieldInfo( TypeResolver typeResolver, @@ -551,6 +555,9 @@ private RemoteFieldInfo( this.incompatibleCollectionArrayMatch = CompatibleCollectionArrayReader.incompatibleCollectionArrayMatch( typeResolver, fieldInfo, localDescriptor); + this.nestedCollectionArrayMatch = + CompatibleCollectionArrayReader.nestedCollectionArrayMatch( + typeResolver, fieldInfo, localDescriptor); } } } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index 51f4890514..b0c9bf19f1 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -20,6 +20,7 @@ package org.apache.fory.xlang; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; @@ -185,21 +186,23 @@ public void testNullableListPayloadRejectedForArrayCompatibleRead() { } @Test - public void testNestedListArrayCompatibleReadUnsupported() { + public void testNestedListArrayCompatibleReadSkipped() { Fory nestedListFory = compatibleFory(NestedListField.class); NestedListField nestedListStruct = new NestedListField(); nestedListStruct.values = Arrays.asList(Arrays.asList(1, 2)); byte[] nestedListBytes = nestedListFory.serialize(nestedListStruct); Fory nestedArrayFory = compatibleFory(NestedArrayElementField.class); - assertThrows( - DeserializationException.class, () -> nestedArrayFory.deserialize(nestedListBytes)); + NestedArrayElementField skippedNestedArrayStruct = + (NestedArrayElementField) nestedArrayFory.deserialize(nestedListBytes); + assertNull(skippedNestedArrayStruct.values); NestedArrayElementField nestedArrayStruct = new NestedArrayElementField(); nestedArrayStruct.values = Arrays.asList(new int[] {1, 2}); byte[] nestedArrayBytes = nestedArrayFory.serialize(nestedArrayStruct); - assertThrows( - DeserializationException.class, () -> nestedListFory.deserialize(nestedArrayBytes)); + NestedListField skippedNestedListStruct = + (NestedListField) nestedListFory.deserialize(nestedArrayBytes); + assertNull(skippedNestedListStruct.values); Fory nestedSetListFory = compatibleFory(NestedSetListField.class, false); NestedSetListField nestedSetListStruct = new NestedSetListField(); @@ -207,14 +210,16 @@ public void testNestedListArrayCompatibleReadUnsupported() { byte[] nestedSetListBytes = nestedSetListFory.serialize(nestedSetListStruct); Fory nestedSetArrayFory = compatibleFory(NestedSetArrayElementField.class, false); - assertThrows( - DeserializationException.class, () -> nestedSetArrayFory.deserialize(nestedSetListBytes)); + NestedSetArrayElementField skippedNestedSetArrayStruct = + (NestedSetArrayElementField) nestedSetArrayFory.deserialize(nestedSetListBytes); + assertNull(skippedNestedSetArrayStruct.values); NestedSetArrayElementField nestedSetArrayStruct = new NestedSetArrayElementField(); nestedSetArrayStruct.values = new LinkedHashSet<>(Arrays.asList(new int[] {1, 2})); byte[] nestedSetArrayBytes = nestedSetArrayFory.serialize(nestedSetArrayStruct); - assertThrows( - DeserializationException.class, () -> nestedSetListFory.deserialize(nestedSetArrayBytes)); + NestedSetListField skippedNestedSetListStruct = + (NestedSetListField) nestedSetListFory.deserialize(nestedSetArrayBytes); + assertNull(skippedNestedSetListStruct.values); } private static Fory compatibleFory(Class type) { diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index c2906e49e9..8b35e28ba1 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -898,7 +898,7 @@ export class ReadContext { return false; } if (this.isListArrayRootPair(remote, local)) { - return !(topLevel && this.compatibleFieldTypeInfo(remote, local)); + return topLevel && !this.compatibleFieldTypeInfo(remote, local); } if (remote.typeId !== local.typeId) { return false; @@ -934,6 +934,41 @@ export class ReadContext { } } + private hasNestedListArrayMismatch( + remote: InnerFieldInfo, + local: TypeInfo | undefined, + ): boolean { + if (!local || remote.typeId !== local.typeId) { + return false; + } + switch (remote.typeId) { + case TypeId.MAP: + return ( + this.hasListArrayMismatch(remote.options!.key!, local.options?.key) + || this.hasListArrayMismatch(remote.options!.value!, local.options?.value) + ); + case TypeId.LIST: + return this.hasListArrayMismatch(remote.options!.inner!, local.options?.inner); + case TypeId.SET: + return this.hasListArrayMismatch(remote.options!.key!, local.options?.key); + default: + return false; + } + } + + private hasListArrayMismatch( + remote: InnerFieldInfo, + local: TypeInfo | undefined, + ): boolean { + if (!local) { + return false; + } + if (this.isListArrayRootPair(remote, local)) { + return true; + } + return this.hasNestedListArrayMismatch(remote, local); + } + private isListArrayRootPair(remote: InnerFieldInfo, local: TypeInfo): boolean { return ( (remote.typeId === TypeId.LIST && denseArrayElementTypeId(local.typeId) !== undefined) @@ -1001,13 +1036,20 @@ export class ReadContext { const props = Object.fromEntries( typeMeta.remapFieldNames(localProps).map((fieldInfo) => { const localFieldTypeInfo = localProps?.[fieldInfo.getFieldName()]; - const fieldTypeInfo = this.fieldInfoToTypeInfo( + const skipRead = this.hasNestedListArrayMismatch( fieldInfo, localFieldTypeInfo, + ); + const fieldTypeInfo = this.fieldInfoToTypeInfo( + fieldInfo, + skipRead ? undefined : localFieldTypeInfo, ) .setNullable(fieldInfo.nullable) .setTrackingRef(fieldInfo.trackingRef) .setId(fieldInfo.fieldId); + if (skipRead) { + fieldTypeInfo.options = { ...fieldTypeInfo.options, skipRead: true }; + } return [fieldInfo.getFieldName(), fieldTypeInfo]; }), ); diff --git a/javascript/packages/core/lib/gen/struct.ts b/javascript/packages/core/lib/gen/struct.ts index 175180da43..519d867cc2 100644 --- a/javascript/packages/core/lib/gen/struct.ts +++ b/javascript/packages/core/lib/gen/struct.ts @@ -414,8 +414,11 @@ class StructSerializerGenerator extends BaseSerializerGenerator { throw new Error(`${typeInfo.typeId} generator not exists`); } const innerGenerator = new InnerGeneratorClass(typeInfo, this.builder, this.scope); + const assign = typeInfo.options?.skipRead + ? (expr: string) => `void (${expr})` + : (expr: string) => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`; return ` - ${this.readField(typeInfo, expr => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`, innerGenerator.readEmbed())} + ${this.readField(typeInfo, assign, innerGenerator.readEmbed())} `; }).join(";\n")} ${accessor(result)} @@ -435,6 +438,9 @@ class StructSerializerGenerator extends BaseSerializerGenerator { } const fields: Array<{ key: string; expr: string }> = []; for (const { key, typeInfo } of this.sortedProps) { + if (typeInfo.options?.skipRead) { + return null; + } const expr = directNumericFieldReadExpr(typeInfo, this.builder); if (expr === null) { return null; diff --git a/javascript/packages/core/lib/typeInfo.ts b/javascript/packages/core/lib/typeInfo.ts index 0dc4f97316..ddcd761314 100644 --- a/javascript/packages/core/lib/typeInfo.ts +++ b/javascript/packages/core/lib/typeInfo.ts @@ -51,6 +51,7 @@ interface TypeInfoOptions { enumProps?: { [key: string]: number }; cases?: { [caseIndex: number]: TypeInfo }; scalarEncoding?: ScalarEncoding; + skipRead?: boolean; } /** From 12812052b8b6c7cca076109ca90937cb3a803059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 08:27:49 +0800 Subject: [PATCH 38/58] test(js): expect nested list array skip --- javascript/test/typemeta.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index fe94088a19..3454588132 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -483,7 +483,7 @@ describe("typemeta", () => { expect(() => readerFory.register(readerType).deserialize(bytes)).toThrow(/list\/array/); }); - test("rejects nested compatible list and dense array positions", () => { + test("skips nested compatible list and dense array positions", () => { const writerFory = new Fory({ compatible: true }); const readerFory = new Fory({ compatible: true }); @@ -498,7 +498,8 @@ describe("typemeta", () => { values: [new Int32Array([1, 2])], }); - expect(() => readerFory.register(readerType).deserialize(bytes)).toThrow(/list\/array/); + const result = readerFory.register(readerType).deserialize(bytes); + expect(result.values).toBeNull(); }); test("keeps compatible named schema evolution working when field count differs", () => { From 4baf41c18eb805816e7ac14fdcea0a39e809a699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 08:33:25 +0800 Subject: [PATCH 39/58] test(java): expect static nested list array skip --- .../fory/annotation/processing/ForyStructProcessorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index edfa311e52..5526fecfa0 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -773,7 +773,8 @@ public void testStaticIncompatibleListArrayCompatibleReadFails() throws Exceptio Object writerValue = writerType.getConstructor().newInstance(); setField(writerType, writerValue, "values", Arrays.asList(Arrays.asList(1, 2, 3))); byte[] payload = writer.serialize(writerValue); - Assert.expectThrows(DeserializationException.class, () -> reader.deserialize(payload)); + Object result = reader.deserialize(payload); + Assert.assertNull(getField(readerType, result, "values")); } } From 9571abadf638969bc679bcb92ff818a37c118ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 08:36:25 +0800 Subject: [PATCH 40/58] ci: avoid caching rust tool binaries --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 062b5dc8cc..bfb79cd67b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -728,6 +728,7 @@ jobs: uses: Swatinem/rust-cache@v2 with: workspaces: rust + cache-bin: false - name: Run Rust CI run: python ./ci/run_ci.py rust @@ -756,6 +757,7 @@ jobs: uses: Swatinem/rust-cache@v2 with: workspaces: rust + cache-bin: false - name: Run Rust Xlang Test env: FORY_RUST_JAVA_CI: "1" From 165f1aca6fbbd443e6b0cda4a52aab39678c71ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 08:40:04 +0800 Subject: [PATCH 41/58] docs: fix generated java evolution annotation --- docs/compiler/generated-code.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/compiler/generated-code.md b/docs/compiler/generated-code.md index a47be8a511..693b939756 100644 --- a/docs/compiler/generated-code.md +++ b/docs/compiler/generated-code.md @@ -155,14 +155,14 @@ public class Person { ``` When a message or inherited schema option sets `evolving=false`, the Java generator emits -`@ForyStruct(evolving = Evolution.DISABLED)` and imports `ForyStruct.Evolution` so the generated +`@ForyStruct(evolution = Evolution.DISABLED)` and imports `ForyStruct.Evolution` so the generated class uses fixed-schema struct encoding: ```java import org.apache.fory.annotation.ForyStruct; import org.apache.fory.annotation.ForyStruct.Evolution; -@ForyStruct(evolving = Evolution.DISABLED) +@ForyStruct(evolution = Evolution.DISABLED) public class StableMessage { ... } ``` From ee901d924446d64ea9037b372b718453c138d3e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 08:48:04 +0800 Subject: [PATCH 42/58] style: apply python formatting --- python/pyfory/meta/typedef.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py index 91a00bf5f8..6de7ad268b 100644 --- a/python/pyfory/meta/typedef.py +++ b/python/pyfory/meta/typedef.py @@ -631,19 +631,13 @@ def __repr__(self): } -def _list_array_element_type_matches( - list_field_type: FieldType, array_field_type: FieldType, require_non_nullable_elements: bool -) -> bool: +def _list_array_element_type_matches(list_field_type: FieldType, array_field_type: FieldType, require_non_nullable_elements: bool) -> bool: array_element_type_id = _ARRAY_ELEMENT_TYPE_IDS.get(array_field_type.type_id) if list_field_type.type_id != TypeId.LIST or array_element_type_id is None: return False - if require_non_nullable_elements and ( - list_field_type.element_type.is_nullable or list_field_type.element_type.is_tracking_ref - ): + if require_non_nullable_elements and (list_field_type.element_type.is_nullable or list_field_type.element_type.is_tracking_ref): return False - return _list_element_type_matches_array_element( - list_field_type.element_type.type_id, array_element_type_id - ) + return _list_element_type_matches_array_element(list_field_type.element_type.type_id, array_element_type_id) def _list_element_type_matches_array_element(list_element_type_id: TypeId, array_element_type_id: TypeId) -> bool: From 61cd4edbaf879e10af4f4a43a537c9b32840acbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 09:24:51 +0800 Subject: [PATCH 43/58] perf(java): speed up static generated serializers --- benchmarks/java/pom.xml | 5 + .../apache/fory/benchmark/XlangBenchmark.java | 24 ++ .../StaticSerializerSourceWriter.java | 380 +++++++++++++++++- 3 files changed, 395 insertions(+), 14 deletions(-) diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index eaaba8100b..a326e9b1f1 100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -252,6 +252,11 @@ jmh-generator-annprocess ${jmh.version} + + org.apache.fory + fory-annotation-processor + ${project.version} + diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/XlangBenchmark.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/XlangBenchmark.java index 2ad374033d..08051e40f8 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/XlangBenchmark.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/XlangBenchmark.java @@ -40,6 +40,8 @@ import org.apache.fory.benchmark.xlang.generated.FBSSampleList; import org.apache.fory.config.Int32Encoding; import org.apache.fory.integration_tests.state.generated.ProtoMessage; +import org.apache.fory.serializer.Serializer; +import org.apache.fory.serializer.StaticGeneratedStructSerializer; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.CompilerControl; @@ -48,6 +50,7 @@ import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -64,6 +67,9 @@ public class XlangBenchmark { @State(Scope.Thread) public static class XlangState { + @Param({"true", "false"}) + public boolean codegen; + public Fory fory; public NumericStruct numericStruct; @@ -107,6 +113,7 @@ public void setup() { Fory.builder() .withXlang(true) .withCompatible(true) + .withCodegen(codegen) .withRefTracking(false) .withClassVersionCheck(false) .requireClassRegistration(true) @@ -152,10 +159,27 @@ public void setup() { } private void verifySetup() { + verifyForySerializerMode(NumericStruct.class); + verifyForySerializerMode(Sample.class); + verifyForySerializerMode(MediaContent.class); fory.deserialize(foryNumericStructBytes); fromProtoStruct(protobufNumericStructBytes); fromFlatBufferNumericStruct(flatbufferNumericStructBuffer); } + + private void verifyForySerializerMode(Class type) { + Serializer serializer = fory.getTypeResolver().getSerializer(type); + boolean staticSerializer = serializer instanceof StaticGeneratedStructSerializer; + if (staticSerializer == codegen) { + throw new IllegalStateException( + "Unexpected serializer for " + + type.getName() + + " with codegen=" + + codegen + + ": " + + serializer.getClass().getName()); + } + } } private static void registerForyTypes(Fory fory) { diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index bd027d4a43..5ee5d7b08c 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -66,6 +66,7 @@ private void writeHeader() { builder.append("import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo;\n"); builder.append("import org.apache.fory.serializer.StaticGeneratedStructSerializer;\n"); builder.append("import org.apache.fory.type.Descriptor;\n"); + builder.append("import org.apache.fory.type.DispatchId;\n"); builder.append("import org.apache.fory.type.Types;\n\n"); } @@ -265,18 +266,25 @@ private void writeWriteGroup( .append("Fields(WriteContext writeContext, ") .append(struct.typeName) .append(" value) {\n"); + if (groupName.equals("BuildIn") && hasDirectWriteField()) { + builder.append(" MemoryBuffer buffer = writeContext.getBuffer();\n"); + } builder.append(" for (int i = 0; i < ").append(fieldsName).append(".length; i++) {\n"); builder.append(" SerializationFieldInfo fieldInfo = ").append(fieldsName).append("[i];\n"); builder.append(" switch (").append(idsName).append("[i]) {\n"); for (SourceField field : struct.fields) { builder.append(" case ").append(field.id).append(":\n"); appendDebugWrite("before", "fieldInfo", 10); - builder - .append(" ") - .append(helperName) - .append("(writeContext, fieldInfo, ") - .append(field.readExpression("value")) - .append(");\n"); + if (groupName.equals("BuildIn") && canEmitDirectWriteField(field)) { + appendDirectWrite(field); + } else { + builder + .append(" ") + .append(helperName) + .append("(writeContext, fieldInfo, ") + .append(field.readExpression("value")) + .append(");\n"); + } appendDebugWrite("after", "fieldInfo", 10); builder.append(" break;\n"); } @@ -312,21 +320,41 @@ private void writeReadBeanGroup( .append("Fields(ReadContext readContext, ") .append(struct.typeName) .append(" value) {\n"); + if (groupName.equals("BuildIn") && hasDirectReadField()) { + builder.append(" MemoryBuffer buffer = readContext.getBuffer();\n"); + } builder.append(" for (int i = 0; i < ").append(fieldsName).append(".length; i++) {\n"); builder.append(" SerializationFieldInfo fieldInfo = ").append(fieldsName).append("[i];\n"); appendDebugRead("before", "fieldInfo", 6); - builder - .append(" Object fieldValue = ") - .append(helperName) - .append("(readContext, fieldInfo);\n"); + if (!(groupName.equals("BuildIn") && hasDirectReadField())) { + builder + .append(" Object fieldValue = ") + .append(helperName) + .append("(readContext, fieldInfo);\n"); + } appendDebugRead("after", "fieldInfo", 6); builder.append(" switch (").append(idsName).append("[i]) {\n"); for (SourceField field : struct.fields) { builder.append(" case ").append(field.id).append(":\n"); - builder - .append(" ") - .append(field.writeStatement("value", field.castExpression("fieldValue"))) - .append("\n"); + if (groupName.equals("BuildIn") && canEmitDirectReadField(field)) { + appendDirectRead(field); + } else { + String fieldValueName = "fieldValue" + field.id; + if (groupName.equals("BuildIn") && hasDirectReadField()) { + builder + .append(" Object ") + .append(fieldValueName) + .append(" = ") + .append(helperName) + .append("(readContext, fieldInfo);\n"); + } else { + fieldValueName = "fieldValue"; + } + builder + .append(" ") + .append(field.writeStatement("value", field.castExpression(fieldValueName))) + .append("\n"); + } builder.append(" break;\n"); } builder.append(" default:\n"); @@ -339,6 +367,330 @@ private void writeReadBeanGroup( builder.append(" }\n\n"); } + private boolean hasDirectWriteField() { + for (SourceField field : struct.fields) { + if (canEmitDirectWriteField(field)) { + return true; + } + } + return false; + } + + private boolean hasDirectReadField() { + for (SourceField field : struct.fields) { + if (canEmitDirectReadField(field)) { + return true; + } + } + return false; + } + + private boolean canEmitDirectWriteField(SourceField field) { + if (canEmitDirectStringField(field)) { + return true; + } + if (canEmitDirectArrayField(field)) { + return true; + } + return field.typeNode.primitive && primitiveWriteCases(field) != null; + } + + private boolean canEmitDirectReadField(SourceField field) { + if (canEmitDirectStringField(field)) { + return true; + } + if (canEmitDirectArrayField(field)) { + return true; + } + return field.typeNode.primitive + && (exactPrimitiveReadExpression(field) != null || primitiveReadCases(field) != null); + } + + private boolean canEmitDirectStringField(SourceField field) { + return field.erasedType.equals("java.lang.String") && !field.nullable && !field.trackingRef; + } + + private boolean canEmitDirectArrayField(SourceField field) { + SourceTypeNode componentType = field.typeNode.componentType; + return field.arrayType + && componentType != null + && componentType.primitive + && !field.nullable + && !field.trackingRef; + } + + private void appendDirectWrite(SourceField field) { + if (canEmitDirectStringField(field)) { + builder + .append(" writeContext.writeString(") + .append(field.readExpression("value")) + .append(");\n"); + return; + } + if (canEmitDirectArrayField(field)) { + builder + .append(" fieldInfo.serializer.write(writeContext, ") + .append(field.readExpression("value")) + .append(");\n"); + return; + } + String exactWrite = exactPrimitiveWriteStatement(field, field.readExpression("value")); + if (exactWrite != null) { + builder.append(" ").append(exactWrite).append("\n"); + return; + } + builder.append(" switch (fieldInfo.dispatchId) {\n"); + String[][] cases = primitiveWriteCases(field); + for (String[] writeCase : cases) { + builder.append(" case DispatchId.").append(writeCase[0]).append(":\n"); + builder + .append(" buffer.") + .append(writeCase[1]) + .append("(") + .append(writeCase[2].replace("$value", field.readExpression("value"))) + .append(");\n"); + builder.append(" break;\n"); + } + builder.append(" default:\n"); + builder + .append(" writeBuildInFieldValue(writeContext, fieldInfo, ") + .append(field.readExpression("value")) + .append(");\n"); + builder.append(" }\n"); + } + + private void appendDirectRead(SourceField field) { + if (canEmitDirectStringField(field)) { + builder + .append(" ") + .append(field.writeStatement("value", "readContext.readString()")) + .append("\n"); + return; + } + if (canEmitDirectArrayField(field)) { + builder.append(" readContext.preserveRefId(-1);\n"); + builder + .append(" ") + .append( + field.writeStatement( + "value", "(" + field.erasedType + ") readContext.readNonRef(fieldInfo.typeInfo)")) + .append("\n"); + return; + } + String exactRead = exactPrimitiveReadExpression(field); + if (exactRead == null) { + appendPrimitiveReadSwitch(field); + return; + } + builder.append(" ").append(field.writeStatement("value", exactRead)).append("\n"); + } + + private void appendPrimitiveReadSwitch(SourceField field) { + builder.append(" switch (fieldInfo.dispatchId) {\n"); + String[][] cases = primitiveReadCases(field); + for (String[] readCase : cases) { + builder.append(" case DispatchId.").append(readCase[0]).append(":\n"); + builder + .append(" ") + .append(field.writeStatement("value", readCase[1])) + .append("\n"); + builder.append(" break;\n"); + } + builder.append(" default:\n"); + builder + .append(" Object fieldValue") + .append(field.id) + .append(" = readBuildInFieldValue(readContext, fieldInfo);\n"); + builder + .append(" ") + .append(field.writeStatement("value", field.castExpression("fieldValue" + field.id))) + .append("\n"); + builder.append(" }\n"); + } + + private String[][] primitiveWriteCases(SourceField field) { + switch (field.erasedType) { + case "boolean": + return new String[][] {{"BOOL", "writeBoolean", "$value"}}; + case "byte": + return new String[][] {{"INT8", "writeByte", "$value"}}; + case "char": + return new String[][] {{"CHAR", "writeChar", "$value"}}; + case "short": + return new String[][] {{"INT16", "writeInt16", "$value"}}; + case "int": + return new String[][] { + {"INT32", "writeInt32", "$value"}, + {"VARINT32", "writeVarInt32", "$value"}, + {"UINT8", "writeByte", "$value"}, + {"UINT16", "writeInt16", "(short) $value"}, + {"UINT32", "writeInt32", "$value"}, + {"VAR_UINT32", "writeVarUInt32", "$value"} + }; + case "long": + return new String[][] { + {"INT64", "writeInt64", "$value"}, + {"UINT64", "writeInt64", "$value"}, + {"VARINT64", "writeVarInt64", "$value"}, + {"TAGGED_INT64", "writeTaggedInt64", "$value"}, + {"VAR_UINT64", "writeVarUInt64", "$value"}, + {"TAGGED_UINT64", "writeTaggedUInt64", "$value"} + }; + case "float": + return new String[][] {{"FLOAT32", "writeFloat32", "$value"}}; + case "double": + return new String[][] {{"FLOAT64", "writeFloat64", "$value"}}; + default: + return null; + } + } + + private String[][] primitiveReadCases(SourceField field) { + switch (field.erasedType) { + case "boolean": + return new String[][] {{"BOOL", "buffer.readBoolean()"}}; + case "byte": + return new String[][] {{"INT8", "buffer.readByte()"}}; + case "char": + return new String[][] {{"CHAR", "buffer.readChar()"}}; + case "short": + return new String[][] {{"INT16", "buffer.readInt16()"}}; + case "int": + return new String[][] { + {"INT32", "buffer.readInt32()"}, + {"VARINT32", "buffer.readVarInt32()"}, + {"UINT8", "buffer.readByte() & 0xFF"}, + {"UINT16", "buffer.readInt16() & 0xFFFF"}, + {"UINT32", "buffer.readInt32()"}, + {"VAR_UINT32", "buffer.readVarUInt32()"} + }; + case "long": + return new String[][] { + {"INT64", "buffer.readInt64()"}, + {"UINT64", "buffer.readInt64()"}, + {"VARINT64", "buffer.readVarInt64()"}, + {"TAGGED_INT64", "buffer.readTaggedInt64()"}, + {"VAR_UINT64", "buffer.readVarUInt64()"}, + {"TAGGED_UINT64", "buffer.readTaggedUInt64()"} + }; + case "float": + return new String[][] {{"FLOAT32", "buffer.readFloat32()"}}; + case "double": + return new String[][] {{"FLOAT64", "buffer.readFloat64()"}}; + default: + return null; + } + } + + private String exactPrimitiveWriteStatement(SourceField field, String valueExpression) { + String typeId = exactPrimitiveTypeId(field); + if (typeId == null) { + return null; + } + switch (typeId) { + case "BOOL": + return "buffer.writeBoolean(" + valueExpression + ");"; + case "INT8": + return "buffer.writeByte(" + valueExpression + ");"; + case "INT16": + return "buffer.writeInt16(" + valueExpression + ");"; + case "INT32": + return "buffer.writeInt32(" + valueExpression + ");"; + case "VARINT32": + return "buffer.writeVarInt32(" + valueExpression + ");"; + case "UINT8": + return "buffer.writeByte(" + valueExpression + ");"; + case "UINT16": + return "buffer.writeInt16((short) " + valueExpression + ");"; + case "UINT32": + return "buffer.writeInt32((int) " + valueExpression + ");"; + case "VAR_UINT32": + return "buffer.writeVarUInt32((int) " + valueExpression + ");"; + case "INT64": + case "UINT64": + return "buffer.writeInt64(" + valueExpression + ");"; + case "VARINT64": + return "buffer.writeVarInt64(" + valueExpression + ");"; + case "TAGGED_INT64": + return "buffer.writeTaggedInt64(" + valueExpression + ");"; + case "VAR_UINT64": + return "buffer.writeVarUInt64(" + valueExpression + ");"; + case "TAGGED_UINT64": + return "buffer.writeTaggedUInt64(" + valueExpression + ");"; + case "FLOAT32": + return "buffer.writeFloat32(" + valueExpression + ");"; + case "FLOAT64": + return "buffer.writeFloat64(" + valueExpression + ");"; + default: + return null; + } + } + + private String exactPrimitiveReadExpression(SourceField field) { + String typeId = exactPrimitiveTypeId(field); + if (typeId == null) { + return null; + } + switch (typeId) { + case "BOOL": + return "buffer.readBoolean()"; + case "INT8": + return "buffer.readByte()"; + case "INT16": + return "buffer.readInt16()"; + case "INT32": + return "buffer.readInt32()"; + case "VARINT32": + return "buffer.readVarInt32()"; + case "UINT8": + return "buffer.readByte() & 0xFF"; + case "UINT16": + return "buffer.readInt16() & 0xFFFF"; + case "UINT32": + return field.erasedType.equals("long") + ? "Integer.toUnsignedLong(buffer.readInt32())" + : "buffer.readInt32()"; + case "VAR_UINT32": + return field.erasedType.equals("long") + ? "Integer.toUnsignedLong(buffer.readVarUInt32())" + : "buffer.readVarUInt32()"; + case "INT64": + case "UINT64": + return "buffer.readInt64()"; + case "VARINT64": + return "buffer.readVarInt64()"; + case "TAGGED_INT64": + return "buffer.readTaggedInt64()"; + case "VAR_UINT64": + return "buffer.readVarUInt64()"; + case "TAGGED_UINT64": + return "buffer.readTaggedUInt64()"; + case "FLOAT32": + return "buffer.readFloat32()"; + case "FLOAT64": + return "buffer.readFloat64()"; + default: + return null; + } + } + + private String exactPrimitiveTypeId(SourceField field) { + String meta = field.typeNode.typeExtMeta; + if (meta == null) { + return null; + } + String prefix = "meta(Types."; + int start = meta.indexOf(prefix); + if (start < 0) { + return null; + } + int end = meta.indexOf(',', start + prefix.length()); + if (end < 0) { + return null; + } + return meta.substring(start + prefix.length(), end); + } + private void writeReadRecordGroup( String groupName, String fieldsName, String idsName, String helperName) { builder From 33fb0ccde3a013d34a98f1f6659b0435182ff59c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 10:00:53 +0800 Subject: [PATCH 44/58] fix(xlang): allow nullable list schema array reads --- .../serialization/struct_compatible_test.cc | 9 +- cpp/fory/serialization/struct_serializer.h | 6 - cpp/fory/serialization/xlang_test_main.cc | 5 +- csharp/src/Fory/TypeMeta.cs | 9 -- csharp/tests/Fory.Tests/ForyRuntimeTests.cs | 7 +- csharp/tests/Fory.XlangPeer/Program.cs | 4 +- .../specification/xlang_serialization_spec.md | 14 +- go/fory/fory_compatible_test.go | 7 +- go/fory/struct_init.go | 3 - go/fory/tests/xlang/xlang_test_main.go | 7 +- .../java/org/apache/fory/meta/FieldInfo.java | 3 +- .../CompatibleCollectionArrayReader.java | 12 +- .../org/apache/fory/xlang/XlangTestBase.java | 13 +- javascript/packages/core/lib/context.ts | 122 ++++++++++++------ javascript/test/crossLanguage.test.ts | 8 +- javascript/test/typemeta.test.ts | 69 +++++----- python/pyfory/meta/typedef.py | 12 +- python/pyfory/tests/test_typedef_encoding.py | 5 +- python/pyfory/tests/xlang_test_main.py | 7 +- rust/fory-core/src/serializer/collection.rs | 5 - rust/tests/tests/compatible/test_struct.rs | 10 +- rust/tests/tests/test_cross_language.rs | 12 +- swift/Sources/Fory/FieldCodecs.swift | 3 - swift/Sources/Fory/TypeMeta.swift | 18 +-- .../Tests/ForyTests/CompatibilityTests.swift | 7 +- swift/Tests/ForyXlangTests/main.swift | 6 +- 26 files changed, 198 insertions(+), 185 deletions(-) diff --git a/cpp/fory/serialization/struct_compatible_test.cc b/cpp/fory/serialization/struct_compatible_test.cc index 0078342618..b05d6006e5 100644 --- a/cpp/fory/serialization/struct_compatible_test.cc +++ b/cpp/fory/serialization/struct_compatible_test.cc @@ -491,7 +491,8 @@ TEST(SchemaEvolutionTest, ImmediateArrayFieldCanReadIntoListCarrier) { EXPECT_EQ(decoded.value().values, (std::vector{4, 5, 6})); } -TEST(SchemaEvolutionTest, NullableListElementsCannotReadIntoArrayCarrier) { +TEST(SchemaEvolutionTest, + NullableListSchemaWithoutNullElementsCanReadIntoArrayCarrier) { auto writer = Fory::builder().compatible(true).xlang(true).build(); auto reader = Fory::builder().compatible(true).xlang(true).build(); @@ -506,10 +507,8 @@ TEST(SchemaEvolutionTest, NullableListElementsCannotReadIntoArrayCarrier) { auto decoded = reader.deserialize(payload.data(), payload.size()); - ASSERT_FALSE(decoded.ok()); - EXPECT_NE(decoded.error().to_string().find("non-null elements"), - std::string::npos) - << decoded.error().to_string(); + ASSERT_TRUE(decoded.ok()) << decoded.error().to_string(); + EXPECT_EQ(decoded.value().values, (std::vector{1, 2})); bytes = writer.serialize(CompatibleNullableListField{{1, std::nullopt}}); ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 6664ad26a6..178ee464ee 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -3292,12 +3292,6 @@ void read_single_field_by_index_compatible(T &obj, ReadContext &ctx, return; } const auto &remote_element_type = remote_field_type.generics[0]; - if (FORY_PREDICT_FALSE(remote_element_type.nullable || - remote_element_type.track_ref)) { - ctx.set_error(Error::invalid_data( - "compatible list to array field requires non-null elements")); - return; - } constexpr int8_t child = configured_node_child(); FieldType result = read_configured_list_data_as_array_field< FieldType, T, Index, 0, child>(ctx, remote_element_type.type_id); diff --git a/cpp/fory/serialization/xlang_test_main.cc b/cpp/fory/serialization/xlang_test_main.cc index 93d78f1760..370c7f6d3b 100644 --- a/cpp/fory/serialization/xlang_test_main.cc +++ b/cpp/fory/serialization/xlang_test_main.cc @@ -2348,7 +2348,10 @@ void run_test_list_array_compatible_nullable_list_to_array_error( Buffer buffer = make_buffer(bytes); auto result = fory.deserialize(buffer); if (result.ok()) { - fail("Expected nullable list payload to fail compatible array read"); + std::vector out; + append_serialized(fory, result.value(), out); + write_file(data_file, out); + return; } write_file(data_file, bytes); } diff --git a/csharp/src/Fory/TypeMeta.cs b/csharp/src/Fory/TypeMeta.cs index 6bdb8fc6ce..dba55d1174 100644 --- a/csharp/src/Fory/TypeMeta.cs +++ b/csharp/src/Fory/TypeMeta.cs @@ -808,13 +808,6 @@ private static void ThrowIfUnsupportedListArrayMismatch( { return; } - if (remote.TypeId == (uint)global::Apache.Fory.TypeId.List && - TryPackedArrayElementTypeId(local.TypeId).HasValue && - remote.Generics.Count == 1 && - (remote.Generics[0].Nullable || remote.Generics[0].TrackRef)) - { - throw new InvalidDataException("compatible list to array field requires non-null elements"); - } throw new InvalidDataException("unsupported compatible list/array schema mismatch"); } } @@ -854,8 +847,6 @@ private static bool IsCompatibleListArrayFieldPair(TypeMetaFieldType remote, Typ bool remoteListLocalArray = remote.TypeId == (uint)global::Apache.Fory.TypeId.List && localArrayElementTypeId.HasValue && remote.Generics.Count == 1 && - !remote.Generics[0].Nullable && - !remote.Generics[0].TrackRef && CompatibleScalarTypeId(localArrayElementTypeId.Value) == CompatibleScalarTypeId(remote.Generics[0].TypeId); if (remoteListLocalArray) diff --git a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs index 038058e94f..0057e1f32a 100644 --- a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs +++ b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs @@ -1072,7 +1072,7 @@ public void CompatibleReadSupportsUInt32ListArrayFieldPairs() } [Fact] - public void CompatibleReadRejectsNullableListElementsIntoArrayCarrier() + public void CompatibleReadAllowsNullableListSchemaWithoutNullElementsIntoArrayCarrier() { ForyRuntime writer = ForyRuntime.Builder().Compatible(true).Build(); writer.Register(308); @@ -1080,9 +1080,8 @@ public void CompatibleReadRejectsNullableListElementsIntoArrayCarrier() reader.Register(308); byte[] nonNullPayload = writer.Serialize(new CompatibleNullableListSchema { Values = [1, 2] }); - InvalidDataException nonNullException = - Assert.Throws(() => reader.Deserialize(nonNullPayload)); - Assert.Contains("compatible list to array field requires non-null elements", nonNullException.Message); + CompatibleArraySchema nonNullDecoded = reader.Deserialize(nonNullPayload); + Assert.Equal([1, 2], nonNullDecoded.Values); byte[] payload = writer.Serialize(new CompatibleNullableListSchema { Values = [1, null] }); InvalidDataException exception = diff --git a/csharp/tests/Fory.XlangPeer/Program.cs b/csharp/tests/Fory.XlangPeer/Program.cs index 2fad7dfbf3..f5110f4f36 100644 --- a/csharp/tests/Fory.XlangPeer/Program.cs +++ b/csharp/tests/Fory.XlangPeer/Program.cs @@ -844,13 +844,13 @@ private static byte[] CaseListArrayCompatibleNullableListToArrayError(byte[] inp ReadOnlySequence sequence = new(input); try { - _ = fory.Deserialize(ref sequence); + CompatibleInt32ArrayField value = fory.Deserialize(ref sequence); + return fory.Serialize(value); } catch (Apache.Fory.InvalidDataException) { return input; } - throw new InvalidOperationException("Expected nullable list payload to fail compatible array read"); } private static byte[] CaseOneEnumFieldSchema(byte[] input) diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index 516622ae64..1c9a424931 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -195,10 +195,14 @@ and `array` as distinct kinds. The adaptation is limited to the immediate schema of the matched compatible field. It does not apply when `list` or `array` appears inside another field type, including collection elements, map keys or values, array elements, -union alternatives, or other generic/container positions. When a peer `list` -payload declares nullable or ref-tracked elements, a local matched `array` -field must raise a compatible-read error. Null list elements must not be coerced -to dense-array default values. +union alternatives, or other generic/container positions. A peer `list` +TypeDef element may be declared nullable or ref-tracked; that declaration alone +does not prevent a local matched `array` field from reading the value. The +reader must decide from the collection payload. If the payload actually carries +a null element or reference-tracked element encoding that cannot be represented +as a dense array element value, the local `array` field must raise a +compatible-read error. Null list elements must not be coerced to dense-array +default values. Users can also provide meta hints for fields of a type, or the type whole. Here is an example in java which use annotation to provide such information. @@ -1169,7 +1173,7 @@ The elements header is a single byte that encodes metadata about the collection | Bit | Name | Value | Meaning when SET (1) | Meaning when UNSET (0) | | --- | ----------------- | ----- | --------------------------------------- | --------------------------------------- | | 0 | track_ref | 0x01 | Track references for elements | Don't track element references | -| 1 | has_null | 0x02 | Collection may contain null elements | No null elements (skip null checks) | +| 1 | has_null | 0x02 | Payload contains null element markers | No null elements (skip null checks) | | 2 | is_decl_elem_type | 0x04 | Elements are the declared generic type | Element types differ from declared type | | 3 | is_same_type | 0x08 | All elements have the same runtime type | Elements have different runtime types | diff --git a/go/fory/fory_compatible_test.go b/go/fory/fory_compatible_test.go index 25d4c6aa4e..4a78cd5c32 100644 --- a/go/fory/fory_compatible_test.go +++ b/go/fory/fory_compatible_test.go @@ -603,14 +603,17 @@ func TestCompatibleSerializationScenarios(t *testing.T) { }, }, { - name: "NullableInt32ListWithoutNullsDoesNotMatchArray", + name: "NullableInt32ListWithoutNullsMatchesArray", tag: "Int32Sequence", writeType: NullableInt32ListPayloadDataClass{}, readType: Int32ArrayPayloadDataClass{}, input: NullableInt32ListPayloadDataClass{ Payload: []*int32{ptr(int32(1)), ptr(int32(2)), ptr(int32(3))}, }, - unmarshalErrContains: "compatible list to array field requires non-null elements", + assertFunc: func(t *testing.T, input any, output any) { + out := output.(Int32ArrayPayloadDataClass) + assert.Equal(t, [3]int32{1, 2, 3}, out.Payload) + }, }, { name: "NullableInt32ListPayloadDoesNotMatchArray", diff --git a/go/fory/struct_init.go b/go/fory/struct_init.go index a20b0d9154..87f814c3e4 100644 --- a/go/fory/struct_init.go +++ b/go/fory/struct_init.go @@ -534,9 +534,6 @@ func (s *structSerializer) initFieldsFromTypeDef(typeResolver *TypeResolver) err shouldRead = true fieldType = localType } else if defTypeId == LIST && localFieldSpec != nil && compatibleListFieldHasPrimitiveArrayShape(def.typeSpec, localFieldSpec.Type, localType) { - if def.typeSpec.Element.Nullable || def.typeSpec.Element.TrackRef { - return fmt.Errorf("field %s: compatible list to array field requires non-null elements", def.name) - } shouldRead = true usesCompatibleCollectionArrayReader = true fieldType = localType diff --git a/go/fory/tests/xlang/xlang_test_main.go b/go/fory/tests/xlang/xlang_test_main.go index 5dfba1f1a3..e7978e96c4 100644 --- a/go/fory/tests/xlang/xlang_test_main.go +++ b/go/fory/tests/xlang/xlang_test_main.go @@ -1657,7 +1657,12 @@ func testListArrayCompatibleNullableListToArrayError() { var result CompatibleInt32ArrayField if err := f.Deserialize(data, &result); err == nil { - panic("Expected nullable list payload to fail compatible array read") + serialized, err := f.Serialize(&result) + if err != nil { + panic(fmt.Sprintf("Failed to serialize compatible array field: %v", err)) + } + writeFile(dataFile, serialized) + return } writeFile(dataFile, data) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index 50197b029c..d12a936539 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -239,10 +239,9 @@ private boolean isTopLevelListArrayCompatibleReadPair( FieldTypes.FieldType localFieldType = FieldTypes.buildFieldType(resolver, localField); int peerListElementTypeId = listElementTypeId(fieldType); if (peerListElementTypeId != Types.UNKNOWN) { - int nonNullablePeerListElementTypeId = nonNullableListElementTypeId(fieldType); int localArrayTypeId = arrayTypeId(localFieldType); return localArrayTypeId != Types.UNKNOWN - && localArrayTypeId == denseArrayTypeId(nonNullablePeerListElementTypeId); + && localArrayTypeId == denseArrayTypeId(peerListElementTypeId); } int peerArrayTypeId = arrayTypeId(fieldType); if (peerArrayTypeId != Types.UNKNOWN) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index ff5e32ae56..b854babcd0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -88,16 +88,16 @@ static ReadAction readAction(TypeResolver resolver, Descriptor descriptor) { FieldTypes.FieldType localFieldType = FieldTypes.buildFieldType(resolver, field); int peerListElementTypeId = listElementTypeId(descriptor); if (peerListElementTypeId != Types.UNKNOWN) { - int nonNullablePeerListElementTypeId = nonNullableListElementTypeId(descriptor); + // Element nullable/ref flags in TypeDef describe what the peer schema can encode, not what + // this payload actually contains. Dense-array compatibility is decided here by element type; + // readListPayloadAsPrimitiveArray rejects payloads that carry null/ref element markers. int localArrayTypeId = arrayTypeId(localFieldType); if (localArrayTypeId != Types.UNKNOWN - && localArrayTypeId == denseArrayTypeId(nonNullablePeerListElementTypeId)) { + && localArrayTypeId == denseArrayTypeId(peerListElementTypeId)) { return new ReadAction( - READ_LIST_TO_ARRAY, - localArrayTypeId, - nonNullablePeerListElementTypeId, - field.getType()); + READ_LIST_TO_ARRAY, localArrayTypeId, peerListElementTypeId, field.getType()); } + int nonNullablePeerListElementTypeId = nonNullableListElementTypeId(descriptor); int localListElementTypeId = nonNullableListElementTypeId(localFieldType); int peerArrayTypeId = denseArrayTypeId(peerListElementTypeId); // List-to-array and list-to-list materialize through a dense primitive array, so they cannot diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index 77193acf52..43a4fcc477 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -1880,14 +1880,18 @@ protected void testListArrayCompatibleRead(boolean enableCodegen) throws java.io buffer = MemoryBuffer.newHeapBuffer(256); nullableListFory.serialize(buffer, nullableListWithoutNulls); byte[] nullableListWithoutNullsPayload = buffer.getBytes(0, buffer.writerIndex()); - Assert.expectThrows( - DeserializationException.class, - () -> arrayFory.deserialize(MemoryUtils.wrap(nullableListWithoutNullsPayload))); + XlangCompatibleInt32ArrayField nullableListWithoutNullsArray = + (XlangCompatibleInt32ArrayField) + arrayFory.deserialize(MemoryUtils.wrap(nullableListWithoutNullsPayload)); + Assert.assertEquals(nullableListWithoutNullsArray.values, new int[] {1, 2, 3}); ctx = prepareExecution( "test_list_array_compatible_nullable_list_to_array_error", nullableListWithoutNullsPayload); runPeer(ctx); + nullableListWithoutNullsArray = + (XlangCompatibleInt32ArrayField) arrayFory.deserialize(readBuffer(ctx.dataFile())); + Assert.assertEquals(nullableListWithoutNullsArray.values, new int[] {1, 2, 3}); XlangCompatibleNullableInt32ListField nullableListValue = newCompatibleNullableInt32ListField(1, null, 3); @@ -1901,6 +1905,9 @@ protected void testListArrayCompatibleRead(boolean enableCodegen) throws java.io prepareExecution( "test_list_array_compatible_nullable_list_to_array_error", nullablePayload); runPeer(ctx); + Path nullableDataFile = ctx.dataFile(); + Assert.expectThrows( + DeserializationException.class, () -> arrayFory.deserialize(readBuffer(nullableDataFile))); } @Test(groups = "xlang", dataProvider = "enableCodegenParallel") diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 8b35e28ba1..e7a19251be 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -47,7 +47,9 @@ type RegeneratedReadSerializerCacheEntry = { serializers: Map; }; -function remoteListElementType(fieldInfo: InnerFieldInfo): InnerFieldInfo | undefined { +function remoteListElementType( + fieldInfo: InnerFieldInfo, +): InnerFieldInfo | undefined { if (fieldInfo.typeId !== TypeId.LIST) { return undefined; } @@ -367,8 +369,8 @@ export class WriteContext { checkCollectionSize(size: number) { if (size > this._maxCollectionSize) { throw new Error( - `Collection size ${size} exceeds maxCollectionSize ${this._maxCollectionSize}. ` - + "The data may be malicious, or increase maxCollectionSize if needed.", + `Collection size ${size} exceeds maxCollectionSize ${this._maxCollectionSize}. ` + + "The data may be malicious, or increase maxCollectionSize if needed.", ); } } @@ -376,8 +378,8 @@ export class WriteContext { checkBinarySize(size: number) { if (size > this._maxBinarySize) { throw new Error( - `Binary size ${size} exceeds maxBinarySize ${this._maxBinarySize}. ` - + "The data may be malicious, or increase maxBinarySize if needed.", + `Binary size ${size} exceeds maxBinarySize ${this._maxBinarySize}. ` + + "The data may be malicious, or increase maxBinarySize if needed.", ); } } @@ -584,13 +586,20 @@ export class ReadContext { return headerHigh * 0x100000 + (headerLow >>> 12); } - private findRecentTypeMeta(headerLow: number, headerHigh: number): TypeMeta | null { + private findRecentTypeMeta( + headerLow: number, + headerHigh: number, + ): TypeMeta | null { const lows = this.recentTypeMetaHeaderLows; const highs = this.recentTypeMetaHeaderHighs; const metas = this.recentTypeMetas; for (let i = 0; i < metas.length; i++) { const typeMeta = metas[i]; - if (typeMeta !== null && lows[i] === headerLow && highs[i] === headerHigh) { + if ( + typeMeta !== null && + lows[i] === headerLow && + highs[i] === headerHigh + ) { return typeMeta; } } @@ -647,8 +656,8 @@ export class ReadContext { this._depth++; if (this._depth > this._maxDepth) { throw new Error( - `Deserialization depth limit exceeded: ${this._depth} > ${this._maxDepth}. ` - + "The data may be malicious, or increase maxDepth if needed.", + `Deserialization depth limit exceeded: ${this._depth} > ${this._maxDepth}. ` + + "The data may be malicious, or increase maxDepth if needed.", ); } } @@ -660,8 +669,8 @@ export class ReadContext { checkCollectionSize(size: number) { if (size > this._maxCollectionSize) { throw new Error( - `Collection size ${size} exceeds maxCollectionSize ${this._maxCollectionSize}. ` - + "The data may be malicious, or increase maxCollectionSize if needed.", + `Collection size ${size} exceeds maxCollectionSize ${this._maxCollectionSize}. ` + + "The data may be malicious, or increase maxCollectionSize if needed.", ); } } @@ -669,8 +678,8 @@ export class ReadContext { checkBinarySize(size: number) { if (size > this._maxBinarySize) { throw new Error( - `Binary size ${size} exceeds maxBinarySize ${this._maxBinarySize}. ` - + "The data may be malicious, or increase maxBinarySize if needed.", + `Binary size ${size} exceeds maxBinarySize ${this._maxBinarySize}. ` + + "The data may be malicious, or increase maxBinarySize if needed.", ); } } @@ -693,9 +702,9 @@ export class ReadContext { headerHigh: number, ): TypeMeta { if ( - this.lastTypeMeta !== null - && this.lastTypeMetaHeaderLow === headerLow - && this.lastTypeMetaHeaderHigh === headerHigh + this.lastTypeMeta !== null && + this.lastTypeMetaHeaderLow === headerLow && + this.lastTypeMetaHeaderHigh === headerHigh ) { TypeMeta.skipBodyByHeaderLow(this.reader, headerLow); this.typeMeta[dynamicTypeId] = this.lastTypeMeta; @@ -775,7 +784,11 @@ export class ReadContext { } else { const headerLow = this.reader.readUint32(); const headerHigh = this.reader.readUint32(); - typeMeta = this.readTypeMetaFromHeader(idOrLen >> 1, headerLow, headerHigh); + typeMeta = this.readTypeMetaFromHeader( + idOrLen >> 1, + headerLow, + headerHigh, + ); remoteHash = ReadContext.typeMetaHeaderHash(headerLow, headerHigh); } if (expectedHash !== remoteHash) { @@ -790,12 +803,21 @@ export class ReadContext { topLevel = true, ): TypeInfo { if (topLevel && fallbackTypeInfo) { - const compatible = this.compatibleFieldTypeInfo(fieldInfo, fallbackTypeInfo); + const compatible = this.compatibleFieldTypeInfo( + fieldInfo, + fallbackTypeInfo, + ); if (compatible) { return compatible; } } - if (this.hasUnsupportedListArrayMismatch(fieldInfo, fallbackTypeInfo, topLevel)) { + if ( + this.hasUnsupportedListArrayMismatch( + fieldInfo, + fallbackTypeInfo, + topLevel, + ) + ) { throw new Error("unsupported compatible list/array schema mismatch"); } switch (fieldInfo.typeId) { @@ -872,17 +894,15 @@ export class ReadContext { if (compatibleArrayElementTypeId(remoteElement.typeId) !== localElement) { return undefined; } - if (remoteElement.nullable === true || remoteElement.trackingRef === true) { - return undefined; - } return compatibleListToArrayTypeInfo(remoteElement, localElement); } const remoteArrayElement = denseArrayElementTypeId(remote.typeId); if ( - remoteArrayElement !== undefined - && local.typeId === TypeId.LIST - && local.options?.inner - && compatibleArrayElementTypeId(local.options.inner.typeId) === remoteArrayElement + remoteArrayElement !== undefined && + local.typeId === TypeId.LIST && + local.options?.inner && + compatibleArrayElementTypeId(local.options.inner.typeId) === + remoteArrayElement ) { return compatibleArrayToListTypeInfo(remoteArrayElement); } @@ -910,8 +930,8 @@ export class ReadContext { remote.options!.key!, local.options?.key, false, - ) - || this.hasUnsupportedListArrayMismatch( + ) || + this.hasUnsupportedListArrayMismatch( remote.options!.value!, local.options?.value, false, @@ -944,13 +964,22 @@ export class ReadContext { switch (remote.typeId) { case TypeId.MAP: return ( - this.hasListArrayMismatch(remote.options!.key!, local.options?.key) - || this.hasListArrayMismatch(remote.options!.value!, local.options?.value) + this.hasListArrayMismatch(remote.options!.key!, local.options?.key) || + this.hasListArrayMismatch( + remote.options!.value!, + local.options?.value, + ) ); case TypeId.LIST: - return this.hasListArrayMismatch(remote.options!.inner!, local.options?.inner); + return this.hasListArrayMismatch( + remote.options!.inner!, + local.options?.inner, + ); case TypeId.SET: - return this.hasListArrayMismatch(remote.options!.key!, local.options?.key); + return this.hasListArrayMismatch( + remote.options!.key!, + local.options?.key, + ); default: return false; } @@ -969,10 +998,15 @@ export class ReadContext { return this.hasNestedListArrayMismatch(remote, local); } - private isListArrayRootPair(remote: InnerFieldInfo, local: TypeInfo): boolean { + private isListArrayRootPair( + remote: InnerFieldInfo, + local: TypeInfo, + ): boolean { return ( - (remote.typeId === TypeId.LIST && denseArrayElementTypeId(local.typeId) !== undefined) - || (denseArrayElementTypeId(remote.typeId) !== undefined && local.typeId === TypeId.LIST) + (remote.typeId === TypeId.LIST && + denseArrayElementTypeId(local.typeId) !== undefined) || + (denseArrayElementTypeId(remote.typeId) !== undefined && + local.typeId === TypeId.LIST) ); } @@ -983,9 +1017,9 @@ export class ReadContext { const localHash = original.getHash(); let entry = this.regeneratedReadSerializers.get(original); if ( - entry === undefined - || entry.localTypeInfo !== localTypeInfo - || entry.localHash !== localHash + entry === undefined || + entry.localTypeInfo !== localTypeInfo || + entry.localHash !== localHash ) { entry = { localHash, @@ -1013,9 +1047,10 @@ export class ReadContext { ); } } - const cacheEntry = original === undefined - ? undefined - : this.getRegeneratedReadSerializerCache(original); + const cacheEntry = + original === undefined + ? undefined + : this.getRegeneratedReadSerializerCache(original); const remoteHash = typeMeta.getHash(); const cached = cacheEntry?.serializers.get(remoteHash); if (cached !== undefined) { @@ -1061,8 +1096,9 @@ export class ReadContext { ? this.typeResolver.generateReadSerializer(typeInfo) : this.typeResolver.regenerateReadSerializer(typeInfo); if ( - cacheEntry !== undefined - && cacheEntry.serializers.size < ReadContext.MAX_CACHED_REGENERATED_READ_SERIALIZER + cacheEntry !== undefined && + cacheEntry.serializers.size < + ReadContext.MAX_CACHED_REGENERATED_READ_SERIALIZER ) { cacheEntry.serializers.set(remoteHash, serializer); } diff --git a/javascript/test/crossLanguage.test.ts b/javascript/test/crossLanguage.test.ts index e94b96ea3f..a0eea92390 100644 --- a/javascript/test/crossLanguage.test.ts +++ b/javascript/test/crossLanguage.test.ts @@ -1427,8 +1427,12 @@ describe("bool", () => { }), ); - expect(() => serializer.deserialize(content)).toThrow(); - writeToFile(content); + try { + const value = serializer.deserialize(content); + writeToFile(serializer.serialize(value) as Buffer); + } catch { + writeToFile(content); + } }); test("test_one_enum_field_schema", () => { diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index 3454588132..69b0a7410e 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -129,11 +129,7 @@ describe("typemeta", () => { const bodyOnlyHash = bodyOnlyHeaderHashBits(bytes.subarray(bodyOffset)); expect(header & HEADER_HASH_MASK).not.toBe(bodyOnlyHash); - view.setBigUint64( - 0, - bodyOnlyHash | (header & LOW_HEADER_BITS_MASK), - true, - ); + view.setBigUint64(0, bodyOnlyHash | (header & LOW_HEADER_BITS_MASK), true); const reader = new BinaryReader({}); reader.reset(malformed); @@ -171,9 +167,10 @@ describe("typemeta", () => { } as any, config, ); - (context as any).typeMetaCache.set(Number(header >> 32n), new Map([ - [Number(header & 0xffffffffn), typeMeta], - ])); + (context as any).typeMetaCache.set( + Number(header >> 32n), + new Map([[Number(header & 0xffffffffn), typeMeta]]), + ); context.reset(writer.dump()); expect(context.readTypeMeta()).toBe(typeMeta); @@ -229,9 +226,11 @@ describe("typemeta", () => { value: Type.int32().setId(1), }); - const changedBytes = changedWriterFory.register(changedWriterType).serialize({ - value: "hello", - }); + const changedBytes = changedWriterFory + .register(changedWriterType) + .serialize({ + value: "hello", + }); const localBytes = localWriterFory.register(localWriterType).serialize({ value: 123, }); @@ -267,9 +266,12 @@ describe("typemeta", () => { test("regenerated read serializers keep getTypeInfo", () => { const fory = new Fory({ compatible: true }); const serializer = (fory as any).typeResolver.regenerateReadSerializer( - Type.struct({ namespace: "example", typeName: "repro_struct" }, { - value: Type.int32(), - }), + Type.struct( + { namespace: "example", typeName: "repro_struct" }, + { + value: Type.int32(), + }, + ), ); expect(typeof serializer.getTypeInfo).toBe("function"); @@ -291,9 +293,10 @@ describe("typemeta", () => { const localChildType = Type.struct(7311, { value: Type.int32().setId(1), }); - const createParentType = () => Type.struct(7312, { - child: Type.struct(7311).setId(1), - }); + const createParentType = () => + Type.struct(7312, { + child: Type.struct(7311).setId(1), + }); stringWriterFory.register(stringChildType); boolWriterFory.register(boolChildType); @@ -306,8 +309,8 @@ describe("typemeta", () => { const reader = readerFory.register(createParentType()); const typeResolver = (readerFory as any).typeResolver; - const generateReadSerializer - = typeResolver.generateReadSerializer.bind(typeResolver); + const generateReadSerializer = + typeResolver.generateReadSerializer.bind(typeResolver); let generatedReaders = 0; typeResolver.generateReadSerializer = (typeInfo: any) => { generatedReaders++; @@ -409,8 +412,14 @@ describe("typemeta", () => { expect(result.bools).toBeInstanceOf(BoolArray); expect(Array.from(result.bools)).toEqual([true, false]); expect(result.float16s).toBeInstanceOf(Float16Array as any); - expect(Array.from(result.float16s as Iterable)[0]).toBeCloseTo(1.5, 1); - expect(Array.from(result.float16s as Iterable)[1]).toBeCloseTo(-2, 1); + expect(Array.from(result.float16s as Iterable)[0]).toBeCloseTo( + 1.5, + 1, + ); + expect(Array.from(result.float16s as Iterable)[1]).toBeCloseTo( + -2, + 1, + ); expect(result.bfloat16s).toBeInstanceOf(BFloat16Array); expect(Array.from(result.bfloat16s as Iterable)).toEqual([1.5, -2]); }); @@ -435,7 +444,7 @@ describe("typemeta", () => { expect(result).toEqual({ values: [1, 2, 3] }); }); - test("rejects compatible list to dense array when payload has nullable elements", () => { + test("adapts nullable list schema to dense array when payload has no null elements", () => { const writerFory = new Fory({ compatible: true }); const readerFory = new Fory({ compatible: true }); @@ -453,15 +462,14 @@ describe("typemeta", () => { values: [1, 2, 3], }); const reader = readerFory.register(readerType); - expect(() => reader.deserialize(nonNullBytes)).toThrow( - "unsupported compatible list/array schema mismatch", - ); + const nonNullResult = reader.deserialize(nonNullBytes); + expect(Array.from(nonNullResult.values)).toEqual([1, 2, 3]); const nullableBytes = serializer.serialize({ values: [1, null, 3], }); expect(() => reader.deserialize(nullableBytes)).toThrow( - "unsupported compatible list/array schema mismatch", + "compatible list-to-array field cannot read nullable or ref-tracked elements", ); }); @@ -480,7 +488,9 @@ describe("typemeta", () => { values: ["1", "2"], }); - expect(() => readerFory.register(readerType).deserialize(bytes)).toThrow(/list\/array/); + expect(() => readerFory.register(readerType).deserialize(bytes)).toThrow( + /list\/array/, + ); }); test("skips nested compatible list and dense array positions", () => { @@ -720,10 +730,7 @@ function typeMetaBodyOffset(bytes: Uint8Array) { function bodyOnlyHeaderHashBits(buffer: Uint8Array) { const hash = x64hash128(buffer, 47); - let header = BigInt.asIntN( - 64, - hash.getBigInt64(0, false) << HASH_SHIFT_BITS, - ); + let header = BigInt.asIntN(64, hash.getBigInt64(0, false) << HASH_SHIFT_BITS); if (header < 0n) { header = -header; } diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py index 6de7ad268b..1ea5b624a4 100644 --- a/python/pyfory/meta/typedef.py +++ b/python/pyfory/meta/typedef.py @@ -631,12 +631,10 @@ def __repr__(self): } -def _list_array_element_type_matches(list_field_type: FieldType, array_field_type: FieldType, require_non_nullable_elements: bool) -> bool: +def _list_array_element_type_matches(list_field_type: FieldType, array_field_type: FieldType) -> bool: array_element_type_id = _ARRAY_ELEMENT_TYPE_IDS.get(array_field_type.type_id) if list_field_type.type_id != TypeId.LIST or array_element_type_id is None: return False - if require_non_nullable_elements and (list_field_type.element_type.is_nullable or list_field_type.element_type.is_tracking_ref): - return False return _list_element_type_matches_array_element(list_field_type.element_type.type_id, array_element_type_id) @@ -651,11 +649,9 @@ def _is_root_list_array_pair(remote_field_type: FieldType, local_field_type: Fie if local_field_type is None: return False if remote_field_type.type_id == TypeId.LIST and local_field_type.type_id in _ARRAY_TYPE_IDS: - return _list_array_element_type_matches(remote_field_type, local_field_type, True) + return _list_array_element_type_matches(remote_field_type, local_field_type) if local_field_type.type_id == TypeId.LIST and remote_field_type.type_id in _ARRAY_TYPE_IDS: - # A dense remote array can feed a nullable local list; only list-to-array requires rejecting - # nullable/ref-tracked elements because the local dense array has no carrier for them. - return _list_array_element_type_matches(local_field_type, remote_field_type, False) + return _list_array_element_type_matches(local_field_type, remote_field_type) return False @@ -674,7 +670,7 @@ def _remote_list_to_local_array_allowed(remote_field_type: FieldType, local_fiel return ( remote_field_type.type_id == TypeId.LIST and local_field_type.type_id in _ARRAY_TYPE_IDS - and _list_array_element_type_matches(remote_field_type, local_field_type, True) + and _list_array_element_type_matches(remote_field_type, local_field_type) ) diff --git a/python/pyfory/tests/test_typedef_encoding.py b/python/pyfory/tests/test_typedef_encoding.py index bd0ee8ca49..b25d067cbf 100644 --- a/python/pyfory/tests/test_typedef_encoding.py +++ b/python/pyfory/tests/test_typedef_encoding.py @@ -602,8 +602,9 @@ def test_compatible_nullable_int32_list_payload_rejects_array_read(): _register_int32_payload(writer, NullableInt32ListPayload) _register_int32_payload(reader, Int32ArrayPayload) - with pytest.raises(TypeNotCompatibleError): - reader.deserialize(writer.serialize(NullableInt32ListPayload(payload=[1, 2, 3]))) + decoded = reader.deserialize(writer.serialize(NullableInt32ListPayload(payload=[1, 2, 3]))) + assert isinstance(decoded, Int32ArrayPayload) + assert list(decoded.payload) == [1, 2, 3] with pytest.raises(TypeNotCompatibleError): reader.deserialize(writer.serialize(NullableInt32ListPayload(payload=[1, None, 3]))) diff --git a/python/pyfory/tests/xlang_test_main.py b/python/pyfory/tests/xlang_test_main.py index d9c189e3ec..1478934656 100644 --- a/python/pyfory/tests/xlang_test_main.py +++ b/python/pyfory/tests/xlang_test_main.py @@ -956,12 +956,11 @@ def test_list_array_compatible_nullable_list_to_array_error(): fory = pyfory.Fory(xlang=True, compatible=True) fory.register_type(CompatibleInt32ArrayField, type_id=901) try: - fory.deserialize(data_bytes) + value = fory.deserialize(data_bytes) except TypeNotCompatibleError: - pass + new_bytes = data_bytes else: - raise AssertionError("Expected nullable list payload to fail compatible array read") - new_bytes = data_bytes + new_bytes = fory.serialize(value) with open(data_file, "wb") as f: f.write(new_bytes) diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs index b3506e817b..9968ba9873 100644 --- a/rust/fory-core/src/serializer/collection.rs +++ b/rust/fory-core/src/serializer/collection.rs @@ -835,11 +835,6 @@ where && !remote_field_type.generics.is_empty() && list_element_type_matches_array_shape(remote_field_type, local_field_type) { - if remote_field_type.generics[0].nullable || remote_field_type.generics[0].track_ref { - return Err(Error::type_error( - "compatible list to array field requires non-null elements", - )); - } if field_ref_mode(remote_field_type) != RefMode::None { let ref_flag = context.reader.read_i8()?; if ref_flag == RefFlag::Null as i8 { diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index 55d2972531..52361b3538 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -144,14 +144,8 @@ fn compatible_list_array_field_pairs() { payload: vec![Some(1), Some(2), Some(3)], }) .unwrap(); - let err = reader - .deserialize::(&bytes) - .expect_err("expected nullable list schema to fail compatible array read"); - assert!( - err.to_string() - .contains("compatible list to array field requires non-null elements"), - "{err}" - ); + let decoded: ArrayPayload = reader.deserialize(&bytes).unwrap(); + assert_eq!(decoded.payload, vec![1, 2, 3]); let bytes = writer .serialize(&NullableListPayload { diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index fcf5f0d261..918d846426 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -1456,13 +1456,11 @@ fn test_list_array_compatible_nullable_list_to_array_error() { let mut fory = Fory::builder().compatible(true).xlang(true).build(); fory.register::(901).unwrap(); - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - fory.deserialize::(&bytes) - })); - assert!( - result.is_err() || result.is_ok_and(|value| value.is_err()), - "Expected nullable list payload to fail compatible array read" - ); + if let Ok(value) = fory.deserialize::(&bytes) { + let new_bytes = fory.serialize(&value).unwrap(); + fs::write(&data_file_path, new_bytes).unwrap(); + return; + } fs::write(&data_file_path, bytes).unwrap(); } diff --git a/swift/Sources/Fory/FieldCodecs.swift b/swift/Sources/Fory/FieldCodecs.swift index 16cec20891..9d48cd6e74 100644 --- a/swift/Sources/Fory/FieldCodecs.swift +++ b/swift/Sources/Fory/FieldCodecs.swift @@ -711,9 +711,6 @@ public enum ArrayFieldCodec: FieldCodec { let element = remoteFieldType.generics.first, let localArrayTypeID = packedArrayTypeID(for: ElementCodec.self), TypeId.listElementTypeID(element.typeID, matchesDenseArrayTypeID: localArrayTypeID.rawValue) { - if element.nullable || element.trackRef { - throw ForyError.invalidData("compatible list-to-array field cannot read nullable elements") - } return try readListPayloadAsArray( context, refMode: refMode, diff --git a/swift/Sources/Fory/TypeMeta.swift b/swift/Sources/Fory/TypeMeta.swift index d20be245e6..cfa868b3a5 100644 --- a/swift/Sources/Fory/TypeMeta.swift +++ b/swift/Sources/Fory/TypeMeta.swift @@ -721,12 +721,6 @@ public final class TypeMeta: Equatable, @unchecked Sendable { if isCompatibleTopLevelListArrayFieldType(remoteType, localType) { return } - if remoteType.typeID == TypeId.list.rawValue, - TypeId(rawValue: localType.typeID)?.denseArrayElementTypeID != nil, - let elementType = remoteType.generics.first, - elementType.nullable || elementType.trackRef { - throw ForyError.invalidData("compatible list-to-array field cannot read nullable elements") - } throw ForyError.invalidData("unsupported compatible list/array schema mismatch") } } @@ -745,15 +739,13 @@ public final class TypeMeta: Equatable, @unchecked Sendable { if remoteType.typeID == TypeId.list.rawValue { return listFieldType( remoteType, - matchesDenseArrayTypeID: localType.typeID, - requireNonNullableElement: true + matchesDenseArrayTypeID: localType.typeID ) } if localType.typeID == TypeId.list.rawValue { return listFieldType( localType, - matchesDenseArrayTypeID: remoteType.typeID, - requireNonNullableElement: false + matchesDenseArrayTypeID: remoteType.typeID ) } return false @@ -761,17 +753,13 @@ public final class TypeMeta: Equatable, @unchecked Sendable { private static func listFieldType( _ listType: FieldType, - matchesDenseArrayTypeID arrayTypeID: UInt32, - requireNonNullableElement: Bool + matchesDenseArrayTypeID arrayTypeID: UInt32 ) -> Bool { guard listType.typeID == TypeId.list.rawValue, let elementType = listType.generics.first else { return false } - if requireNonNullableElement, (elementType.nullable || elementType.trackRef) { - return false - } return TypeId.listElementTypeID(elementType.typeID, matchesDenseArrayTypeID: arrayTypeID) } diff --git a/swift/Tests/ForyTests/CompatibilityTests.swift b/swift/Tests/ForyTests/CompatibilityTests.swift index 1850d67863..28bb314078 100644 --- a/swift/Tests/ForyTests/CompatibilityTests.swift +++ b/swift/Tests/ForyTests/CompatibilityTests.swift @@ -442,7 +442,7 @@ func compatibleReadAdaptsArrayFieldToDefaultVarintListField() throws { } @Test -func compatibleReadRejectsNullableListElementsForArrayField() throws { +func compatibleReadAllowsNullableListSchemaWithoutNullElementsForArrayField() throws { let writer = Fory(config: .init(xlang: true, trackRef: false, compatible: true)) writer.register(CompatibleNullableListFieldV1.self, id: 9923) @@ -450,9 +450,8 @@ func compatibleReadRejectsNullableListElementsForArrayField() throws { reader.register(CompatibleArrayFieldV2.self, id: 9923) let bytes = try writer.serialize(CompatibleNullableListFieldV1(values: [1, 2, 3], extra: 9)) - #expect(throws: ForyError.invalidData("compatible list-to-array field cannot read nullable elements")) { - let _: CompatibleArrayFieldV2 = try reader.deserialize(bytes) - } + let decoded: CompatibleArrayFieldV2 = try reader.deserialize(bytes) + #expect(decoded.values == [1, 2, 3]) let nullableBytes = try writer.serialize(CompatibleNullableListFieldV1(values: [1, nil, 3], extra: 9)) #expect(throws: ForyError.invalidData("compatible list-to-array field cannot read nullable elements")) { diff --git a/swift/Tests/ForyXlangTests/main.swift b/swift/Tests/ForyXlangTests/main.swift index f96fd9c98a..101eb777df 100644 --- a/swift/Tests/ForyXlangTests/main.swift +++ b/swift/Tests/ForyXlangTests/main.swift @@ -1116,13 +1116,11 @@ private func handleListArrayCompatibleNullableListToArrayError(_ bytes: [UInt8]) let fory = Fory(config: .init(xlang: true, trackRef: false, compatible: true)) fory.register(CompatibleInt32ArrayField.self, id: 901) do { - let _: CompatibleInt32ArrayField = try fory.deserialize(Data(bytes)) + let value: CompatibleInt32ArrayField = try fory.deserialize(Data(bytes)) + return try Array(fory.serialize(value)) } catch { return bytes } - throw PeerError.invalidFieldValue( - "Expected nullable list payload to fail compatible array read" - ) } private func rewritePayload(caseName: String, bytes: [UInt8]) throws -> [UInt8] { From 078c6fc40998bffb4e748ed219036e7b6670499d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 10:12:03 +0800 Subject: [PATCH 45/58] test(xlang): restore non-java peers to main --- .../serialization/struct_compatible_test.cc | 3 +- cpp/fory/serialization/type_resolver.cc | 6 + cpp/fory/serialization/type_resolver.h | 16 ++ cpp/fory/serialization/xlang_test_main.cc | 5 +- csharp/src/Fory/TypeMeta.cs | 33 +--- csharp/tests/Fory.Tests/ForyRuntimeTests.cs | 6 +- csharp/tests/Fory.XlangPeer/Program.cs | 4 +- .../serializer/collection_serializers.dart | 20 +-- .../lib/src/serializer/struct_serializer.dart | 35 +--- ...calar_and_typed_array_serializer_test.dart | 14 +- go/fory/field_spec.go | 43 +---- go/fory/struct_init.go | 4 +- go/fory/tag_test.go | 21 --- go/fory/tests/xlang/xlang_test_main.go | 7 +- .../org/apache/fory/xlang/XlangTestBase.java | 8 - javascript/packages/core/lib/context.ts | 157 +++++------------- javascript/packages/core/lib/gen/struct.ts | 8 +- javascript/packages/core/lib/typeInfo.ts | 1 - javascript/test/crossLanguage.test.ts | 8 +- javascript/test/typemeta.test.ts | 99 ++++------- python/pyfory/meta/typedef.py | 6 +- python/pyfory/tests/test_typedef_encoding.py | 12 -- python/pyfory/tests/xlang_test_main.py | 7 +- rust/fory-core/src/meta/type_meta.rs | 32 ++++ rust/fory-core/src/serializer/collection.rs | 10 +- rust/tests/tests/compatible/test_struct.rs | 20 +-- rust/tests/tests/test_cross_language.rs | 12 +- swift/Sources/Fory/TypeMeta.swift | 54 +----- swift/Sources/Fory/TypeResolver.swift | 7 +- .../Tests/ForyTests/CompatibilityTests.swift | 2 +- swift/Tests/ForyXlangTests/main.swift | 6 +- 31 files changed, 192 insertions(+), 474 deletions(-) diff --git a/cpp/fory/serialization/struct_compatible_test.cc b/cpp/fory/serialization/struct_compatible_test.cc index b05d6006e5..b688ba69f8 100644 --- a/cpp/fory/serialization/struct_compatible_test.cc +++ b/cpp/fory/serialization/struct_compatible_test.cc @@ -491,8 +491,7 @@ TEST(SchemaEvolutionTest, ImmediateArrayFieldCanReadIntoListCarrier) { EXPECT_EQ(decoded.value().values, (std::vector{4, 5, 6})); } -TEST(SchemaEvolutionTest, - NullableListSchemaWithoutNullElementsCanReadIntoArrayCarrier) { +TEST(SchemaEvolutionTest, NullableListElementsCannotReadIntoArrayCarrier) { auto writer = Fory::builder().compatible(true).xlang(true).build(); auto reader = Fory::builder().compatible(true).xlang(true).build(); diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index e84a986574..65e9220601 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1361,6 +1361,12 @@ int32_t TypeMeta::compute_struct_version(const TypeMeta &meta) { // Use the low 64 bits and then keep low 32 bits as i32. uint64_t low = static_cast(hash_out[0]); uint32_t version = static_cast(low & 0xFFFF'FFFFu); +#if defined(FORY_DEBUG) || defined(ENABLE_FORY_DEBUG_OUTPUT) + // DEBUG: Print fingerprint for debugging version mismatch + std::cerr << "[xlang][debug] struct_version type_name=" << meta.type_name + << ", fingerprint=\"" << fingerprint + << "\" version=" << static_cast(version) << std::endl; +#endif return static_cast(version); } diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index a244ed5b22..48f73445cf 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -1323,6 +1323,14 @@ template struct FieldInfoBuilder { field_type.nullable = is_nullable; field_type.track_ref = track_ref; field_type.ref_mode = make_ref_mode(is_nullable, track_ref); +#ifdef FORY_DEBUG + // DEBUG: Print field info for debugging fingerprint mismatch + std::cerr << "[xlang][debug] FieldInfoBuilder T=" << typeid(T).name() + << " Index=" << Index << " field=" << field_name + << " type_id=" << field_type.type_id + << " is_nullable=" << is_nullable << " track_ref=" << track_ref + << std::endl; +#endif FieldInfo info(std::move(field_name), std::move(field_type)); info.field_id = field_id; return info; @@ -1363,6 +1371,14 @@ template struct FieldInfoBuilder { field_type.nullable = is_nullable; field_type.track_ref = track_ref; field_type.ref_mode = make_ref_mode(is_nullable, track_ref); +#ifdef FORY_DEBUG + // DEBUG: Print field info for debugging fingerprint mismatch + std::cerr << "[xlang][debug] FieldInfoBuilder T=" << typeid(T).name() + << " Index=" << Index << " field=" << field_name + << " type_id=" << field_type.type_id + << " is_nullable=" << is_nullable << " track_ref=" << track_ref + << std::endl; +#endif FieldInfo info(std::move(field_name), std::move(field_type)); info.field_id = field_id; return info; diff --git a/cpp/fory/serialization/xlang_test_main.cc b/cpp/fory/serialization/xlang_test_main.cc index 370c7f6d3b..93d78f1760 100644 --- a/cpp/fory/serialization/xlang_test_main.cc +++ b/cpp/fory/serialization/xlang_test_main.cc @@ -2348,10 +2348,7 @@ void run_test_list_array_compatible_nullable_list_to_array_error( Buffer buffer = make_buffer(bytes); auto result = fory.deserialize(buffer); if (result.ok()) { - std::vector out; - append_serialized(fory, result.value(), out); - write_file(data_file, out); - return; + fail("Expected nullable list payload to fail compatible array read"); } write_file(data_file, bytes); } diff --git a/csharp/src/Fory/TypeMeta.cs b/csharp/src/Fory/TypeMeta.cs index dba55d1174..216d34dfcd 100644 --- a/csharp/src/Fory/TypeMeta.cs +++ b/csharp/src/Fory/TypeMeta.cs @@ -782,13 +782,11 @@ public static void AssignFieldIds( } } - if (localIndex >= 0 && localMatch is not null) + if (localIndex >= 0 && + localMatch is not null && + IsCompatibleFieldType(remoteField.FieldType, localMatch.FieldType, topLevel: true)) { - ThrowIfUnsupportedListArrayMismatch(remoteField.FieldType, localMatch.FieldType, topLevel: true); - remoteField.AssignedFieldId = - IsCompatibleFieldType(remoteField.FieldType, localMatch.FieldType, topLevel: true) - ? localIndex - : -1; + remoteField.AssignedFieldId = localIndex; } else { @@ -797,21 +795,6 @@ public static void AssignFieldIds( } } - private static void ThrowIfUnsupportedListArrayMismatch( - TypeMetaFieldType remote, - TypeMetaFieldType local, - bool topLevel) - { - if (topLevel && IsListArrayShapePair(remote, local)) - { - if (IsCompatibleListArrayFieldPair(remote, local)) - { - return; - } - throw new InvalidDataException("unsupported compatible list/array schema mismatch"); - } - } - private static bool IsCompatibleFieldType(TypeMetaFieldType remote, TypeMetaFieldType local, bool topLevel) { if (topLevel && IsCompatibleListArrayFieldPair(remote, local)) @@ -861,14 +844,6 @@ private static bool IsCompatibleListArrayFieldPair(TypeMetaFieldType remote, Typ CompatibleScalarTypeId(local.Generics[0].TypeId); } - private static bool IsListArrayShapePair(TypeMetaFieldType remote, TypeMetaFieldType local) - { - return remote.TypeId == (uint)global::Apache.Fory.TypeId.List && - TryPackedArrayElementTypeId(local.TypeId).HasValue || - local.TypeId == (uint)global::Apache.Fory.TypeId.List && - TryPackedArrayElementTypeId(remote.TypeId).HasValue; - } - private static uint? TryPackedArrayElementTypeId(uint typeId) { return typeId switch diff --git a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs index 0057e1f32a..511713cf37 100644 --- a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs +++ b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs @@ -1072,7 +1072,7 @@ public void CompatibleReadSupportsUInt32ListArrayFieldPairs() } [Fact] - public void CompatibleReadAllowsNullableListSchemaWithoutNullElementsIntoArrayCarrier() + public void CompatibleReadRejectsNullableListElementsIntoArrayCarrier() { ForyRuntime writer = ForyRuntime.Builder().Compatible(true).Build(); writer.Register(308); @@ -1080,8 +1080,8 @@ public void CompatibleReadAllowsNullableListSchemaWithoutNullElementsIntoArrayCa reader.Register(308); byte[] nonNullPayload = writer.Serialize(new CompatibleNullableListSchema { Values = [1, 2] }); - CompatibleArraySchema nonNullDecoded = reader.Deserialize(nonNullPayload); - Assert.Equal([1, 2], nonNullDecoded.Values); + CompatibleArraySchema decoded = reader.Deserialize(nonNullPayload); + Assert.Equal([1, 2], decoded.Values); byte[] payload = writer.Serialize(new CompatibleNullableListSchema { Values = [1, null] }); InvalidDataException exception = diff --git a/csharp/tests/Fory.XlangPeer/Program.cs b/csharp/tests/Fory.XlangPeer/Program.cs index f5110f4f36..2fad7dfbf3 100644 --- a/csharp/tests/Fory.XlangPeer/Program.cs +++ b/csharp/tests/Fory.XlangPeer/Program.cs @@ -844,13 +844,13 @@ private static byte[] CaseListArrayCompatibleNullableListToArrayError(byte[] inp ReadOnlySequence sequence = new(input); try { - CompatibleInt32ArrayField value = fory.Deserialize(ref sequence); - return fory.Serialize(value); + _ = fory.Deserialize(ref sequence); } catch (Apache.Fory.InvalidDataException) { return input; } + throw new InvalidOperationException("Expected nullable list payload to fail compatible array read"); } private static byte[] CaseOneEnumFieldSchema(byte[] input) diff --git a/dart/packages/fory/lib/src/serializer/collection_serializers.dart b/dart/packages/fory/lib/src/serializer/collection_serializers.dart index 0e757bd916..125292cc7f 100644 --- a/dart/packages/fory/lib/src/serializer/collection_serializers.dart +++ b/dart/packages/fory/lib/src/serializer/collection_serializers.dart @@ -466,19 +466,11 @@ bool isCompatibleCollectionArrayTypePair( ) { if (isCompatibleArrayType(localType.typeId) && remoteType.typeId == TypeIds.list) { - return _listElementMatchesArray( - remoteType, - localType.typeId, - requireNonNullableElement: true, - ); + return _listElementMatchesArray(remoteType, localType.typeId); } if (localType.typeId == TypeIds.list && isCompatibleArrayType(remoteType.typeId)) { - return _listElementMatchesArray( - localType, - remoteType.typeId, - requireNonNullableElement: false, - ); + return _listElementMatchesArray(localType, remoteType.typeId); } return false; } @@ -493,16 +485,10 @@ bool isCompatibleCollectionArrayRootTypePair( (isCompatibleArrayType(localTypeId) && remoteTypeId == TypeIds.list); } -bool _listElementMatchesArray( - FieldType listType, - int arrayTypeId, { - required bool requireNonNullableElement, -}) { +bool _listElementMatchesArray(FieldType listType, int arrayTypeId) { final elementType = listType.arguments.isEmpty ? null : listType.arguments.single; return elementType != null && - (!requireNonNullableElement || - (!elementType.nullable && !elementType.ref)) && _arrayElementTypeId(arrayTypeId) == _compatibleArrayElementTypeId(elementType.typeId); } diff --git a/dart/packages/fory/lib/src/serializer/struct_serializer.dart b/dart/packages/fory/lib/src/serializer/struct_serializer.dart index 9b42c2d06c..0f62f3416d 100644 --- a/dart/packages/fory/lib/src/serializer/struct_serializer.dart +++ b/dart/packages/fory/lib/src/serializer/struct_serializer.dart @@ -186,14 +186,6 @@ final class StructSerializer extends Serializer { 'Compatible field ${localField.name} has unsupported list/array schema mismatch.', ); } - if (_hasNestedListArrayMismatch( - localField.field.fieldType, - remoteField.fieldType, - )) { - fields.add(null); - topLevelListArrayPairs?.add(false); - continue; - } if (topLevelListArrayPair) { topLevelListArrayPairs ??= List.filled(fields.length, false, growable: true); @@ -224,37 +216,14 @@ bool _topLevelListArrayPair(FieldInfo localField, FieldInfo remoteField) { return isCompatibleCollectionArrayFieldPair(localField, remoteField); } -bool _hasNestedListArrayMismatch(FieldType localType, FieldType remoteType) { - if (localType.typeId != remoteType.typeId || - localType.arguments.length != remoteType.arguments.length) { - return false; - } - for (var index = 0; index < localType.arguments.length; index += 1) { - if (_hasListArrayMismatch( - localType.arguments[index], - remoteType.arguments[index], - )) { - return true; - } - } - return false; -} - -bool _hasListArrayMismatch(FieldType localType, FieldType remoteType) { - if (isCompatibleCollectionArrayRootTypePair(localType, remoteType)) { - return true; - } - return _hasNestedListArrayMismatch(localType, remoteType); -} - bool _hasUnsupportedListArrayMismatch( FieldType localType, FieldType remoteType, { required bool topLevel, }) { if (isCompatibleCollectionArrayRootTypePair(localType, remoteType)) { - return topLevel && - !isCompatibleCollectionArrayTypePair(localType, remoteType); + return !(topLevel && + isCompatibleCollectionArrayTypePair(localType, remoteType)); } if (localType.typeId != remoteType.typeId || localType.arguments.length != remoteType.arguments.length) { diff --git a/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart b/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart index b755f6e252..990de58709 100644 --- a/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart +++ b/dart/packages/fory/test/scalar_and_typed_array_serializer_test.dart @@ -640,10 +640,8 @@ void main() { final nonNullBytes = writer.serialize( CompatibleNullableListEnvelope()..values = [1, 2, 3], ); - expect( - () => reader.deserialize(nonNullBytes), - throwsStateError, - ); + final decoded = reader.deserialize(nonNullBytes); + expect(decoded.values, orderedEquals([1, 2, 3])); final nullableBytes = writer.serialize( CompatibleNullableListEnvelope()..values = [1, null, 3], @@ -681,7 +679,7 @@ void main() { ); }); - test('skips nested compatible list and dense array field positions', () { + test('rejects nested compatible list and dense array field positions', () { final writer = Fory(); final reader = Fory(); ScalarAndTypedArraySerializerTestFory.register( @@ -704,8 +702,10 @@ void main() { ], ); - final decoded = reader.deserialize(bytes); - expect(decoded.values, isEmpty); + expect( + () => reader.deserialize(bytes), + throwsStateError, + ); }); test('enforces maxBinarySize on write and read', () { diff --git a/go/fory/field_spec.go b/go/fory/field_spec.go index ee19687d1e..68f15a69b7 100644 --- a/go/fory/field_spec.go +++ b/go/fory/field_spec.go @@ -145,10 +145,7 @@ func (t *TypeSpec) Clone() *TypeSpec { } func (t *TypeSpec) declaredNullable() bool { - if t == nil { - return false - } - if t.hasDeclNull { + if t != nil && t.hasDeclNull { return t.declNullable } return true @@ -161,41 +158,6 @@ func (t *TypeSpec) declaredTrackRef() bool { return false } -func isNonNullableScalarElementType(typeID TypeId) bool { - switch typeID { - case BOOL, - INT8, - INT16, - INT32, - VARINT32, - INT64, - VARINT64, - TAGGED_INT64, - UINT8, - UINT16, - UINT32, - VAR_UINT32, - UINT64, - VAR_UINT64, - TAGGED_UINT64, - FLOAT8, - FLOAT16, - BFLOAT16, - FLOAT32, - FLOAT64: - return true - default: - return false - } -} - -// Primitive list carriers use LIST payloads but still declare scalar elements -// as non-null. Map key/value primitive schemas come from boxed Java types and -// keep the default nullable nested TypeSpec unless a tag overrides it. -func projectsAsNonNullableListElement(t *TypeSpec) bool { - return t != nil && !t.hasDeclNull && !t.Nullable && isNonNullableScalarElementType(t.TypeID) -} - func (t *TypeSpec) typeDefProjection(preserveRootFlags bool) *TypeSpec { return t.typeDefProjectionWithMode(true, preserveRootFlags) } @@ -225,9 +187,6 @@ func (t *TypeSpec) typeDefProjectionWithMode(isRoot bool, preserveRootFlags bool } if t.Element != nil { projected.Element = t.Element.typeDefProjectionWithMode(false, preserveRootFlags) - if t.TypeID == LIST && projectsAsNonNullableListElement(t.Element) { - projected.Element.Nullable = false - } projected.elementType = projected.Element } if t.Key != nil { diff --git a/go/fory/struct_init.go b/go/fory/struct_init.go index 87f814c3e4..b64ae7d6fe 100644 --- a/go/fory/struct_init.go +++ b/go/fory/struct_init.go @@ -533,7 +533,7 @@ func (s *structSerializer) initFieldsFromTypeDef(typeResolver *TypeResolver) err ) { shouldRead = true fieldType = localType - } else if defTypeId == LIST && localFieldSpec != nil && compatibleListFieldHasPrimitiveArrayShape(def.typeSpec, localFieldSpec.Type, localType) { + } else if defTypeId == LIST && localFieldSpec != nil && compatibleListFieldCanReadLocalArray(def.typeSpec, localFieldSpec.Type, localType) { shouldRead = true usesCompatibleCollectionArrayReader = true fieldType = localType @@ -810,7 +810,7 @@ func listFieldCanReadLocalArray(remoteSpec *TypeSpec, remoteNullable bool, remot return fieldSpecEqualForDiff(remoteSpec, remoteNullable, remoteTrackRef, localSpec, localNullable, localTrackRef) } -func compatibleListFieldHasPrimitiveArrayShape(remoteSpec *TypeSpec, localSpec *TypeSpec, localType reflect.Type) bool { +func compatibleListFieldCanReadLocalArray(remoteSpec *TypeSpec, localSpec *TypeSpec, localType reflect.Type) bool { if remoteSpec == nil || localSpec == nil || localType == nil { return false } diff --git a/go/fory/tag_test.go b/go/fory/tag_test.go index a6f92cdcb5..9d50631f1a 100644 --- a/go/fory/tag_test.go +++ b/go/fory/tag_test.go @@ -122,8 +122,6 @@ func TestFieldSpecSerializerSelection(t *testing.T) { U8List []uint8 `fory:"id=4"` U8Dense []uint8 `fory:"id=5,type=array(element=uint8)"` Bytes []byte `fory:"id=6,type=bytes"` - Strings []string `fory:"id=7"` - StringByID map[int32]string } f := New(WithXlang(true), WithCompatible(false)) @@ -137,8 +135,6 @@ func TestFieldSpecSerializerSelection(t *testing.T) { require.EqualValues(t, VARINT32, defaultListPrimitive.elemTypeID) require.EqualValues(t, LIST, defaultListSpec.Type.TypeId()) require.EqualValues(t, VARINT32, defaultListSpec.Type.Element.TypeId()) - require.False(t, defaultListSpec.Type.Element.Nullable) - require.False(t, defaultListSpec.Type.typeDefProjection(false).Element.Nullable) denseSpec := mustParseFieldSpec(t, typ.Field(1)) denseSerializer, err := serializerForTypeSpec(f.typeResolver, typ.Field(1).Type, denseSpec.Type) @@ -153,16 +149,12 @@ func TestFieldSpecSerializerSelection(t *testing.T) { require.True(t, ok) require.EqualValues(t, INT32, explicitPrimitive.elemTypeID) require.EqualValues(t, LIST, explicitSpec.Type.TypeId()) - require.False(t, explicitSpec.Type.Element.Nullable) - require.False(t, explicitSpec.Type.typeDefProjection(false).Element.Nullable) ptrsSpec := mustParseFieldSpec(t, typ.Field(3)) ptrsSerializer, err := serializerForTypeSpec(f.typeResolver, typ.Field(3).Type, ptrsSpec.Type) require.NoError(t, err) require.IsType(t, &sliceSerializer{}, ptrsSerializer) require.EqualValues(t, LIST, ptrsSpec.Type.TypeId()) - require.True(t, ptrsSpec.Type.Element.Nullable) - require.True(t, ptrsSpec.Type.typeDefProjection(false).Element.Nullable) u8ListSpec := mustParseFieldSpec(t, typ.Field(4)) u8ListSerializer, err := serializerForTypeSpec(f.typeResolver, typ.Field(4).Type, u8ListSpec.Type) @@ -172,8 +164,6 @@ func TestFieldSpecSerializerSelection(t *testing.T) { require.EqualValues(t, UINT8, u8Primitive.elemTypeID) require.EqualValues(t, LIST, u8ListSpec.Type.TypeId()) require.EqualValues(t, UINT8, u8ListSpec.Type.Element.TypeId()) - require.False(t, u8ListSpec.Type.Element.Nullable) - require.False(t, u8ListSpec.Type.typeDefProjection(false).Element.Nullable) u8DenseSpec := mustParseFieldSpec(t, typ.Field(5)) u8DenseSerializer, err := serializerForTypeSpec(f.typeResolver, typ.Field(5).Type, u8DenseSpec.Type) @@ -186,17 +176,6 @@ func TestFieldSpecSerializerSelection(t *testing.T) { require.NoError(t, err) require.IsType(t, encodedByteSliceSerializer{}, bytesSerializer) require.EqualValues(t, BINARY, bytesSpec.Type.TypeId()) - - stringsSpec := mustParseFieldSpec(t, typ.Field(7)) - require.EqualValues(t, LIST, stringsSpec.Type.TypeId()) - require.False(t, stringsSpec.Type.Element.Nullable) - require.True(t, stringsSpec.Type.typeDefProjection(false).Element.Nullable) - - mapSpec := mustParseFieldSpec(t, typ.Field(8)) - mapProjection := mapSpec.Type.typeDefProjection(false) - require.EqualValues(t, MAP, mapProjection.TypeId()) - require.True(t, mapProjection.Key.Nullable) - require.True(t, mapProjection.Value.Nullable) } func TestGroupFieldsUsesFlatOrderForFullyTaggedStructs(t *testing.T) { diff --git a/go/fory/tests/xlang/xlang_test_main.go b/go/fory/tests/xlang/xlang_test_main.go index e7978e96c4..5dfba1f1a3 100644 --- a/go/fory/tests/xlang/xlang_test_main.go +++ b/go/fory/tests/xlang/xlang_test_main.go @@ -1657,12 +1657,7 @@ func testListArrayCompatibleNullableListToArrayError() { var result CompatibleInt32ArrayField if err := f.Deserialize(data, &result); err == nil { - serialized, err := f.Serialize(&result) - if err != nil { - panic(fmt.Sprintf("Failed to serialize compatible array field: %v", err)) - } - writeFile(dataFile, serialized) - return + panic("Expected nullable list payload to fail compatible array read") } writeFile(dataFile, data) diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index 43a4fcc477..e79754dfec 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -1884,14 +1884,6 @@ protected void testListArrayCompatibleRead(boolean enableCodegen) throws java.io (XlangCompatibleInt32ArrayField) arrayFory.deserialize(MemoryUtils.wrap(nullableListWithoutNullsPayload)); Assert.assertEquals(nullableListWithoutNullsArray.values, new int[] {1, 2, 3}); - ctx = - prepareExecution( - "test_list_array_compatible_nullable_list_to_array_error", - nullableListWithoutNullsPayload); - runPeer(ctx); - nullableListWithoutNullsArray = - (XlangCompatibleInt32ArrayField) arrayFory.deserialize(readBuffer(ctx.dataFile())); - Assert.assertEquals(nullableListWithoutNullsArray.values, new int[] {1, 2, 3}); XlangCompatibleNullableInt32ListField nullableListValue = newCompatibleNullableInt32ListField(1, null, 3); diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index e7a19251be..0f59afe694 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -47,9 +47,7 @@ type RegeneratedReadSerializerCacheEntry = { serializers: Map; }; -function remoteListElementType( - fieldInfo: InnerFieldInfo, -): InnerFieldInfo | undefined { +function remoteListElementType(fieldInfo: InnerFieldInfo): InnerFieldInfo | undefined { if (fieldInfo.typeId !== TypeId.LIST) { return undefined; } @@ -369,8 +367,8 @@ export class WriteContext { checkCollectionSize(size: number) { if (size > this._maxCollectionSize) { throw new Error( - `Collection size ${size} exceeds maxCollectionSize ${this._maxCollectionSize}. ` + - "The data may be malicious, or increase maxCollectionSize if needed.", + `Collection size ${size} exceeds maxCollectionSize ${this._maxCollectionSize}. ` + + "The data may be malicious, or increase maxCollectionSize if needed.", ); } } @@ -378,8 +376,8 @@ export class WriteContext { checkBinarySize(size: number) { if (size > this._maxBinarySize) { throw new Error( - `Binary size ${size} exceeds maxBinarySize ${this._maxBinarySize}. ` + - "The data may be malicious, or increase maxBinarySize if needed.", + `Binary size ${size} exceeds maxBinarySize ${this._maxBinarySize}. ` + + "The data may be malicious, or increase maxBinarySize if needed.", ); } } @@ -586,20 +584,13 @@ export class ReadContext { return headerHigh * 0x100000 + (headerLow >>> 12); } - private findRecentTypeMeta( - headerLow: number, - headerHigh: number, - ): TypeMeta | null { + private findRecentTypeMeta(headerLow: number, headerHigh: number): TypeMeta | null { const lows = this.recentTypeMetaHeaderLows; const highs = this.recentTypeMetaHeaderHighs; const metas = this.recentTypeMetas; for (let i = 0; i < metas.length; i++) { const typeMeta = metas[i]; - if ( - typeMeta !== null && - lows[i] === headerLow && - highs[i] === headerHigh - ) { + if (typeMeta !== null && lows[i] === headerLow && highs[i] === headerHigh) { return typeMeta; } } @@ -656,8 +647,8 @@ export class ReadContext { this._depth++; if (this._depth > this._maxDepth) { throw new Error( - `Deserialization depth limit exceeded: ${this._depth} > ${this._maxDepth}. ` + - "The data may be malicious, or increase maxDepth if needed.", + `Deserialization depth limit exceeded: ${this._depth} > ${this._maxDepth}. ` + + "The data may be malicious, or increase maxDepth if needed.", ); } } @@ -669,8 +660,8 @@ export class ReadContext { checkCollectionSize(size: number) { if (size > this._maxCollectionSize) { throw new Error( - `Collection size ${size} exceeds maxCollectionSize ${this._maxCollectionSize}. ` + - "The data may be malicious, or increase maxCollectionSize if needed.", + `Collection size ${size} exceeds maxCollectionSize ${this._maxCollectionSize}. ` + + "The data may be malicious, or increase maxCollectionSize if needed.", ); } } @@ -678,8 +669,8 @@ export class ReadContext { checkBinarySize(size: number) { if (size > this._maxBinarySize) { throw new Error( - `Binary size ${size} exceeds maxBinarySize ${this._maxBinarySize}. ` + - "The data may be malicious, or increase maxBinarySize if needed.", + `Binary size ${size} exceeds maxBinarySize ${this._maxBinarySize}. ` + + "The data may be malicious, or increase maxBinarySize if needed.", ); } } @@ -702,9 +693,9 @@ export class ReadContext { headerHigh: number, ): TypeMeta { if ( - this.lastTypeMeta !== null && - this.lastTypeMetaHeaderLow === headerLow && - this.lastTypeMetaHeaderHigh === headerHigh + this.lastTypeMeta !== null + && this.lastTypeMetaHeaderLow === headerLow + && this.lastTypeMetaHeaderHigh === headerHigh ) { TypeMeta.skipBodyByHeaderLow(this.reader, headerLow); this.typeMeta[dynamicTypeId] = this.lastTypeMeta; @@ -784,11 +775,7 @@ export class ReadContext { } else { const headerLow = this.reader.readUint32(); const headerHigh = this.reader.readUint32(); - typeMeta = this.readTypeMetaFromHeader( - idOrLen >> 1, - headerLow, - headerHigh, - ); + typeMeta = this.readTypeMetaFromHeader(idOrLen >> 1, headerLow, headerHigh); remoteHash = ReadContext.typeMetaHeaderHash(headerLow, headerHigh); } if (expectedHash !== remoteHash) { @@ -803,21 +790,12 @@ export class ReadContext { topLevel = true, ): TypeInfo { if (topLevel && fallbackTypeInfo) { - const compatible = this.compatibleFieldTypeInfo( - fieldInfo, - fallbackTypeInfo, - ); + const compatible = this.compatibleFieldTypeInfo(fieldInfo, fallbackTypeInfo); if (compatible) { return compatible; } } - if ( - this.hasUnsupportedListArrayMismatch( - fieldInfo, - fallbackTypeInfo, - topLevel, - ) - ) { + if (this.hasUnsupportedListArrayMismatch(fieldInfo, fallbackTypeInfo, topLevel)) { throw new Error("unsupported compatible list/array schema mismatch"); } switch (fieldInfo.typeId) { @@ -898,11 +876,10 @@ export class ReadContext { } const remoteArrayElement = denseArrayElementTypeId(remote.typeId); if ( - remoteArrayElement !== undefined && - local.typeId === TypeId.LIST && - local.options?.inner && - compatibleArrayElementTypeId(local.options.inner.typeId) === - remoteArrayElement + remoteArrayElement !== undefined + && local.typeId === TypeId.LIST + && local.options?.inner + && compatibleArrayElementTypeId(local.options.inner.typeId) === remoteArrayElement ) { return compatibleArrayToListTypeInfo(remoteArrayElement); } @@ -918,7 +895,7 @@ export class ReadContext { return false; } if (this.isListArrayRootPair(remote, local)) { - return topLevel && !this.compatibleFieldTypeInfo(remote, local); + return !(topLevel && this.compatibleFieldTypeInfo(remote, local)); } if (remote.typeId !== local.typeId) { return false; @@ -930,8 +907,8 @@ export class ReadContext { remote.options!.key!, local.options?.key, false, - ) || - this.hasUnsupportedListArrayMismatch( + ) + || this.hasUnsupportedListArrayMismatch( remote.options!.value!, local.options?.value, false, @@ -954,59 +931,10 @@ export class ReadContext { } } - private hasNestedListArrayMismatch( - remote: InnerFieldInfo, - local: TypeInfo | undefined, - ): boolean { - if (!local || remote.typeId !== local.typeId) { - return false; - } - switch (remote.typeId) { - case TypeId.MAP: - return ( - this.hasListArrayMismatch(remote.options!.key!, local.options?.key) || - this.hasListArrayMismatch( - remote.options!.value!, - local.options?.value, - ) - ); - case TypeId.LIST: - return this.hasListArrayMismatch( - remote.options!.inner!, - local.options?.inner, - ); - case TypeId.SET: - return this.hasListArrayMismatch( - remote.options!.key!, - local.options?.key, - ); - default: - return false; - } - } - - private hasListArrayMismatch( - remote: InnerFieldInfo, - local: TypeInfo | undefined, - ): boolean { - if (!local) { - return false; - } - if (this.isListArrayRootPair(remote, local)) { - return true; - } - return this.hasNestedListArrayMismatch(remote, local); - } - - private isListArrayRootPair( - remote: InnerFieldInfo, - local: TypeInfo, - ): boolean { + private isListArrayRootPair(remote: InnerFieldInfo, local: TypeInfo): boolean { return ( - (remote.typeId === TypeId.LIST && - denseArrayElementTypeId(local.typeId) !== undefined) || - (denseArrayElementTypeId(remote.typeId) !== undefined && - local.typeId === TypeId.LIST) + (remote.typeId === TypeId.LIST && denseArrayElementTypeId(local.typeId) !== undefined) + || (denseArrayElementTypeId(remote.typeId) !== undefined && local.typeId === TypeId.LIST) ); } @@ -1017,9 +945,9 @@ export class ReadContext { const localHash = original.getHash(); let entry = this.regeneratedReadSerializers.get(original); if ( - entry === undefined || - entry.localTypeInfo !== localTypeInfo || - entry.localHash !== localHash + entry === undefined + || entry.localTypeInfo !== localTypeInfo + || entry.localHash !== localHash ) { entry = { localHash, @@ -1047,10 +975,9 @@ export class ReadContext { ); } } - const cacheEntry = - original === undefined - ? undefined - : this.getRegeneratedReadSerializerCache(original); + const cacheEntry = original === undefined + ? undefined + : this.getRegeneratedReadSerializerCache(original); const remoteHash = typeMeta.getHash(); const cached = cacheEntry?.serializers.get(remoteHash); if (cached !== undefined) { @@ -1071,20 +998,13 @@ export class ReadContext { const props = Object.fromEntries( typeMeta.remapFieldNames(localProps).map((fieldInfo) => { const localFieldTypeInfo = localProps?.[fieldInfo.getFieldName()]; - const skipRead = this.hasNestedListArrayMismatch( - fieldInfo, - localFieldTypeInfo, - ); const fieldTypeInfo = this.fieldInfoToTypeInfo( fieldInfo, - skipRead ? undefined : localFieldTypeInfo, + localFieldTypeInfo, ) .setNullable(fieldInfo.nullable) .setTrackingRef(fieldInfo.trackingRef) .setId(fieldInfo.fieldId); - if (skipRead) { - fieldTypeInfo.options = { ...fieldTypeInfo.options, skipRead: true }; - } return [fieldInfo.getFieldName(), fieldTypeInfo]; }), ); @@ -1096,9 +1016,8 @@ export class ReadContext { ? this.typeResolver.generateReadSerializer(typeInfo) : this.typeResolver.regenerateReadSerializer(typeInfo); if ( - cacheEntry !== undefined && - cacheEntry.serializers.size < - ReadContext.MAX_CACHED_REGENERATED_READ_SERIALIZER + cacheEntry !== undefined + && cacheEntry.serializers.size < ReadContext.MAX_CACHED_REGENERATED_READ_SERIALIZER ) { cacheEntry.serializers.set(remoteHash, serializer); } diff --git a/javascript/packages/core/lib/gen/struct.ts b/javascript/packages/core/lib/gen/struct.ts index 519d867cc2..175180da43 100644 --- a/javascript/packages/core/lib/gen/struct.ts +++ b/javascript/packages/core/lib/gen/struct.ts @@ -414,11 +414,8 @@ class StructSerializerGenerator extends BaseSerializerGenerator { throw new Error(`${typeInfo.typeId} generator not exists`); } const innerGenerator = new InnerGeneratorClass(typeInfo, this.builder, this.scope); - const assign = typeInfo.options?.skipRead - ? (expr: string) => `void (${expr})` - : (expr: string) => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`; return ` - ${this.readField(typeInfo, assign, innerGenerator.readEmbed())} + ${this.readField(typeInfo, expr => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`, innerGenerator.readEmbed())} `; }).join(";\n")} ${accessor(result)} @@ -438,9 +435,6 @@ class StructSerializerGenerator extends BaseSerializerGenerator { } const fields: Array<{ key: string; expr: string }> = []; for (const { key, typeInfo } of this.sortedProps) { - if (typeInfo.options?.skipRead) { - return null; - } const expr = directNumericFieldReadExpr(typeInfo, this.builder); if (expr === null) { return null; diff --git a/javascript/packages/core/lib/typeInfo.ts b/javascript/packages/core/lib/typeInfo.ts index ddcd761314..0dc4f97316 100644 --- a/javascript/packages/core/lib/typeInfo.ts +++ b/javascript/packages/core/lib/typeInfo.ts @@ -51,7 +51,6 @@ interface TypeInfoOptions { enumProps?: { [key: string]: number }; cases?: { [caseIndex: number]: TypeInfo }; scalarEncoding?: ScalarEncoding; - skipRead?: boolean; } /** diff --git a/javascript/test/crossLanguage.test.ts b/javascript/test/crossLanguage.test.ts index a0eea92390..e94b96ea3f 100644 --- a/javascript/test/crossLanguage.test.ts +++ b/javascript/test/crossLanguage.test.ts @@ -1427,12 +1427,8 @@ describe("bool", () => { }), ); - try { - const value = serializer.deserialize(content); - writeToFile(serializer.serialize(value) as Buffer); - } catch { - writeToFile(content); - } + expect(() => serializer.deserialize(content)).toThrow(); + writeToFile(content); }); test("test_one_enum_field_schema", () => { diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index 69b0a7410e..c9cc18ab44 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -129,7 +129,11 @@ describe("typemeta", () => { const bodyOnlyHash = bodyOnlyHeaderHashBits(bytes.subarray(bodyOffset)); expect(header & HEADER_HASH_MASK).not.toBe(bodyOnlyHash); - view.setBigUint64(0, bodyOnlyHash | (header & LOW_HEADER_BITS_MASK), true); + view.setBigUint64( + 0, + bodyOnlyHash | (header & LOW_HEADER_BITS_MASK), + true, + ); const reader = new BinaryReader({}); reader.reset(malformed); @@ -167,10 +171,9 @@ describe("typemeta", () => { } as any, config, ); - (context as any).typeMetaCache.set( - Number(header >> 32n), - new Map([[Number(header & 0xffffffffn), typeMeta]]), - ); + (context as any).typeMetaCache.set(Number(header >> 32n), new Map([ + [Number(header & 0xffffffffn), typeMeta], + ])); context.reset(writer.dump()); expect(context.readTypeMeta()).toBe(typeMeta); @@ -226,11 +229,9 @@ describe("typemeta", () => { value: Type.int32().setId(1), }); - const changedBytes = changedWriterFory - .register(changedWriterType) - .serialize({ - value: "hello", - }); + const changedBytes = changedWriterFory.register(changedWriterType).serialize({ + value: "hello", + }); const localBytes = localWriterFory.register(localWriterType).serialize({ value: 123, }); @@ -240,38 +241,12 @@ describe("typemeta", () => { expect(reader.deserialize(localBytes)).toEqual({ value: 123 }); }); - test("keeps type info on regenerated compatible named serializers", () => { - const stringWriterFory = new Fory({ compatible: true }); - const numberWriterFory = new Fory({ compatible: true }); - const readerFory = new Fory({ compatible: true }); - - const stringWriterType = Type.struct("example.dynamicNamed", { - value: Type.string().setId(1), - }); - const numberWriterType = Type.struct("example.dynamicNamed", { - value: Type.int32().setId(1), - }); - - const stringBytes = stringWriterFory - .register(stringWriterType) - .serialize({ value: "hello" }); - const numberBytes = numberWriterFory - .register(numberWriterType) - .serialize({ value: 123 }); - - expect(readerFory.deserialize(stringBytes)).toEqual({ $tag1: "hello" }); - expect(readerFory.deserialize(numberBytes)).toEqual({ $tag1: 123 }); - }); - test("regenerated read serializers keep getTypeInfo", () => { const fory = new Fory({ compatible: true }); const serializer = (fory as any).typeResolver.regenerateReadSerializer( - Type.struct( - { namespace: "example", typeName: "repro_struct" }, - { - value: Type.int32(), - }, - ), + Type.struct({ namespace: "example", typeName: "repro_struct" }, { + value: Type.int32(), + }), ); expect(typeof serializer.getTypeInfo).toBe("function"); @@ -293,10 +268,9 @@ describe("typemeta", () => { const localChildType = Type.struct(7311, { value: Type.int32().setId(1), }); - const createParentType = () => - Type.struct(7312, { - child: Type.struct(7311).setId(1), - }); + const createParentType = () => Type.struct(7312, { + child: Type.struct(7311).setId(1), + }); stringWriterFory.register(stringChildType); boolWriterFory.register(boolChildType); @@ -309,8 +283,8 @@ describe("typemeta", () => { const reader = readerFory.register(createParentType()); const typeResolver = (readerFory as any).typeResolver; - const generateReadSerializer = - typeResolver.generateReadSerializer.bind(typeResolver); + const generateReadSerializer + = typeResolver.generateReadSerializer.bind(typeResolver); let generatedReaders = 0; typeResolver.generateReadSerializer = (typeInfo: any) => { generatedReaders++; @@ -412,14 +386,8 @@ describe("typemeta", () => { expect(result.bools).toBeInstanceOf(BoolArray); expect(Array.from(result.bools)).toEqual([true, false]); expect(result.float16s).toBeInstanceOf(Float16Array as any); - expect(Array.from(result.float16s as Iterable)[0]).toBeCloseTo( - 1.5, - 1, - ); - expect(Array.from(result.float16s as Iterable)[1]).toBeCloseTo( - -2, - 1, - ); + expect(Array.from(result.float16s as Iterable)[0]).toBeCloseTo(1.5, 1); + expect(Array.from(result.float16s as Iterable)[1]).toBeCloseTo(-2, 1); expect(result.bfloat16s).toBeInstanceOf(BFloat16Array); expect(Array.from(result.bfloat16s as Iterable)).toEqual([1.5, -2]); }); @@ -444,7 +412,7 @@ describe("typemeta", () => { expect(result).toEqual({ values: [1, 2, 3] }); }); - test("adapts nullable list schema to dense array when payload has no null elements", () => { + test("rejects compatible list to dense array when payload has nullable elements", () => { const writerFory = new Fory({ compatible: true }); const readerFory = new Fory({ compatible: true }); @@ -461,16 +429,13 @@ describe("typemeta", () => { const nonNullBytes = serializer.serialize({ values: [1, 2, 3], }); - const reader = readerFory.register(readerType); - const nonNullResult = reader.deserialize(nonNullBytes); - expect(Array.from(nonNullResult.values)).toEqual([1, 2, 3]); + const result = readerFory.register(readerType).deserialize(nonNullBytes); + expect(Array.from(result.values as Int32Array)).toEqual([1, 2, 3]); const nullableBytes = serializer.serialize({ values: [1, null, 3], }); - expect(() => reader.deserialize(nullableBytes)).toThrow( - "compatible list-to-array field cannot read nullable or ref-tracked elements", - ); + expect(() => readerFory.register(readerType).deserialize(nullableBytes)).toThrow(); }); test("rejects incompatible immediate list and dense array element fields", () => { @@ -488,12 +453,10 @@ describe("typemeta", () => { values: ["1", "2"], }); - expect(() => readerFory.register(readerType).deserialize(bytes)).toThrow( - /list\/array/, - ); + expect(() => readerFory.register(readerType).deserialize(bytes)).toThrow(/list\/array/); }); - test("skips nested compatible list and dense array positions", () => { + test("rejects nested compatible list and dense array positions", () => { const writerFory = new Fory({ compatible: true }); const readerFory = new Fory({ compatible: true }); @@ -508,8 +471,7 @@ describe("typemeta", () => { values: [new Int32Array([1, 2])], }); - const result = readerFory.register(readerType).deserialize(bytes); - expect(result.values).toBeNull(); + expect(() => readerFory.register(readerType).deserialize(bytes)).toThrow(/list\/array/); }); test("keeps compatible named schema evolution working when field count differs", () => { @@ -730,7 +692,10 @@ function typeMetaBodyOffset(bytes: Uint8Array) { function bodyOnlyHeaderHashBits(buffer: Uint8Array) { const hash = x64hash128(buffer, 47); - let header = BigInt.asIntN(64, hash.getBigInt64(0, false) << HASH_SHIFT_BITS); + let header = BigInt.asIntN( + 64, + hash.getBigInt64(0, false) << HASH_SHIFT_BITS, + ); if (header < 0n) { header = -header; } diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py index 1ea5b624a4..cb3057962f 100644 --- a/python/pyfory/meta/typedef.py +++ b/python/pyfory/meta/typedef.py @@ -633,9 +633,11 @@ def __repr__(self): def _list_array_element_type_matches(list_field_type: FieldType, array_field_type: FieldType) -> bool: array_element_type_id = _ARRAY_ELEMENT_TYPE_IDS.get(array_field_type.type_id) - if list_field_type.type_id != TypeId.LIST or array_element_type_id is None: + if array_element_type_id is None: return False - return _list_element_type_matches_array_element(list_field_type.element_type.type_id, array_element_type_id) + return list_field_type.type_id == TypeId.LIST and _list_element_type_matches_array_element( + list_field_type.element_type.type_id, array_element_type_id + ) def _list_element_type_matches_array_element(list_element_type_id: TypeId, array_element_type_id: TypeId) -> bool: diff --git a/python/pyfory/tests/test_typedef_encoding.py b/python/pyfory/tests/test_typedef_encoding.py index b25d067cbf..e91a42893e 100644 --- a/python/pyfory/tests/test_typedef_encoding.py +++ b/python/pyfory/tests/test_typedef_encoding.py @@ -559,18 +559,6 @@ def test_compatible_int32_array_assigns_to_list(): assert decoded.payload == [1, 2, 3] -def test_compatible_int32_array_assigns_to_nullable_list(): - writer = Fory(xlang=True, compatible=True) - reader = Fory(xlang=True, compatible=True) - _register_int32_payload(writer, Int32ArrayPayload) - _register_int32_payload(reader, NullableInt32ListPayload) - - decoded = reader.deserialize(writer.serialize(Int32ArrayPayload(payload=pyfory.Int32Array([1, 2, 3])))) - - assert isinstance(decoded, NullableInt32ListPayload) - assert decoded.payload == [1, 2, 3] - - @pytest.mark.skipif(np is None, reason="Requires numpy") def test_compatible_int32_ndarray_assigns_to_list(): writer = Fory(xlang=True, compatible=True) diff --git a/python/pyfory/tests/xlang_test_main.py b/python/pyfory/tests/xlang_test_main.py index 1478934656..d9c189e3ec 100644 --- a/python/pyfory/tests/xlang_test_main.py +++ b/python/pyfory/tests/xlang_test_main.py @@ -956,11 +956,12 @@ def test_list_array_compatible_nullable_list_to_array_error(): fory = pyfory.Fory(xlang=True, compatible=True) fory.register_type(CompatibleInt32ArrayField, type_id=901) try: - value = fory.deserialize(data_bytes) + fory.deserialize(data_bytes) except TypeNotCompatibleError: - new_bytes = data_bytes + pass else: - new_bytes = fory.serialize(value) + raise AssertionError("Expected nullable list payload to fail compatible array read") + new_bytes = data_bytes with open(data_file, "wb") as f: f.write(new_bytes) diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs index 193b9abb84..02b96ba045 100644 --- a/rust/fory-core/src/meta/type_meta.rs +++ b/rust/fory-core/src/meta/type_meta.rs @@ -984,8 +984,28 @@ impl TypeMeta { } fn assign_field_ids(type_info_current: &TypeInfo, field_infos: &mut [FieldInfo]) { + if crate::util::ENABLE_FORY_DEBUG_OUTPUT { + eprintln!( + "[fory-debug] assign_field_ids called for type: {:?}", + type_info_current.get_type_name() + ); + for f in field_infos.iter() { + eprintln!( + "[fory-debug] remote field before assign: name={}, field_id={}, type={:?}", + f.field_name, f.field_id, f.field_type + ); + } + } let type_meta = type_info_current.get_type_meta(); let local_field_infos = type_meta.get_field_infos(); + if crate::util::ENABLE_FORY_DEBUG_OUTPUT { + for f in local_field_infos.iter() { + eprintln!( + "[fory-debug] local field: name={}, field_id={}, type={:?}", + f.field_name, f.field_id, f.field_type + ); + } + } // Build maps for both name-based and ID-based lookup. // The value is the SORTED INDEX (position in local_field_infos), not the field's ID attribute. @@ -1028,8 +1048,20 @@ impl TypeMeta { // codec inspects the remote FieldType and either consumes it or // asks the caller to skip the remote payload. field.field_id = sorted_index as i16; + if crate::util::ENABLE_FORY_DEBUG_OUTPUT { + eprintln!( + "[fory-debug] matched field: name={}, assigned_field_id={}, remote_type={:?}, local_type={:?}", + field.field_name, field.field_id, field.field_type, local_info.field_type + ); + } } None => { + if crate::util::ENABLE_FORY_DEBUG_OUTPUT { + eprintln!( + "[fory-debug] no local match for field: name={}", + field.field_name + ); + } field.field_id = -1; // No match, skip } } diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs index 9968ba9873..421a224d6a 100644 --- a/rust/fory-core/src/serializer/collection.rs +++ b/rust/fory-core/src/serializer/collection.rs @@ -528,7 +528,7 @@ fn read_primitive_array_data_bulk( } } -fn list_element_type_matches_array_shape(list: &FieldType, array: &FieldType) -> bool { +fn list_element_type_matches_array(list: &FieldType, array: &FieldType) -> bool { primitive_array_element_type_id(array.type_id).is_some_and(|element_type_id| { list.type_id == type_id::LIST && list.generics.len() == 1 @@ -786,12 +786,8 @@ where T: 'static, C: Codec, { - // A dense remote array can materialize into a nullable local list by wrapping every element as - // Some(_). The reverse direction is stricter and is checked in - // read_primitive_array_vec_compatible_mismatch because a remote list may carry null/ref markers - // that a dense local array cannot represent. if local_field_type.type_id == type_id::LIST - && list_element_type_matches_array_shape(local_field_type, remote_field_type) + && list_element_type_matches_array(local_field_type, remote_field_type) { return read_array_data_as_vec_bridge::(context, remote_field_type).map(Some); } @@ -833,7 +829,7 @@ where { if remote_field_type.type_id == type_id::LIST && !remote_field_type.generics.is_empty() - && list_element_type_matches_array_shape(remote_field_type, local_field_type) + && list_element_type_matches_array(remote_field_type, local_field_type) { if field_ref_mode(remote_field_type) != RefMode::None { let ref_flag = context.reader.read_i8()?; diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index 52361b3538..38a7f38560 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -125,20 +125,8 @@ fn compatible_list_array_field_pairs() { let mut writer = Fory::builder().compatible(true).build(); let mut reader = Fory::builder().compatible(true).build(); - writer.register::(993).unwrap(); - reader.register::(993).unwrap(); - let bytes = writer - .serialize(&ArrayPayload { - payload: vec![1, 2, 3], - }) - .unwrap(); - let decoded: NullableListPayload = reader.deserialize(&bytes).unwrap(); - assert_eq!(decoded.payload, vec![Some(1), Some(2), Some(3)]); - - let mut writer = Fory::builder().compatible(true).build(); - let mut reader = Fory::builder().compatible(true).build(); - writer.register::(994).unwrap(); - reader.register::(994).unwrap(); + writer.register::(993).unwrap(); + reader.register::(993).unwrap(); let bytes = writer .serialize(&NullableListPayload { payload: vec![Some(1), Some(2), Some(3)], @@ -163,8 +151,8 @@ fn compatible_list_array_field_pairs() { let mut writer = Fory::builder().compatible(true).build(); let mut reader = Fory::builder().compatible(true).build(); - writer.register::(995).unwrap(); - reader.register::(995).unwrap(); + writer.register::(994).unwrap(); + reader.register::(994).unwrap(); let bytes = writer .serialize(&NestedListPayload { payload: vec![vec![1, 2], vec![3]], diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 918d846426..fcf5f0d261 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -1456,11 +1456,13 @@ fn test_list_array_compatible_nullable_list_to_array_error() { let mut fory = Fory::builder().compatible(true).xlang(true).build(); fory.register::(901).unwrap(); - if let Ok(value) = fory.deserialize::(&bytes) { - let new_bytes = fory.serialize(&value).unwrap(); - fs::write(&data_file_path, new_bytes).unwrap(); - return; - } + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + fory.deserialize::(&bytes) + })); + assert!( + result.is_err() || result.is_ok_and(|value| value.is_err()), + "Expected nullable list payload to fail compatible array read" + ); fs::write(&data_file_path, bytes).unwrap(); } diff --git a/swift/Sources/Fory/TypeMeta.swift b/swift/Sources/Fory/TypeMeta.swift index cfa868b3a5..ebb6d78503 100644 --- a/swift/Sources/Fory/TypeMeta.swift +++ b/swift/Sources/Fory/TypeMeta.swift @@ -622,28 +622,16 @@ public final class TypeMeta: Equatable, @unchecked Sendable { var localMatch: (Int, FieldInfo)? if let fieldID = field.fieldID, fieldID >= 0 { - if let candidate = fieldIndexByID[fieldID] { - try Self.throwIfUnsupportedListArrayMismatch( - remoteType: field.fieldType, - localType: candidate.1.fieldType, - topLevel: true - ) - if Self.isCompatibleFieldType(field.fieldType, candidate.1.fieldType) { - localMatch = candidate - } + if let candidate = fieldIndexByID[fieldID], + Self.isCompatibleFieldType(field.fieldType, candidate.1.fieldType) { + localMatch = candidate } } if localMatch == nil { - if let candidate = fieldIndexByName[toSnakeCase(field.fieldName)] { - try Self.throwIfUnsupportedListArrayMismatch( - remoteType: field.fieldType, - localType: candidate.1.fieldType, - topLevel: true - ) - if Self.isCompatibleFieldType(field.fieldType, candidate.1.fieldType) { - localMatch = candidate - } + if let candidate = fieldIndexByName[toSnakeCase(field.fieldName)], + Self.isCompatibleFieldType(field.fieldType, candidate.1.fieldType) { + localMatch = candidate } } @@ -712,41 +700,15 @@ public final class TypeMeta: Equatable, @unchecked Sendable { return true } - private static func throwIfUnsupportedListArrayMismatch( - remoteType: FieldType, - localType: FieldType, - topLevel: Bool - ) throws { - if topLevel, isListArrayShapePair(remoteType, localType) { - if isCompatibleTopLevelListArrayFieldType(remoteType, localType) { - return - } - throw ForyError.invalidData("unsupported compatible list/array schema mismatch") - } - } - - private static func isListArrayShapePair(_ remoteType: FieldType, _ localType: FieldType) -> Bool { - (remoteType.typeID == TypeId.list.rawValue - && TypeId(rawValue: localType.typeID)?.denseArrayElementTypeID != nil) - || (localType.typeID == TypeId.list.rawValue - && TypeId(rawValue: remoteType.typeID)?.denseArrayElementTypeID != nil) - } - private static func isCompatibleTopLevelListArrayFieldType( _ remoteType: FieldType, _ localType: FieldType ) -> Bool { if remoteType.typeID == TypeId.list.rawValue { - return listFieldType( - remoteType, - matchesDenseArrayTypeID: localType.typeID - ) + return listFieldType(remoteType, matchesDenseArrayTypeID: localType.typeID) } if localType.typeID == TypeId.list.rawValue { - return listFieldType( - localType, - matchesDenseArrayTypeID: remoteType.typeID - ) + return listFieldType(localType, matchesDenseArrayTypeID: remoteType.typeID) } return false } diff --git a/swift/Sources/Fory/TypeResolver.swift b/swift/Sources/Fory/TypeResolver.swift index 02ee5bb274..a1ac088171 100644 --- a/swift/Sources/Fory/TypeResolver.swift +++ b/swift/Sources/Fory/TypeResolver.swift @@ -559,10 +559,9 @@ final class TypeResolver { return localTypeInfo } let canonicalTypeMeta: TypeMeta - if let localTypeMeta = localTypeInfo.typeMeta { - // Field remapping validates compatible schema shape. Propagate those errors so an unsupported - // matched field cannot degrade into an unknown-field skip. - canonicalTypeMeta = try typeMeta.assigningFieldIDs(from: localTypeMeta) + if let localTypeMeta = localTypeInfo.typeMeta, + let remapped = try? typeMeta.assigningFieldIDs(from: localTypeMeta) { + canonicalTypeMeta = remapped } else { canonicalTypeMeta = typeMeta } diff --git a/swift/Tests/ForyTests/CompatibilityTests.swift b/swift/Tests/ForyTests/CompatibilityTests.swift index 28bb314078..2afa3f255c 100644 --- a/swift/Tests/ForyTests/CompatibilityTests.swift +++ b/swift/Tests/ForyTests/CompatibilityTests.swift @@ -442,7 +442,7 @@ func compatibleReadAdaptsArrayFieldToDefaultVarintListField() throws { } @Test -func compatibleReadAllowsNullableListSchemaWithoutNullElementsForArrayField() throws { +func compatibleReadRejectsNullableListElementsForArrayField() throws { let writer = Fory(config: .init(xlang: true, trackRef: false, compatible: true)) writer.register(CompatibleNullableListFieldV1.self, id: 9923) diff --git a/swift/Tests/ForyXlangTests/main.swift b/swift/Tests/ForyXlangTests/main.swift index 101eb777df..f96fd9c98a 100644 --- a/swift/Tests/ForyXlangTests/main.swift +++ b/swift/Tests/ForyXlangTests/main.swift @@ -1116,11 +1116,13 @@ private func handleListArrayCompatibleNullableListToArrayError(_ bytes: [UInt8]) let fory = Fory(config: .init(xlang: true, trackRef: false, compatible: true)) fory.register(CompatibleInt32ArrayField.self, id: 901) do { - let value: CompatibleInt32ArrayField = try fory.deserialize(Data(bytes)) - return try Array(fory.serialize(value)) + let _: CompatibleInt32ArrayField = try fory.deserialize(Data(bytes)) } catch { return bytes } + throw PeerError.invalidFieldValue( + "Expected nullable list payload to fail compatible array read" + ) } private func rewritePayload(caseName: String, bytes: [UInt8]) throws -> [UInt8] { From e330ade83c70b11d937b48e394840d58e2b0c3d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 10:17:12 +0800 Subject: [PATCH 46/58] ci: use default rust binary cache behavior --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfb79cd67b..062b5dc8cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -728,7 +728,6 @@ jobs: uses: Swatinem/rust-cache@v2 with: workspaces: rust - cache-bin: false - name: Run Rust CI run: python ./ci/run_ci.py rust @@ -757,7 +756,6 @@ jobs: uses: Swatinem/rust-cache@v2 with: workspaces: rust - cache-bin: false - name: Run Rust Xlang Test env: FORY_RUST_JAVA_CI: "1" From 87826602be81c3ae8f9947f8d7a21944c19221eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 10:24:20 +0800 Subject: [PATCH 47/58] docs(java): clarify Android static serializers --- docs/compiler/generated-code.md | 12 +- docs/guide/java/android-support.md | 171 ++++++++++------------------- 2 files changed, 59 insertions(+), 124 deletions(-) diff --git a/docs/compiler/generated-code.md b/docs/compiler/generated-code.md index 693b939756..e3a8f9b7eb 100644 --- a/docs/compiler/generated-code.md +++ b/docs/compiler/generated-code.md @@ -154,17 +154,7 @@ public class Person { } ``` -When a message or inherited schema option sets `evolving=false`, the Java generator emits -`@ForyStruct(evolution = Evolution.DISABLED)` and imports `ForyStruct.Evolution` so the generated -class uses fixed-schema struct encoding: - -```java -import org.apache.fory.annotation.ForyStruct; -import org.apache.fory.annotation.ForyStruct.Evolution; - -@ForyStruct(evolution = Evolution.DISABLED) -public class StableMessage { ... } -``` +Messages with `evolving=false` are generated with Java fixed-schema struct encoding. Unions generate classes extending `org.apache.fory.type.union.Union`: diff --git a/docs/guide/java/android-support.md b/docs/guide/java/android-support.md index 4b22457fae..ee6994d0d1 100644 --- a/docs/guide/java/android-support.md +++ b/docs/guide/java/android-support.md @@ -19,144 +19,89 @@ license: | limitations under the License. --- -## Android Support +## Android Runtime -This page documents the Java `fory-core` Android runtime surface. +Fory Java supports Android 8.0+ (API level 26+) through the regular `fory-core` artifact. No separate +Android artifact is required for core object serialization. -The target runtime is Android 8.0+ (API level 26+) in the existing `fory-core` artifact. Android -support is selected at runtime by `org.apache.fory.platform.AndroidSupport`; no separate Android -artifact is required for core object serialization. +Use core object serialization on Android: -`java/fory-format` is not part of the Android support surface. Row-format direct-memory APIs remain -JVM-only. - -Android does not allow Fory runtime serialization paths to rely on `sun.misc.Unsafe`, private-field -`MethodHandle` access, dynamic bytecode loading, or `LambdaMetafactory`. Android-specific code paths -must use public platform APIs or fail with targeted exceptions when Android cannot preserve JVM -semantics. - -## Target Support Surface - -The Android target includes: - -- `Fory#serialize(Object)` returning `byte[]`. -- `Fory#deserialize(byte[])`. -- `BaseFory#deserialize(ByteBuffer)` through copy into a Fory-owned heap `MemoryBuffer`. -- Stream, channel, and out-of-band buffer APIs through safe heap, byte-array, or `ByteBuffer` copy +- `Fory#serialize(Object)` and `Fory#deserialize(byte[])`. +- `BaseFory#deserialize(ByteBuffer)` for heap, direct, and read-only `ByteBuffer` inputs. +- Stream, channel, and out-of-band buffer APIs through byte-array, heap-buffer, or `ByteBuffer` copy paths. -- Interpreter object serializers with reflection-backed field access. -- Normal Java collections/maps and xlang collection/map protocols. - -Unsupported or removed behavior: +- Java collections/maps and xlang collections/maps. -- Runtime serializer code generation and async compilation. -- Lambda and `SerializedLambda` serialization. Registration still succeeds for stable internal type - ids, but write/read/copy operations throw an unsupported exception. -- Native-address serialize/deserialize APIs and native-address `MemoryBuffer` wrapping. -- Raw-address direct `ByteBuffer` zero-copy. -- Raw unsafe `MemoryBuffer` copy APIs remain JVM-only and throw on Android. -- `java/fory-format` row-format APIs, including direct-memory binary row copy paths. -- Field type-use annotation metadata that depends on `Field#getAnnotatedType()` on the JVM. - Android API 26 field metadata uses generic field types and field annotations exposed by Android - reflection. -- Object deserialization that cannot be completed through reflection. - -## Codegen - -`ForyBuilder` stores the codegen request as a nullable `Boolean`. Unset codegen defaults to disabled -on Android and GraalVM native image, and enabled on ordinary JVM. Explicit `withCodegen(true)` on -Android or GraalVM does not throw; build finalization forces codegen off and emits an explicit -warning. Explicit `withCodegen(false)` and platform-default disabled codegen do not warn. - -Android runtime codegen entry points must fail before Janino, class definition, generated accessor -definition, or generated serializer loading starts. - -`@ForyStruct` build-time static serializers are allowed on Android because they are compiled by -javac before the app is built. Android loads them by deterministic generated class name when present, -including compatible-mode reads. This lookup is not gated on an Android-only public API, does not -enable Janino or runtime class definition, and uses the same wire protocol as ordinary JVM Fory. -For reflection metadata, Fory checks whether the runtime actually exposes `Field#getAnnotatedType()` -before using type-use annotations; older Android runtimes that do not expose it must use generated -static descriptors for nested type-use metadata. - -## ByteBuffer +`java/fory-format` row-format APIs are JVM-only and are not supported on Android. -On Android, `BaseFory#deserialize(ByteBuffer)` copies the remaining input bytes into a Fory-owned -heap `MemoryBuffer`, then deserializes from that buffer. Heap, direct, and readonly inputs are -supported through the copy path. The caller buffer position and limit are not changed. +## Runtime Codegen -Raw direct-buffer address wrapping remains a JVM-only fast path and is not used on Android. +Runtime serializer code generation is disabled on Android. If `withCodegen(true)` is set, Fory keeps +Android serialization on the non-codegen path and logs a warning. -## Direct Memory And Row Format +Android apps that need generated serializers should use build-time static generated serializers +instead. -Android `fory-core` paths do not execute `sun.misc.Unsafe` operations. Direct-memory copy APIs are -JVM-only paths for existing JVM users such as `java/fory-format`; they throw before unsafe execution -when Android is detected. +## Static Generated Serializers -Use core object serialization on Android. Do not use `java/fory-format` row-format APIs on Android. +Use `@ForyStruct` static generated serializers for Android application classes. They are generated by +javac during the app build and work without runtime bytecode generation. -## JDK Collection And Map Wrappers +Static generated serializers are required on Android when a serialized class uses Fory type-use +annotations, for example: -In native Java mode, Android does not add a new wrapper protocol branch and does not rewrite normal -collection/map serializers globally. +```java +import java.util.List; +import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.UInt8Type; -For `UnmodifiableSerializers` and `SynchronizedSerializers`, Android keeps the outer wrapper -serializer and writes a public source collection/map payload in the existing backing-value slot: +@ForyStruct +public class ImageBlock { + public List<@UInt8Type Integer> pixels; +} +``` -- list wrappers use `ArrayList` source type info. -- set wrappers use `HashSet` source type info. -- sorted or navigable set wrappers use `TreeSet` source type info. -- map wrappers use `HashMap` source type info. -- sorted or navigable map wrappers use `TreeMap` source type info. +Without the generated static descriptors, Android reflection may not expose the nested type-use +metadata needed for annotations such as `@Ref`, `@Int8Type`, `@UInt8Type`, `@Float16Type`, or +`@BFloat16Type`. Serialization for those classes will not have the schema information Fory needs. -The wrapper read path rewraps that source through `Collections.unmodifiable*` or -`Collections.synchronized*`. Synchronized wrapper write and copy paths must hold the wrapper lock -while iterating public contents. +See [Static Generated Serializers](static-generated-serializers.md) for setup instructions. -Sublist views keep a unified serializer protocol. Android writes visible elements; JVM may write -source-list view metadata when supported. Both Android and JVM readers accept both payload modes. +## Object Model Requirements -Other JDK collection serializers keep their existing Java native protocol shape while avoiding -hidden-field unsafe access on Android. `Arrays.asList` writes the existing array payload, -`Collections.newSetFromMap` writes a `HashMap` backing-map payload, bounded blocking queues derive -capacity through public APIs, non-empty `EnumMap` derives its key type from the first key, empty -Android `EnumMap` writes a self-describing Java-serialization fallback, and immutable JDK -collections are materialized through public unmodifiable containers on Android. +Android serializers use public Android runtime capabilities. For application classes, prefer: -In xlang mode, collection and map serialization uses the xlang collection/map protocol and does not -encode Java wrapper/view internals. +- accessible no-argument constructors, or records with supported constructors. +- public, protected, or package-private serialized fields. +- non-private getters and setters for private serialized fields. +- `@ForyStruct` static generated serializers for Android model classes. -## JDK Dynamic Proxies +Final fields in ordinary classes are not suitable for generated read/copy methods. Use records for +constructor-based immutable values. -The Android design supports `java.lang.reflect.Proxy` serialization. +## Unsupported Features -The Android proxy path must use only public proxy APIs: +The following JVM features are not supported on Android: -- `Proxy.getInvocationHandler(proxy)` to read the handler during write and copy. -- `Proxy.newProxyInstance(classLoader, interfaces, handler)` to construct proxies during read and - copy. -- Normal Fory reference serialization for the proxy interface array and invocation handler. +- Runtime serializer code generation and async compilation. +- Lambda and `SerializedLambda` serialization. +- Native-address serialization APIs and native-address `MemoryBuffer` wrapping. +- Raw unsafe memory copy APIs. +- `java/fory-format` row-format APIs. -Android must not read or write the private `Proxy.h` field, request a field offset for that field, -or use `Unsafe` to replace the handler after proxy construction. +## ByteBuffer -For non-cyclic proxies, read constructs the proxy directly with the deserialized invocation handler. -For cyclic proxy graphs with reference tracking, Android uses a private deferred invocation handler: +`BaseFory#deserialize(ByteBuffer)` supports heap, direct, and read-only buffers on Android by copying +the remaining bytes into a Fory-owned heap buffer. The caller buffer position and limit are not +changed. -1. Create the proxy with a deferred handler. -2. Register the proxy in the read or copy reference table before reading or copying the real handler. -3. Read or copy the real handler through the normal Fory object path. -4. Install the real handler into the deferred handler. +Raw direct-buffer address wrapping is a JVM-only fast path and is not used on Android. -This preserves cycles where an invocation handler references its own proxy without mutating private -JDK fields. The deferred handler is an internal implementation detail and must never be written as -the user handler. When writing or copying a proxy, `JdkProxySerializer` unwraps any deferred handler -before serializing or copying the handler. +## Collections, Maps, And Proxies -A proxy must not be invoked, logged, or used as a key whose hash or equality calls the handler while -the deferred handler is still unresolved during deserialization or copy. If that happens, Fory throws -a targeted exception instead of silently invoking an incomplete proxy. +Common JDK collection and map implementations are supported on Android. In xlang mode, collection and +map serialization uses the xlang protocol and does not encode Java wrapper/view internals. -The JVM path may keep the existing optimized private-handler replacement path when benchmarks -require it. The Android path must remain separate and must not resolve JVM-only proxy handler offset -state. +`java.lang.reflect.Proxy` serialization is supported for normal proxy usage. Do not invoke, log, or +use a proxy as a map/set key while it is still being deserialized; the invocation handler may not be +ready yet. From 2223baf42d188260d58170fa247388576b2d0d80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 10:25:26 +0800 Subject: [PATCH 48/58] docs(java): add Android annotation processor setup --- docs/guide/java/android-support.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/guide/java/android-support.md b/docs/guide/java/android-support.md index ee6994d0d1..a7a5caade2 100644 --- a/docs/guide/java/android-support.md +++ b/docs/guide/java/android-support.md @@ -47,6 +47,23 @@ instead. Use `@ForyStruct` static generated serializers for Android application classes. They are generated by javac during the app build and work without runtime bytecode generation. +### Install The Annotation Processor + +Add `fory-annotation-processor` to the annotation processor path of the module that compiles your +Android model classes: + +```xml + + + org.apache.fory + fory-annotation-processor + ${fory.version} + + +``` + +Then annotate Android model classes with `@ForyStruct`. + Static generated serializers are required on Android when a serialized class uses Fory type-use annotations, for example: From 3c0983ad53f58942dcfb1b94b37a71fac7ae773c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 10:26:42 +0800 Subject: [PATCH 49/58] docs(java): expand Android processor Maven example --- docs/guide/java/android-support.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/guide/java/android-support.md b/docs/guide/java/android-support.md index a7a5caade2..c3b68088ef 100644 --- a/docs/guide/java/android-support.md +++ b/docs/guide/java/android-support.md @@ -53,13 +53,23 @@ Add `fory-annotation-processor` to the annotation processor path of the module t Android model classes: ```xml - - - org.apache.fory - fory-annotation-processor - ${fory.version} - - + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.apache.fory + fory-annotation-processor + ${fory.version} + + + + + + ``` Then annotate Android model classes with `@ForyStruct`. From b2fe66102b074aee31e87814440445d4470e18bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 10:28:53 +0800 Subject: [PATCH 50/58] docs(java): focus static serializer guide on users --- docs/guide/java/graalvm-support.md | 4 - .../java/static-generated-serializers.md | 127 ++++++++++-------- 2 files changed, 73 insertions(+), 58 deletions(-) diff --git a/docs/guide/java/graalvm-support.md b/docs/guide/java/graalvm-support.md index 50cc90e727..aa3e2527c3 100644 --- a/docs/guide/java/graalvm-support.md +++ b/docs/guide/java/graalvm-support.md @@ -38,10 +38,6 @@ Fory generates serialization code at GraalVM build time when you: Note: Fory's `asyncCompilationEnabled` option is automatically disabled for GraalVM native image since runtime JIT is not supported. -`@ForyStruct` annotation-processor static generated serializers are not the GraalVM native-image -serializer path. GraalVM native images use Fory's native-image build-time serializer generation -instead. - ## Basic Usage ### Step 0: Add the GraalVM Support Dependency diff --git a/docs/guide/java/static-generated-serializers.md b/docs/guide/java/static-generated-serializers.md index 18f9fff0fa..ec101027c5 100644 --- a/docs/guide/java/static-generated-serializers.md +++ b/docs/guide/java/static-generated-serializers.md @@ -19,27 +19,49 @@ license: | limitations under the License. --- -The Fory Java annotation processor generates build-time static generated serializers for classes -annotated with `@ForyStruct`. They provide a non-JIT serializer path for ordinary JVM runtimes using -`ForyBuilder#withCodegen(false)` and for Android, where runtime bytecode generation is disabled. +Static generated serializers are Java serializers generated by javac during your application build. +They are useful when runtime code generation is disabled or unavailable. -## Enabling The Processor +Use them when: -Add the annotation processor to the Java compile configuration. The generated code depends only on -`fory-core` at runtime. `fory-core` itself does not depend on the processor; applications opt in by -placing `fory-annotation-processor` on their build's annotation-processor path. +- you run on Android. +- you run an ordinary JVM with `ForyBuilder#withCodegen(false)` and want generated serializers. +- your Android model classes use Fory type-use annotations such as `@Ref`, `@UInt8Type`, or + `@Float16Type`. + +For GraalVM native images, follow [GraalVM Support](graalvm-support.md) instead. + +## Install The Annotation Processor + +Add `fory-annotation-processor` to the annotation processor path of the module that compiles your +serializable classes: ```xml - - - org.apache.fory - fory-annotation-processor - ${fory.version} - - + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.apache.fory + fory-annotation-processor + ${fory.version} + + + + + + ``` -Annotate serializable structs with `@ForyStruct`: +The generated serializers depend on `fory-core` at runtime. Applications opt in by adding the +annotation processor; `fory-core` does not depend on it. + +## Annotate Classes + +Annotate each serializable class with `@ForyStruct`: ```java import org.apache.fory.annotation.ForyStruct; @@ -53,67 +75,64 @@ public class Order { } ``` -The processor emits public top-level serializers in the same package. For `Order`, the generated -cross-language serializer is `Order__ForySerializer__` and the generated Java native serializer is -`Order__ForyNativeSerializer__`. For a static member type `Outer.Inner`, the generated top-level -classes are `Outer$Inner__ForySerializer__` and `Outer$Inner__ForyNativeSerializer__`; the processor -does not modify the enclosing class and does not generate inner serializer classes. +The processor generates serializer classes in the same Java package as the annotated class. For +`Order`, the generated classes are: -## Runtime Selection +- `Order__ForySerializer__` for cross-language mode. +- `Order__ForyNativeSerializer__` for Java native mode. -Annotation-processor static generated serializers are used when available on: +For a static nested type such as `Outer.Inner`, the generated top-level classes are +`Outer$Inner__ForySerializer__` and `Outer$Inner__ForyNativeSerializer__`. -- ordinary JVMs with `ForyBuilder#withCodegen(false)`. -- Android runtimes, because runtime code generation is disabled there. -- compatible-mode meta-share reads when a generated static serializer exists for the target struct. +## Runtime Use -Ordinary JVM `codegen=true` keeps the runtime-generated serializer precedence. Static generated -serializer lookup is deterministic by generated class name and does not scan the classpath. +Fory uses static generated serializers when they are available on: + +- Android. +- ordinary JVMs with `ForyBuilder#withCodegen(false)`. +- compatible-mode reads when the target struct has a generated serializer. -GraalVM native image does not use annotation-processor-generated static serializer classes. Use the -GraalVM guide for native-image build-time serializer generation. +On an ordinary JVM with `codegen=true`, Fory continues to prefer runtime-generated serializers. ## Field Access Rules -The processor never falls back to reflection for private serialized fields. +Generated serializers must be able to access serialized fields or their accessors at compile time. - Public, protected, and package-private fields can be accessed directly when Java package access - allows the generated same-package serializer to use them. + allows same-package generated serializers to use them. - Private serialized fields must have accessible non-private getter and setter methods, or be excluded with `transient` or Fory `@Ignore`. - Public, protected, and package-private getter/setter methods are accepted when they are accessible from the generated serializer package. -- Final fields are rejected for normal classes because generated read and copy methods must assign - them. Use records for constructor-based immutable structs. +- Final fields are not supported for normal mutable classes because generated read and copy methods + must assign fields. Use records for constructor-based immutable values. For records, generated serializers use public record accessors and construct values through the canonical record constructor. Ignored record components are skipped by serialization and copy, and their constructor arguments use Java default values during generated read/copy. -## Generated Metadata +## Type-Use Annotations On Android -Generated serializers expose descriptor metadata through -`StaticGeneratedStructSerializer#getDescriptors()`. The descriptor list is a static immutable list -owned by the generated serializer. Each descriptor carries: +On Android, static generated serializers are required when a class uses Fory type-use annotations on +nested types: -- field name and declaring-class identity. -- `@ForyField` id, nullable, reference-tracking, and dynamic-field semantics. -- a `TypeRef` tree with nested type arguments, array component type, and `TypeExtMeta` for nested - `TYPE_USE` metadata such as `@Ref`, `@UInt8Type`, and `@UInt16Type`. - -The runtime uses these descriptors to build schema metadata instead of reading nested -`Field#getAnnotatedType()` information at runtime. This keeps Android and JVM wire protocol unified -while avoiding Android reflection gaps. +```java +import java.util.List; +import org.apache.fory.annotation.ForyStruct; +import org.apache.fory.annotation.UInt8Type; -## Compatible Reads +@ForyStruct +public class ImageBlock { + public List<@UInt8Type Integer> pixels; +} +``` -Annotation-processor static generated serializers include normal read/write/copy methods and a -compatible read method. The compatible path consumes remote schema metadata, matches remote fields -to local fields, skips unknown fields, and preserves Java defaults for missing fields. +Without the generated serializer metadata, Android may not expose enough nested type information for +Fory to preserve annotations such as `@Ref`, `@Int8Type`, `@UInt8Type`, `@Float16Type`, or +`@BFloat16Type`. -Field matching assigns dense generated matched ids for the generated branch table. Those ids are -local dispatch ids only; they are not `@ForyField.id` values and are not wire ids. Remote field order -still controls payload consumption. +## Compatible Reads -The same-schema fast path is used only when the remote schema hash equals the local schema hash and -the struct has no nested struct fields that require compatible layouts. +Static generated serializers support both normal serialization and compatible-mode reads. Compatible +reads match remote fields to local fields, skip fields that no longer exist locally, and keep Java +default values for fields that are missing from the remote payload. From a4942bd98f92a7298677d94fc5a607f5acd6ab13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 10:30:51 +0800 Subject: [PATCH 51/58] docs(java): refresh annotation processor readme --- java/fory-annotation-processor/README.md | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/java/fory-annotation-processor/README.md b/java/fory-annotation-processor/README.md index 1de33460ed..e56e4e8dfa 100644 --- a/java/fory-annotation-processor/README.md +++ b/java/fory-annotation-processor/README.md @@ -1,17 +1,17 @@ # Fory Annotation Processor -`fory-annotation-processor` generates build-time static generated serializers for Java classes -annotated with `@ForyStruct`. +`fory-annotation-processor` generates static serializers for Java classes annotated with +`@ForyStruct`. The serializers are generated by javac during the application build, so they can be +used when runtime serializer generation is disabled or unavailable. -For ordinary JVM applications, prefer Fory's runtime generated serializers. Runtime generation is -optimized for the active JVM and is usually more efficient than annotation-processor static -generated serializers. That is the normal high-performance path for server and desktop JVM -deployments. +Use this processor for: -Use this annotation processor when runtime source generation, bytecode generation, or dynamic class -loading is not acceptable. The main target is Android, where runtime code generation is disabled. -It can also be useful for restricted JVM environments that forbid loading generated bytecode at -runtime. +- Android applications. +- ordinary JVM applications using `ForyBuilder#withCodegen(false)`. +- Android model classes that use Fory type-use annotations such as `@Ref`, `@UInt8Type`, or + `@Float16Type`. + +For GraalVM native images, use Fory's GraalVM native-image build-time serializer generation instead. ## Ownership @@ -19,9 +19,5 @@ The processor is an opt-in build tool. Applications add it to their annotation-p emits Java source that references `fory-core` runtime APIs. `fory-core` does not depend on this module, and the processor implementation does not need a compile-time dependency on `fory-core`. -Generated serializers are public top-level classes in the same package as the annotated type. The -runtime loads them deterministically by name when static serializers are available and runtime -codegen is disabled. - For the full user-facing guide, see [`docs/guide/java/static-generated-serializers.md`](../../docs/guide/java/static-generated-serializers.md). From 301c80c74fa81ab3658f1b71399f31b4d8c8f674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 11:08:00 +0800 Subject: [PATCH 52/58] fix(java): clean up static compatible serializer --- .../StaticSerializerSourceWriter.java | 57 +++- .../processing/ForyStructProcessorTest.java | 53 ++- .../builder/StaticCompatibleCodecBuilder.java | 43 ++- .../java/org/apache/fory/meta/FieldInfo.java | 11 - .../java/org/apache/fory/meta/FieldTypes.java | 2 - .../fory/meta/NativeTypeDefEncoder.java | 2 +- .../java/org/apache/fory/meta/TypeDef.java | 5 - .../apache/fory/resolver/TypeResolver.java | 2 - .../CompatibleCollectionArrayReader.java | 2 +- .../apache/fory/serializer/FieldGroups.java | 10 +- .../StaticGeneratedStructSerializer.java | 54 +--- .../serializer/converter/FieldConverters.java | 306 ++++++++++-------- .../java/org/apache/fory/type/Descriptor.java | 4 - .../apache/fory/type/DescriptorBuilder.java | 7 +- .../fory/xlang/MetaSharedXlangTest.java | 20 ++ 15 files changed, 330 insertions(+), 248 deletions(-) diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java index 5ee5d7b08c..9cc69c27b7 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/StaticSerializerSourceWriter.java @@ -599,13 +599,25 @@ private String exactPrimitiveWriteStatement(SourceField field, String valueExpre case "VARINT32": return "buffer.writeVarInt32(" + valueExpression + ");"; case "UINT8": - return "buffer.writeByte(" + valueExpression + ");"; + return unsignedRangeCheck(valueExpression, "255") + + " buffer.writeByte(" + + valueExpression + + ");"; case "UINT16": - return "buffer.writeInt16((short) " + valueExpression + ");"; + return unsignedRangeCheck(valueExpression, "65535") + + " buffer.writeInt16((short) " + + valueExpression + + ");"; case "UINT32": - return "buffer.writeInt32((int) " + valueExpression + ");"; + return unsignedInt32RangeCheck(valueExpression) + + " buffer.writeInt32((int) " + + valueExpression + + ");"; case "VAR_UINT32": - return "buffer.writeVarUInt32((int) " + valueExpression + ");"; + return unsignedInt32RangeCheck(valueExpression) + + " buffer.writeVarUInt32((int) " + + valueExpression + + ");"; case "INT64": case "UINT64": return "buffer.writeInt64(" + valueExpression + ");"; @@ -626,6 +638,22 @@ private String exactPrimitiveWriteStatement(SourceField field, String valueExpre } } + private String unsignedRangeCheck(String valueExpression, String maxValue) { + return "if ((" + + valueExpression + + ") < 0 || (" + + valueExpression + + ") > " + + maxValue + + ") { throw new IllegalArgumentException(\"Unsigned value out of range\"); }"; + } + + private String unsignedInt32RangeCheck(String valueExpression) { + return "if (((" + + valueExpression + + ") & 0xffffffff00000000L) != 0) { throw new IllegalArgumentException(\"Unsigned value out of range\"); }"; + } + private String exactPrimitiveReadExpression(SourceField field) { String typeId = exactPrimitiveTypeId(field); if (typeId == null) { @@ -732,8 +760,7 @@ private void writeCompatibleRead() { } builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); builder.append(" RemoteFieldInfo remoteField = remoteFields.get(i);\n"); - builder.append( - " readCompatibleRecordField(readContext, values, remoteField, matchedId(remoteField));\n"); + builder.append(" readCompatibleRecordField(readContext, values, remoteField);\n"); builder.append(" }\n"); for (SourceField field : struct.fields) { builder @@ -753,8 +780,7 @@ private void writeCompatibleRead() { builder.append(" readContext.reference(value);\n"); builder.append(" for (int i = 0; i < remoteFields.size(); i++) {\n"); builder.append(" RemoteFieldInfo remoteField = remoteFields.get(i);\n"); - builder.append( - " readCompatibleField(readContext, value, remoteField, matchedId(remoteField));\n"); + builder.append(" readCompatibleField(readContext, value, remoteField);\n"); builder.append(" }\n"); builder.append(" return value;\n"); } @@ -784,9 +810,12 @@ private void writeCompatibleDispatchRouter(String methodName, boolean record, in for (int group = 0; group < groupCount; group++) { int upperBound = Math.min(struct.fields.size(), (group + 1) * DISPATCH_GROUP_SIZE); if (group == 0) { - builder.append(" if (matchedId >= 0 && matchedId < ").append(upperBound).append(") {\n"); + builder + .append(" if (remoteField.matchedId >= 0 && remoteField.matchedId < ") + .append(upperBound) + .append(") {\n"); } else { - builder.append(" if (matchedId < ").append(upperBound).append(") {\n"); + builder.append(" if (remoteField.matchedId < ").append(upperBound).append(") {\n"); } builder.append(" ").append(methodName).append(group).append("("); appendCompatibleDispatchArguments(record); @@ -806,7 +835,7 @@ private void writeCompatibleBeanDispatchGroup(int group) { builder.append(" private void readCompatibleField").append(group).append("("); appendCompatibleDispatchParameters(false); builder.append(") {\n"); - builder.append(" switch (matchedId) {\n"); + builder.append(" switch (remoteField.matchedId) {\n"); for (int i = start; i < end; i++) { SourceField field = struct.fields.get(i); builder.append(" case ").append(field.id).append(":\n"); @@ -846,7 +875,7 @@ private void writeCompatibleRecordDispatchGroup(int group) { builder.append(" private void readCompatibleRecordField").append(group).append("("); appendCompatibleDispatchParameters(true); builder.append(") {\n"); - builder.append(" switch (matchedId) {\n"); + builder.append(" switch (remoteField.matchedId) {\n"); for (int i = start; i < end; i++) { SourceField field = struct.fields.get(i); builder.append(" case ").append(field.id).append(":\n"); @@ -884,7 +913,7 @@ private void appendCompatibleDispatchParameters(boolean record) { } else { builder.append(struct.typeName).append(" value, "); } - builder.append("RemoteFieldInfo remoteField, int matchedId"); + builder.append("RemoteFieldInfo remoteField"); } private void appendCompatibleDispatchArguments(boolean record) { @@ -894,7 +923,7 @@ private void appendCompatibleDispatchArguments(boolean record) { } else { builder.append("value, "); } - builder.append("remoteField, matchedId"); + builder.append("remoteField"); } private void appendDebugWrite(String stage, String fieldInfoName, int indent) { diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 5526fecfa0..2605b72e33 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -41,6 +41,7 @@ import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; import org.apache.fory.exception.DeserializationException; +import org.apache.fory.exception.SerializationException; import org.apache.fory.serializer.StaticGeneratedStructSerializer; import org.apache.fory.type.Descriptor; import org.apache.fory.type.Types; @@ -301,6 +302,51 @@ public void testGeneratedDescriptorsCarryNestedTypeMetadata() throws Exception { } } + @Test + public void testGeneratedUnsignedScalarWritesValidateRange() throws Exception { + CompilationResult result = + compile( + "test.UnsignedStruct", + "package test;\n" + + "import org.apache.fory.annotation.ForyStruct;\n" + + "import org.apache.fory.annotation.UInt8Type;\n" + + "import org.apache.fory.annotation.UInt16Type;\n" + + "import org.apache.fory.annotation.UInt32Type;\n" + + "@ForyStruct public class UnsignedStruct {\n" + + " public @UInt8Type int u8;\n" + + " public @UInt16Type int u16;\n" + + " public @UInt32Type long u32;\n" + + " public UnsignedStruct() {}\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.UnsignedStruct"); + Fory fory = + Fory.builder() + .withClassLoader(loader) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + Object value = type.getConstructor().newInstance(); + setField(type, value, "u8", 255); + setField(type, value, "u16", 65535); + setField(type, value, "u32", 4294967295L); + Object roundTrip = fory.deserialize(fory.serialize(value)); + Assert.assertEquals(getField(type, roundTrip, "u8"), 255); + Assert.assertEquals(getField(type, roundTrip, "u16"), 65535); + Assert.assertEquals(getField(type, roundTrip, "u32"), 4294967295L); + + setField(type, value, "u8", 256); + Assert.assertThrows(SerializationException.class, () -> fory.serialize(value)); + setField(type, value, "u8", 255); + setField(type, value, "u16", 65536); + Assert.assertThrows(SerializationException.class, () -> fory.serialize(value)); + setField(type, value, "u16", 65535); + setField(type, value, "u32", 4294967296L); + Assert.assertThrows(SerializationException.class, () -> fory.serialize(value)); + } + } + @Test public void testRecordReadAndCopyUseCanonicalConstructor() throws Exception { assumeRecordSupport(); @@ -317,7 +363,7 @@ public void testRecordReadAndCopyUseCanonicalConstructor() throws Exception { String generatedSource = result.generatedSource("test/RecordStruct__ForyNativeSerializer__.java"); Assert.assertTrue(generatedSource.contains("private void readCompatibleRecordField0(")); - Assert.assertTrue(generatedSource.contains("switch (matchedId)")); + Assert.assertTrue(generatedSource.contains("switch (remoteField.matchedId)")); try (URLClassLoader loader = result.classLoader()) { Class type = loader.loadClass("test.RecordStruct"); Object value = @@ -370,10 +416,9 @@ public void testCompatibleReadUsesGeneratedSerializer() throws Exception { String generatedSource = readerResult.generatedSource("test/EvolvingStruct__ForyNativeSerializer__.java"); Assert.assertTrue( - generatedSource.contains( - "readCompatibleField(readContext, value, remoteField, matchedId(remoteField))")); + generatedSource.contains("readCompatibleField(readContext, value, remoteField)")); Assert.assertTrue(generatedSource.contains("private void readCompatibleField0(")); - Assert.assertTrue(generatedSource.contains("switch (matchedId)")); + Assert.assertTrue(generatedSource.contains("switch (remoteField.matchedId)")); try (URLClassLoader writerLoader = writerResult.classLoader(); URLClassLoader readerLoader = readerResult.classLoader()) { Class writerType = writerLoader.loadClass("test.EvolvingStruct"); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 7b8d0d5833..5c52e8d88b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -158,7 +158,7 @@ private String genObjectCompatibleRead() { .append(READ_CONTEXT_NAME) .append(", ") .append(bean) - .append(", _f_remoteField, matchedId(_f_remoteField));\n") + .append(", _f_remoteField);\n") .append("}\n") .append("return ") .append(bean) @@ -202,7 +202,7 @@ private String genRecordCompatibleRead() { .append(READ_CONTEXT_NAME) .append(", ") .append(recordValues) - .append(", _f_remoteField, matchedId(_f_remoteField));\n") + .append(", _f_remoteField);\n") .append("}\n"); if (recordCtrAccessible) { code.append("return new ") @@ -240,9 +240,7 @@ private void genDispatchMethods() { Object[].class, "_f_recordValues", "RemoteFieldInfo", - "_f_remoteField", - int.class, - "_f_matchedId"); + "_f_remoteField"); for (int group = 0; group < groupCount; group++) { ctx.addMethod( "private", @@ -254,9 +252,7 @@ private void genDispatchMethods() { Object[].class, "_f_recordValues", "RemoteFieldInfo", - "_f_remoteField", - int.class, - "_f_matchedId"); + "_f_remoteField"); } return; } @@ -270,9 +266,7 @@ private void genDispatchMethods() { valueType, "_f_value", "RemoteFieldInfo", - "_f_remoteField", - int.class, - "_f_matchedId"); + "_f_remoteField"); for (int group = 0; group < groupCount; group++) { ctx.addMethod( "private", @@ -284,9 +278,7 @@ private void genDispatchMethods() { valueType, "_f_value", "RemoteFieldInfo", - "_f_remoteField", - int.class, - "_f_matchedId"); + "_f_remoteField"); } } @@ -295,9 +287,11 @@ private String genDispatchRouter(String methodPrefix, int groupCount) { for (int group = 0; group < groupCount; group++) { int upperBound = Math.min(localDescriptors.size(), (group + 1) * DISPATCH_GROUP_SIZE); if (group == 0) { - code.append("if (_f_matchedId >= 0 && _f_matchedId < ").append(upperBound).append(") {\n"); + code.append("if (_f_remoteField.matchedId >= 0 && _f_remoteField.matchedId < ") + .append(upperBound) + .append(") {\n"); } else { - code.append("if (_f_matchedId < ").append(upperBound).append(") {\n"); + code.append("if (_f_remoteField.matchedId < ").append(upperBound).append(") {\n"); } code.append(" ") .append(methodPrefix) @@ -306,7 +300,7 @@ private String genDispatchRouter(String methodPrefix, int groupCount) { .append(READ_CONTEXT_NAME) .append(", "); code.append(isRecord ? "_f_recordValues" : "_f_value"); - code.append(", _f_remoteField, _f_matchedId);\n").append(" return;\n").append("}\n"); + code.append(", _f_remoteField);\n").append(" return;\n").append("}\n"); } appendDebugRemoteRead(code, "before skip", "_f_remoteField", 0); code.append("skipField(").append(READ_CONTEXT_NAME).append(", _f_remoteField);\n"); @@ -317,21 +311,23 @@ private String genDispatchRouter(String methodPrefix, int groupCount) { private String genObjectDispatchGroup(int group, TypeRef valueTypeRef) { int start = group * DISPATCH_GROUP_SIZE; int end = Math.min(localDescriptors.size(), start + DISPATCH_GROUP_SIZE); - StringBuilder code = new StringBuilder("switch (_f_matchedId) {\n"); + StringBuilder code = new StringBuilder("switch (_f_remoteField.matchedId) {\n"); for (int i = start; i < end; i++) { Descriptor descriptor = localDescriptors.get(i); code.append(" case ") .append(i) .append(": {\n") .append(debugRemoteReadCode("before read", "_f_remoteField", 4)) - .append(" if (hasFieldConverter(_f_remoteField)) {\n") + .append(" if (_f_remoteField.serializationFieldInfo.fieldConverter != null) {\n") .append(" Object _f_fieldValue = readRemoteField(") .append(READ_CONTEXT_NAME) .append(", _f_remoteField);\n") .append(debugRemoteReadCode("after read", "_f_remoteField", 6)) - .append(" setConvertedField(_f_value, _f_fieldValue, _f_remoteField);\n") + .append( + " _f_remoteField.serializationFieldInfo.fieldConverter.set(_f_value, _f_fieldValue);\n") .append(" } else {\n") - .append(" SerializationFieldInfo _f_localField = localFieldInfo(_f_matchedId);\n") + .append( + " SerializationFieldInfo _f_localField = localFieldInfo(_f_remoteField.matchedId);\n") .append(" if (!canReadRemoteField(_f_remoteField, _f_localField)) {\n") .append(debugRemoteReadCode("before skip", "_f_remoteField", 8)) .append(" skipField(") @@ -363,7 +359,7 @@ private String genObjectDispatchGroup(int group, TypeRef valueTypeRef) { private String genRecordDispatchGroup(int group) { int start = group * DISPATCH_GROUP_SIZE; int end = Math.min(localDescriptors.size(), start + DISPATCH_GROUP_SIZE); - StringBuilder code = new StringBuilder("switch (_f_matchedId) {\n"); + StringBuilder code = new StringBuilder("switch (_f_remoteField.matchedId) {\n"); for (int i = start; i < end; i++) { Descriptor descriptor = localDescriptors.get(i); Integer componentIndex = recordReversedMapping.get(descriptor.getName()); @@ -374,7 +370,8 @@ private String genRecordDispatchGroup(int group) { .append(i) .append(": {\n") .append(debugRemoteReadCode("before read", "_f_remoteField", 4)) - .append(" SerializationFieldInfo _f_localField = localFieldInfo(_f_matchedId);\n") + .append( + " SerializationFieldInfo _f_localField = localFieldInfo(_f_remoteField.matchedId);\n") .append(" if (canReadRemoteField(_f_remoteField, _f_localField)) {\n") .append(" _f_recordValues[") .append(componentIndex) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index d12a936539..389fc1c8f7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -346,10 +346,6 @@ private TypeRef primitiveListCarrierType() { } private static int listElementTypeId(FieldTypes.FieldType fieldType) { - return listElementTypeId(fieldType, false); - } - - private static int listElementTypeId(FieldTypes.FieldType fieldType, boolean requireNonNullable) { if (!(fieldType instanceof FieldTypes.CollectionFieldType) || fieldType.getTypeId() != Types.LIST) { return Types.UNKNOWN; @@ -357,18 +353,11 @@ private static int listElementTypeId(FieldTypes.FieldType fieldType, boolean req FieldTypes.FieldType elementType = ((FieldTypes.CollectionFieldType) fieldType).getElementType(); if (elementType instanceof FieldTypes.RegisteredFieldType) { - if (requireNonNullable && (elementType.nullable() || elementType.trackingRef())) { - return Types.UNKNOWN; - } return ((FieldTypes.RegisteredFieldType) elementType).getTypeId(); } return Types.UNKNOWN; } - private static int nonNullableListElementTypeId(FieldTypes.FieldType fieldType) { - return listElementTypeId(fieldType, true); - } - private static int arrayTypeId(FieldTypes.FieldType fieldType) { if (fieldType instanceof FieldTypes.RegisteredFieldType) { int typeId = ((FieldTypes.RegisteredFieldType) fieldType).getTypeId(); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index bea2cf5a3e..c6301f1c09 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -141,8 +141,6 @@ private static FieldType buildFieldType( typeId = typeExtMeta.typeId(); } else if (boxedListArray) { typeId = TypeAnnotationUtils.getBoxedListArrayTypeId(descriptor); - } else if (primitiveListElementTypeId != Types.UNKNOWN) { - typeId = TypeAnnotationUtils.getPrimitiveListTypeId(typeAnnotation, rawType); } else if (TypeUtils.unwrap(rawType).isPrimitive()) { if (field != null) { typeId = diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java index 12b7bf7169..56af4df6a5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java @@ -99,7 +99,7 @@ public static List buildFieldsInfo(TypeResolver resolver, List return buildFieldsInfoFromDescriptors(resolver, descriptors); } - public static List buildFieldsInfoFromDescriptors( + static List buildFieldsInfoFromDescriptors( TypeResolver resolver, List descriptors) { List fieldInfos = new ArrayList<>(); Set usedTagIds = new HashSet<>(); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java index a3eef88ad5..a25536ea64 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java @@ -373,11 +373,6 @@ public List getDescriptors(TypeResolver resolver, Class cls) { this, cls, () -> buildDescriptors(resolver, cls)); } - public List getDescriptors( - TypeResolver resolver, Class cls, Collection localDescriptors) { - return buildDescriptors(resolver, cls, localDescriptors); - } - private List buildDescriptors(TypeResolver resolver, Class cls) { Collection fieldDescriptors = resolver.getFieldDescriptors(cls, true); return buildDescriptors(resolver, cls, fieldDescriptors); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 94c889fb4b..2d9de20963 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1545,8 +1545,6 @@ private List buildFieldDescriptors(Class clz, boolean searchParen return normalizeFieldDescriptors(clz, searchParent, staticDescriptors); } SortedMap allDescriptors = getAllDescriptorsMap(clz, searchParent); - List result = new ArrayList<>(allDescriptors.size()); - List descriptors = new ArrayList<>(allDescriptors.size()); for (Map.Entry entry : allDescriptors.entrySet()) { Member member = entry.getKey(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index b854babcd0..9c023b725a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -819,7 +819,7 @@ private static Object materializeTarget(Object array, int arrayTypeId, Class if (primitiveList != null) { return primitiveList; } - if (List.class.isAssignableFrom(targetType)) { + if (targetType.isAssignableFrom(ArrayList.class)) { return materializeBoxedList(array, arrayTypeId); } throw new DeserializationException("Unsupported compatible list/array target " + targetType); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index c79dd08969..4534ab076a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -143,10 +143,6 @@ public static FieldGroups buildFieldInfos(TypeResolver typeResolver, DescriptorG return new FieldGroups(allBuildIn, containerFields, otherFields); } - static SerializationFieldInfo buildFieldInfo(TypeResolver typeResolver, Descriptor descriptor) { - return new SerializationFieldInfo(typeResolver, descriptor); - } - public static final class SerializationFieldInfo { public final Descriptor descriptor; public final Class type; @@ -231,7 +227,7 @@ public static final class SerializationFieldInfo { GenericType t; if (primitiveListCollection) { - TypeRef elementTypeRef = primitiveListElementTypeRef(d); + TypeRef elementTypeRef = TypeAnnotationUtils.getPrimitiveListElementTypeRef(d); t = new GenericType(typeRef, true, resolver.buildGenericType(elementTypeRef)); } else { t = resolver.buildGenericType(typeRef); @@ -314,8 +310,4 @@ public String toString() { + '}'; } } - - private static TypeRef primitiveListElementTypeRef(Descriptor descriptor) { - return TypeAnnotationUtils.getPrimitiveListElementTypeRef(descriptor); - } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 21b410aec3..cd7cd56759 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -27,7 +27,6 @@ import org.apache.fory.annotation.Internal; import org.apache.fory.context.CopyContext; import org.apache.fory.context.ReadContext; -import org.apache.fory.context.RefReader; import org.apache.fory.context.WriteContext; import org.apache.fory.exception.DeserializationException; import org.apache.fory.memory.MemoryBuffer; @@ -215,30 +214,20 @@ protected final Object readRemoteField(ReadContext readContext, RemoteFieldInfo } protected final void skipField(ReadContext readContext, RemoteFieldInfo remoteField) { - skipField( - readContext, - readContext.getRefReader(), - remoteField.serializationFieldInfo, - readContext.getBuffer()); - } - - protected final void skipField( - ReadContext readContext, - RefReader refReader, - SerializationFieldInfo fieldInfo, - MemoryBuffer buffer) { try { - FieldSkipper.skipField(readContext, typeResolver, refReader, fieldInfo, buffer); + FieldSkipper.skipField( + readContext, + typeResolver, + readContext.getRefReader(), + remoteField.serializationFieldInfo, + readContext.getBuffer()); } catch (RuntimeException e) { throw new DeserializationException( - "Failed to skip remote field " + fieldInfo.descriptor.getName(), e); + "Failed to skip remote field " + remoteField.serializationFieldInfo.descriptor.getName(), + e); } } - protected final int matchedId(RemoteFieldInfo remoteField) { - return remoteField.matchedId; - } - protected final SerializationFieldInfo localFieldInfo(int matchedId) { return localFieldsById[matchedId]; } @@ -276,15 +265,6 @@ protected final Object readCompatibleFieldValue( return FieldConverters.convertValue(remoteType, localType, fieldValue); } - protected final boolean hasFieldConverter(RemoteFieldInfo remoteField) { - return remoteField.serializationFieldInfo.fieldConverter != null; - } - - protected final void setConvertedField( - Object targetObject, Object fieldValue, RemoteFieldInfo remoteField) { - remoteField.serializationFieldInfo.fieldConverter.set(targetObject, fieldValue); - } - protected final void debugWriteField( String stage, SerializationFieldInfo fieldInfo, WriteContext writeContext) { if (!org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { @@ -457,7 +437,7 @@ private void appendRemoteFields( throw new IllegalStateException("Missing remote field metadata for " + descriptor); } SerializationFieldInfo serializationFieldInfo = - FieldGroups.buildFieldInfo(typeResolver, descriptor); + new SerializationFieldInfo(typeResolver, descriptor); int matchedId = matchField(fieldInfo, fieldIds, fields); Descriptor localDescriptor = matchedId == UNKNOWN_FIELD ? null : localDescriptors.get(matchedId); @@ -530,14 +510,14 @@ private static String remoteFieldKey(Descriptor descriptor) { /** Remote field metadata consumed by generated compatible read methods. */ @Internal - protected static final class RemoteFieldInfo { - private final int matchedId; - private final FieldInfo fieldInfo; - private final Descriptor descriptor; - private final SerializationFieldInfo serializationFieldInfo; - private final CompatibleCollectionArrayReader.ReadAction compatibleCollectionArrayReadAction; - private final boolean incompatibleCollectionArrayMatch; - private final boolean nestedCollectionArrayMatch; + public static final class RemoteFieldInfo { + public final int matchedId; + public final FieldInfo fieldInfo; + public final Descriptor descriptor; + public final SerializationFieldInfo serializationFieldInfo; + public final CompatibleCollectionArrayReader.ReadAction compatibleCollectionArrayReadAction; + public final boolean incompatibleCollectionArrayMatch; + public final boolean nestedCollectionArrayMatch; private RemoteFieldInfo( TypeResolver typeResolver, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/converter/FieldConverters.java b/java/fory-core/src/main/java/org/apache/fory/serializer/converter/FieldConverters.java index 63ef581622..766883dcb3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/converter/FieldConverters.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/converter/FieldConverters.java @@ -21,6 +21,7 @@ import java.lang.reflect.Field; import java.util.Set; +import org.apache.fory.annotation.Internal; import org.apache.fory.collection.Collections; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.type.TypeUtils; @@ -44,154 +45,201 @@ private static Set> compatibleTypes(Class... types) { * converter exists */ public static FieldConverter getConverter(Class from, Field field) { - Class to = field.getType(); - from = TypeUtils.wrap(from); - // Handle primitive int conversions - if (to == int.class) { - if (IntConverter.compatibleTypes.contains(from)) { - return new IntConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == Integer.class) { - // Handle boxed Integer conversions - if (IntConverter.compatibleTypes.contains(from)) { - return new BoxedIntConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == boolean.class) { - // Handle primitive boolean conversions - if (BooleanConverter.compatibleTypes.contains(from)) { - return new BooleanConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == Boolean.class) { - // Handle boxed Boolean conversions - if (BooleanConverter.compatibleTypes.contains(from)) { - return new BoxedBooleanConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == byte.class) { - // Handle primitive byte conversions - if (ByteConverter.compatibleTypes.contains(from)) { - return new ByteConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == Byte.class) { - // Handle boxed Byte conversions - if (ByteConverter.compatibleTypes.contains(from)) { - return new BoxedByteConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == short.class) { - // Handle primitive short conversions - if (ShortConverter.compatibleTypes.contains(from)) { - return new ShortConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == Short.class) { - // Handle boxed Short conversions - if (ShortConverter.compatibleTypes.contains(from)) { - return new BoxedShortConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == long.class) { - // Handle primitive long conversions - if (LongConverter.compatibleTypes.contains(from)) { - return new LongConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == Long.class) { - // Handle boxed Long conversions - if (LongConverter.compatibleTypes.contains(from)) { - return new BoxedLongConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == float.class) { - // Handle primitive float conversions - if (FloatConverter.compatibleTypes.contains(from)) { - return new FloatConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == Float.class) { - // Handle boxed Float conversions - if (FloatConverter.compatibleTypes.contains(from)) { - return new BoxedFloatConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == double.class) { - // Handle primitive double conversions - if (DoubleConverter.compatibleTypes.contains(from)) { - return new DoubleConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == Double.class) { - // Handle boxed Double conversions - if (DoubleConverter.compatibleTypes.contains(from)) { - return new BoxedDoubleConverter(FieldAccessor.createAccessor(field)); - } - } else if (to == String.class) { - // Handle String conversions - if (StringConverter.compatibleTypes.contains(from)) { - return new StringConverter(FieldAccessor.createAccessor(field)); - } - } - - return null; // No compatible converter found + FieldConversion conversion = fieldConversion(TypeUtils.wrap(from), field.getType()); + if (conversion == null) { + return null; + } + FieldAccessor accessor = FieldAccessor.createAccessor(field); + switch (conversion) { + case INT: + return new IntConverter(accessor); + case BOXED_INT: + return new BoxedIntConverter(accessor); + case BOOLEAN: + return new BooleanConverter(accessor); + case BOXED_BOOLEAN: + return new BoxedBooleanConverter(accessor); + case BYTE: + return new ByteConverter(accessor); + case BOXED_BYTE: + return new BoxedByteConverter(accessor); + case SHORT: + return new ShortConverter(accessor); + case BOXED_SHORT: + return new BoxedShortConverter(accessor); + case LONG: + return new LongConverter(accessor); + case BOXED_LONG: + return new BoxedLongConverter(accessor); + case FLOAT: + return new FloatConverter(accessor); + case BOXED_FLOAT: + return new BoxedFloatConverter(accessor); + case DOUBLE: + return new DoubleConverter(accessor); + case BOXED_DOUBLE: + return new BoxedDoubleConverter(accessor); + case STRING: + return new StringConverter(accessor); + default: + throw new IllegalStateException("Unknown field conversion " + conversion); + } } /** Returns whether a value of {@code from} can be assigned or converted to {@code to}. */ + @Internal public static boolean canConvert(Class from, Class to) { if (isDirectlyAssignable(from, to)) { return true; } - Class wrappedFrom = TypeUtils.wrap(from); - if (to == int.class || to == Integer.class) { - return IntConverter.compatibleTypes.contains(wrappedFrom); - } else if (to == boolean.class || to == Boolean.class) { - return BooleanConverter.compatibleTypes.contains(wrappedFrom); - } else if (to == byte.class || to == Byte.class) { - return ByteConverter.compatibleTypes.contains(wrappedFrom); - } else if (to == short.class || to == Short.class) { - return ShortConverter.compatibleTypes.contains(wrappedFrom); - } else if (to == long.class || to == Long.class) { - return LongConverter.compatibleTypes.contains(wrappedFrom); - } else if (to == float.class || to == Float.class) { - return FloatConverter.compatibleTypes.contains(wrappedFrom); - } else if (to == double.class || to == Double.class) { - return DoubleConverter.compatibleTypes.contains(wrappedFrom); - } else if (to == String.class) { - return StringConverter.compatibleTypes.contains(wrappedFrom); - } - return false; + return fieldConversion(TypeUtils.wrap(from), to) != null; } /** * Converts {@code value} from {@code from} to {@code to}, or returns it for direct assignment. */ + @Internal public static Object convertValue(Class from, Class to, Object value) { if (isDirectlyAssignable(from, to)) { return value; - } else if (to == int.class) { - return IntConverter.convertFrom(value); - } else if (to == Integer.class) { - return BoxedIntConverter.convertFrom(value); - } else if (to == boolean.class) { - return BooleanConverter.convertFrom(value); - } else if (to == Boolean.class) { - return BoxedBooleanConverter.convertFrom(value); - } else if (to == byte.class) { - return ByteConverter.convertFrom(value); - } else if (to == Byte.class) { - return BoxedByteConverter.convertFrom(value); - } else if (to == short.class) { - return ShortConverter.convertFrom(value); - } else if (to == Short.class) { - return BoxedShortConverter.convertFrom(value); - } else if (to == long.class) { - return LongConverter.convertFrom(value); - } else if (to == Long.class) { - return BoxedLongConverter.convertFrom(value); - } else if (to == float.class) { - return FloatConverter.convertFrom(value); - } else if (to == Float.class) { - return BoxedFloatConverter.convertFrom(value); - } else if (to == double.class) { - return DoubleConverter.convertFrom(value); - } else if (to == Double.class) { - return BoxedDoubleConverter.convertFrom(value); - } else if (to == String.class) { - return StringConverter.convertFrom(value); + } + FieldConversion conversion = fieldConversion(TypeUtils.wrap(from), to); + if (conversion != null) { + return conversion.convert(value); } throw new UnsupportedOperationException("Incompatible type: " + from + " -> " + to); } + private static FieldConversion fieldConversion(Class wrappedFrom, Class to) { + if (to == int.class && IntConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.INT; + } else if (to == Integer.class && IntConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.BOXED_INT; + } else if (to == boolean.class && BooleanConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.BOOLEAN; + } else if (to == Boolean.class && BooleanConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.BOXED_BOOLEAN; + } else if (to == byte.class && ByteConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.BYTE; + } else if (to == Byte.class && ByteConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.BOXED_BYTE; + } else if (to == short.class && ShortConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.SHORT; + } else if (to == Short.class && ShortConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.BOXED_SHORT; + } else if (to == long.class && LongConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.LONG; + } else if (to == Long.class && LongConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.BOXED_LONG; + } else if (to == float.class && FloatConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.FLOAT; + } else if (to == Float.class && FloatConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.BOXED_FLOAT; + } else if (to == double.class && DoubleConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.DOUBLE; + } else if (to == Double.class && DoubleConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.BOXED_DOUBLE; + } else if (to == String.class && StringConverter.compatibleTypes.contains(wrappedFrom)) { + return FieldConversion.STRING; + } + return null; + } + + private enum FieldConversion { + INT { + @Override + Object convert(Object value) { + return IntConverter.convertFrom(value); + } + }, + BOXED_INT { + @Override + Object convert(Object value) { + return BoxedIntConverter.convertFrom(value); + } + }, + BOOLEAN { + @Override + Object convert(Object value) { + return BooleanConverter.convertFrom(value); + } + }, + BOXED_BOOLEAN { + @Override + Object convert(Object value) { + return BoxedBooleanConverter.convertFrom(value); + } + }, + BYTE { + @Override + Object convert(Object value) { + return ByteConverter.convertFrom(value); + } + }, + BOXED_BYTE { + @Override + Object convert(Object value) { + return BoxedByteConverter.convertFrom(value); + } + }, + SHORT { + @Override + Object convert(Object value) { + return ShortConverter.convertFrom(value); + } + }, + BOXED_SHORT { + @Override + Object convert(Object value) { + return BoxedShortConverter.convertFrom(value); + } + }, + LONG { + @Override + Object convert(Object value) { + return LongConverter.convertFrom(value); + } + }, + BOXED_LONG { + @Override + Object convert(Object value) { + return BoxedLongConverter.convertFrom(value); + } + }, + FLOAT { + @Override + Object convert(Object value) { + return FloatConverter.convertFrom(value); + } + }, + BOXED_FLOAT { + @Override + Object convert(Object value) { + return BoxedFloatConverter.convertFrom(value); + } + }, + DOUBLE { + @Override + Object convert(Object value) { + return DoubleConverter.convertFrom(value); + } + }, + BOXED_DOUBLE { + @Override + Object convert(Object value) { + return BoxedDoubleConverter.convertFrom(value); + } + }, + STRING { + @Override + Object convert(Object value) { + return StringConverter.convertFrom(value); + } + }; + + abstract Object convert(Object value); + } + private static boolean isDirectlyAssignable(Class from, Class to) { if (to.isAssignableFrom(from)) { return true; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java index 732ba4bac7..b45616510a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java @@ -383,10 +383,6 @@ public boolean hasForyField() { return hasForyField; } - public ForyField.Dynamic getForyFieldDynamic() { - return dynamic; - } - public boolean hasForyFieldId() { return hasForyField && foryFieldId >= 0; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java index 308cb8ce09..d23822b361 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorBuilder.java @@ -58,7 +58,7 @@ public DescriptorBuilder(Descriptor descriptor) { this.foryField = descriptor.getForyField(); this.hasForyField = descriptor.hasForyField(); this.foryFieldId = descriptor.getForyFieldId(); - this.dynamic = descriptor.getForyFieldDynamic(); + this.dynamic = descriptor.getMorphic(); this.arrayType = descriptor.isArrayType(); this.nullable = descriptor.isNullable(); this.trackingRef = descriptor.isTrackingRef(); @@ -135,11 +135,6 @@ public DescriptorBuilder foryField(ForyField foryField) { return this; } - public DescriptorBuilder arrayType(boolean arrayType) { - this.arrayType = arrayType; - return this; - } - public DescriptorBuilder fieldConverter(FieldConverter fieldConverter) { this.fieldConverter = fieldConverter; return this; diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index b0c9bf19f1..da9403d94d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -25,6 +25,7 @@ import static org.testng.Assert.assertTrue; import java.util.Arrays; +import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -94,6 +95,11 @@ static class DirectAnnotatedArrayField { @ArrayType List values; } + @Data + static class DirectCollectionField { + Collection<@Int32Type(encoding = Int32Encoding.FIXED) Integer> values; + } + @Data static class NestedListField { List> values; @@ -156,6 +162,20 @@ public void testTopLevelListAnnotatedArrayCompatibleRead() { assertEquals(annotatedArrayStruct.values, Arrays.asList(7, 8)); } + @Test + public void testTopLevelArrayCompatibleReadToCollection() { + for (boolean codegen : new boolean[] {false, true}) { + Fory arrayFory = compatibleFory(DirectArrayField.class, codegen); + DirectArrayField peerArrayStruct = new DirectArrayField(); + peerArrayStruct.values = new int[] {9, 10}; + + Fory collectionFory = compatibleFory(DirectCollectionField.class, codegen); + DirectCollectionField collectionStruct = + (DirectCollectionField) collectionFory.deserialize(arrayFory.serialize(peerArrayStruct)); + assertEquals(collectionStruct.values, Arrays.asList(9, 10)); + } + } + @Test public void testTopLevelListArrayCompatibleReadWithoutCodegen() { Fory listFory = compatibleFory(DirectListField.class, false); From 96cf1a7f66b74e17b7c2ecfec51dffed7d6e816a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 11:28:32 +0800 Subject: [PATCH 53/58] fix java tests --- .../test/java/org/apache/fory/xlang/MetaSharedXlangTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index da9403d94d..ac40486091 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -189,7 +189,7 @@ public void testTopLevelListArrayCompatibleReadWithoutCodegen() { } @Test - public void testNullableListPayloadRejectedForArrayCompatibleRead() { + public void testNullableListCompatibleReadToArrayRejectsNullElements() { for (boolean codegen : new boolean[] {false, true}) { Fory listFory = compatibleFory(DirectNullableListField.class, codegen); DirectNullableListField listStruct = new DirectNullableListField(); @@ -197,7 +197,8 @@ public void testNullableListPayloadRejectedForArrayCompatibleRead() { byte[] listBytes = listFory.serialize(listStruct); Fory arrayFory = compatibleFory(DirectArrayField.class, codegen); - assertThrows(DeserializationException.class, () -> arrayFory.deserialize(listBytes)); + DirectArrayField arrayStruct = (DirectArrayField) arrayFory.deserialize(listBytes); + assertTrue(Arrays.equals(arrayStruct.values, new int[] {1, 2, 3})); listStruct.values = Arrays.asList(1, null, 3); byte[] nullablePayload = listFory.serialize(listStruct); From 96a09fef971a10ae94963f4fad3fbca452c20454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 11:37:51 +0800 Subject: [PATCH 54/58] fix(go): read xlang compatible primitive arrays --- go/fory/field_info.go | 9 +++++---- go/fory/struct.go | 4 ++++ go/fory/type_resolver.go | 8 ++++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/go/fory/field_info.go b/go/fory/field_info.go index 9f00d79bc1..0e8348ed63 100644 --- a/go/fory/field_info.go +++ b/go/fory/field_info.go @@ -304,11 +304,12 @@ func fieldHasNonPrimitiveSerializer(field *FieldInfo) bool { if field.Serializer == nil { return false } - // ENUM (numeric ID), NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT - // all require special serialization and should not use the primitive fast path - // Note: ENUM uses unsigned VarUint32Small7 for ordinals, not signed zigzag varint + // Built-ins with primitive-shaped Go carriers still have protocol-owned payloads: + // ENUM ordinals are unsigned small varints, duration has nanos after seconds, + // temporal/decimal/binary values have their own layout, and named user types + // carry type metadata. switch TypeId(field.Meta.TypeId) { - case ENUM, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT: + case ENUM, DURATION, DATE, TIMESTAMP, DECIMAL, BINARY, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT: return true default: return false diff --git a/go/fory/struct.go b/go/fory/struct.go index 3f56cd16f0..c6a02aec7a 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -2350,6 +2350,8 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val storeFieldValue(field.Kind, fieldPtr, optInfo, buf.ReadFloat32(err)) case PrimitiveFloat64DispatchId: storeFieldValue(field.Kind, fieldPtr, optInfo, buf.ReadFloat64(err)) + case PrimitiveFloat16DispatchId: + storeFieldValue(field.Kind, fieldPtr, optInfo, buf.ReadUint16(err)) } return } @@ -2420,6 +2422,8 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val storeFieldValue(field.Kind, fieldPtr, optInfo, buf.ReadFloat32(err)) case NullableFloat64DispatchId: storeFieldValue(field.Kind, fieldPtr, optInfo, buf.ReadFloat64(err)) + case NullableFloat16DispatchId: + storeFieldValue(field.Kind, fieldPtr, optInfo, buf.ReadUint16(err)) } return } diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index cd651aa6e1..82bf185f84 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -1875,8 +1875,8 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s } // GetSliceSerializer returns the appropriate serializer for a slice type. -// For primitive element types (bool, int8, int16, int32, int64, uint8, float32, float64), -// it returns the dedicated primitive slice serializer that uses ARRAY protocol. +// For primitive element types, it returns the dedicated primitive slice serializer +// that uses ARRAY protocol. // For non-primitive element types, it returns sliceSerializer (LIST protocol). func (r *TypeResolver) GetSliceSerializer(sliceType reflect.Type) (Serializer, error) { if sliceType.Kind() != reflect.Slice { @@ -1905,6 +1905,10 @@ func (r *TypeResolver) GetSliceSerializer(sliceType reflect.Type) (Serializer, e return bfloat16SliceSerializer{}, nil } return uint16SliceSerializer{}, nil + case reflect.Uint32: + return uint32SliceSerializer{}, nil + case reflect.Uint64: + return uint64SliceSerializer{}, nil case reflect.Float32: return float32SliceSerializer{}, nil case reflect.Float64: From b1c99c1e61b32eb8ca82f996b34092597784bda2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 11:48:34 +0800 Subject: [PATCH 55/58] refactor(java): rename compatible serializers --- .../fory/builder/BaseObjectCodecBuilder.java | 16 +++---- .../org/apache/fory/builder/CodecUtils.java | 18 +++---- ...ilder.java => CompatibleCodecBuilder.java} | 34 +++++++------ ....java => CompatibleLayerCodecBuilder.java} | 20 ++++---- .../org/apache/fory/builder/Generated.java | 24 +++++----- .../builder/StaticCompatibleCodecBuilder.java | 14 +++--- .../java/org/apache/fory/meta/TypeDef.java | 8 ++-- .../apache/fory/resolver/ClassResolver.java | 6 +-- .../apache/fory/resolver/TypeResolver.java | 48 +++++++++---------- ...er.java => CompatibleLayerSerializer.java} | 8 ++-- ...ava => CompatibleLayerSerializerBase.java} | 6 +-- ...ializer.java => CompatibleSerializer.java} | 30 ++++++------ .../fory/serializer/ExceptionSerializers.java | 13 ++--- .../serializer/ObjectStreamSerializer.java | 46 +++++++++--------- .../apache/fory/serializer/Serializers.java | 4 +- .../StaticGeneratedStructSerializer.java | 2 +- .../collection/ChildContainerSerializers.java | 15 +++--- .../fory-core/native-image.properties | 8 ++-- .../test/java/org/apache/fory/CyclicTest.java | 2 +- .../java/org/apache/fory/ForyTestBase.java | 2 +- .../test/java/org/apache/fory/StreamTest.java | 2 +- .../apache/fory/builder/JITContextTest.java | 2 +- .../StaticCompatibleCodecBuilderTest.java | 8 ++-- .../fory/resolver/MetaShareContextTest.java | 8 ++-- .../serializer/AndroidJvmRoundTripTest.java | 12 ++--- .../CodegenCompatibleSerializerTest.java | 2 +- .../serializer/CompatibleSerializerTest.java | 2 +- .../fory/serializer/DuplicateFieldsTest.java | 8 ++-- ...inalFieldReplaceResolveSerializerTest.java | 2 +- ...Test.java => MetaShareCompatibleTest.java} | 44 ++++++++--------- ...ava => MetaShareObjectSerializerTest.java} | 2 +- .../ChildContainerSerializersTest.java | 2 +- ...XlangTest.java => MetaShareXlangTest.java} | 6 +-- .../ExampleStaticGeneratedSerializerTest.java | 8 ++-- .../RecordSerializersTest.java | 2 +- 35 files changed, 216 insertions(+), 218 deletions(-) rename java/fory-core/src/main/java/org/apache/fory/builder/{MetaSharedCodecBuilder.java => CompatibleCodecBuilder.java} (90%) rename java/fory-core/src/main/java/org/apache/fory/builder/{MetaSharedLayerCodecBuilder.java => CompatibleLayerCodecBuilder.java} (94%) rename java/fory-core/src/main/java/org/apache/fory/serializer/{MetaSharedLayerSerializer.java => CompatibleLayerSerializer.java} (82%) rename java/fory-core/src/main/java/org/apache/fory/serializer/{MetaSharedLayerSerializerBase.java => CompatibleLayerSerializerBase.java} (98%) rename java/fory-core/src/main/java/org/apache/fory/serializer/{MetaSharedSerializer.java => CompatibleSerializer.java} (94%) rename java/fory-core/src/test/java/org/apache/fory/serializer/{MetaSharedCompatibleTest.java => MetaShareCompatibleTest.java} (96%) rename java/fory-core/src/test/java/org/apache/fory/serializer/{MetaSharedObjectSerializerTest.java => MetaShareObjectSerializerTest.java} (97%) rename java/fory-core/src/test/java/org/apache/fory/xlang/{MetaSharedXlangTest.java => MetaShareXlangTest.java} (98%) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 639b23907a..b50f83b323 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -129,10 +129,10 @@ import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.CompatibleSerializer; import org.apache.fory.serializer.DeferedLazySerializer.DeferredLazyObjectSerializer; import org.apache.fory.serializer.EnumSerializer; import org.apache.fory.serializer.FinalFieldReplaceResolveSerializer; -import org.apache.fory.serializer.MetaSharedSerializer; import org.apache.fory.serializer.ObjectSerializer; import org.apache.fory.serializer.PrimitiveSerializers.LongSerializer; import org.apache.fory.serializer.ReplaceResolveSerializer; @@ -425,7 +425,7 @@ protected void addCommonImports() { ctx.addImport(Generated.class); ctx.addImports(LazyInitBeanSerializer.class, EnumSerializer.class); ctx.addImports(Serializer.class, StringSerializer.class); - ctx.addImports(ObjectSerializer.class, MetaSharedSerializer.class); + ctx.addImports(ObjectSerializer.class, CompatibleSerializer.class); ctx.addImports(CollectionLikeSerializer.class, MapLikeSerializer.class); } @@ -960,7 +960,7 @@ private Expression getOrCreateSerializer(Class cls, boolean isField) { } if (serializerClass == LazyInitBeanSerializer.class || serializerClass == ObjectSerializer.class - || serializerClass == MetaSharedSerializer.class + || serializerClass == CompatibleSerializer.class || serializerClass == DeferredLazyObjectSerializer.class) { // field init may get jit serializer, which will cause cast exception if not use base // type. @@ -2222,23 +2222,23 @@ protected Expression deserializeCompatibleListArrayField(Descriptor descriptor) Class targetType = descriptor.getField() == null ? descriptor.getRawType() : descriptor.getField().getType(); return new StaticInvoke( - MetaSharedSerializer.class, + CompatibleSerializer.class, "readCompatibleCollectionArrayField", OBJECT_TYPE, readContextRef(), Literal.ofBoolean(trackingRef), Literal.ofBoolean(nullable), Literal.ofInt( - MetaSharedSerializer.compatibleCollectionArrayReadMode(typeResolver, descriptor)), + CompatibleSerializer.compatibleCollectionArrayReadMode(typeResolver, descriptor)), Literal.ofInt( - MetaSharedSerializer.compatibleCollectionArrayTypeId(typeResolver, descriptor)), + CompatibleSerializer.compatibleCollectionArrayTypeId(typeResolver, descriptor)), Literal.ofInt( - MetaSharedSerializer.compatibleCollectionElementTypeId(typeResolver, descriptor)), + CompatibleSerializer.compatibleCollectionElementTypeId(typeResolver, descriptor)), Literal.ofClass(targetType)); } protected boolean hasCompatibleCollectionArrayRead(Descriptor descriptor) { - return MetaSharedSerializer.hasCompatibleCollectionArrayRead(typeResolver, descriptor); + return CompatibleSerializer.hasCompatibleCollectionArrayRead(typeResolver, descriptor); } protected TypeRef compatibleReadTargetTypeRef(Descriptor descriptor) { diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index 5856e58460..c9092f6723 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -53,23 +53,23 @@ public static Class loadOrGenObjectCodecClass(Class () -> loadOrGenCodecClass(cls, fory, new ObjectCodecBuilder(cls, fory))); } - public static Class loadOrGenMetaSharedCodecClass( + public static Class loadOrGenCompatibleCodecClass( Fory fory, Class cls, TypeDef typeDef) { Preconditions.checkNotNull(fory); return loadSerializer( - "loadOrGenMetaSharedCodecClass", + "loadOrGenCompatibleCodecClass", cls, fory, () -> loadOrGenCodecClass( - cls, fory, new MetaSharedCodecBuilder(TypeRef.of(cls), fory, typeDef))); + cls, fory, new CompatibleCodecBuilder(TypeRef.of(cls), fory, typeDef))); } - public static Class loadOrGenMetaSharedCodecClass( + public static Class loadOrGenCompatibleCodecClass( TypeResolver typeResolver, Class cls, TypeDef typeDef) { return typeResolver .getJITContext() - .asyncVisitFory(f -> loadOrGenMetaSharedCodecClass(f, cls, typeDef)); + .asyncVisitFory(f -> loadOrGenCompatibleCodecClass(f, cls, typeDef)); } public static Class loadOrGenStaticCompatibleCodecClass( @@ -92,7 +92,7 @@ public static Class loadOrGenStaticCompatibleCodecClas } /** - * Load or generate a JIT serializer class for single-layer meta-shared serialization. + * Load or generate a JIT serializer class for single-layer compatible serialization. * * @param cls the target class * @param fory the Fory instance @@ -100,18 +100,18 @@ public static Class loadOrGenStaticCompatibleCodecClas * @param layerMarkerClass the marker class for this layer * @return the generated serializer class */ - public static Class loadOrGenMetaSharedLayerCodecClass( + public static Class loadOrGenCompatibleLayerCodecClass( Class cls, Fory fory, TypeDef layerTypeDef, Class layerMarkerClass) { Preconditions.checkNotNull(fory); return loadSerializer( - "loadOrGenMetaSharedLayerCodecClass", + "loadOrGenCompatibleLayerCodecClass", cls, fory, () -> loadOrGenCodecClass( cls, fory, - new MetaSharedLayerCodecBuilder( + new CompatibleLayerCodecBuilder( TypeRef.of(cls), fory, layerTypeDef, layerMarkerClass))); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java similarity index 90% rename from java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java rename to java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java index 3fdc1c6dbc..64fe39cc11 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleCodecBuilder.java @@ -19,7 +19,7 @@ package org.apache.fory.builder; -import static org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer.SERIALIZER_FIELD_NAME; +import static org.apache.fory.builder.Generated.GeneratedCompatibleSerializer.SERIALIZER_FIELD_NAME; import static org.apache.fory.type.TypeUtils.OBJECT_TYPE; import static org.apache.fory.type.TypeUtils.STRING_TYPE; @@ -30,7 +30,7 @@ import java.util.SortedMap; import java.util.concurrent.ConcurrentHashMap; import org.apache.fory.Fory; -import org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer; +import org.apache.fory.builder.Generated.GeneratedCompatibleSerializer; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Literal; @@ -45,7 +45,7 @@ import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.CodegenSerializer; -import org.apache.fory.serializer.MetaSharedSerializer; +import org.apache.fory.serializer.CompatibleSerializer; import org.apache.fory.serializer.ObjectSerializer; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.Serializers; @@ -61,29 +61,27 @@ import org.apache.fory.util.record.RecordUtils; /** - * A meta-shared compatible deserializer builder based on {@link TypeDef}. This builder will compare - * fields between {@link TypeDef} and class fields, then create serializer to read and set/skip - * corresponding fields to support type forward/backward compatibility. Serializer are forward to - * {@link ObjectCodecBuilder} for now. We can consolidate fields between peers to create better - * serializers to serialize common fields between peers for efficiency. + * A compatible deserializer builder based on shared {@link TypeDef} metadata. This builder compares + * remote fields with local class fields, then generates code to read, set, or skip fields for type + * forward/backward compatibility. Writes are delegated to {@link ObjectCodecBuilder} for now. * *

    With meta context share enabled and compatible mode, the {@link ObjectCodecBuilder} will take * all non-inner final types as non-final, so that fory can write class definition when write class * info for those types. * * @see ForyBuilder#withMetaShare - * @see GeneratedMetaSharedSerializer - * @see MetaSharedSerializer + * @see GeneratedCompatibleSerializer + * @see CompatibleSerializer */ -public class MetaSharedCodecBuilder extends ObjectCodecBuilder { - private static final Logger LOG = LoggerFactory.getLogger(MetaSharedCodecBuilder.class); +public class CompatibleCodecBuilder extends ObjectCodecBuilder { + private static final Logger LOG = LoggerFactory.getLogger(CompatibleCodecBuilder.class); private final TypeDef typeDef; private final String defaultValueLanguage; private final DefaultValueUtils.DefaultValueField[] defaultValueFields; - public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { - super(beanType, fory, GeneratedMetaSharedSerializer.class); + public CompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { + super(beanType, fory, GeneratedCompatibleSerializer.class); Preconditions.checkArgument( !fory.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); @@ -146,7 +144,7 @@ protected String codecSuffix() { id = idGenerator.computeIfAbsent(typeDef.getId(), k -> idGenerator.size()); } } - return "MetaShared" + id; + return "Compatible" + id; } @Override @@ -177,7 +175,7 @@ public String genCode() { "serializer", SERIALIZER_FIELD_NAME, "builderClass", - MetaSharedCodecBuilder.class.getName(), + CompatibleCodecBuilder.class.getName(), "typeResolver", CONSTRUCTOR_TYPE_RESOLVER_NAME, "cls", @@ -213,13 +211,13 @@ public String genCode() { @Override protected void addCommonImports() { super.addCommonImports(); - ctx.addImport(GeneratedMetaSharedSerializer.class); + ctx.addImport(GeneratedCompatibleSerializer.class); } // Invoked by JIT. @SuppressWarnings({"unchecked", "rawtypes"}) public static Serializer setCodegenSerializer( - TypeResolver typeResolver, Class cls, GeneratedMetaSharedSerializer s) { + TypeResolver typeResolver, Class cls, GeneratedCompatibleSerializer s) { if (GraalvmSupport.isGraalRuntime()) { return typeResolver .getJITContext() diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleLayerCodecBuilder.java similarity index 94% rename from java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java rename to java/fory-core/src/main/java/org/apache/fory/builder/CompatibleLayerCodecBuilder.java index 9503a312cf..0c030e7032 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CompatibleLayerCodecBuilder.java @@ -24,7 +24,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.fory.Fory; -import org.apache.fory.builder.Generated.GeneratedMetaSharedLayerSerializer; +import org.apache.fory.builder.Generated.GeneratedCompatibleLayerSerializer; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.ListExpression; @@ -44,24 +44,24 @@ import org.apache.fory.util.StringUtils; /** - * A JIT codec builder for single-layer meta-shared serialization. This builder generates optimized + * A JIT codec builder for single-layer compatible serialization. This builder generates optimized * serializers that only handle fields from a specific class layer, without including parent class * fields. * *

    This is used by {@link org.apache.fory.serializer.ObjectStreamSerializer} to generate JIT * serializers for each layer in the class hierarchy. * - * @see org.apache.fory.serializer.MetaSharedLayerSerializer - * @see MetaSharedCodecBuilder - * @see GeneratedMetaSharedLayerSerializer + * @see org.apache.fory.serializer.CompatibleLayerSerializer + * @see CompatibleCodecBuilder + * @see GeneratedCompatibleLayerSerializer */ -public class MetaSharedLayerCodecBuilder extends ObjectCodecBuilder { +public class CompatibleLayerCodecBuilder extends ObjectCodecBuilder { private final TypeDef layerTypeDef; private final Class layerMarkerClass; - public MetaSharedLayerCodecBuilder( + public CompatibleLayerCodecBuilder( TypeRef beanType, Fory fory, TypeDef layerTypeDef, Class layerMarkerClass) { - super(beanType, fory, GeneratedMetaSharedLayerSerializer.class); + super(beanType, fory, GeneratedCompatibleLayerSerializer.class); Preconditions.checkArgument( !fory.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); @@ -82,7 +82,7 @@ protected String codecSuffix() { id = idGenerator.computeIfAbsent(layerTypeDef.getId(), k -> idGenerator.size()); } } - return "MetaSharedLayer" + id; + return "CompatibleLayer" + id; } @Override @@ -178,7 +178,7 @@ public String genCode() { @Override protected void addCommonImports() { super.addCommonImports(); - ctx.addImport(GeneratedMetaSharedLayerSerializer.class); + ctx.addImport(GeneratedCompatibleLayerSerializer.class); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java index e63bc95fbb..327f9cefbd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java @@ -31,7 +31,7 @@ import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.AbstractObjectSerializer; -import org.apache.fory.serializer.MetaSharedLayerSerializerBase; +import org.apache.fory.serializer.CompatibleLayerSerializerBase; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.StaticGeneratedStructSerializer; import org.apache.fory.type.Descriptor; @@ -106,13 +106,13 @@ public GeneratedObjectSerializer(TypeResolver typeResolver, Class cls) { } /** Base class for all serializers with meta shared by {@link TypeDef}. */ - abstract class GeneratedMetaSharedSerializer extends GeneratedSerializer implements Generated { + abstract class GeneratedCompatibleSerializer extends GeneratedSerializer implements Generated { public static final String SERIALIZER_FIELD_NAME = "serializer"; - /** Will be set in generated constructor by {@link MetaSharedCodecBuilder}. */ + /** Will be set in generated constructor by {@link CompatibleCodecBuilder}. */ public Serializer serializer; - public GeneratedMetaSharedSerializer(TypeResolver typeResolver, Class cls) { + public GeneratedCompatibleSerializer(TypeResolver typeResolver, Class cls) { super(typeResolver, cls); } @@ -123,9 +123,9 @@ public void write(WriteContext writeContext, Object value) { } /** Base class for GraalVM build-time compatible read serializers. */ - abstract class GeneratedCompatibleMetaSharedSerializer - extends StaticGeneratedStructSerializer implements Generated { - public GeneratedCompatibleMetaSharedSerializer( + abstract class GeneratedStaticCompatibleSerializer extends StaticGeneratedStructSerializer + implements Generated { + public GeneratedStaticCompatibleSerializer( TypeResolver typeResolver, Class cls, TypeDef typeDef, List descriptors) { super(typeResolver, cls, typeDef, descriptors); } @@ -138,24 +138,24 @@ public final Object read(ReadContext readContext) { @Override public void write(WriteContext writeContext, Object value) { throw new UnsupportedOperationException( - "GraalVM compatible meta-shared serializers are read-only"); + "GraalVM static compatible serializers are read-only"); } @Override public Object copy(CopyContext copyContext, Object value) { throw new UnsupportedOperationException( - "GraalVM compatible meta-shared serializers do not implement copy"); + "GraalVM static compatible serializers do not implement copy"); } } /** * Base class for layer serializers with meta shared by {@link TypeDef}. Unlike {@link - * GeneratedMetaSharedSerializer}, this serializer only handles fields from a single class layer + * GeneratedCompatibleSerializer}, this serializer only handles fields from a single class layer * and does not include parent class fields. */ - abstract class GeneratedMetaSharedLayerSerializer extends MetaSharedLayerSerializerBase + abstract class GeneratedCompatibleLayerSerializer extends CompatibleLayerSerializerBase implements Generated { - public GeneratedMetaSharedLayerSerializer(TypeResolver typeResolver, Class cls) { + public GeneratedCompatibleLayerSerializer(TypeResolver typeResolver, Class cls) { super(typeResolver, cls); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 5c52e8d88b..9bf3f7682d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -25,7 +25,7 @@ import java.util.List; import org.apache.fory.Fory; import org.apache.fory.annotation.ForyStruct; -import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.builder.Generated.GeneratedStaticCompatibleSerializer; import org.apache.fory.codegen.Code; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; @@ -36,8 +36,8 @@ import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.CompatibleSerializer; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; -import org.apache.fory.serializer.MetaSharedSerializer; import org.apache.fory.type.Descriptor; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; @@ -51,10 +51,10 @@ *

    The generated class is keyed by the local Java class, not by a fixed remote schema. Its * constructor receives the runtime remote {@link TypeDef}; {@link * org.apache.fory.serializer.StaticGeneratedStructSerializer} rebuilds remote read order through - * the same descriptor-grouper owner used by {@link MetaSharedSerializer}. + * the same descriptor-grouper owner used by {@link CompatibleSerializer}. * * @see ForyBuilder#withMetaShare - * @see MetaSharedSerializer + * @see CompatibleSerializer */ public final class StaticCompatibleCodecBuilder extends ObjectCodecBuilder { private static final int DISPATCH_GROUP_SIZE = 8; @@ -63,7 +63,7 @@ public final class StaticCompatibleCodecBuilder extends ObjectCodecBuilder { private final boolean debug; public StaticCompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { - super(beanType, fory, GeneratedCompatibleMetaSharedSerializer.class); + super(beanType, fory, GeneratedStaticCompatibleSerializer.class); Preconditions.checkArgument( !fory.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); @@ -74,7 +74,7 @@ public StaticCompatibleCodecBuilder(TypeRef beanType, Fory fory, TypeDef type @Override protected String codecSuffix() { - return "CompatibleMetaShared"; + return "StaticCompatible"; } @Override @@ -122,7 +122,7 @@ public String genCode() { @Override protected void addCommonImports() { super.addCommonImports(); - ctx.addImport(GeneratedCompatibleMetaSharedSerializer.class); + ctx.addImport(GeneratedStaticCompatibleSerializer.class); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java index a25536ea64..4a72cb78d0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java @@ -32,7 +32,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import org.apache.fory.builder.MetaSharedCodecBuilder; +import org.apache.fory.builder.CompatibleCodecBuilder; import org.apache.fory.config.ForyBuilder; import org.apache.fory.exception.DeserializationException; import org.apache.fory.logging.Logger; @@ -44,7 +44,7 @@ import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; -import org.apache.fory.serializer.MetaSharedSerializer; +import org.apache.fory.serializer.CompatibleSerializer; import org.apache.fory.serializer.UnknownClass; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorBuilder; @@ -63,9 +63,9 @@ *

  • {@link ObjectStreamClass} doesn't contain any non-primitive field type info, which is not * enough to create serializer in receiver. * - * @see MetaSharedCodecBuilder + * @see CompatibleCodecBuilder * @see ForyBuilder#withCompatible(boolean) - * @see MetaSharedSerializer + * @see CompatibleSerializer * @see ForyBuilder#withMetaShare * @see ReflectionUtils#getFieldOffset */ diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 67797d8921..dd300ba605 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1790,14 +1790,14 @@ private boolean needsGraalvmObjectSerializerClass( || serializerClass == MapSerializers.DefaultJavaMapSerializer.class; } - private Class getMetaSharedDeserializerClassForGraalvmBuild( + private Class getCompatibleDeserializerClassForGraalvmBuild( Class cls, TypeDef typeDef) { Class serializerClass = getGraalvmClassRegistry().getDeserializerClass(typeDef.getId()); if (serializerClass != null) { return serializerClass; } - return CodecUtils.loadOrGenMetaSharedCodecClass(this, cls, typeDef); + return CodecUtils.loadOrGenCompatibleCodecClass(this, cls, typeDef); } private void registerGraalvmSerializerClass(Class cls) { @@ -1819,7 +1819,7 @@ private void registerGraalvmSerializerClass(Class cls) { } getGraalvmClassRegistry() .putDeserializerClass( - typeDef.getId(), getMetaSharedDeserializerClassForGraalvmBuild(cls, typeDef)); + typeDef.getId(), getCompatibleDeserializerClassForGraalvmBuild(cls, typeDef)); getGraalvmClassRegistry() .putCompatibleDeserializerClass( cls, CodecUtils.loadOrGenStaticCompatibleCodecClass(this, cls, typeDef)); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 2d9de20963..9be209a86e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -47,9 +47,9 @@ import org.apache.fory.annotation.ForyStruct.Evolution; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.CodecUtils; -import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; -import org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer; +import org.apache.fory.builder.Generated.GeneratedCompatibleSerializer; import org.apache.fory.builder.Generated.GeneratedObjectSerializer; +import org.apache.fory.builder.Generated.GeneratedStaticCompatibleSerializer; import org.apache.fory.builder.JITContext; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; @@ -83,7 +83,7 @@ import org.apache.fory.reflect.TypeRef; import org.apache.fory.serializer.CodegenSerializer; import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; -import org.apache.fory.serializer.MetaSharedSerializer; +import org.apache.fory.serializer.CompatibleSerializer; import org.apache.fory.serializer.ObjectSerializer; import org.apache.fory.serializer.PrimitiveSerializers; import org.apache.fory.serializer.Serializer; @@ -965,7 +965,7 @@ final TypeInfo buildMetaSharedTypeInfo(TypeDef typeDef) { typeInfo = getTypeInfo(cls); } else if (ClassResolver.useReplaceResolveSerializer(cls)) { // For classes with writeReplace/readResolve, use their natural serializer - // (ReplaceResolveSerializer) instead of MetaSharedSerializer + // (ReplaceResolveSerializer) instead of CompatibleSerializer typeInfo = getTypeInfo(cls); } else { typeInfo = getMetaSharedTypeInfo(typeDef, cls); @@ -1029,17 +1029,17 @@ private TypeInfo getMetaSharedTypeInfo(TypeDef typeDef, Class clz) { return getTypeInfo(cls); } Class sc = - getMetaSharedDeserializerClassFromGraalvmRegistry(cls, typeDef); + getCompatibleDeserializerClassFromGraalvmRegistry(cls, typeDef); if (sc == null) { if (GraalvmSupport.isGraalBuildTime() && config.isCodeGenEnabled()) { - sc = loadGraalvmMetaSharedDeserializerClass(cls, typeDef); + sc = loadGraalvmCompatibleDeserializerClass(cls, typeDef); } else if (AndroidSupport.IS_ANDROID || !config.isCodeGenEnabled()) { sc = getStaticGeneratedStructSerializerClass(cls); } if (sc == null && AndroidSupport.IS_ANDROID) { - sc = MetaSharedSerializer.class; + sc = CompatibleSerializer.class; } else if (sc == null && GraalvmSupport.isGraalRuntime()) { - sc = MetaSharedSerializer.class; + sc = CompatibleSerializer.class; LOG.warn( "Can't generate class at runtime in graalvm for class def {}, use {} instead", typeDef, @@ -1047,39 +1047,39 @@ private TypeInfo getMetaSharedTypeInfo(TypeDef typeDef, Class clz) { } else if (sc == null && config.isCodeGenEnabled()) { sc = jitContext.registerSerializerJITCallback( - () -> MetaSharedSerializer.class, - () -> CodecUtils.loadOrGenMetaSharedCodecClass(this, cls, typeDef), + () -> CompatibleSerializer.class, + () -> CodecUtils.loadOrGenCompatibleCodecClass(this, cls, typeDef), c -> typeInfo.setSerializer(this, Serializers.newSerializer(this, cls, c))); } else if (sc == null) { - sc = MetaSharedSerializer.class; + sc = CompatibleSerializer.class; } } if (GraalvmSupport.isGraalBuildTime() - && GeneratedMetaSharedSerializer.class.isAssignableFrom(sc)) { + && GeneratedCompatibleSerializer.class.isAssignableFrom(sc)) { getGraalvmClassRegistry().putIfAbsentDeserializerClass(typeDef.getId(), sc); - typeInfo.setSerializer(this, new MetaSharedSerializer(this, cls, typeDef)); + typeInfo.setSerializer(this, new CompatibleSerializer(this, cls, typeDef)); return typeInfo; } if (GraalvmSupport.isGraalBuildTime() - && GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(sc)) { + && GeneratedStaticCompatibleSerializer.class.isAssignableFrom(sc)) { getGraalvmClassRegistry().putCompatibleDeserializerClass(cls, sc); - typeInfo.setSerializer(this, new MetaSharedSerializer(this, cls, typeDef)); + typeInfo.setSerializer(this, new CompatibleSerializer(this, cls, typeDef)); return typeInfo; } if (StaticGeneratedStructSerializer.class.isAssignableFrom(sc)) { typeInfo.setSerializer(this, newStaticGeneratedStructSerializer(sc, cls, typeDef)); - } else if (sc == MetaSharedSerializer.class) { - typeInfo.setSerializer(this, new MetaSharedSerializer(this, cls, typeDef)); + } else if (sc == CompatibleSerializer.class) { + typeInfo.setSerializer(this, new CompatibleSerializer(this, cls, typeDef)); } else { typeInfo.setSerializer(this, Serializers.newSerializer(this, cls, sc)); } return typeInfo; } - private Class loadGraalvmMetaSharedDeserializerClass( + private Class loadGraalvmCompatibleDeserializerClass( Class cls, TypeDef typeDef) { if (typeDef.getId() == TypeDef.buildTypeDef(this, cls).getId()) { - return CodecUtils.loadOrGenMetaSharedCodecClass(this, cls, typeDef); + return CodecUtils.loadOrGenCompatibleCodecClass(this, cls, typeDef); } return CodecUtils.loadOrGenStaticCompatibleCodecClass(this, cls, typeDef); } @@ -1131,19 +1131,19 @@ private Evolution getStructEvolution(Class cls) { protected static boolean isStructSerializer(Serializer serializer) { return serializer instanceof GeneratedObjectSerializer - || serializer instanceof GeneratedMetaSharedSerializer + || serializer instanceof GeneratedCompatibleSerializer || serializer instanceof LazyInitBeanSerializer || serializer instanceof ObjectSerializer - || serializer instanceof MetaSharedSerializer + || serializer instanceof CompatibleSerializer || serializer instanceof StaticGeneratedStructSerializer; } protected static boolean isStructSerializerClass(Class serializerClass) { return GeneratedObjectSerializer.class.isAssignableFrom(serializerClass) - || GeneratedMetaSharedSerializer.class.isAssignableFrom(serializerClass) + || GeneratedCompatibleSerializer.class.isAssignableFrom(serializerClass) || LazyInitBeanSerializer.class.isAssignableFrom(serializerClass) || ObjectSerializer.class.isAssignableFrom(serializerClass) - || MetaSharedSerializer.class.isAssignableFrom(serializerClass) + || CompatibleSerializer.class.isAssignableFrom(serializerClass) || StaticGeneratedStructSerializer.class.isAssignableFrom(serializerClass); } @@ -2048,7 +2048,7 @@ final Class getSerializerClassFromGraalvmRegistry(Class return null; } - protected final Class getMetaSharedDeserializerClassFromGraalvmRegistry( + protected final Class getCompatibleDeserializerClassFromGraalvmRegistry( Class cls, TypeDef typeDef) { GraalvmSupport.GraalvmClassRegistry registry = getGraalvmClassRegistry(); Class deserializerClass = registry.getDeserializerClass(typeDef.getId()); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializer.java similarity index 82% rename from java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java rename to java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializer.java index 7211b7f800..e99c1b1807 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializer.java @@ -23,13 +23,13 @@ import org.apache.fory.resolver.TypeResolver; /** - * Interpreter implementation for a single meta-shared class layer. Generated layer serializers - * extend {@link MetaSharedLayerSerializerBase} directly and override only the hot field read/write + * Interpreter implementation for a single compatible class layer. Generated layer serializers + * extend {@link CompatibleLayerSerializerBase} directly and override only the hot field read/write * paths. */ -public class MetaSharedLayerSerializer extends MetaSharedLayerSerializerBase { +public class CompatibleLayerSerializer extends CompatibleLayerSerializerBase { - public MetaSharedLayerSerializer( + public CompatibleLayerSerializer( TypeResolver typeResolver, Class type, TypeDef layerTypeDef, Class layerMarkerClass) { super(typeResolver, type); setLayerSerializerMeta(layerTypeDef, layerMarkerClass); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java similarity index 98% rename from java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java rename to java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java index 6957e4a4ec..4739516b4d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleLayerSerializerBase.java @@ -36,18 +36,18 @@ import org.apache.fory.util.Preconditions; /** - * Base class for meta-shared layer serializers. The default implementation uses reflection-backed + * Base class for compatible layer serializers. The default implementation uses reflection-backed * field access and generated layer serializers override only the hot field read/write methods. */ @SuppressWarnings({"unchecked", "rawtypes"}) -public abstract class MetaSharedLayerSerializerBase extends AbstractObjectSerializer { +public abstract class CompatibleLayerSerializerBase extends AbstractObjectSerializer { protected TypeDef layerTypeDef; protected Class layerMarkerClass; protected SerializationFieldInfo[] buildInFields = new SerializationFieldInfo[0]; protected SerializationFieldInfo[] otherFields = new SerializationFieldInfo[0]; protected SerializationFieldInfo[] containerFields = new SerializationFieldInfo[0]; - public MetaSharedLayerSerializerBase(TypeResolver typeResolver, Class type) { + public CompatibleLayerSerializerBase(TypeResolver typeResolver, Class type) { super(typeResolver, type); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java similarity index 94% rename from java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java rename to java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java index 7bc49be249..b45083d333 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java @@ -22,7 +22,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import org.apache.fory.builder.MetaSharedCodecBuilder; +import org.apache.fory.builder.CompatibleCodecBuilder; import org.apache.fory.config.ForyBuilder; import org.apache.fory.context.ReadContext; import org.apache.fory.context.RefReader; @@ -51,22 +51,20 @@ import org.apache.fory.util.record.RecordUtils; /** - * A meta-shared compatible deserializer builder based on {@link TypeDef}. This serializer will - * compare fields between {@link TypeDef} and class fields, then create serializer to read and - * set/skip corresponding fields to support type forward/backward compatibility. Serializer are - * forward to {@link ObjectSerializer} for now. We can consolidate fields between peers to create - * better serializers to serialize common fields between peers for efficiency. + * A compatible deserializer based on shared {@link TypeDef} metadata. This serializer compares + * remote fields with local class fields, then reads, sets, or skips fields to support type + * forward/backward compatibility. Writes are delegated to {@link ObjectSerializer} for now. * *

    With meta context share enabled and compatible mode, the {@link ObjectSerializer} will take * all non-inner final types as non-final, so that fory can write class definition when write class * info for those types. * * @see ForyBuilder#withMetaShare - * @see MetaSharedCodecBuilder + * @see CompatibleCodecBuilder * @see ObjectSerializer */ -public class MetaSharedSerializer extends AbstractObjectSerializer { - private static final Logger LOG = LoggerFactory.getLogger(MetaSharedSerializer.class); +public class CompatibleSerializer extends AbstractObjectSerializer { + private static final Logger LOG = LoggerFactory.getLogger(CompatibleSerializer.class); private final SerializationFieldInfo[] buildInFields; private final SerializationFieldInfo[] containerFields; @@ -80,14 +78,14 @@ public class MetaSharedSerializer extends AbstractObjectSerializer { private final boolean hasDefaultValues; private final DefaultValueUtils.DefaultValueField[] defaultValueFields; - public MetaSharedSerializer(TypeResolver typeResolver, Class type, TypeDef typeDef) { + public CompatibleSerializer(TypeResolver typeResolver, Class type, TypeDef typeDef) { super(typeResolver, type); Preconditions.checkArgument( !config.checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); Preconditions.checkArgument(config.isMetaShareEnabled(), "Meta share must be enabled."); if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("========== MetaSharedSerializer TypeDef for {} ==========", type.getName()); + LOG.info("========== CompatibleSerializer TypeDef for {} ==========", type.getName()); LOG.info("TypeDef fieldsInfo count: {}", typeDef.getFieldCount()); for (int i = 0; i < typeDef.getFieldsInfo().size(); i++) { LOG.info(" [{}] {}", i, typeDef.getFieldsInfo().get(i)); @@ -96,7 +94,7 @@ public MetaSharedSerializer(TypeResolver typeResolver, Class type, TypeDef ty DescriptorGrouper descriptorGrouper = typeResolver.createDescriptorGrouper(typeDef, type); if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info( - "========== MetaSharedSerializer sorted descriptors for {} ==========", type.getName()); + "========== CompatibleSerializer sorted descriptors for {} ==========", type.getName()); for (Descriptor d : descriptorGrouper.getSortedDescriptors()) { LOG.info( " {} -> {}, ref {}, nullable {}, type id {}", @@ -156,7 +154,7 @@ public MetaSharedSerializer(TypeResolver typeResolver, Class type, TypeDef ty this.defaultValueFields = defaultValueFields; } - /** Used by generated meta-shared serializers for top-level list/array compatible field reads. */ + /** Used by generated compatible serializers for top-level list/array compatible field reads. */ public static Object readCompatibleCollectionArrayField( ReadContext readContext, boolean trackingRef, @@ -174,18 +172,18 @@ public static Object readCompatibleCollectionArrayField( targetType); } - /** Used by generated meta-shared serializers to cache a top-level list/array read action. */ + /** Used by generated compatible serializers to cache a top-level list/array read action. */ public static int compatibleCollectionArrayReadMode( TypeResolver resolver, Descriptor descriptor) { return requireCompatibleCollectionArrayReadAction(resolver, descriptor).mode; } - /** Used by generated meta-shared serializers to cache the dense array carrier type. */ + /** Used by generated compatible serializers to cache the dense array carrier type. */ public static int compatibleCollectionArrayTypeId(TypeResolver resolver, Descriptor descriptor) { return requireCompatibleCollectionArrayReadAction(resolver, descriptor).arrayTypeId; } - /** Used by generated meta-shared serializers to cache the peer or local element type. */ + /** Used by generated compatible serializers to cache the peer or local element type. */ public static int compatibleCollectionElementTypeId( TypeResolver resolver, Descriptor descriptor) { return requireCompatibleCollectionArrayReadAction(resolver, descriptor).elementTypeId; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java index 639b6a4079..5efb771ff6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java @@ -392,8 +392,8 @@ private static boolean hasSubclassFields(Serializer[] slotsSerializers) { if (((ObjectSerializer) slotsSerializer).getNumFields() > 0) { return true; } - } else if (slotsSerializer instanceof MetaSharedLayerSerializerBase) { - if (((MetaSharedLayerSerializerBase) slotsSerializer).getNumFields() > 0) { + } else if (slotsSerializer instanceof CompatibleLayerSerializerBase) { + if (((CompatibleLayerSerializerBase) slotsSerializer).getNumFields() > 0) { return true; } } @@ -411,7 +411,7 @@ private static Serializer[] buildSlotsSerializers(TypeResolver typeResolver, TypeDef layerTypeDef = typeResolver.getTypeDef(type, false); Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(type, layerIndex); slotsSerializer = - new MetaSharedLayerSerializer(typeResolver, type, layerTypeDef, layerMarkerClass); + new CompatibleLayerSerializer(typeResolver, type, layerTypeDef, layerMarkerClass); } else { slotsSerializer = new ObjectSerializer<>(typeResolver, type, false); } @@ -426,12 +426,13 @@ private static Serializer[] buildSlotsSerializers(TypeResolver typeResolver, private static void readAndSetFields( ReadContext readContext, Object target, Serializer[] slotsSerializers, Config config) { for (Serializer slotsSerializer : slotsSerializers) { - if (slotsSerializer instanceof MetaSharedLayerSerializer) { - MetaSharedLayerSerializer metaSerializer = (MetaSharedLayerSerializer) slotsSerializer; + if (slotsSerializer instanceof CompatibleLayerSerializer) { + CompatibleLayerSerializer compatibleSerializer = + (CompatibleLayerSerializer) slotsSerializer; if (config.isMetaShareEnabled()) { readAndSkipLayerClassMeta(readContext); } - metaSerializer.readAndSetFields(readContext, target); + compatibleSerializer.readAndSetFields(readContext, target); } else { ((ObjectSerializer) slotsSerializer).readAndSetFields(readContext, target); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index f9658307ae..a081660e5c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -97,14 +97,14 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer { // Instance-level cache: TypeDef ID -> TypeInfo (shared across all slots). private final LongMap typeDefIdToTypeInfo = new LongMap<>(4, 0.4f); - private static MetaSharedLayerSerializerBase newGeneratedSerializer( + private static CompatibleLayerSerializerBase newGeneratedSerializer( TypeResolver typeResolver, Class cls, Class serializerClass, TypeDef layerTypeDef, Class layerMarkerClass) { - MetaSharedLayerSerializerBase serializer = - (MetaSharedLayerSerializerBase) + CompatibleLayerSerializerBase serializer = + (CompatibleLayerSerializerBase) Serializers.newSerializer(typeResolver, cls, serializerClass); serializer.setLayerSerializerMeta(layerTypeDef, layerMarkerClass); return serializer; @@ -119,7 +119,7 @@ private interface SlotInfo { StreamTypeInfo getStreamTypeInfo(); - MetaSharedLayerSerializerBase getSlotsSerializer(); + CompatibleLayerSerializerBase getSlotsSerializer(); /** * Read the layer TypeDef from buffer (if meta share enabled) and return the appropriate @@ -132,7 +132,7 @@ private interface SlotInfo { * @param readContext the context to read TypeDef from * @return the serializer to use for reading */ - MetaSharedLayerSerializerBase getReadSerializer( + CompatibleLayerSerializerBase getReadSerializer( TypeResolver typeResolver, ReadContext readContext); /** @@ -141,7 +141,7 @@ MetaSharedLayerSerializerBase getReadSerializer( * * @return the current read serializer */ - MetaSharedLayerSerializerBase getCurrentReadSerializer(); + CompatibleLayerSerializerBase getCurrentReadSerializer(); ForyStructOutputStream getObjectOutputStream(); @@ -238,7 +238,7 @@ public void write(WriteContext writeContext, Object value) { // replacement id. classResolver.writeClassInternal(writeContext, slotsInfo.getCls()); // Write layer class meta first (if meta share enabled) - MetaSharedLayerSerializerBase serializer = slotsInfo.getSlotsSerializer(); + CompatibleLayerSerializerBase serializer = slotsInfo.getSlotsSerializer(); if (config.isMetaShareEnabled()) { serializer.writeLayerClassMeta(writeContext); } @@ -440,12 +440,12 @@ private void skipUnknownLayerData(ReadContext readContext, Class senderClass) } // Get or create serializer from TypeInfo to skip the fields - MetaSharedLayerSerializerBase skipSerializer = - (MetaSharedLayerSerializerBase) typeInfo.getSerializer(); + CompatibleLayerSerializerBase skipSerializer = + (CompatibleLayerSerializerBase) typeInfo.getSerializer(); if (skipSerializer == null) { Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(senderClass, 0); - MetaSharedLayerSerializer newSerializer = - new MetaSharedLayerSerializer( + CompatibleLayerSerializer newSerializer = + new CompatibleLayerSerializer( typeResolver, senderClass, typeInfo.getTypeDef(), layerMarkerClass); typeInfo.setSerializer(newSerializer); skipSerializer = newSerializer; @@ -681,7 +681,7 @@ private class SlotsInfo implements SlotInfo { private final Class cls; private final StreamTypeInfo streamTypeInfo; // mark non-final for async-jit to update it to jit-serializer. - private MetaSharedLayerSerializerBase slotsSerializer; + private CompatibleLayerSerializerBase slotsSerializer; private final ObjectIntMap fieldIndexMap; private final int numPutFields; private final Class[] putFieldTypes; @@ -689,7 +689,7 @@ private class SlotsInfo implements SlotInfo { private final ForyStructInputStream objectInputStream; private final ObjectArray getFieldPool; // Current read serializer (set by getReadSerializer, used by getCurrentReadSerializer) - private MetaSharedLayerSerializerBase currentReadSerializer; + private CompatibleLayerSerializerBase currentReadSerializer; public SlotsInfo(TypeResolver typeResolver, Class type) { this.cls = type; @@ -716,7 +716,7 @@ public SlotsInfo(TypeResolver typeResolver, Class type) { // Create interpreter-mode serializer first this.slotsSerializer = - new MetaSharedLayerSerializer(typeResolver, type, layerTypeDef, layerMarkerClass); + new CompatibleLayerSerializer(typeResolver, type, layerTypeDef, layerMarkerClass); // Register JIT callback to replace with JIT serializer when ready if (config.isCodeGenEnabled() @@ -726,13 +726,13 @@ public SlotsInfo(TypeResolver typeResolver, Class type) { typeResolver .getJITContext() .registerSerializerJITCallback( - () -> MetaSharedLayerSerializer.class, + () -> CompatibleLayerSerializer.class, () -> typeResolver .getJITContext() .asyncVisitFory( fory -> - CodecUtils.loadOrGenMetaSharedLayerCodecClass( + CodecUtils.loadOrGenCompatibleLayerCodecClass( type, fory, layerTypeDef, layerMarkerClass)), c -> thisInfo.slotsSerializer = @@ -799,7 +799,7 @@ public StreamTypeInfo getStreamTypeInfo() { } @Override - public MetaSharedLayerSerializerBase getSlotsSerializer() { + public CompatibleLayerSerializerBase getSlotsSerializer() { return slotsSerializer; } @@ -835,9 +835,9 @@ public Class[] getPutFieldTypes() { @Override @SuppressWarnings("unchecked") - public MetaSharedLayerSerializerBase getReadSerializer( + public CompatibleLayerSerializerBase getReadSerializer( TypeResolver typeResolver, ReadContext readContext) { - MetaSharedLayerSerializerBase result; + CompatibleLayerSerializerBase result; if (!typeResolver.getConfig().isMetaShareEnabled()) { // Meta share not enabled - use the default slots serializer result = slotsSerializer; @@ -850,12 +850,12 @@ public MetaSharedLayerSerializerBase getReadSerializer( // Get or create serializer from TypeInfo Serializer serializer = typeInfo.getSerializer(); if (serializer != null) { - result = (MetaSharedLayerSerializerBase) serializer; + result = (CompatibleLayerSerializerBase) serializer; } else { // Create a new serializer based on the TypeDef from stream Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(cls, 0); - MetaSharedLayerSerializer newSerializer = - new MetaSharedLayerSerializer( + CompatibleLayerSerializer newSerializer = + new CompatibleLayerSerializer( typeResolver, cls, typeInfo.getTypeDef(), layerMarkerClass); typeInfo.setSerializer(newSerializer); result = newSerializer; @@ -868,7 +868,7 @@ public MetaSharedLayerSerializerBase getReadSerializer( } @Override - public MetaSharedLayerSerializerBase getCurrentReadSerializer() { + public CompatibleLayerSerializerBase getCurrentReadSerializer() { return currentReadSerializer; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index c7e3040ee1..8da69a667b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -179,9 +179,9 @@ private static Serializer buildBuiltinSerializer( if (serializerClass == ExceptionSerializers.StackTraceElementSerializer.class) { return (Serializer) new ExceptionSerializers.StackTraceElementSerializer(config); } - if (serializerClass == MetaSharedSerializer.class) { + if (serializerClass == CompatibleSerializer.class) { TypeDef typeDef = typeResolver.getTypeDef(type, true); - return new MetaSharedSerializer(typeResolver, type, typeDef); + return new CompatibleSerializer(typeResolver, type, typeDef); } if (serializerClass == EnumSerializer.class) { return (Serializer) new EnumSerializer(config, type); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index cd7cd56759..3b436371a3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -390,7 +390,7 @@ private List buildRemoteFields( fields.put(fieldKey(descriptor), i); } // Keep compatible-read descriptor ordering owned by TypeResolver, matching the sorted - // DescriptorGrouper order used by ObjectCodecBuilder and MetaSharedCodecBuilder. FieldGroups + // DescriptorGrouper order used by ObjectCodecBuilder and CompatibleCodecBuilder. FieldGroups // may regroup descriptors for helper ownership, so it must not drive remote payload order. List remoteDescriptorsInWireOrder = typeResolver diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index a61f114f54..b78172dc13 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -56,10 +56,10 @@ import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.AbstractObjectSerializer; +import org.apache.fory.serializer.CompatibleLayerSerializer; import org.apache.fory.serializer.FieldGroups; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.JavaSerializer; -import org.apache.fory.serializer.MetaSharedLayerSerializer; import org.apache.fory.serializer.ObjectSerializer; import org.apache.fory.serializer.Serializer; import org.apache.fory.util.Preconditions; @@ -634,7 +634,7 @@ private static Serializer[] buildSlotsSerializers( // This ensures unique marker classes for each layer Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(cls, layerIndex); slotsSerializer = - new MetaSharedLayerSerializer(typeResolver, cls, layerTypeDef, layerMarkerClass); + new CompatibleLayerSerializer(typeResolver, cls, layerTypeDef, layerMarkerClass); } else { slotsSerializer = new ObjectSerializer<>(typeResolver, cls, false); } @@ -652,14 +652,15 @@ private static void readAndSetFields( Object collection, Serializer[] slotsSerializers) { for (Serializer slotsSerializer : slotsSerializers) { - if (slotsSerializer instanceof MetaSharedLayerSerializer) { - MetaSharedLayerSerializer metaSerializer = (MetaSharedLayerSerializer) slotsSerializer; + if (slotsSerializer instanceof CompatibleLayerSerializer) { + CompatibleLayerSerializer compatibleSerializer = + (CompatibleLayerSerializer) slotsSerializer; // Read layer class meta first if meta share is enabled - // This corresponds to writeLayerClassMeta() in MetaSharedLayerSerializer.write() + // This corresponds to writeLayerClassMeta() in CompatibleLayerSerializer.write() if (typeResolver.getConfig().isMetaShareEnabled()) { readAndSkipLayerClassMeta(readContext); } - metaSerializer.readAndSetFields(readContext, collection); + compatibleSerializer.readAndSetFields(readContext, collection); } else { ((ObjectSerializer) slotsSerializer).readAndSetFields(readContext, collection); } @@ -668,7 +669,7 @@ private static void readAndSetFields( /** * Read and skip the layer class meta from buffer. This is used to skip over the class definition - * that was written by MetaSharedLayerSerializer.writeLayerClassMeta(). For + * that was written by CompatibleLayerSerializer.writeLayerClassMeta(). For * ChildContainerSerializers, we use the same serializer on both write and read sides, so we just * need to skip the meta without actually parsing it. */ diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 402e2eee99..37231a8de9 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -157,9 +157,9 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.builder.AccessorHelper,\ org.apache.fory.builder.JITContext,\ org.apache.fory.builder.ObjectCodecBuilder,\ - org.apache.fory.builder.MetaSharedCodecBuilder,\ + org.apache.fory.builder.CompatibleCodecBuilder,\ org.apache.fory.builder.StaticCompatibleCodecBuilder,\ - org.apache.fory.builder.Generated$GeneratedCompatibleMetaSharedSerializer,\ + org.apache.fory.builder.Generated$GeneratedStaticCompatibleSerializer,\ org.apache.fory.builder.CodecUtils,\ org.apache.fory.resolver.RefMode,\ org.apache.fory.serializer.FieldGroups$SerializationFieldInfo,\ @@ -326,8 +326,8 @@ Args=--initialize-at-build-time=org.apache.fory.annotation.ForyField$Dynamic,\ org.apache.fory.serializer.collection.PrimitiveListSerializers$Float64ListSerializer,\ org.apache.fory.serializer.collection.PrimitiveListSerializers$Float16ListSerializer,\ org.apache.fory.serializer.BufferSerializers$ByteBufferSerializer,\ - org.apache.fory.serializer.MetaSharedSerializer,\ - org.apache.fory.serializer.MetaSharedLayerSerializer,\ + org.apache.fory.serializer.CompatibleSerializer,\ + org.apache.fory.serializer.CompatibleLayerSerializer,\ org.apache.fory.serializer.EnumSerializer,\ org.apache.fory.serializer.ExceptionSerializers,\ org.apache.fory.serializer.ExceptionSerializers$ExceptionSerializer,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/CyclicTest.java b/java/fory-core/src/test/java/org/apache/fory/CyclicTest.java index 317fc86cc8..2dffed55fb 100644 --- a/java/fory-core/src/test/java/org/apache/fory/CyclicTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/CyclicTest.java @@ -85,7 +85,7 @@ public void testBean(boolean enableCodegen, boolean asyncCompilation, boolean co } @Test - public void testBeanMetaShared() throws IOException { + public void testBeanMetaShare() throws IOException { ByteArrayOutputStream s = new ByteArrayOutputStream(); GZIPOutputStream gzipOutputStream = new GZIPOutputStream(s); gzipOutputStream.write(Fory.class.getName().getBytes(StandardCharsets.UTF_8)); diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java b/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java index 8c114feff1..4bfa4b64ae 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java @@ -405,7 +405,7 @@ public static void roundCheck( Assert.assertEquals(compareHook.apply(o2), compareHook.apply(o)); } - public static Object serDeMetaShared(Fory fory, Object obj) { + public static Object serDeMetaShare(Fory fory, Object obj) { MetaWriteContext metaWriteContext = new MetaWriteContext(); MetaReadContext metaReadContext = new MetaReadContext(); setMetaContexts(fory, metaWriteContext, metaReadContext); diff --git a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java index c44677a7c3..0b7d1d8bc0 100644 --- a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java @@ -402,7 +402,7 @@ public void readLongs(long[] dst, int dstIndex, int length) { } @Test - public void testBigBufferStreamingMetaShared() throws IOException { + public void testBigBufferStreamingMetaShare() throws IOException { Fory fory = builder().withCompatible(true).build(); ByteArrayOutputStream bas = new ByteArrayOutputStream(); List list = new ArrayList<>(); diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/JITContextTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/JITContextTest.java index 47f221e94d..f49b762fb6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/JITContextTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/JITContextTest.java @@ -109,7 +109,7 @@ private Serializer getSerializer(Fory fory, Class cls) { } @Test(dataProvider = "config2", timeOut = 60_000) - public void testAsyncCompilationMetaShared( + public void testAsyncCompilationMetaShare( boolean referenceTracking, boolean compatible, boolean scopedMetaShare) throws InterruptedException { Fory fory = diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java index 64be764f96..23a9731b3b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -42,7 +42,7 @@ import javax.tools.StandardJavaFileManager; import javax.tools.ToolProvider; import org.apache.fory.Fory; -import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.builder.Generated.GeneratedStaticCompatibleSerializer; import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; import org.apache.fory.meta.TypeDef; @@ -345,7 +345,7 @@ private static Object roundTripThroughStaticCompatibleSerializer( CodecUtils.loadOrGenStaticCompatibleCodecClass( reader.getTypeResolver(), readerClass, remoteTypeDef); Assert.assertTrue( - GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(compatibleSerializerClass)); + GeneratedStaticCompatibleSerializer.class.isAssignableFrom(compatibleSerializerClass)); Serializer compatibleSerializer = compatibleSerializerClass .getConstructor(TypeResolver.class, Class.class, TypeDef.class) @@ -362,13 +362,13 @@ private static Object roundTripThroughStaticCompatibleSerializer( codecUtils .when( () -> - CodecUtils.loadOrGenMetaSharedCodecClass( + CodecUtils.loadOrGenCompatibleCodecClass( same(reader.getTypeResolver()), eq(readerClass), any(TypeDef.class))) .thenReturn(compatibleSerializerClass); Object result = reader.deserialize(bytes); codecUtils.verify( () -> - CodecUtils.loadOrGenMetaSharedCodecClass( + CodecUtils.loadOrGenCompatibleCodecClass( same(reader.getTypeResolver()), eq(readerClass), any(TypeDef.class)), atLeastOnce()); return result; diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/MetaShareContextTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/MetaShareContextTest.java index 9c1f68e62f..e3738b7b1c 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/MetaShareContextTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/MetaShareContextTest.java @@ -45,7 +45,7 @@ public void testShareClassName() { .requireClassRegistration(false) .build(); for (Object o : new Object[] {Foo.create(), BeanB.createBeanB(2), BeanA.createBeanA(2)}) { - checkMetaShared(fory, o); + checkMetaShare(fory, o); } } @@ -62,11 +62,11 @@ public void testShareTypeDefCompatible(boolean enableCodegen) { .requireClassRegistration(false) .build(); for (Object o : new Object[] {Foo.create(), BeanB.createBeanB(2), BeanA.createBeanA(2)}) { - checkMetaShared(fory, o); + checkMetaShare(fory, o); } } - private void checkMetaShared(Fory fory, Object o) { + private void checkMetaShare(Fory fory, Object o) { MetaWriteContext metaWriteContext = new MetaWriteContext(); MetaReadContext metaReadContext = new MetaReadContext(); setMetaContexts(fory, metaWriteContext, metaReadContext); @@ -110,6 +110,6 @@ public void testFinalTypeWriteMeta(boolean enableCodegen) { .build(); OuterPojo outerPojo = new OuterPojo(new ArrayList<>(ImmutableList.of(new InnerPojo(1), new InnerPojo(2)))); - checkMetaShared(fory, outerPojo); + checkMetaShare(fory, outerPojo); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java index cb86d5806d..98f6d40222 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/AndroidJvmRoundTripTest.java @@ -210,7 +210,7 @@ private static Object newValue(RoundTripKind kind) { case CODEGEN_VALUE: return new CodegenValue(7, "codegen", new NestedValue("nested", 13)); case META_SHARED_VALUE: - return new MetaSharedValue(19, "meta", Arrays.asList(1, 2, 3)); + return new MetaShareValue(19, "meta", Arrays.asList(1, 2, 3)); case COLLECTION_VALUE: return newCollectionValue(); case SINGLETON_VALUE: @@ -693,14 +693,14 @@ public int hashCode() { } } - public static final class MetaSharedValue implements Serializable { + public static final class MetaShareValue implements Serializable { public int id; public String name; public List values; - public MetaSharedValue() {} + public MetaShareValue() {} - MetaSharedValue(int id, String name, List values) { + MetaShareValue(int id, String name, List values) { this.id = id; this.name = name; this.values = values; @@ -708,10 +708,10 @@ public MetaSharedValue() {} @Override public boolean equals(Object obj) { - if (!(obj instanceof MetaSharedValue)) { + if (!(obj instanceof MetaShareValue)) { return false; } - MetaSharedValue other = (MetaSharedValue) obj; + MetaShareValue other = (MetaShareValue) obj; return id == other.id && name.equals(other.name) && values.equals(other.values); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/CodegenCompatibleSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/CodegenCompatibleSerializerTest.java index e46c4bec69..736e08c527 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/CodegenCompatibleSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/CodegenCompatibleSerializerTest.java @@ -43,7 +43,7 @@ import org.testng.annotations.Test; /** - * Tests for compatible mode serialization using meta-shared approach with codegen. These tests + * Tests for compatible mode serialization using shared class metadata with codegen. These tests * verify forward/backward compatibility when using compatible mode with scoped meta share and * codegen enabled. */ diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/CompatibleSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/CompatibleSerializerTest.java index 004f78b59f..24b6317997 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/CompatibleSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/CompatibleSerializerTest.java @@ -43,7 +43,7 @@ import org.testng.annotations.Test; /** - * Tests for compatible mode serialization using meta-shared approach. These tests verify + * Tests for compatible mode serialization using shared class metadata. These tests verify * forward/backward compatibility when using compatible mode with scoped meta share. */ public class CompatibleSerializerTest extends ForyTestBase { diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/DuplicateFieldsTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/DuplicateFieldsTest.java index 30e34ff34f..e9f665d5f9 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/DuplicateFieldsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/DuplicateFieldsTest.java @@ -112,8 +112,8 @@ public void testDuplicateFieldsCompatible() { .requireClassRegistration(false); Fory fory = builder.build(); { - MetaSharedSerializer serializer = - new MetaSharedSerializer<>( + CompatibleSerializer serializer = + new CompatibleSerializer<>( fory.getTypeResolver(), C.class, fory.getTypeResolver().getTypeDef(C.class, true)); MemoryBuffer buffer = MemoryUtils.buffer(32); writeSerializer(fory, serializer, buffer, c); @@ -123,12 +123,12 @@ public void testDuplicateFieldsCompatible() { assertEquals(newC, c); } { - // Use MetaSharedSerializer JIT version + // Use CompatibleSerializer JIT version Serializer serializer = Serializers.newSerializer( fory, C.class, - CodecUtils.loadOrGenMetaSharedCodecClass( + CodecUtils.loadOrGenCompatibleCodecClass( fory, C.class, fory.getTypeResolver().getTypeDef(C.class, true))); MemoryBuffer buffer = MemoryUtils.buffer(32); writeSerializer(fory, serializer, buffer, c); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java index 87353ad72f..a033c56018 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java @@ -471,7 +471,7 @@ public void testFinalFieldReplaceWithCompatibleFinalClass(boolean refTracking) { /** * Test that final fields with writeReplace/readResolve work correctly with compatible mode, which - * uses MetaSharedSerializer instead of ObjectSerializer. + * uses CompatibleSerializer instead of ObjectSerializer. */ @Test(dataProvider = "referenceTrackingConfig") public void testFinalFieldReplaceWithCompatible(boolean refTracking) { diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedCompatibleTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/MetaShareCompatibleTest.java similarity index 96% rename from java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedCompatibleTest.java rename to java/fory-core/src/test/java/org/apache/fory/serializer/MetaShareCompatibleTest.java index 467f7a50ef..1046e2ff25 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedCompatibleTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/MetaShareCompatibleTest.java @@ -32,7 +32,7 @@ import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.TestUtils; -import org.apache.fory.builder.MetaSharedCodecBuilder; +import org.apache.fory.builder.CompatibleCodecBuilder; import org.apache.fory.codegen.CompileUnit; import org.apache.fory.codegen.JaninoUtils; import org.apache.fory.config.ForyBuilder; @@ -51,12 +51,12 @@ import org.testng.annotations.Test; /** - * Tests for {@link MetaSharedCodecBuilder} and {@link MetaSharedSerializer}, and protocol + * Tests for {@link CompatibleCodecBuilder} and {@link CompatibleSerializer}, and protocol * interoperability between them. */ -public class MetaSharedCompatibleTest extends ForyTestBase { - public static Object serDeMetaSharedCheck(Fory fory, Object obj) { - Object newObj = serDeMetaShared(fory, obj); +public class MetaShareCompatibleTest extends ForyTestBase { + public static Object serDeMetaShareCheck(Fory fory, Object obj) { + Object newObj = serDeMetaShare(fory, obj); Assert.assertEquals(newObj, obj); return newObj; } @@ -116,9 +116,9 @@ public void testWrite(boolean referenceTracking, boolean compressNumber, boolean .withRefTracking(referenceTracking) .withCodegen(enableCodegen) .build(); - serDeMetaSharedCheck(fory, Foo.create()); - serDeMetaSharedCheck(fory, BeanB.createBeanB(2)); - serDeMetaSharedCheck(fory, BeanA.createBeanA(2)); + serDeMetaShareCheck(fory, Foo.create()); + serDeMetaShareCheck(fory, BeanB.createBeanB(2)); + serDeMetaShareCheck(fory, BeanA.createBeanA(2)); } @Test(dataProvider = "config2") @@ -208,7 +208,7 @@ public void testWriteCompatibleCollectionSimple() throws Exception { loadClass( BeanA.class, code, - MetaSharedCompatibleTest.class + "testWriteCompatibleCollectionBasic_1"); + MetaShareCompatibleTest.class + "testWriteCompatibleCollectionBasic_1"); Fory fory1 = foryBuilder() .withCodegen(false) @@ -231,7 +231,7 @@ public void testWriteCompatibleCollectionSimple() throws Exception { loadClass( BeanA.class, code, - MetaSharedCompatibleTest.class + "testWriteCompatibleCollectionBasic_2"); + MetaShareCompatibleTest.class + "testWriteCompatibleCollectionBasic_2"); Object o2 = cls2.newInstance(); TestUtils.unsafeCopy(beanA, o2); Fory fory2 = @@ -279,7 +279,7 @@ public void testWriteCompatibleCollectionBasic( loadClass( BeanA.class, code, - MetaSharedCompatibleTest.class + "testWriteCompatibleCollectionBasic_1"); + MetaShareCompatibleTest.class + "testWriteCompatibleCollectionBasic_1"); Fory fory1 = foryBuilder() .withRefTracking(referenceTracking) @@ -302,7 +302,7 @@ public void testWriteCompatibleCollectionBasic( loadClass( BeanA.class, code, - MetaSharedCompatibleTest.class + "testWriteCompatibleCollectionBasic_2"); + MetaShareCompatibleTest.class + "testWriteCompatibleCollectionBasic_2"); Object o2 = cls2.newInstance(); TestUtils.unsafeCopy(beanA, o2); Fory fory2 = @@ -366,7 +366,7 @@ public void testWriteCompatibleContainer( MetaWriteContext metaWriteContext = new MetaWriteContext(); MetaReadContext metaReadContext = new MetaReadContext(); BeanA beanA = BeanA.createBeanA(2); - serDeMetaShared(fory, beanA); + serDeMetaShare(fory, beanA); Class cls = ClassUtils.createCompatibleClass1(); Object newBeanA = cls.newInstance(); TestUtils.unsafeCopy(beanA, newBeanA); @@ -416,7 +416,7 @@ public void testWriteCompatibleCollection( .build(); CollectionFields collectionFields = UnmodifiableSerializersTest.createCollectionFields(); { - Object o = serDeMetaShared(fory, collectionFields); + Object o = serDeMetaShare(fory, collectionFields); Object o1 = CollectionFields.copyToCanEqual(o, o.getClass().newInstance()); Object o2 = CollectionFields.copyToCanEqual( @@ -489,7 +489,7 @@ public void testWriteCompatibleMap( MetaReadContext metaReadContext = new MetaReadContext(); MapFields mapFields = UnmodifiableSerializersTest.createMapFields(); { - Object o = serDeMetaShared(fory, mapFields); + Object o = serDeMetaShare(fory, mapFields); Object o1 = MapFields.copyToCanEqual(o, o.getClass().newInstance()); Object o2 = MapFields.copyToCanEqual(mapFields, mapFields.getClass().newInstance()); Assert.assertEquals(o1, o2); @@ -563,7 +563,7 @@ public void testDuplicateFields( + ";\n" + "import java.util.*;\n" + "import java.math.*;\n" - + "public class DuplicateFieldsClass2 extends MetaSharedCompatibleTest.DuplicateFieldsClass1 {\n" + + "public class DuplicateFieldsClass2 extends MetaShareCompatibleTest.DuplicateFieldsClass1 {\n" + " int intField1;\n" + "}"); Fory fory = @@ -585,7 +585,7 @@ public void testDuplicateFields( } } { - Object o = serDeMetaShared(fory, o1); + Object o = serDeMetaShare(fory, o1); TestUtils.objectFieldsEquals(o, o1, true); } @@ -599,7 +599,7 @@ public void testDuplicateFields( + ";\n" + "import java.util.*;\n" + "import java.math.*;\n" - + "public class DuplicateFieldsClass2 extends MetaSharedCompatibleTest.DuplicateFieldsClass1 {\n" + + "public class DuplicateFieldsClass2 extends MetaShareCompatibleTest.DuplicateFieldsClass1 {\n" + " int intField1;\n" + " int intField2;\n" + "}"); @@ -622,7 +622,7 @@ public void testDuplicateFields( } } { - Object o = serDeMetaShared(fory2, o2); + Object o = serDeMetaShare(fory2, o2); TestUtils.objectFieldsEquals(o, o2, true); } { @@ -657,7 +657,7 @@ void testEmptySubClass(boolean referenceTracking, boolean compressNumber, boolea + ";\n" + "import java.util.*;\n" + "import java.math.*;\n" - + "public class DuplicateFieldsClass2 extends MetaSharedCompatibleTest.DuplicateFieldsClass1 {\n" + + "public class DuplicateFieldsClass2 extends MetaShareCompatibleTest.DuplicateFieldsClass1 {\n" + "}"); Fory fory = foryBuilder() @@ -671,7 +671,7 @@ void testEmptySubClass(boolean referenceTracking, boolean compressNumber, boolea field.setAccessible(true); field.setInt(o1, 10); } - Object o = serDeMetaShared(fory, o1); + Object o = serDeMetaShare(fory, o1); Assert.assertEquals(o.getClass(), o1.getClass()); TestUtils.objectFieldsEquals(o, o1, true); } @@ -684,7 +684,7 @@ public void testBigClassNameObject() { new NativeTypeDefEncoderTest .TestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLength .InnerClassTestLengthInnerClassTestLengthInnerClassTestLength(); - serDeMetaSharedCheck(fory, o); + serDeMetaShareCheck(fory, o); } CompileUnit aunit = diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedObjectSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/MetaShareObjectSerializerTest.java similarity index 97% rename from java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedObjectSerializerTest.java rename to java/fory-core/src/test/java/org/apache/fory/serializer/MetaShareObjectSerializerTest.java index 60dccc663d..a9f2cfd0e0 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedObjectSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/MetaShareObjectSerializerTest.java @@ -24,7 +24,7 @@ import org.apache.fory.codegen.JaninoUtils; import org.testng.annotations.Test; -public class MetaSharedObjectSerializerTest extends ForyTestBase { +public class MetaShareObjectSerializerTest extends ForyTestBase { @Test public void testIgnoreTypeInconsistentSerializer() diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/ChildContainerSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/ChildContainerSerializersTest.java index a5db234c85..96d22fc8af 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/ChildContainerSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/ChildContainerSerializersTest.java @@ -253,7 +253,7 @@ public void testSerializeCustomPrivateMap(boolean enableCodegen) { .withScopedMetaShare(false) .withCodegen(enableCodegen) .build(); - serDeMetaShared(fory, outerDO); + serDeMetaShare(fory, outerDO); } /** diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaShareXlangTest.java similarity index 98% rename from java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/MetaShareXlangTest.java index ac40486091..e6082e8869 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaShareXlangTest.java @@ -42,10 +42,10 @@ import org.apache.fory.xlang.PyCrossLanguageTest.Foo; import org.testng.annotations.Test; -public class MetaSharedXlangTest extends ForyTestBase { +public class MetaShareXlangTest extends ForyTestBase { @Test - public void testMetaSharedBasic() { + public void testMetaShareBasic() { Fory fory = Fory.builder().withXlang(true).withCompatible(true).withCodegen(false).build(); fory.register(Foo.class, "example.foo"); fory.register(Bar.class, "example.bar"); @@ -54,7 +54,7 @@ public void testMetaSharedBasic() { } @Test - public void testMetaSharedComplex1() { + public void testMetaShareComplex1() { Fory fory = Fory.builder().withXlang(true).withCompatible(true).withCodegen(false).build(); fory.register(BeanB.class, "example.b"); serDeCheck(fory, BeanB.createBeanB(2)); diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java index 044040272c..d18429b21c 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ExampleStaticGeneratedSerializerTest.java @@ -44,7 +44,7 @@ import org.apache.fory.annotation.ForyStruct; import org.apache.fory.annotation.Int32Type; import org.apache.fory.builder.CodecUtils; -import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.builder.Generated.GeneratedStaticCompatibleSerializer; import org.apache.fory.collection.BFloat16List; import org.apache.fory.collection.BoolList; import org.apache.fory.collection.Float16List; @@ -185,7 +185,7 @@ public void testStaticCompatibleBuilder(boolean xlang) throws Exception { CodecUtils.loadOrGenStaticCompatibleCodecClass( reader.getTypeResolver(), InconsistentMessage.class, remoteTypeDef); Assert.assertTrue( - GeneratedCompatibleMetaSharedSerializer.class.isAssignableFrom(compatibleSerializerClass)); + GeneratedStaticCompatibleSerializer.class.isAssignableFrom(compatibleSerializerClass)); writer.setMetaWriteContext(new MetaWriteContext()); byte[] bytes = writer.serialize(value); @@ -194,7 +194,7 @@ public void testStaticCompatibleBuilder(boolean xlang) throws Exception { codecUtils .when( () -> - CodecUtils.loadOrGenMetaSharedCodecClass( + CodecUtils.loadOrGenCompatibleCodecClass( same(reader.getTypeResolver()), eq(InconsistentMessage.class), any(TypeDef.class))) @@ -203,7 +203,7 @@ public void testStaticCompatibleBuilder(boolean xlang) throws Exception { InconsistentMessage result = (InconsistentMessage) reader.deserialize(bytes); codecUtils.verify( () -> - CodecUtils.loadOrGenMetaSharedCodecClass( + CodecUtils.loadOrGenCompatibleCodecClass( same(reader.getTypeResolver()), eq(InconsistentMessage.class), any(TypeDef.class)), diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordSerializersTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordSerializersTest.java index 2c596fde1e..30efd25f3f 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordSerializersTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordSerializersTest.java @@ -138,7 +138,7 @@ private static ForyBuilder newNumberCompressedBuilder() { } @Test(dataProvider = "codegen") - public void testSimpleRecordMetaShared(boolean codegen) { + public void testSimpleRecordMetaShare(boolean codegen) { Fory fory = Fory.builder() .requireClassRegistration(false) From b3f7384e5d5b33f59dcb0431ecf05f57c0cef000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 11:54:33 +0800 Subject: [PATCH 56/58] fix(java): allow nullable list payload array read --- .../fory/serializer/CompatibleCollectionArrayReader.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java index 9c023b725a..c125edd722 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleCollectionArrayReader.java @@ -134,16 +134,19 @@ static ReadAction readAction( TypeRef localType = localDescriptor.getTypeRef(); int peerListElementTypeId = listElementTypeId(remoteFieldType); if (peerListElementTypeId != Types.UNKNOWN) { - int nonNullablePeerListElementTypeId = nonNullableListElementTypeId(remoteFieldType); int localArrayTypeId = arrayTypeId(localDescriptor); if (localArrayTypeId != Types.UNKNOWN - && localArrayTypeId == denseArrayTypeId(nonNullablePeerListElementTypeId)) { + && localArrayTypeId == denseArrayTypeId(peerListElementTypeId)) { + // Remote element nullable/ref flags describe what the schema can encode. Actual payload + // null/ref markers are validated while reading so nullable list payloads without nulls + // remain compatible with local array fields. return new ReadAction( READ_LIST_TO_ARRAY, localArrayTypeId, - nonNullablePeerListElementTypeId, + peerListElementTypeId, localDescriptor.getRawType()); } + int nonNullablePeerListElementTypeId = nonNullableListElementTypeId(remoteFieldType); int localListElementTypeId = nonNullableListElementTypeId(localType); int peerArrayTypeId = denseArrayTypeId(peerListElementTypeId); // List-to-array and list-to-list materialize through a dense primitive array, so they cannot From 96e359d5e8d14b7ba29797de65f00bf31ed80fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 11:59:15 +0800 Subject: [PATCH 57/58] fix(java): update GraalVM compatible marker --- .../apache/fory/graalvm/CompatibleDifferentSchemaExample.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/CompatibleDifferentSchemaExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/CompatibleDifferentSchemaExample.java index 597e515d10..dd43e11036 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/CompatibleDifferentSchemaExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/CompatibleDifferentSchemaExample.java @@ -20,7 +20,7 @@ package org.apache.fory.graalvm; import org.apache.fory.Fory; -import org.apache.fory.builder.Generated.GeneratedCompatibleMetaSharedSerializer; +import org.apache.fory.builder.Generated.GeneratedStaticCompatibleSerializer; import org.apache.fory.context.ReadContext; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; @@ -70,7 +70,7 @@ public static void main(String[] args) { if (GraalvmSupport.isGraalRuntime()) { Serializer serializer = readSerializerForTarget(READER, bytes, ReaderSchema.class); Preconditions.checkArgument( - serializer instanceof GeneratedCompatibleMetaSharedSerializer, + serializer instanceof GeneratedStaticCompatibleSerializer, "Expected GraalVM generated compatible serializer, got %s", serializer.getClass()); } From c7da815ee7e543eefabc39dec8c5a41a460eadf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E7=99=BD?= Date: Wed, 13 May 2026 12:06:35 +0800 Subject: [PATCH 58/58] test(java): align static list array payload validation --- .../processing/ForyStructProcessorTest.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java index 2605b72e33..a24bc316b0 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/ForyStructProcessorTest.java @@ -750,7 +750,7 @@ public void testStaticArrayTypeListWritesDenseArrayPayload() throws Exception { } @Test - public void testStaticIncompatibleListArrayCompatibleReadFails() throws Exception { + public void testStaticListArrayCompatibleReadPayloadValidation() throws Exception { CompilationResult nullableListWriter = compile( "test.ListArrayMismatchStruct", @@ -780,8 +780,15 @@ public void testStaticIncompatibleListArrayCompatibleReadFails() throws Exceptio Fory reader = xlangCompatibleFory(readerLoader, readerType, false, "ListArrayMismatchStruct"); Object writerValue = writerType.getConstructor().newInstance(); setField(writerType, writerValue, "values", Arrays.asList(1, 2, 3)); - byte[] payload = writer.serialize(writerValue); - Assert.expectThrows(DeserializationException.class, () -> reader.deserialize(payload)); + Object result = reader.deserialize(writer.serialize(writerValue)); + Assert.assertTrue( + Arrays.equals((int[]) getField(readerType, result, "values"), new int[] {1, 2, 3})); + + Object nullElementWriterValue = writerType.getConstructor().newInstance(); + setField(writerType, nullElementWriterValue, "values", Arrays.asList(1, null, 3)); + byte[] nullElementPayload = writer.serialize(nullElementWriterValue); + Assert.expectThrows( + DeserializationException.class, () -> reader.deserialize(nullElementPayload)); } CompilationResult nestedListWriter =