feat(java): enhance ForyField annotation with tag ID support for optimized serialization

java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java

@@ -28,9 +28,31 @@
 @Target({ElementType.FIELD, ElementType.METHOD})
 public @interface ForyField {
 
-  /** Whether field is nullable, default false. */
+  /**
+   * Field tag ID for schema evolution mode (REQUIRED).
+   *
+   * <ul>
+   *   <li>When >= 0: Uses this numeric ID instead of field name string for compact encoding
+   *   <li>When -1: Explicitly opt-out of tag ID, use field name with meta string encoding
+   * </ul>
+   *
+   * <p>Must be unique within the class (except -1) and stable across versions.
+   */
+  int id();
+
+  /**
+   * Whether this field can be null. When set to false (default), Fory skips writing the null flag
+   * (saves 1 byte). When set to true, Fory writes null flag for nullable fields. Default: false
+   * (field is non-nullable, aligned with xlang protocol defaults)
+   */
   boolean nullable() default false;
 
-  /** Whether field need trackingRef, default false. */
-  boolean trackingRef() default false;
+  /**
+   * Whether to track references for this field. When set to false (default): - Avoids adding the
+   * object to IdentityMap (saves hash map overhead) - Skips writing ref tracking flag (saves 1 byte
+   * when combined with nullable=false) When set to true, enables reference tracking for
+   * shared/circular references. Default: false (no reference tracking, aligned with xlang protocol
+   * defaults)
+   */
+  boolean ref() default false;
 }

java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java

@@ -405,7 +405,14 @@ protected Expression serializeField(
     TypeRef<?> typeRef = descriptor.getTypeRef();
     boolean nullable = descriptor.isNullable();
 
+    boolean useRefTracking;
     if (needWriteRef(typeRef)) {
+      useRefTracking = descriptor.isTrackingRef();
+    } else {
+      useRefTracking = false;
+    }
+
+    if (useRefTracking) {
       return new If(
           not(writeRefOrNull(buffer, fieldValue)),
           serializeForNotNullForField(fieldValue, buffer, typeRef, null, false));
@@ -1768,20 +1775,46 @@ protected Expression deserializeField(
     TypeRef<?> typeRef = descriptor.getTypeRef();
     boolean nullable = descriptor.isNullable();
 
+    boolean typeNeedsRef = needWriteRef(typeRef);
+    boolean useRefTracking;
     if (needWriteRef(typeRef)) {
+      useRefTracking = descriptor.isTrackingRef();
+    } else {
+      useRefTracking = false;
+    }
+
+    if (useRefTracking) {
       return readRef(buffer, callback, () -> deserializeForNotNullForField(buffer, typeRef, null));
     } else {
       if (!nullable) {
         Expression value = deserializeForNotNullForField(buffer, typeRef, null);
-        // Should put value expr ahead to avoid generated code in wrong scope.
+
+        if (typeNeedsRef) {
+          // When a field explicitly disables ref tracking (@ForyField(trackingRef=false))
+          // but the type normally needs ref tracking (e.g., collections),
+          // we need to preserve a -1 id so that when the deserializer calls reference(),
+          // it will pop this -1 and skip the setReadObject call.
+          Expression preserveStubRefId =
+              new Invoke(refResolverRef, "preserveRefId", new Literal(-1, PRIMITIVE_INT_TYPE));
+          return new ListExpression(preserveStubRefId, value, callback.apply(value));
+        }
         return new ListExpression(value, callback.apply(value));
       }
-      return readNullable(
-          buffer,
-          typeRef,
-          callback,
-          () -> deserializeForNotNullForField(buffer, typeRef, null),
-          true);
+
+      Expression readNullableExpr =
+          readNullable(
+              buffer,
+              typeRef,
+              callback,
+              () -> deserializeForNotNullForField(buffer, typeRef, null),
+              true);
+
+      if (typeNeedsRef) {
+        Expression preserveStubRefId =
+            new Invoke(refResolverRef, "preserveRefId", new Literal(-1, PRIMITIVE_INT_TYPE));
+        return new ListExpression(preserveStubRefId, readNullableExpr);
+      }
+      return readNullableExpr;
     }
   }
 

java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java

@@ -40,8 +40,10 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.stream.Collectors;
 import org.apache.fory.Fory;
+import org.apache.fory.annotation.ForyField;
 import org.apache.fory.builder.MetaSharedCodecBuilder;
 import org.apache.fory.collection.Tuple2;
 import org.apache.fory.config.CompatibleMode;
@@ -61,6 +63,7 @@
 import org.apache.fory.serializer.converter.FieldConverter;
 import org.apache.fory.serializer.converter.FieldConverters;
 import org.apache.fory.type.Descriptor;
+import org.apache.fory.type.DescriptorBuilder;
 import org.apache.fory.type.FinalObjectTypeStub;
 import org.apache.fory.type.GenericType;
 import org.apache.fory.type.TypeUtils;
@@ -349,17 +352,76 @@ public List<Descriptor> getDescriptors(TypeResolver resolver, Class<?> cls) {
       SortedMap<Member, Descriptor> allDescriptorsMap =
           resolver.getFory().getClassResolver().getAllDescriptorsMap(cls, true);
       Map<String, Descriptor> descriptorsMap = new HashMap<>();
+      Map<Short, Descriptor> fieldIdToDescriptorMap = new HashMap<>();
+      Map<Member, Descriptor>[] newDescriptors = new Map[] {null};
+
       for (Map.Entry<Member, Descriptor> e : allDescriptorsMap.entrySet()) {
-        if (descriptorsMap.put(
-                e.getKey().getDeclaringClass().getName() + "." + e.getKey().getName(), e.getValue())
-            != null) {
+        String fullName = e.getKey().getDeclaringClass().getName() + "." + e.getKey().getName();
+        Descriptor desc = e.getValue();
+        if (descriptorsMap.put(fullName, desc) != null) {
           throw new IllegalStateException("Duplicate key");
         }
+
+        if (e.getKey() instanceof Field) {
+          boolean refTracking = resolver.getFory().trackingRef();
+          ForyField foryField = desc.getForyField();
+          // update ref tracking if
+          // - global ref tracking is disabled but field is tracking ref (@ForyField#ref set)
+          // - global ref tracking is enabled but field is not tracking ref (@ForyField#ref not set)
+          boolean needsUpdate =
+              (refTracking && foryField == null && !desc.isTrackingRef())
+                  || (foryField != null && desc.isTrackingRef());
+
+          if (needsUpdate) {
+            if (newDescriptors[0] == null) {
+              newDescriptors[0] = new HashMap<>();
+            }
+            boolean newTrackingRef = refTracking && foryField == null;
+            Descriptor newDescriptor =
+                new DescriptorBuilder(desc).trackingRef(newTrackingRef).build();
+
+            descriptorsMap.put(fullName, newDescriptor);
+            desc = newDescriptor;
+            newDescriptors[0].put(e.getKey(), newDescriptor);
+          }
+        }
+
+        // If the field has @ForyField annotation with field ID, index by field ID
+        if (desc.getForyField() != null) {
+          int fieldId = desc.getForyField().id();
+          if (fieldId >= 0) {
+            if (fieldIdToDescriptorMap.containsKey((short) fieldId)) {
+              throw new IllegalArgumentException(
+                  "Duplicate field id "
+                      + fieldId
+                      + " for field "
+                      + desc.getName()
+                      + " in class "
+                      + cls.getName());
+            }
+            fieldIdToDescriptorMap.put((short) fieldId, desc);
+          }
+        }
       }
+
+      if (newDescriptors[0] != null) {
+        SortedMap<Member, Descriptor> allDescriptorsCopy = new TreeMap<>(allDescriptorsMap);
+        allDescriptorsCopy.putAll(newDescriptors[0]);
+        resolver.getFory().getClassResolver().updateDescriptorsCache(cls, true, allDescriptorsCopy);
+      }
+
       descriptors = new ArrayList<>(fieldsInfo.size());
       for (FieldInfo fieldInfo : fieldsInfo) {
-        Descriptor descriptor =
-            descriptorsMap.get(fieldInfo.getDefinedClass() + "." + fieldInfo.getFieldName());
+        Descriptor descriptor;
+
+        // Try to match by field ID first if the FieldInfo has an ID
+        if (fieldInfo.hasFieldId()) {
+          descriptor = fieldIdToDescriptorMap.get(fieldInfo.getFieldId());
+        } else {
+          descriptor =
+              descriptorsMap.get(fieldInfo.getDefinedClass() + "." + fieldInfo.getFieldName());
+        }
+
         Descriptor newDesc = fieldInfo.toDescriptor(resolver, descriptor);
         Class<?> rawType = newDesc.getRawType();
         FieldType fieldType = fieldInfo.getFieldType();
@@ -422,10 +484,18 @@ public static class FieldInfo implements Serializable {
 
     private final FieldType fieldType;
 
+    /** Field ID for schema evolution, -1 means no field ID (use field name). */
+    private final short fieldId;
+
     FieldInfo(String definedClass, String fieldName, FieldType fieldType) {
+      this(definedClass, fieldName, fieldType, (short) -1);
+    }
+
+    FieldInfo(String definedClass, String fieldName, FieldType fieldType, short fieldId) {
       this.definedClass = definedClass;
       this.fieldName = fieldName;
       this.fieldType = fieldType;
+      this.fieldId = fieldId;
     }
 
     /** Returns classname of current field defined. */
@@ -439,13 +509,13 @@ public String getFieldName() {
     }
 
     /** Returns whether field is annotated by an unsigned int id. */
-    public boolean hasTag() {
-      return false;
+    public boolean hasFieldId() {
+      return fieldId >= 0;
     }
 
-    /** Returns annotated tag id for the field. */
-    public short getTag() {
-      return -1;
+    /** Returns annotated field-id for the field. */
+    public short getFieldId() {
+      return fieldId;
     }
 
     /** Returns type of current field. */
@@ -465,13 +535,15 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) {
         if (typeRef.equals(declared)) {
           return descriptor;
         } else {
+          // TODO fix return here
           descriptor.copyWithTypeName(typeRef.getType().getTypeName());
         }
       }
       // This field doesn't exist in peer class, so any legal modifier will be OK.
       // Use constant instead of reflection to avoid GraalVM native image issues.
       int stubModifiers = Modifier.PRIVATE | Modifier.FINAL;
-      return new Descriptor(typeRef, fieldName, stubModifiers, definedClass);
+      return new Descriptor(
+          typeRef, fieldName, stubModifiers, definedClass, resolver.needToWriteRef(typeRef));
     }
 
     @Override
@@ -483,14 +555,15 @@ public boolean equals(Object o) {
         return false;
       }
       FieldInfo fieldInfo = (FieldInfo) o;
-      return Objects.equals(definedClass, fieldInfo.definedClass)
+      return fieldId == fieldInfo.fieldId
+          && Objects.equals(definedClass, fieldInfo.definedClass)
           && Objects.equals(fieldName, fieldInfo.fieldName)
           && Objects.equals(fieldType, fieldInfo.fieldType);
     }
 
     @Override
     public int hashCode() {
-      return Objects.hash(definedClass, fieldName, fieldType);
+      return Objects.hash(definedClass, fieldName, fieldType, fieldId);
     }
 
     @Override
@@ -502,6 +575,7 @@ public String toString() {
           + ", fieldName='"
           + fieldName
           + '\''
+          + (fieldId >= 0 ? ", fieldID=" + fieldId : "")
           + ", fieldType="
           + fieldType
           + '}';
@@ -1089,11 +1163,12 @@ public String toString() {
   static FieldType buildFieldType(TypeResolver resolver, Field field) {
     Preconditions.checkNotNull(field);
     GenericType genericType = resolver.buildGenericType(field.getGenericType());
-    return buildFieldType(resolver, genericType);
+    return buildFieldType(resolver, field, genericType);
   }
 
   /** Build field type from generics, nested generics will be extracted too. */
-  private static FieldType buildFieldType(TypeResolver resolver, GenericType genericType) {
+  private static FieldType buildFieldType(
+      TypeResolver resolver, Field field, GenericType genericType) {
     Preconditions.checkNotNull(genericType);
     Class<?> rawType = genericType.getCls();
     boolean isXlang = resolver.getFory().isCrossLanguage();
@@ -1108,8 +1183,17 @@ private static FieldType buildFieldType(TypeResolver resolver, GenericType gener
     }
     boolean isMonomorphic = genericType.isMonomorphic();
     boolean trackingRef = genericType.trackingRef(resolver);
-    // TODO support @Nullable/ForyField annotation
     boolean nullable = !genericType.getCls().isPrimitive();
+
+    // Apply @ForyField annotation if present
+    if (field != null) {
+      ForyField foryField = field.getAnnotation(ForyField.class);
+      if (foryField != null) {
+        nullable = foryField.nullable();
+        trackingRef = foryField.ref();
+      }
+    }
+
     if (COLLECTION_TYPE.isSupertypeOf(genericType.getTypeRef())) {
       return new CollectionFieldType(
           xtypeId,
@@ -1118,6 +1202,7 @@ private static FieldType buildFieldType(TypeResolver resolver, GenericType gener
           trackingRef,
           buildFieldType(
               resolver,
+              null, // nested fields don't have Field reference
               genericType.getTypeParameter0() == null
                   ? GenericType.build(Object.class)
                   : genericType.getTypeParameter0()));
@@ -1129,11 +1214,13 @@ private static FieldType buildFieldType(TypeResolver resolver, GenericType gener
           trackingRef,
           buildFieldType(
               resolver,
+              null, // nested fields don't have Field reference
               genericType.getTypeParameter0() == null
                   ? GenericType.build(Object.class)
                   : genericType.getTypeParameter0()),
           buildFieldType(
               resolver,
+              null, // nested fields don't have Field reference
               genericType.getTypeParameter1() == null
                   ? GenericType.build(Object.class)
                   : genericType.getTypeParameter1()));
@@ -1160,15 +1247,15 @@ private static FieldType buildFieldType(TypeResolver resolver, GenericType gener
                 isMonomorphic,
                 nullable,
                 trackingRef,
-                buildFieldType(resolver, GenericType.build(elemType)));
+                buildFieldType(resolver, null, GenericType.build(elemType)));
           }
           Tuple2<Class<?>, Integer> info = TypeUtils.getArrayComponentInfo(rawType);
           return new ArrayFieldType(
               xtypeId,
               isMonomorphic,
               nullable,
               trackingRef,
-              buildFieldType(resolver, GenericType.build(info.f0)),
+              buildFieldType(resolver, null, GenericType.build(info.f0)),
               info.f1);
         }
         return new ObjectFieldType(xtypeId, isMonomorphic, nullable, trackingRef);