Add support for options in CRuby, JRuby and FFI (#14594)

Rewrrte and extension of #12828, with additional work for JRuby. Partially fixes #1198 by adding support for custom options. Handling of extensions will be handled in a follow up.

Also includes these unrelated fixes:
* Removes code echo between `google/protobuf/repeated_field.rb` and `google/protobuf/ffi/repeated_field.rb` by `require`'ing the former in the latter.
* Adds missing calles to `testFrozen()` from methods of `RepeatedField` under JRuby that mutate.
* Various typos in comments.

Closes #14594

COPYBARA_INTEGRATE_REVIEW=#14594 from protocolbuffers:add-support-for-options-in-ruby 16cc9e3
PiperOrigin-RevId: 580848874

ruby/ext/google/protobuf_c/defs.c

@@ -44,7 +44,7 @@ static VALUE rb_str_maybe_null(const char* s) {
   }
   return rb_str_new2(s);
 }
-
+static ID options_instancevar_interned;
 // -----------------------------------------------------------------------------
 // DescriptorPool.
 // -----------------------------------------------------------------------------
@@ -192,6 +192,7 @@ static void DescriptorPool_register(VALUE module) {
 
   rb_gc_register_address(&generated_pool);
   generated_pool = rb_class_new_instance(0, NULL, klass);
+  options_instancevar_interned = rb_intern("options");
 }
 
 // -----------------------------------------------------------------------------
@@ -226,6 +227,35 @@ static Descriptor* ruby_to_Descriptor(VALUE val) {
   return ret;
 }
 
+// Decode and return a frozen instance of a Descriptor Option for the given pool
+static VALUE decode_options(VALUE self, const char* option_type, int size,
+                            const char* bytes, VALUE descriptor_pool) {
+  VALUE options_rb = rb_ivar_get(self, options_instancevar_interned);
+  if (options_rb != Qnil) {
+    return options_rb;
+  }
+
+  static const char* prefix = "google.protobuf.";
+  char fullname
+      [/*strlen(prefix)*/ 16 +
+       /*strln(longest option type supported e.g. "MessageOptions")*/ 14 +
+       /*null terminator*/ 1];
+
+  snprintf(fullname, sizeof(fullname), "%s%s", prefix, option_type);
+  const upb_MessageDef* msgdef = upb_DefPool_FindMessageByName(
+      ruby_to_DescriptorPool(descriptor_pool)->symtab, fullname);
+  if (!msgdef) {
+    rb_raise(rb_eRuntimeError, "Cannot find %s in DescriptorPool", option_type);
+  }
+
+  VALUE desc_rb = get_msgdef_obj(descriptor_pool, msgdef);
+  const Descriptor* desc = ruby_to_Descriptor(desc_rb);
+
+  options_rb = Message_decode_bytes(size, bytes, 0, desc->klass, true);
+  rb_ivar_set(self, options_instancevar_interned, options_rb);
+  return options_rb;
+}
+
 /*
  * call-seq:
  *     Descriptor.new => descriptor
@@ -374,6 +404,26 @@ static VALUE Descriptor_msgclass(VALUE _self) {
   return self->klass;
 }
 
+/*
+ * call-seq:
+ *     Descriptor.options => options
+ *
+ * Returns the `MessageOptions` for this `Descriptor`.
+ */
+static VALUE Descriptor_options(VALUE _self) {
+  Descriptor* self = ruby_to_Descriptor(_self);
+  const google_protobuf_MessageOptions* opts =
+      upb_MessageDef_Options(self->msgdef);
+  upb_Arena* arena = upb_Arena_New();
+  size_t size;
+  char* serialized =
+      google_protobuf_MessageOptions_serialize(opts, arena, &size);
+  VALUE message_options = decode_options(_self, "MessageOptions", size,
+                                         serialized, self->descriptor_pool);
+  upb_Arena_Free(arena);
+  return message_options;
+}
+
 static void Descriptor_register(VALUE module) {
   VALUE klass = rb_define_class_under(module, "Descriptor", rb_cObject);
   rb_define_alloc_func(klass, Descriptor_alloc);
@@ -385,6 +435,7 @@ static void Descriptor_register(VALUE module) {
   rb_define_method(klass, "msgclass", Descriptor_msgclass, 0);
   rb_define_method(klass, "name", Descriptor_name, 0);
   rb_define_method(klass, "file_descriptor", Descriptor_file_descriptor, 0);
+  rb_define_method(klass, "options", Descriptor_options, 0);
   rb_include_module(klass, rb_mEnumerable);
   rb_gc_register_address(&cDescriptor);
   cDescriptor = klass;
@@ -484,12 +535,31 @@ static VALUE FileDescriptor_syntax(VALUE _self) {
   }
 }
 
+/*
+ * call-seq:
+ *     FileDescriptor.options => options
+ *
+ * Returns the `FileOptions` for this `FileDescriptor`.
+ */
+static VALUE FileDescriptor_options(VALUE _self) {
+  FileDescriptor* self = ruby_to_FileDescriptor(_self);
+  const google_protobuf_FileOptions* opts = upb_FileDef_Options(self->filedef);
+  upb_Arena* arena = upb_Arena_New();
+  size_t size;
+  char* serialized = google_protobuf_FileOptions_serialize(opts, arena, &size);
+  VALUE file_options = decode_options(_self, "FileOptions", size, serialized,
+                                      self->descriptor_pool);
+  upb_Arena_Free(arena);
+  return file_options;
+}
+
 static void FileDescriptor_register(VALUE module) {
   VALUE klass = rb_define_class_under(module, "FileDescriptor", rb_cObject);
   rb_define_alloc_func(klass, FileDescriptor_alloc);
   rb_define_method(klass, "initialize", FileDescriptor_initialize, 3);
   rb_define_method(klass, "name", FileDescriptor_name, 0);
   rb_define_method(klass, "syntax", FileDescriptor_syntax, 0);
+  rb_define_method(klass, "options", FileDescriptor_options, 0);
   rb_gc_register_address(&cFileDescriptor);
   cFileDescriptor = klass;
 }
@@ -540,7 +610,7 @@ static VALUE FieldDescriptor_alloc(VALUE klass) {
 
 /*
  * call-seq:
- *    EnumDescriptor.new(c_only_cookie, pool, ptr) => EnumDescriptor
+ *    FieldDescriptor.new(c_only_cookie, pool, ptr) => FieldDescriptor
  *
  * Creates a descriptor wrapper object.  May only be called from C.
  */
@@ -841,6 +911,25 @@ static VALUE FieldDescriptor_set(VALUE _self, VALUE msg_rb, VALUE value) {
   return Qnil;
 }
 
+/*
+ * call-seq:
+ *     FieldDescriptor.options => options
+ *
+ * Returns the `FieldOptions` for this `FieldDescriptor`.
+ */
+static VALUE FieldDescriptor_options(VALUE _self) {
+  FieldDescriptor* self = ruby_to_FieldDescriptor(_self);
+  const google_protobuf_FieldOptions* opts =
+      upb_FieldDef_Options(self->fielddef);
+  upb_Arena* arena = upb_Arena_New();
+  size_t size;
+  char* serialized = google_protobuf_FieldOptions_serialize(opts, arena, &size);
+  VALUE field_options = decode_options(_self, "FieldOptions", size, serialized,
+                                       self->descriptor_pool);
+  upb_Arena_Free(arena);
+  return field_options;
+}
+
 static void FieldDescriptor_register(VALUE module) {
   VALUE klass = rb_define_class_under(module, "FieldDescriptor", rb_cObject);
   rb_define_alloc_func(klass, FieldDescriptor_alloc);
@@ -857,6 +946,7 @@ static void FieldDescriptor_register(VALUE module) {
   rb_define_method(klass, "clear", FieldDescriptor_clear, 1);
   rb_define_method(klass, "get", FieldDescriptor_get, 1);
   rb_define_method(klass, "set", FieldDescriptor_set, 2);
+  rb_define_method(klass, "options", FieldDescriptor_options, 0);
   rb_gc_register_address(&cFieldDescriptor);
   cFieldDescriptor = klass;
 }
@@ -956,12 +1046,32 @@ static VALUE OneofDescriptor_each(VALUE _self) {
   return Qnil;
 }
 
+/*
+ * call-seq:
+ *     OneofDescriptor.options => options
+ *
+ * Returns the `OneofOptions` for this `OneofDescriptor`.
+ */
+static VALUE OneOfDescriptor_options(VALUE _self) {
+  OneofDescriptor* self = ruby_to_OneofDescriptor(_self);
+  const google_protobuf_OneofOptions* opts =
+      upb_OneofDef_Options(self->oneofdef);
+  upb_Arena* arena = upb_Arena_New();
+  size_t size;
+  char* serialized = google_protobuf_OneofOptions_serialize(opts, arena, &size);
+  VALUE oneof_options = decode_options(_self, "OneofOptions", size, serialized,
+                                       self->descriptor_pool);
+  upb_Arena_Free(arena);
+  return oneof_options;
+}
+
 static void OneofDescriptor_register(VALUE module) {
   VALUE klass = rb_define_class_under(module, "OneofDescriptor", rb_cObject);
   rb_define_alloc_func(klass, OneofDescriptor_alloc);
   rb_define_method(klass, "initialize", OneofDescriptor_initialize, 3);
   rb_define_method(klass, "name", OneofDescriptor_name, 0);
   rb_define_method(klass, "each", OneofDescriptor_each, 0);
+  rb_define_method(klass, "options", OneOfDescriptor_options, 0);
   rb_include_module(klass, rb_mEnumerable);
   rb_gc_register_address(&cOneofDescriptor);
   cOneofDescriptor = klass;
@@ -1131,6 +1241,24 @@ static VALUE EnumDescriptor_enummodule(VALUE _self) {
   return self->module;
 }
 
+/*
+ * call-seq:
+ *     EnumDescriptor.options => options
+ *
+ * Returns the `EnumOptions` for this `EnumDescriptor`.
+ */
+static VALUE EnumDescriptor_options(VALUE _self) {
+  EnumDescriptor* self = ruby_to_EnumDescriptor(_self);
+  const google_protobuf_EnumOptions* opts = upb_EnumDef_Options(self->enumdef);
+  upb_Arena* arena = upb_Arena_New();
+  size_t size;
+  char* serialized = google_protobuf_EnumOptions_serialize(opts, arena, &size);
+  VALUE enum_options = decode_options(_self, "EnumOptions", size, serialized,
+                                      self->descriptor_pool);
+  upb_Arena_Free(arena);
+  return enum_options;
+}
+
 static void EnumDescriptor_register(VALUE module) {
   VALUE klass = rb_define_class_under(module, "EnumDescriptor", rb_cObject);
   rb_define_alloc_func(klass, EnumDescriptor_alloc);
@@ -1141,6 +1269,7 @@ static void EnumDescriptor_register(VALUE module) {
   rb_define_method(klass, "each", EnumDescriptor_each, 0);
   rb_define_method(klass, "enummodule", EnumDescriptor_enummodule, 0);
   rb_define_method(klass, "file_descriptor", EnumDescriptor_file_descriptor, 0);
+  rb_define_method(klass, "options", EnumDescriptor_options, 0);
   rb_include_module(klass, rb_mEnumerable);
   rb_gc_register_address(&cEnumDescriptor);
   cEnumDescriptor = klass;

ruby/ext/google/protobuf_c/glue.c

@@ -14,8 +14,43 @@
 upb_Arena* Arena_create() { return upb_Arena_Init(NULL, 0, &upb_alloc_global); }
 
 google_protobuf_FileDescriptorProto* FileDescriptorProto_parse(
-    const char* serialized_file_proto, size_t length) {
-  upb_Arena* arena = Arena_create();
+    const char* serialized_file_proto, size_t length, upb_Arena* arena) {
   return google_protobuf_FileDescriptorProto_parse(serialized_file_proto,
                                                    length, arena);
 }
+
+char* EnumDescriptor_serialized_options(const upb_EnumDef* enumdef,
+                                        size_t* size, upb_Arena* arena) {
+  const google_protobuf_EnumOptions* opts = upb_EnumDef_Options(enumdef);
+  char* serialized = google_protobuf_EnumOptions_serialize(opts, arena, size);
+  return serialized;
+}
+
+char* FileDescriptor_serialized_options(const upb_FileDef* filedef,
+                                        size_t* size, upb_Arena* arena) {
+  const google_protobuf_FileOptions* opts = upb_FileDef_Options(filedef);
+  char* serialized = google_protobuf_FileOptions_serialize(opts, arena, size);
+  return serialized;
+}
+
+char* Descriptor_serialized_options(const upb_MessageDef* msgdef, size_t* size,
+                                    upb_Arena* arena) {
+  const google_protobuf_MessageOptions* opts = upb_MessageDef_Options(msgdef);
+  char* serialized =
+      google_protobuf_MessageOptions_serialize(opts, arena, size);
+  return serialized;
+}
+
+char* OneOfDescriptor_serialized_options(const upb_OneofDef* oneofdef,
+                                         size_t* size, upb_Arena* arena) {
+  const google_protobuf_OneofOptions* opts = upb_OneofDef_Options(oneofdef);
+  char* serialized = google_protobuf_OneofOptions_serialize(opts, arena, size);
+  return serialized;
+}
+
+char* FieldDescriptor_serialized_options(const upb_FieldDef* fielddef,
+                                         size_t* size, upb_Arena* arena) {
+  const google_protobuf_FieldOptions* opts = upb_FieldDef_Options(fielddef);
+  char* serialized = google_protobuf_FieldOptions_serialize(opts, arena, size);
+  return serialized;
+}

ruby/ext/google/protobuf_c/map.c

@@ -572,6 +572,26 @@ static VALUE Map_freeze(VALUE _self) {
   return _self;
 }
 
+/*
+ * Deep freezes the map and values recursively.
+ * Internal use only.
+ */
+VALUE Map_internal_deep_freeze(VALUE _self) {
+  Map* self = ruby_to_Map(_self);
+  Map_freeze(_self);
+  if (self->value_type_info.type == kUpb_CType_Message) {
+    size_t iter = kUpb_Map_Begin;
+    upb_MessageValue key, val;
+
+    while (upb_Map_Next(self->map, &key, &val, &iter)) {
+      VALUE val_val =
+          Convert_UpbToRuby(val, self->value_type_info, self->arena);
+      Message_internal_deep_freeze(val_val);
+    }
+  }
+  return _self;
+}
+
 /*
  * call-seq:
  *     Map.hash => hash_value

ruby/ext/google/protobuf_c/map.h

@@ -38,4 +38,7 @@ extern VALUE cMap;
 // Call at startup to register all types in this module.
 void Map_register(VALUE module);
 
+// Recursively freeze map
+VALUE Map_internal_deep_freeze(VALUE _self);
+
 #endif  // RUBY_PROTOBUF_MAP_H_

ruby/ext/google/protobuf_c/message.c

@@ -859,6 +859,32 @@ static VALUE Message_freeze(VALUE _self) {
   return _self;
 }
 
+/*
+ * Deep freezes the message object recursively.
+ * Internal use only.
+ */
+VALUE Message_internal_deep_freeze(VALUE _self) {
+  Message* self = ruby_to_Message(_self);
+  Message_freeze(_self);
+
+  int n = upb_MessageDef_FieldCount(self->msgdef);
+  for (int i = 0; i < n; i++) {
+    const upb_FieldDef* f = upb_MessageDef_Field(self->msgdef, i);
+    VALUE field = Message_getfield(_self, f);
+
+    if (field != Qnil) {
+      if (upb_FieldDef_IsMap(f)) {
+        Map_internal_deep_freeze(field);
+      } else if (upb_FieldDef_IsRepeated(f)) {
+        RepeatedField_internal_deep_freeze(field);
+      } else if (upb_FieldDef_IsSubMessage(f)) {
+        Message_internal_deep_freeze(field);
+      }
+    }
+  }
+  return _self;
+}
+
 /*
  * call-seq:
  *     Message.[](index) => value
@@ -911,7 +937,7 @@ static VALUE Message_index_set(VALUE _self, VALUE field_name, VALUE value) {
  *     MessageClass.decode(data, options) => message
  *
  * Decodes the given data (as a string containing bytes in protocol buffers wire
- * format) under the interpretration given by this message class's definition
+ * format) under the interpretation given by this message class's definition
  * and returns a message object with the corresponding field values.
  * @param options [Hash] options for the decoder
  *  recursion_limit: set to maximum decoding depth for message (default is 64)
@@ -942,18 +968,24 @@ static VALUE Message_decode(int argc, VALUE* argv, VALUE klass) {
     rb_raise(rb_eArgError, "Expected string for binary protobuf data.");
   }
 
+  return Message_decode_bytes(RSTRING_LEN(data), RSTRING_PTR(data), options,
+                              klass, /*freeze*/ false);
+}
+
+VALUE Message_decode_bytes(int size, const char* bytes, int options,
+                           VALUE klass, bool freeze) {
   VALUE msg_rb = initialize_rb_class_with_no_args(klass);
   Message* msg = ruby_to_Message(msg_rb);
 
-  upb_DecodeStatus status =
-      upb_Decode(RSTRING_PTR(data), RSTRING_LEN(data), (upb_Message*)msg->msg,
-                 upb_MessageDef_MiniTable(msg->msgdef), NULL, options,
-                 Arena_get(msg->arena));
-
+  upb_DecodeStatus status = upb_Decode(bytes, size, (upb_Message*)msg->msg,
+                                       upb_MessageDef_MiniTable(msg->msgdef),
+                                       NULL, options, Arena_get(msg->arena));
   if (status != kUpb_DecodeStatus_Ok) {
     rb_raise(cParseError, "Error occurred during parsing");
   }
-
+  if (freeze) {
+    Message_internal_deep_freeze(msg_rb);
+  }
   return msg_rb;
 }
 

ruby/ext/google/protobuf_c/message.h

@@ -73,6 +73,13 @@ VALUE build_module_from_enumdesc(VALUE _enumdesc);
 // module.
 VALUE MessageOrEnum_GetDescriptor(VALUE klass);
 
+// Decodes a Message from a byte sequence.
+VALUE Message_decode_bytes(int size, const char* bytes, int options,
+                           VALUE klass, bool freeze);
+
+// Recursively freeze message
+VALUE Message_internal_deep_freeze(VALUE _self);
+
 // Call at startup to register all types in this module.
 void Message_register(VALUE protobuf);