Small fixes for CI, sync with C

cyclonedds/idl/_xt_builder.py

@@ -16,8 +16,6 @@
 from dataclasses import dataclass
 from hashlib import md5
 
-from cyclonedds.internal import uint32_max
-
 from ._typesupport.DDS import XTypes as xt
 from . import types as pt
 from . import annotations as annotate
@@ -27,6 +25,9 @@
 from ._type_normalize import get_extended_type_hints, get_idl_annotations, get_idl_field_annotations, WrapOpt
 
 
+uint32_max = 2 ** 32 - 1
+
+
 def _is_optional(_type: Any) -> bool:
     return isinstance(_type, WrapOpt)
 
@@ -676,7 +677,7 @@ def _xt_type_identifier(cls, name: str, entity: Any, minimal: bool) -> xt.TypeId
                 fully_descriptive = cls._impl_xt_is_fully_descriptive(entity)
                 header = xt.PlainCollectionHeader(
                     equiv_kind=xt.EK_BOTH if fully_descriptive else (xt.EK_MINIMAL if minimal else xt.EK_COMPLETE),
-                    element_flags=xt.MemberFlag()  # TODO: elements can be external, try_construct
+                    element_flags=xt.MemberFlag(TRY_CONSTRUCT1=True)  # TODO: elements can be external, try_construct
                 )
 
                 if entity.max_length is None:
@@ -708,7 +709,7 @@ def _xt_type_identifier(cls, name: str, entity: Any, minimal: bool) -> xt.TypeId
                 fully_descriptive = cls._impl_xt_is_fully_descriptive(entity)
                 header = xt.PlainCollectionHeader(
                     equiv_kind=xt.EK_BOTH if fully_descriptive else (xt.EK_MINIMAL if minimal else xt.EK_COMPLETE),
-                    element_flags=xt.MemberFlag()  # TODO: elements can be external, try_construct
+                    element_flags=xt.MemberFlag(TRY_CONSTRUCT1=True)  # TODO: elements can be external, try_construct
                 )
 
                 inner: Type[Any] = entity
@@ -978,6 +979,10 @@ def _xt_member_flag(cls, entity: Any, name: str, _type: type) -> xt.MemberFlag:
 
         # TRY CONSTRUCT TODO: INVALID, DISCARD, TRIM?
         if "default" in annotations:
+            # 10 = use_default
+            flag.TRY_CONSTRUCT2 = True
+        else:
+            # 01 = discard
             flag.TRY_CONSTRUCT1 = True
 
         if annotations.get("external", False):
@@ -1074,7 +1079,7 @@ def _xt_complete_discriminator_member(cls, entity: Type[IdlUnion]) -> xt.Complet
     @classmethod
     def _xt_common_discriminator_member(cls, entity: Type[IdlUnion], minimal: bool) -> xt.CommonDiscriminatorMember:
         return xt.CommonDiscriminatorMember(
-            member_flags=xt.MemberFlag(IS_KEY=entity.__idl_discriminator_is_key__, IS_MUST_UNDERSTAND=True),
+            member_flags=xt.MemberFlag(IS_KEY=entity.__idl_discriminator_is_key__, IS_MUST_UNDERSTAND=True, TRY_CONSTRUCT1=True),
             type_id=cls._xt_type_identifier("discriminator", entity.__idl_discriminator__, minimal)
         )
 
@@ -1142,15 +1147,15 @@ def _xt_minimal_sequence_type(cls, seq: pt.sequence) -> xt.MinimalSequenceType:
         return xt.MinimalSequenceType(
             collection_flag=xt.TypeFlag(),  # Unused, no flags apply
             header=cls._xt_minimal_collection_header(seq),
-            element=cls._xt_minimal_collection_element(seq.subtype)
+            element=cls._xt_minimal_collection_element(seq.subtype, xt.TypeFlag(TRY_CONSTRUCT1=True))
         )
 
     @classmethod
     def _xt_complete_sequence_type(cls, seq: pt.sequence) -> xt.CompleteSequenceType:
         return xt.CompleteSequenceType(
             collection_flag=xt.TypeFlag(),  # Unused, no flags apply
             header=cls._xt_complete_collection_header(seq),
-            element=cls._xt_complete_collection_element(seq.subtype)
+            element=cls._xt_complete_collection_element(seq.subtype, xt.TypeFlag(TRY_CONSTRUCT1=True))
         )
 
     @classmethod
@@ -1173,15 +1178,15 @@ def _xt_common_collection_header(cls, collection: Union[pt.array, pt.sequence])
         )
 
     @classmethod
-    def _xt_minimal_collection_element(cls, _type: Any) -> xt.MinimalCollectionElement:
+    def _xt_minimal_collection_element(cls, _type: Any, flags: xt.TypeFlag) -> xt.MinimalCollectionElement:
         return xt.MinimalCollectionElement(
-            common=cls._xt_common_collection_element(_type, True)
+            common=cls._xt_common_collection_element(_type, True, flags)
         )
 
     @classmethod
-    def _xt_complete_collection_element(cls, _type: Any) -> xt.CompleteCollectionElement:
+    def _xt_complete_collection_element(cls, _type: Any, flags: xt.TypeFlag) -> xt.CompleteCollectionElement:
         return xt.CompleteCollectionElement(
-            common=cls._xt_common_collection_element(_type, True),
+            common=cls._xt_common_collection_element(_type, True, flags),
             detail=cls._xt_complete_element_detail(_type)
         )
 
@@ -1193,11 +1198,11 @@ def _xt_complete_element_detail(cls, _type: Any) -> xt.CompleteElementDetail:
         )
 
     @classmethod
-    def _xt_common_collection_element(cls, _type: Any, minimal: bool) -> xt.CommonCollectionElement:
+    def _xt_common_collection_element(cls, _type: Any, minimal: bool, flags: xt.TypeFlag) -> xt.CommonCollectionElement:
         # TODO: how to scope this? Annotations can only be on the parent object,
         # how to get the try_construct/external flags here?
         return xt.CommonCollectionElement(
-            element_flags=xt.TypeFlag(),
+            element_flags=flags,
             type=cls._xt_type_identifier("anonymous", _type, minimal)
         )
 
@@ -1210,7 +1215,7 @@ def _xt_minimal_array_type(cls, arr: pt.array) -> xt.MinimalArrayType:
         return xt.MinimalArrayType(
             collection_flag=xt.TypeFlag(),  # Unused, no flags apply
             header=cls._xt_minimal_array_header(arr),
-            element=cls._xt_minimal_collection_element(inner)
+            element=cls._xt_minimal_collection_element(inner, xt.TypeFlag(TRY_CONSTRUCT1=True))
 
         )
 
@@ -1223,7 +1228,7 @@ def _xt_complete_array_type(cls, arr: pt.array) -> xt.CompleteArrayType:
         return xt.CompleteArrayType(
             collection_flag=xt.TypeFlag(),  # Unused, no flags apply
             header=cls._xt_complete_array_header(arr),
-            element=cls._xt_complete_collection_element(inner)
+            element=cls._xt_complete_collection_element(inner, xt.TypeFlag(TRY_CONSTRUCT1=True))
 
         )
 

tests/support_modules/fuzz_tools/checks/keys.py

@@ -84,7 +84,7 @@ def check_py_c_key_equivalence(log: Stream, ctx: FullContext, typename: str, num
             return False
         time.sleep(0.001)
 
-    time.sleep(0.2)
+    time.sleep(0.5)
 
     for sample in samples:
         dw.write(sample)
@@ -96,14 +96,18 @@ def check_py_c_key_equivalence(log: Stream, ctx: FullContext, typename: str, num
     if not hashes:
         log << f"C-app did not return output, stderr:" << log.endl << log.indent
         log << ctx.c_app.last_error << log.endl
+        log << f"stdout:" << log.endl
+        log << ctx.c_app.last_out << log.endl
         log << log.dedent << "Example sample sent:" << log.endl << log.indent
         log << samples[0] << log.endl << samples[0].serialize()
         log << log.dedent
         return False
 
     if len(hashes) != len(samples):
         log << f"C-app did not return as many samples as were sent, stderr:" << log.endl << log.indent
-        log << ctx.c_app.last_error << log.endl << log.dedent
+        log << ctx.c_app.last_error << log.endl
+        log << f"stdout:" << log.endl
+        log << ctx.c_app.last_out << log.endl << log.dedent
         log << log.dedent << "Example sample sent:" << log.endl << log.indent
         log << samples[0] << log.endl << samples[0].serialize()
         success = False

tests/support_modules/fuzz_tools/checks/mutated.py

@@ -149,6 +149,8 @@ def check_mutation_key(log: Stream, ctx: FullContext, typename: str, num_samples
     if not hashes:
         log << f"C-app did not return output, stderr:" << log.endl << log.indent
         log << ctx.c_app.last_error << log.endl
+        log << f"stdout:" << log.endl
+        log << ctx.c_app.last_out << log.endl
         log << log.dedent << "Example Mutated sample sent:" << log.endl << log.indent
         log << samples[0] << samples[0].serialize()
         log << log.dedent << "[Mutated IDL]:" << log.indent << log.endl

tests/support_modules/fuzz_tools/rand_idl/c_app/xtypes_sub.c

@@ -82,6 +82,7 @@ int main(int argc, char **argv)
         hex_buff = (char*) realloc(hex_buff, hex_buff_size);
         tohex(descriptor->type_mapping.data, descriptor->type_mapping.sz, hex_buff, hex_buff_size);
         printf("%s\n", hex_buff);
+        fflush(stdout);
         return 0;
     }
 
@@ -125,7 +126,7 @@ int main(int argc, char **argv)
 
             tohex(keystream.x.m_buffer, keystream.x.m_index, hex_buff, hex_buff_size);
 
-            printf("%s\n", hex_buff);
+            printf("0x%s\n", hex_buff);
 
             seqq++;
         }

tests/support_modules/fuzz_tools/rand_idl/context_containers.py

@@ -22,6 +22,7 @@ class CAppContext:
     xtypes_dynamic_index: str
     running: bool = False
     last_error: str = ""
+    last_out: str = ""
     process: Optional[Popen] = None
 
     def run(self, typename: str, num_samples: int) -> None:
@@ -49,19 +50,21 @@ def description(self, typename: str) -> Optional[Tuple[bytes, bytes]]:
     def result(self) -> Optional[List[bytes]]:
         try:
             out, err = self.process.communicate(timeout=2)
+            self.last_out = out.decode()
         except TimeoutExpired:
             self.process.kill()
             try:
                 out, err = self.process.communicate(timeout=2)
                 self.last_error = err.decode()
+                self.last_out = out.decode()
             except:
                 self.last_error = "Did not manage to grab error output."
             return None
 
         self.last_error = err.decode()
 
         if self.process.returncode == 0:
-            return [bytes.fromhex(b) for b in out.decode().splitlines() if len(b) > 1]
+            return [bytes.fromhex(b[2:]) for b in out.decode().splitlines() if len(b) > 1 and b.startswith('0x')]
         return None