Skip to content

Commit 51039b2

Browse files
kkim-labelboxkkim-labelbox
authored
Merge pull request #1013 from Labelbox/kkim/AL-5296
[AL-5296] Add global key support to ConversationEntity, DocumentEntity, DicomSe…
2 parents f9e4457 + 2f064a0 commit 51039b2

12 files changed

+169
-59
lines changed

‎labelbox/data/serialization/ndjson/objects.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,7 @@ def from_common(cls, segments: List[DICOMObjectAnnotation], data: VideoData,
353353
segments = [NDDicomSegment.from_common(segment) for segment in segments]
354354

355355
return cls(segments=segments,
356-
dataRow=DataRow(id=data.uid),
356+
dataRow=DataRow(id=data.uid, global_key=data.global_key),
357357
name=name,
358358
schema_id=feature_schema_id,
359359
uuid=extra.get('uuid'),
@@ -465,7 +465,7 @@ def from_common(cls,
465465
confidence: Optional[float] = None) -> "NDDocumentEntity":
466466

467467
return cls(text_selections=document_entity.text_selections,
468-
dataRow=DataRow(id=data.uid),
468+
dataRow=DataRow(id=data.uid, global_key=data.global_key),
469469
name=name,
470470
schema_id=feature_schema_id,
471471
uuid=extra.get('uuid'),
@@ -494,7 +494,7 @@ def from_common(
494494
return cls(location=Location(start=conversation_entity.start,
495495
end=conversation_entity.end),
496496
message_id=conversation_entity.message_id,
497-
dataRow=DataRow(id=data.uid),
497+
dataRow=DataRow(id=data.uid, global_key=data.global_key),
498498
name=name,
499499
schema_id=feature_schema_id,
500500
uuid=extra.get('uuid'),

‎tests/data/assets/ndjson/conversation_entity_import_global_key.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[{
2+
"location": {
3+
"start": 67,
4+
"end": 128
5+
},
6+
"messageId": "some-message-id",
7+
"uuid": "5ad9c52f-058d-49c8-a749-3f20b84f8cd4",
8+
"dataRow": {
9+
"globalKey": "05e8ee85-072e-4eb2-b30a-501dee9b0d9d"
10+
},
11+
"name": "some-text-entity",
12+
"schemaId": "cl6xnuwt95lqq07330tbb3mfd",
13+
"classifications": [],
14+
"confidence": 0.53
15+
}]

‎tests/data/assets/ndjson/pdf_document_entity_import.json‎

Lines changed: 0 additions & 25 deletions
This file was deleted.

‎tests/data/assets/ndjson/pdf_import.json‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,4 +97,30 @@
9797
"height": 203.83000000000004,
9898
"width": 0.37999999999999545
9999
}
100-
}]
100+
},
101+
{
102+
"uuid": "f6879f59-d2b5-49c2-aceb-d9e8dc478673",
103+
"dataRow": {
104+
"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"
105+
},
106+
"name": "named_entity",
107+
"schemaId": "cl6xnuwt95lqq07330tbb3mfd",
108+
"classifications": [],
109+
"textSelections": [
110+
{
111+
"groupId": "2f4336f4-a07e-4e0a-a9e1-5629b03b719b",
112+
"tokenIds": [
113+
"3f984bf3-1d61-44f5-b59a-9658a2e3440f",
114+
"3bf00b56-ff12-4e52-8cc1-08dbddb3c3b8",
115+
"6e1c3420-d4b7-4c5a-8fd6-ead43bf73d80",
116+
"87a43d32-af76-4a1d-b262-5c5f4d5ace3a",
117+
"e8606e8a-dfd9-4c49-a635-ad5c879c75d0",
118+
"67c7c19e-4654-425d-bf17-2adb8cf02c30",
119+
"149c5e80-3e07-49a7-ab2d-29ddfe6a38fa",
120+
"b0e94071-2187-461e-8e76-96c58738a52c"
121+
],
122+
"page": 1
123+
}
124+
]
125+
}
126+
]

‎tests/data/assets/ndjson/pdf_import_global_key.json‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,4 +97,29 @@
9797
"height": 203.83000000000004,
9898
"width": 0.37999999999999545
9999
}
100-
}]
100+
},
101+
{
102+
"uuid": "f6879f59-d2b5-49c2-aceb-d9e8dc478673",
103+
"dataRow": {
104+
"globalKey": "05e8ee85-072e-4eb2-b30a-501dee9b0d9d"
105+
},
106+
"name": "named_entity",
107+
"classifications": [],
108+
"textSelections": [
109+
{
110+
"groupId": "2f4336f4-a07e-4e0a-a9e1-5629b03b719b",
111+
"tokenIds": [
112+
"3f984bf3-1d61-44f5-b59a-9658a2e3440f",
113+
"3bf00b56-ff12-4e52-8cc1-08dbddb3c3b8",
114+
"6e1c3420-d4b7-4c5a-8fd6-ead43bf73d80",
115+
"87a43d32-af76-4a1d-b262-5c5f4d5ace3a",
116+
"e8606e8a-dfd9-4c49-a635-ad5c879c75d0",
117+
"67c7c19e-4654-425d-bf17-2adb8cf02c30",
118+
"149c5e80-3e07-49a7-ab2d-29ddfe6a38fa",
119+
"b0e94071-2187-461e-8e76-96c58738a52c"
120+
],
121+
"page": 1
122+
}
123+
]
124+
}
125+
]

‎tests/data/assets/ndjson/pdf_import_name_only.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,4 +90,28 @@
9090
"height": 203.83000000000004,
9191
"width": 0.37999999999999545
9292
}
93+
},
94+
{
95+
"uuid": "f6879f59-d2b5-49c2-aceb-d9e8dc478673",
96+
"dataRow": {
97+
"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"
98+
},
99+
"name": "named_entity",
100+
"classifications": [],
101+
"textSelections": [
102+
{
103+
"groupId": "2f4336f4-a07e-4e0a-a9e1-5629b03b719b",
104+
"tokenIds": [
105+
"3f984bf3-1d61-44f5-b59a-9658a2e3440f",
106+
"3bf00b56-ff12-4e52-8cc1-08dbddb3c3b8",
107+
"6e1c3420-d4b7-4c5a-8fd6-ead43bf73d80",
108+
"87a43d32-af76-4a1d-b262-5c5f4d5ace3a",
109+
"e8606e8a-dfd9-4c49-a635-ad5c879c75d0",
110+
"67c7c19e-4654-425d-bf17-2adb8cf02c30",
111+
"149c5e80-3e07-49a7-ab2d-29ddfe6a38fa",
112+
"b0e94071-2187-461e-8e76-96c58738a52c"
113+
],
114+
"page": 1
115+
}
116+
]
93117
}]

‎tests/data/serialization/ndjson/test_classification.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
import json
2-
from labelbox.data.serialization.ndjson.classification import NDRadio
32

43
from labelbox.data.serialization.ndjson.converter import NDJsonConverter
5-
from labelbox.data.serialization.ndjson.objects import NDLine
64

75

86
def test_classification():

‎tests/data/serialization/ndjson/test_conversation.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import json
2+
3+
import pytest
4+
5+
from labelbox.data.serialization.ndjson.converter import NDJsonConverter
6+
7+
8+
@pytest.mark.parametrize("filename", [
9+
"tests/data/assets/ndjson/conversation_entity_import.json",
10+
"tests/data/assets/ndjson/conversation_entity_without_confidence_import.json"
11+
])
12+
def test_conversation_entity_import(filename: str):
13+
with open(filename, 'r') as file:
14+
data = json.load(file)
15+
res = list(NDJsonConverter.deserialize(data))
16+
res = list(NDJsonConverter.serialize(res))
17+
assert res == data

‎tests/data/serialization/ndjson/test_dicom.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919

2020
label = lb_types.Label(data=lb_types.DicomData(uid="test-uid"),
2121
annotations=dicom_polyline_annotations)
22+
label_with_global_key = lb_types.Label(
23+
data=lb_types.DicomData(global_key="test-global-key"),
24+
annotations=dicom_polyline_annotations)
2225

2326
label_ndjson = {
2427
'classifications': [],
@@ -51,26 +54,78 @@
5154
}],
5255
}
5356

57+
label_ndjson_with_global_key = {
58+
'classifications': [],
59+
'dataRow': {
60+
'globalKey': 'test-global-key'
61+
},
62+
'name':
63+
'dicom_polyline',
64+
'groupKey':
65+
'axial',
66+
'segments': [{
67+
'keyframes': [{
68+
'frame':
69+
2,
70+
'line': [
71+
{
72+
'x': 680.0,
73+
'y': 100.0
74+
},
75+
{
76+
'x': 100.0,
77+
'y': 190.0
78+
},
79+
{
80+
'x': 190.0,
81+
'y': 220.0
82+
},
83+
]
84+
}]
85+
}],
86+
}
87+
5488

5589
def test_serialize_dicom_polyline_annotation():
5690
serialized_label = next(NDJsonConverter().serialize([label]))
5791
serialized_label.pop('uuid')
5892
assert serialized_label == label_ndjson
5993

6094

95+
def test_serialize_dicom_polyline_annotation_with_global_key():
96+
serialized_label = next(NDJsonConverter().serialize([label_with_global_key
97+
]))
98+
serialized_label.pop('uuid')
99+
assert serialized_label == label_ndjson_with_global_key
100+
101+
61102
def test_deserialize_dicom_polyline_annotation():
62103
deserialized_label = next(NDJsonConverter().deserialize([label_ndjson]))
63104
deserialized_label.annotations[0].extra.pop('uuid')
64105
assert deserialized_label == label
65106

66107

108+
def test_deserialize_dicom_polyline_annotation_with_global_key():
109+
deserialized_label = next(NDJsonConverter().deserialize(
110+
[label_ndjson_with_global_key]))
111+
deserialized_label.annotations[0].extra.pop('uuid')
112+
assert deserialized_label == label_with_global_key
113+
114+
67115
def test_serialize_deserialize_dicom_polyline_annotation():
68116
labels = list(NDJsonConverter.deserialize([label_ndjson]))
69117
res = list(NDJsonConverter.serialize(labels))
70118
res[0].pop('uuid')
71119
assert res == [label_ndjson]
72120

73121

122+
def test_serialize_deserialize_dicom_polyline_annotation_with_global_key():
123+
labels = list(NDJsonConverter.deserialize([label_ndjson_with_global_key]))
124+
res = list(NDJsonConverter.serialize(labels))
125+
res[0].pop('uuid')
126+
assert res == [label_ndjson_with_global_key]
127+
128+
74129
def test_deserialize_nd_dicom_segments():
75130
nd_dicom_segments = NDDicomSegments(**label_ndjson)
76131
assert isinstance(nd_dicom_segments, NDDicomSegments)

‎tests/data/serialization/ndjson/test_document.py‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,20 +28,6 @@ def test_pdf():
2828
f.close()
2929

3030

31-
def test_pdf_document_entity():
32-
"""
33-
Tests a pdf file with bbox annotations only
34-
"""
35-
with open('tests/data/assets/ndjson/pdf_document_entity_import.json',
36-
'r') as f:
37-
data = json.load(f)
38-
res = list(NDJsonConverter.deserialize(data))
39-
res = list(NDJsonConverter.serialize(res))
40-
assert [round_dict(x) for x in res] == [round_dict(x) for x in data]
41-
42-
f.close()
43-
44-
4531
def test_pdf_with_name_only():
4632
"""
4733
Tests a pdf file with bbox annotations only

0 commit comments

Comments
 (0)