Skip to content

[AL-2659] Inclusion of optional metadata parameter #629

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 20, 2022
Merged

[AL-2659] Inclusion of optional metadata parameter #629

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fefee3a
update to have optional include metadata field
jtsodapop Jul 14, 2022
26775de
Merge branch 'develop' into jtso/al-2659
jtsodapop Jul 14, 2022
ebb0844
yapf
jtsodapop Jul 14, 2022
9bfaa63
cleanup so we do not need eto unpack response
jtsodapop Jul 15, 2022
a74db7a
yapf
jtsodapop Jul 15, 2022
daded19
Delete test.py
jtsodapop Jul 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 13 additions & 10 deletions labelbox/schema/batch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ def remove_queued_data_rows(self) -> None:
},
experimental=True)

def export_data_rows(self, timeout_seconds=120) -> Generator:
def export_data_rows(self,
timeout_seconds=120,
include_metadata: bool = False) -> Generator:
""" Returns a generator that produces all data rows that are currently
in this batch.

Expand All @@ -92,23 +94,24 @@ def export_data_rows(self, timeout_seconds=120) -> Generator:
LabelboxError: if the export fails or is unable to download within the specified time.
"""
id_param = "batchId"
query_str = """mutation GetBatchDataRowsExportUrlPyApi($%s: ID!)
{exportBatchDataRows(data:{batchId: $%s }) {downloadUrl createdAt status}}
""" % (id_param, id_param)
metadata_param = "includeMetadataInput"
query_str = """mutation GetBatchDataRowsExportUrlPyApi($%s: ID!, $%s: Boolean!)
{exportBatchDataRows(data:{batchId: $%s , includeMetadataInput: $%s}) {downloadUrl createdAt status}}
""" % (id_param, metadata_param, id_param, metadata_param)
sleep_time = 2
while True:
res = self.client.execute(query_str, {id_param: self.uid})
res = self.client.execute(query_str, {
id_param: self.uid,
metadata_param: include_metadata
})
res = res["exportBatchDataRows"]
if res["status"] == "COMPLETE":
download_url = res["downloadUrl"]
response = requests.get(download_url)
response.raise_for_status()
reader = ndjson.reader(StringIO(response.text))
# TODO: Update result to parse metadataFields when resolver returns
return (Entity.DataRow(self.client, {
**result, 'metadataFields': [],
'customMetadata': []
}) for result in reader)
return (
Entity.DataRow(self.client, result) for result in reader)
elif res["status"] == "FAILED":
raise LabelboxError("Data row export failed.")

Expand Down
23 changes: 13 additions & 10 deletions labelbox/schema/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,9 @@ def data_row_for_external_id(self, external_id) -> "DataRow":
external_id)
return data_rows[0]

def export_data_rows(self, timeout_seconds=120) -> Generator:
def export_data_rows(self,
timeout_seconds=120,
include_metadata: bool = False) -> Generator:
""" Returns a generator that produces all data rows that are currently
attached to this dataset.

Expand All @@ -477,23 +479,24 @@ def export_data_rows(self, timeout_seconds=120) -> Generator:
LabelboxError: if the export fails or is unable to download within the specified time.
"""
id_param = "datasetId"
query_str = """mutation GetDatasetDataRowsExportUrlPyApi($%s: ID!)
{exportDatasetDataRows(data:{datasetId: $%s }) {downloadUrl createdAt status}}
""" % (id_param, id_param)
metadata_param = "includeMetadataInput"
query_str = """mutation GetDatasetDataRowsExportUrlPyApi($%s: ID!, $%s: Boolean!)
{exportDatasetDataRows(data:{datasetId: $%s , includeMetadataInput: $%s}) {downloadUrl createdAt status}}
""" % (id_param, metadata_param, id_param, metadata_param)
sleep_time = 2
while True:
res = self.client.execute(query_str, {id_param: self.uid})
res = self.client.execute(query_str, {
id_param: self.uid,
metadata_param: include_metadata
})
res = res["exportDatasetDataRows"]
if res["status"] == "COMPLETE":
download_url = res["downloadUrl"]
response = requests.get(download_url)
response.raise_for_status()
reader = ndjson.reader(StringIO(response.text))
# TODO: Update result to parse metadataFields when resolver returns
return (Entity.DataRow(self.client, {
**result, 'metadataFields': [],
'customMetadata': []
}) for result in reader)
return (
Entity.DataRow(self.client, result) for result in reader)
elif res["status"] == "FAILED":
raise LabelboxError("Data row export failed.")

Expand Down
18 changes: 12 additions & 6 deletions labelbox/schema/project.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,10 @@ def labels(self, datasets=None, order_by=None) -> PaginatedCollection:
return PaginatedCollection(self.client, query_str, {id_param: self.uid},
["project", "labels"], Label)

def export_queued_data_rows(self,
timeout_seconds=120) -> List[Dict[str, str]]:
def export_queued_data_rows(
self,
timeout_seconds=120,
include_metadata: bool = False) -> List[Dict[str, str]]:
""" Returns all data rows that are currently enqueued for this project.

Args:
Expand All @@ -197,12 +199,16 @@ def export_queued_data_rows(self,
LabelboxError: if the export fails or is unable to download within the specified time.
"""
id_param = "projectId"
query_str = """mutation GetQueuedDataRowsExportUrlPyApi($%s: ID!)
{exportQueuedDataRows(data:{projectId: $%s }) {downloadUrl createdAt status} }
""" % (id_param, id_param)
metadata_param = "includeMetadataInput"
query_str = """mutation GetQueuedDataRowsExportUrlPyApi($%s: ID!, $%s: Boolean!)
{exportQueuedDataRows(data:{projectId: $%s , includeMetadataInput: $%s}) {downloadUrl createdAt status} }
""" % (id_param, metadata_param, id_param, metadata_param)
sleep_time = 2
while True:
res = self.client.execute(query_str, {id_param: self.uid})
res = self.client.execute(query_str, {
id_param: self.uid,
metadata_param: include_metadata
})
res = res["exportQueuedDataRows"]
if res["status"] == "COMPLETE":
download_url = res["downloadUrl"]
Expand Down