Skip to content

Commit

Permalink
Changes NGAS reproduce data to use the checksum provided by NGAS if a…
Browse files Browse the repository at this point in the history
…vailable, else use ID data, rather than hashing the entire contents.
  • Loading branch information
pritchardn committed Jul 5, 2022
1 parent 90f0170 commit 93f9645
Showing 1 changed file with 5 additions and 5 deletions.
10 changes: 5 additions & 5 deletions daliuge-engine/dlg/drop.py
Original file line number Diff line number Diff line change
Expand Up @@ -1720,6 +1720,7 @@ class NgasDROP(DataDROP):
ngasConnectTimeout = dlg_int_param("ngasConnectTimeout", 2)
ngasMime = dlg_string_param("ngasMime", "application/octet-stream")
len = dlg_int_param("len", -1)
ngas_checksum = None

def initialize(self, **kwargs):
if self.len == -1:
Expand Down Expand Up @@ -1785,6 +1786,7 @@ def setCompleted(self):
"Setting size of NGASDrop %s to %s", self.fileId, stat["FileSize"]
)
self._size = int(stat["FileSize"])
self.ngas_checksum = str(stat["Checksum"])
except:
# we''ll try this again in case there is some other issue
# try:
Expand All @@ -1808,11 +1810,9 @@ def dataURL(self) -> str:

# Override
def generate_reproduce_data(self):
# TODO: This is a bad implementation. Will need to sort something better out
from .droputils import allDropContents

data = allDropContents(self, self.size)
return {"data_hash": common_hash(data)}
if self.ngas_checksum is None or self.ngas_checksum == '':
return {"fileid": self.ngasFileId, "size": self._size}
return {"data_hash": self.ngas_checksum}


##
Expand Down

0 comments on commit 93f9645

Please sign in to comment.