Merge pull request #86 from datamol-io/fix/checksum

Sort the files and save the .bin model file for backwards compatibility

molfeat/trans/pretrained/hf_transformers.py

@@ -50,6 +50,11 @@ def save(cls, model: HFExperiment, path: str, clean_up: bool = False):
         # we can save both the tokenizer and the model to the same path
         model.model.save_pretrained(local_path)
         model.tokenizer.save_pretrained(local_path)
+
+        # With transformers>=4.35.0, models are by default saved as safetensors.
+        # For backwards compatibility, we also save the model as the older pickle-based format.
+        model.model.save_pretrained(local_path, safe_serialization=False)
+
         dm.fs.copy_dir(local_path, path, force=True, progress=True, leave_progress=False)
         logger.info(f"Model saved to {path}")
         # clean up now

molfeat/utils/commons.py

@@ -47,7 +47,7 @@ def sha256sum(filepath: Union[str, os.PathLike]):
     else:
         files = [filepath]
     file_hash = hashlib.sha256()
-    for filepath in files:
+    for filepath in sorted(files):
         with fsspec.open(filepath) as f:
             file_hash.update(f.read())  # type: ignore
     file_hash = file_hash.hexdigest()