Merge pull request #74 from ClimateImpactLab/version_metadata

Version metadata

datafs/config/constructor.py

@@ -112,8 +112,8 @@ def _generate_manager(manager_config):
         >>> from datafs.managers.manager_dynamo import DynamoDBManager
         >>> assert isinstance(mgr, DynamoDBManager)
         >>>
-        >>> mgr.table_names
-        []
+        >>> 'data-from-yaml' in mgr.table_names
+        False
         >>> mgr.create_archive_table('data-from-yaml')
         >>> print(mgr.table_names[0])
         data-from-yaml

datafs/core/data_archive.py

@@ -27,24 +27,24 @@ def __repr__(self):
     def versioned(self):
         return self._versioned
 
-    @property
-    def latest_version(self):
+
+    def get_latest_version(self):
 
-        versions = self.versions
+        versions = self.get_versions()
 
         if len(versions) == 0:
             return None
 
         else:
             return max(versions)
 
-    @property
-    def versions(self):
+
+    def get_versions(self):
 
         if not self.versioned:
             return [None]
 
-        versions = self.history
+        versions = self.get_history()
 
         if len(versions) == 0:
             return []
@@ -91,7 +91,7 @@ def get_version_path(self, version=None):
 
         if self.versioned:
             if version is None:
-                version = self.latest_version
+                version = self.get_latest_version()
 
             if version is None:
                 return fs.path.join(self.archive_path, str(BumpableVersion()))
@@ -114,44 +114,44 @@ def authority(self):
     def archive_path(self):
         return self._archive_path
 
-    @property
-    def metadata(self):
+
+    def get_metadata(self):
         return self.api.manager.get_metadata(self.archive_name)
 
-    @property
-    def latest_hash(self):
+
+    def get_history(self):
+        return self.api.manager.get_version_history(self.archive_name)
+
+    def get_latest_hash(self):
         return self.api.manager.get_latest_hash(self.archive_name)
 
     def get_version_hash(self, version=None):
         if self.versioned:
             if version is None:
-                version = self.latest_version
+                version = self.get_latest_version()
 
             if version is None:
                 return None
 
-            for ver in self.history:
+            for ver in self.get_history():
                 if BumpableVersion(ver['version']) == version:
                     return ver['checksum']
 
             raise ValueError('Version "{}" not found in archive history'.format(
                 version))
 
         else:
-            return self.latest_hash
+            return self.get_latest_hash()
 
 
-    @property
-    def history(self):
-        return self.api.manager.get_versions(self.archive_name)
-
     def update(
         self, 
         filepath, 
         cache=False, 
         remove=False, 
         bumpversion='patch', 
         prerelease=None, 
+        dependencies=None,
         **kwargs):
         '''
         Enter a new version to a DataArchive
@@ -189,18 +189,19 @@ def update(
 
         '''
 
+        latest_version = self.get_latest_version()
 
-        latest_version = self.latest_version
+        hashval = self.api.hash_file(filepath)
 
-        checksum = self.api.hash_file(filepath)
+        checksum = hashval['checksum']
+        algorithm = hashval['algorithm']
 
-        if checksum['checksum'] == self.latest_hash:
+        if checksum == self.get_latest_hash():
             self.update_metadata(kwargs)
 
             if remove and os.path.isfile(filepath):
                 os.remove(filepath)
 
-            self._update_manager(checksum, kwargs, version=latest_version)
             return
 
         if self.versioned:
@@ -227,21 +228,19 @@ def update(
         else:
             self.authority.upload(filepath, next_path, remove=remove)
 
-        self._update_manager(checksum, kwargs, version=next_version)
+        self._update_manager(
+            archive_metadata=kwargs, 
+            version_metadata=dict(checksum=checksum, algorithm=algorithm, version=next_version, dependencies=dependencies))
 
 
-    def _update_manager(self, checksum, metadata={}, version=None):
-        version_metadata = dict(
-            archive_name=self.archive_name,
-            checksum=checksum,
-            metadata=metadata,
-            user_config=self.api.user_config)
+    def _update_manager(self, archive_metadata={}, version_metadata={}):
+
+        version_metadata['user_config'] = self.api.user_config
 
-        if self.versioned:
-            version_metadata['version'] = str(version)
 
         # update records in self.api.manager
-        self.api.manager.update(**version_metadata)
+        self.api.manager.update(self.archive_name, version_metadata)
+        self.update_metadata(archive_metadata)
 
     def update_metadata(self, metadata):
 
@@ -252,7 +251,7 @@ def update_metadata(self, metadata):
     # File I/O methods
 
     @contextmanager
-    def open(self, mode='r', version=None, bumpversion='patch', prerelease=None, *args, **kwargs):
+    def open(self, mode='r', version=None, bumpversion='patch', prerelease=None, dependencies = None, *args, **kwargs):
         '''
         Opens a file for read/write
 
@@ -285,11 +284,11 @@ def open(self, mode='r', version=None, bumpversion='patch', prerelease=None, *ar
         
         '''
         if version is None:
-            latest_version = self.latest_version
+            latest_version = self.get_latest_version()
             version = latest_version
 
         else:
-            latest_version = self.latest_version
+            latest_version = self.get_latest_version()
 
         version_hash = self.get_version_hash(version)
 
@@ -320,8 +319,8 @@ def open(self, mode='r', version=None, bumpversion='patch', prerelease=None, *ar
         version_check = lambda chk: chk['checksum'] == version_hash
 
         # Updater updates the manager with the latest version number
-        updater = lambda *args, **kwargs: self._update_manager(
-            *args, version=next_version, **kwargs)
+        updater = lambda **kwargs: self._update_manager(
+            version_metadata=dict(version=next_version, dependencies=dependencies, **kwargs))
 
         opener = data_file.open_file(
             self.authority,
@@ -339,7 +338,7 @@ def open(self, mode='r', version=None, bumpversion='patch', prerelease=None, *ar
             yield f
 
     @contextmanager
-    def get_local_path(self, version=None, bumpversion='patch', prerelease=None):
+    def get_local_path(self, version=None, bumpversion='patch', prerelease=None, dependencies=None, *args, **kwargs):
         '''
         Returns a local path for read/write
 
@@ -366,11 +365,11 @@ def get_local_path(self, version=None, bumpversion='patch', prerelease=None):
 
         '''
         if version is None:
-            latest_version = self.latest_version
+            latest_version = self.get_latest_version()
             version = latest_version
 
         else:
-            latest_version = self.latest_version
+            latest_version = self.get_latest_version()
 
         version_hash = self.get_version_hash(version)
 
@@ -401,8 +400,8 @@ def get_local_path(self, version=None, bumpversion='patch', prerelease=None):
         version_check = lambda chk: chk['checksum'] == version_hash
 
         # Updater updates the manager with the latest version number
-        updater = lambda *args, **kwargs: self._update_manager(
-            *args, version=next_version, **kwargs)
+        updater = lambda **kwargs: self._update_manager(
+            version_metadata=dict(version=next_version, dependencies=dependencies, **kwargs))
 
         path = data_file.get_local_path(
             self.authority,
@@ -427,7 +426,7 @@ def download(self, filepath, version=None):
         '''
 
         if version is None:
-            version = self.latest_version
+            version = self.get_latest_version()
 
         dirname, filename= os.path.split(
             os.path.abspath(os.path.expanduser(filepath)))
@@ -474,7 +473,7 @@ def delete(self):
 
         '''
 
-        for version in self.versions:
+        for version in self.get_versions():
             if self.authority.fs.exists(self.get_version_path(version)):
                 self.authority.fs.remove(self.get_version_path(version))
 
@@ -544,7 +543,7 @@ def is_cached(self, version=None):
         '''
 
         if version is None:
-            version = self.latest_version
+            version = self.get_latest_version()
 
         if self.api.cache and self.api.cache.fs.isfile(self.get_version_path(version)):
             return True
@@ -557,7 +556,7 @@ def cache(self, version=None):
             raise ValueError('No cache attached')
 
         if version is None:
-            version = self.latest_version
+            version = self.get_latest_version()
 
         if not self.api.cache.fs.isfile(self.get_version_path(version)):
             data_file._touch(self.api.cache.fs, self.get_version_path(version))
@@ -569,7 +568,24 @@ def cache(self, version=None):
     def remove_from_cache(self, version=None):
 
         if version is None:
-            version = self.latest_version
+            version = self.get_latest_version()
 
         if self.api.cache.fs.isfile(self.get_version_path(version)):
             self.api.cache.fs.remove(self.get_version_path(version))
+
+    def get_dependencies(self, version=None):
+        '''
+        Parameters
+        ----------
+        version: str
+            string representing version number whose dependencies you are looking up
+        '''
+
+        if version is None:
+            raise ValueError('No version provided')
+
+        for i,v in enumerate(self.get_history()):
+            if v['version'] == version:
+                return self.get_history()[i]['dependencies']
+
+

datafs/core/data_file.py

@@ -226,7 +226,7 @@ def open_file(
                         fs.utils.copyfile(
                             write_fs, read_path, authority.fs, write_path)
 
-                    update(checksum=checksum)
+                    update(**checksum)
 
         else:
 
@@ -303,7 +303,7 @@ def get_local_path(
                         _makedirs(authority.fs, fs.path.dirname(write_path))
                         fs.utils.copyfile(
                             write_fs, read_path, authority.fs, write_path)
-                    update(checksum=checksum)
+                    update(**checksum)
 
             else:
                 raise OSError(

datafs/datafs.py

@@ -148,10 +148,10 @@ def upload(ctx, archive_name, filepath, bumpversion='patch', prerelease=None):
     kwargs = parse_args_as_kwargs(ctx.args)
 
     var = ctx.obj.api.get_archive(archive_name)
-    latest_version = var.latest_version
+    latest_version = var.get_latest_version()
 
     var.update(filepath, bumpversion=bumpversion, prerelease=prerelease, **kwargs)
-    new_version = var.latest_version
+    new_version = var.get_latest_version()
 
     if new_version != latest_version:
         bumpmsg = ' version bumped {} --> {}.'.format(latest_version, new_version)
@@ -172,7 +172,7 @@ def download(ctx, archive_name, filepath, version):
     var = ctx.obj.api.get_archive(archive_name)
 
     if version is None:
-        version = var.latest_version
+        version = var.get_latest_version()
 
     var.download(filepath, version=version)
 
@@ -187,7 +187,7 @@ def download(ctx, archive_name, filepath, version):
 @click.pass_context
 def metadata(ctx, archive_name):
     var = ctx.obj.api.get_archive(archive_name)
-    click.echo(pprint.pformat(var.metadata))
+    click.echo(pprint.pformat(var.get_metadata()))
 
 
 @cli.command()