Feature/provenance

.travis.yml

@@ -37,9 +37,16 @@ before_script:
 
 # command to run tests
 script:
-    - coverage run -a --source=quit tests/test_core.py
-    - coverage run -a --source=quit tests/test_conf.py
     - coverage run -a --source=quit tests/test_app.py
+    - coverage run -a --source=quit tests/test_cache.py
+    - coverage run -a --source=quit tests/test_conf.py
+    - coverage run -a --source=quit tests/test_core.py
+    - coverage run -a --source=quit tests/test_endpoint.py
+    - coverage run -a --source=quit tests/test_git.py
+    - coverage run -a --source=quit tests/test_graphs.py
+    - coverage run -a --source=quit tests/test_helpers.py
+    - coverage run -a --source=quit tests/test_namespace.py
+    - coverage run -a --source=quit tests/test_provenance.py
 
 after_success:
     coveralls

Dockerfile

@@ -42,4 +42,7 @@ EXPOSE 80
 # Quit writes its log file to the current directory
 WORKDIR /var/log
 
+# Set default git user
+RUN git config --global user.name QuitStore && git config --global user.email quitstore@example.org
+
 CMD quit --configfile ${QUIT_CONFIGFILE} --port ${QUIT_PORT}

README.md

@@ -80,6 +80,13 @@ Run Quit-Store without versioning activated
 
 Enable garbage collection. With this option activated, git will check for garbage collection after each commit. This may slow down response time but will keep the repository size small.
 
+`-f`, `--features`
+
+This option enables additional features of the store:
+
+- `provenance` - Store provenance information for each revision.
+- `persistance` - Store all internal data as RDF graph.
+
 `-v`, `--verbose` and `-vv`, `--verboseverbose`
 
 Set the loglevel for the standard output to verbose (INFO) respective extra verbose (DEBUG).

config-vocab.ttl

@@ -25,5 +25,20 @@ quit:storeFromDir a rdfs:Property ;
 quit:pathOfGitRepo a rdfs:Property ;
   rdfs:label "Path to the local Git Repository" .
 
+quit:defaultBranch a rdfs:Property ;
+  rdfs:label "Branch used as default" .
+
+quit:globalFile a rdfs:Property ;
+  rdfs:label "File for unassigned graphs" .
+
 quit:linkToGitRemote a rdfs:Property ;
   rdfs:label "Link to the Git Remote" .
+
+quit:Binding a rdfs:Class ;
+  rdfs:label "Information about custom namespaces" .
+
+quit:prefix a rdfs:Property ;
+  rdfs:label "Prefix for a binding" .
+
+quit:namespace a rdfs:Property ;
+  rdfs:label "Actual namespace for a binding" .

config.example.ttl

@@ -1,18 +1,35 @@
-@base <http://my.quit.conf/> .
+@prefix conf: <http://my.quit.conf/> .
 @prefix quit: <http://quit.aksw.org/> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix xsd:  <http://www.w3.org/2001/XMLSchema#> .
 
-<exampleStore> a quit:QuitStore ;
+conf:exampleStore a quit:QuitStore ;
   quit:allowNewGraphs 1 ;
-  quit:pathOfGitRepo "../store.default" .
+  quit:pathOfGitRepo "../store.default" ;
+  quit:defaultBranch "master" ;
+  quit:globalFile "global.nq" .
 
-<exampleDBpedia> a quit:Graph ;
+conf:exampleDBpedia a quit:Graph ;
   quit:graphUri <http://dbpedia.org/> ;
   quit:isVersioned 1 ;
   quit:graphFile "dbpedia.nq" .
 
-<exampleGraph> a quit:Graph ;
+conf:exampleGraph a quit:Graph ;
   quit:graphUri <http://my.quit.graph/> ;
   quit:isVersioned 1 ;
   quit:graphFile "default.nq" .
+
+conf:default a <Graph> ;
+  quit:graphUri <http://aksw.org/NormanRadtke> ;
+  quit:isVersioned 1 ;
+  quit:graphFile "radtke.nq" .
+
+conf:default1 a <Graph> ;
+  quit:graphUri <http://aksw.org/NatanaelArndt> ;
+  quit:isVersioned 1 ;
+  quit:graphFile "arndt.nq" .
+
+conf:aksw a <Binding> ;
+  quit:prefix "aksw" ;
+  quit:namespace <http://aksw.org/> .
+  quit:graphFile "default.nq" .

lol.bat

@@ -0,0 +1,2 @@
+python quit/quit.py -cm localconfig --disable-data-store
+:: --disable-data-store  --disable-provenance-store

pylama.ini

@@ -1,6 +1,6 @@
 [pylama]
 format = pylint
-skip = tests/*,quit/update.py
+skip = tests/*,quit/tools/*
 linters = pep8
 ignore = E402
 

quit/__init__.py

@@ -0,0 +1,20 @@
+import quit.tools.plugin
+
+from rdflib.plugins.sparql.algebra import SequencePath
+
+# from Github: https://github.com/RDFLib/rdflib/issues/617
+# Egregious hack, the SequencePath object doesn't support compare, this implements the __lt__ method
+# so that algebra.py works on sorting in SPARQL queries on e.g. rdf:List paths
+
+
+def sequencePathCompareLt(self, other):
+    return str(self) < str(other)
+
+
+def sequencePathCompareGt(self, other):
+    return str(self) < str(other)
+
+
+setattr(SequencePath, '__lt__', sequencePathCompareLt)
+setattr(SequencePath, '__gt__', sequencePathCompareGt)
+# End egregious hack

quit/benchmark.py

@@ -0,0 +1,17 @@
+import time
+
+
+def benchmark(f):
+
+    def timed(*args, **kw):
+
+        print("starting '%r'" % (f.__name__))
+
+        ts = time.time()
+        result = f(*args, **kw)
+        te = time.time()
+
+        print("ending '%r' after %2.4f sec" % (f.__name__, te - ts))
+        return result
+
+    return timed

quit/cache.py

@@ -0,0 +1,90 @@
+from collections import OrderedDict
+from sortedcontainers import SortedList
+
+
+class Cache:
+
+    def __init__(self, capacity=50):
+        self.stack = OrderedDict()
+        self.capacity = capacity
+
+    def get(self, key):
+        """Get a value from the cache.
+
+        Raises:
+            KeyError if no value was found for the given key
+        """
+        value = self.stack.pop(key)
+        self.stack[key] = value
+        return value
+
+    def set(self, key, value):
+        try:
+            self.stack.pop(key)
+        except KeyError:
+            if len(self.stack) >= self.capacity:
+                self.stack.popitem(last=False)
+        self.stack[key] = value
+
+    def remove(self, key):
+        try:
+            return self.stack.pop(key)
+        except KeyError:
+            return
+
+    def __contains__(self, key):
+        return key in self.stack
+
+    def __iter__(self):
+        return (c for c in self.stack)
+
+    @property
+    def size(self):
+        """
+        Return the size of the cache
+        """
+        return len(self.stack)
+
+
+class FileReference:
+    """A class that manages n-quad files.
+    This class stores inforamtation about the location of a n-quad file and is
+    able to add and delete triples/quads to that file.
+    """
+
+    def __init__(self, path, content):
+        """Initialize a new FileReference instance.
+        Args:
+            filelocation: A string of the filepath.
+            filecontentinmem: Boolean to decide if local filesystem should be used to
+                or if file content should be kept in memory too . (Defaults false)
+        Raises:
+            ValueError: If no file at the filelocation, or in the given directory + filelocation.
+        """
+
+        if isinstance(content, str):
+            content = content.splitlines() or []
+
+        self._path = path
+        self._content = SortedList(content)
+        self._modified = False
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def content(self):
+        return "\n".join(self._content)
+
+    def add(self, data):
+        """Add a quad to the file content."""
+        self._content.add(data)
+
+    def extend(self, data):
+        """Add quads to the file content."""
+        self._content.extend(data)
+
+    def remove(self, data):
+        """Remove quad from the file content."""
+        self._content.remove(data)