added code to get the sha256 hashes of files

This is needed for a new validation of the source server.

The source server will 'ask' for the sha256 sum of files which are new or
modified and osc calculates the sha256 sums for those files and sends them
back to the server.

The server checks the sha256 sums and if dies if something is wrong.

osc/core.py

@@ -22,6 +22,7 @@
 import socket
 import errno
 import shlex
+import hashlib
 
 try:
     from urllib.parse import urlsplit, urlunsplit, urlparse, quote_plus, urlencode, unquote
@@ -1393,7 +1394,7 @@ def commit_get_missing(filelist):
             todo.append(n.get('name'))
         return todo
 
-    def __send_commitlog(self, msg, local_filelist):
+    def __send_commitlog(self, msg, local_filelist, validate=False):
         """send the commitlog and the local filelist to the server"""
         query = {}
         if self.islink() and self.isexpanded():
@@ -1405,6 +1406,8 @@ def __send_commitlog(self, msg, local_filelist):
                 query['linkrev'] = self.get_pulled_srcmd5()
         if self.islinkrepair():
             query['repairlink'] = '1'
+        if validate:
+            query['withvalidate'] = '1'
         return self.commit_filelist(self.apiurl, self.prjname, self.name,
                                     local_filelist, msg, **query)
 
@@ -1444,6 +1447,7 @@ def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch
         todo_send = {}
         todo_delete = []
         real_send = []
+        sha256sums = {}
         for filename in self.filenamelist + [i for i in self.to_be_added if not i in self.filenamelist]:
             if filename.startswith('_service:') or filename.startswith('_service_'):
                 continue
@@ -1454,6 +1458,7 @@ def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch
             elif filename in self.todo:
                 if st in ('A', 'R', 'M'):
                     todo_send[filename] = dgst(os.path.join(self.absdir, filename))
+                    sha256sums[filename] = sha256_dgst(os.path.join(self.absdir, filename))
                     real_send.append(filename)
                     print(statfrmt('Sending', os.path.join(pathn, filename)))
                 elif st in (' ', '!', 'S'):
@@ -1486,7 +1491,21 @@ def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch
 
         print('Transmitting file data', end=' ')
         filelist = self.__generate_commitlist(todo_send)
-        sfilelist = self.__send_commitlog(msg, filelist)
+        sfilelist = self.__send_commitlog(msg, filelist, validate=True)
+        if sfilelist.get('error') and sfilelist.findall('.//entry[@hash]'):
+            name2elem = dict([(e.get('name'), e) for e in filelist.findall('entry')])
+            for entry in sfilelist.findall('.//entry[@hash]'):
+                filename = entry.get('name')
+                fileelem = name2elem.get(filename)
+                if filename not in sha256sums:
+                    msg = 'There is no sha256 sum for file %s.\n' \
+                          'This could be due to an outdated working copy.\n' \
+                          'Please update your working copy with osc update and\n' \
+                          'commit again afterwards.'
+                    print(msg % filename)
+                    return 1
+                fileelem.set('hash', 'sha256:%s' % sha256sums[filename])
+            sfilelist = self.__send_commitlog(msg, filelist)
         send = self.commit_get_missing(sfilelist)
         real_send = [i for i in real_send if not i in send]
         # abort after 3 tries
@@ -4629,6 +4648,18 @@ def dgst(file):
     f.close()
     return s.hexdigest()
 
+def sha256_dgst(file):
+
+    global BUFSIZE
+
+    f = open(file, 'rb')
+    s = hashlib.sha256()
+    while True:
+        buf = f.read(BUFSIZE)
+        if not buf: break
+        s.update(buf)
+    f.close()
+    return s.hexdigest()
 
 def binary(s):
     """return true if a string is binary data using diff's heuristic"""

tests/commit_fixtures/testAddedMissing_lfilelistwithSHA

@@ -0,0 +1 @@
+<directory><entry hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" md5="14758f1afd44c09b7992073ccf00b43d" name="bar" /><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /></directory>

tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA

@@ -0,0 +1,3 @@
+<directory error="missing" name="added_missing">
+  <entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="new"/>
+</directory>

tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum

@@ -0,0 +1,3 @@
+<directory error="missing" name="added_missing">
+  <entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"/>
+</directory>

tests/commit_fixtures/testSimple_lfilelistwithSHA

@@ -0,0 +1 @@
+<directory><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /><entry md5="17b9e9e1a032ed44e7a584dc6303ffa8" name="merge" /><entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="382588b92f5976de693f44c4d6df27b7" name="nochange" /></directory>

tests/commit_fixtures/testSimple_missingfilelistwithSHA

@@ -0,0 +1,3 @@
+<directory error="missing" name="simple">
+  <entry hash="missing" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
+</directory>

tests/commit_fixtures/testSimple_missingfilelistwithSHAsum

@@ -0,0 +1,3 @@
+<directory error="missing" name="simple">
+  <entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
+</directory>

tests/test_commit.py

@@ -25,7 +25,7 @@ def _get_fixtures_dir(self):
     @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
     @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
     @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
           exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@@ -48,7 +48,7 @@ def test_simple(self):
     @GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
     @POST('http://localhost/source/osctest/add?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAddfile_missingfilelist', expfile='testAddfile_lfilelist')
     @PUT('http://localhost/source/osctest/add/add?rev=repository',
          exp='added file\n', text=rev_dummy)
@@ -73,7 +73,7 @@ def test_addfile(self):
     @GET('http://localhost/source/osctest/delete?rev=latest', file='testDeletefile_filesremote')
     @POST('http://localhost/source/osctest/delete?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testDeletefile_cfilesremote', expfile='testDeletefile_lfilelist')
     def test_deletefile(self):
         """delete a file"""
@@ -120,7 +120,7 @@ def test_nochanges(self):
     @GET('http://localhost/source/osctest/multiple?rev=latest', file='testMultiple_filesremote')
     @POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testMultiple_missingfilelist', expfile='testMultiple_lfilelist')
     @PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
     @PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
@@ -149,7 +149,7 @@ def test_multiple(self):
     @GET('http://localhost/source/osctest/multiple?rev=latest', file='testPartial_filesremote')
     @POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testPartial_missingfilelist', expfile='testPartial_lfilelist')
     @PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
     @PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
@@ -176,7 +176,7 @@ def test_partial(self):
     @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
     @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
     @PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n',
         exception=IOError('test exception'), text=rev_dummy)
@@ -194,7 +194,7 @@ def test_interrupt(self):
     @GET('http://localhost/source/osctest/allstates?rev=latest', file='testPartial_filesremote')
     @POST('http://localhost/source/osctest/allstates?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAllStates_missingfilelist', expfile='testAllStates_lfilelist')
     @PUT('http://localhost/source/osctest/allstates/add?rev=repository', exp='added file\n', text=rev_dummy)
     @PUT('http://localhost/source/osctest/allstates/missing?rev=repository', exp='replaced\n', text=rev_dummy)
@@ -224,7 +224,7 @@ def test_allstates(self):
     @GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
     @POST('http://localhost/source/osctest/add?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAddfile_cfilesremote', expfile='testAddfile_lfilelist')
     def test_remoteexists(self):
         """file 'add' should be committed but already exists on the server"""
@@ -245,7 +245,7 @@ def test_remoteexists(self):
     @GET('http://localhost/source/osctest/branch?rev=latest', file='testExpand_filesremote')
     @POST('http://localhost/source/osctest/branch?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
+    @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&withvalidate=1&keeplink=1',
           file='testExpand_missingfilelist', expfile='testExpand_lfilelist')
     @PUT('http://localhost/source/osctest/branch/simple?rev=repository', exp='simple modified file.\n', text=rev_dummy)
     @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
@@ -277,7 +277,7 @@ def test_added_missing(self):
     @GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
     @POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAddedMissing_missingfilelist', expfile='testAddedMissing_lfilelist')
     @PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
     @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
@@ -296,7 +296,7 @@ def test_added_missing2(self):
     @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
     @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
     @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
           exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@@ -312,6 +312,52 @@ def test_commitfilelist_error(self):
         self.assertEqual(sys.stdout.getvalue(), exp)
         self._check_status(p, 'nochange', 'M')
 
-if __name__ == '__main__':
+    @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
+    @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
+          exp='', text='<services />')
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
+          file='testSimple_missingfilelistwithSHA', expfile='testSimple_lfilelist')
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+          file='testSimple_missingfilelistwithSHAsum', expfile='testSimple_lfilelistwithSHA')
+    @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
+          exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+          file='testSimple_cfilesremote', expfile='testSimple_lfilelistwithSHA')
+    def test_simple_sha256(self):
+        """a simple commit (only one modified file)"""
+        self._change_to_pkg('simple')
+        p = osc.core.Package('.')
+        p.commit()
+        exp = 'Sending    nochange\nTransmitting file data .\nCommitted revision 2.\n'
+        self.assertEqual(sys.stdout.getvalue(), exp)
+        self._check_digests('testSimple_cfilesremote')
+        self.assertTrue(os.path.exists('nochange'))
+        self.assertEqual(open('nochange', 'r').read(), open(os.path.join('.osc', 'nochange'), 'r').read())
+        self._check_status(p, 'nochange', ' ')
+        self._check_status(p, 'foo', ' ')
+        self._check_status(p, 'merge', ' ')
+
+    @GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
+    @POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
+          exp='', text='<services />')
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
+          file='testAddedMissing_missingfilelistwithSHA', expfile='testAddedMissing_lfilelist')
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+          file='testAddedMissing_missingfilelistwithSHAsum', expfile='testAddedMissing_lfilelistwithSHA')
+    @PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+          file='testAddedMissing_cfilesremote', expfile='testAddedMissing_lfilelistwithSHA')
+    def test_added_missing2_sha256(self):
+        """commit an added file, another added file missing (but it's not part of the commit)"""
+        self._change_to_pkg('added_missing')
+        p = osc.core.Package('.')
+        p.todo = ['bar']
+        p.commit()
+        exp = 'Sending    bar\nTransmitting file data .\nCommitted revision 2.\n'
+        self.assertEqual(sys.stdout.getvalue(), exp)
+        self._check_status(p, 'add', '!')
+        self._check_status(p, 'bar', ' ')
+
+if  __name__ == '__main__':
     import unittest
     unittest.main()