Merge branch 'check_hash_for_file' of https://github.com/lethliel/osc

Validate the sha256 hashes for new, modified, and replaced files, if the
srcserver supports it.

osc/core.py

@@ -22,6 +22,7 @@
 import socket
 import errno
 import shlex
+import hashlib
 
 try:
     from urllib.parse import urlsplit, urlunsplit, urlparse, quote_plus, urlencode, unquote
@@ -1393,7 +1394,7 @@ def commit_get_missing(filelist):
             todo.append(n.get('name'))
         return todo
 
-    def __send_commitlog(self, msg, local_filelist):
+    def __send_commitlog(self, msg, local_filelist, validate=False):
         """send the commitlog and the local filelist to the server"""
         query = {}
         if self.islink() and self.isexpanded():
@@ -1405,6 +1406,8 @@ def __send_commitlog(self, msg, local_filelist):
                 query['linkrev'] = self.get_pulled_srcmd5()
         if self.islinkrepair():
             query['repairlink'] = '1'
+        if validate:
+            query['withvalidate'] = '1'
         return self.commit_filelist(self.apiurl, self.prjname, self.name,
                                     local_filelist, msg, **query)
 
@@ -1444,6 +1447,7 @@ def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch
         todo_send = {}
         todo_delete = []
         real_send = []
+        sha256sums = {}
         for filename in self.filenamelist + [i for i in self.to_be_added if not i in self.filenamelist]:
             if filename.startswith('_service:') or filename.startswith('_service_'):
                 continue
@@ -1454,6 +1458,7 @@ def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch
             elif filename in self.todo:
                 if st in ('A', 'R', 'M'):
                     todo_send[filename] = dgst(os.path.join(self.absdir, filename))
+                    sha256sums[filename] = sha256_dgst(os.path.join(self.absdir, filename))
                     real_send.append(filename)
                     print(statfrmt('Sending', os.path.join(pathn, filename)))
                 elif st in (' ', '!', 'S'):
@@ -1486,7 +1491,21 @@ def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch
 
         print('Transmitting file data', end=' ')
         filelist = self.__generate_commitlist(todo_send)
-        sfilelist = self.__send_commitlog(msg, filelist)
+        sfilelist = self.__send_commitlog(msg, filelist, validate=True)
+        if sfilelist.get('error') and sfilelist.findall('.//entry[@hash]'):
+            name2elem = dict([(e.get('name'), e) for e in filelist.findall('entry')])
+            for entry in sfilelist.findall('.//entry[@hash]'):
+                filename = entry.get('name')
+                fileelem = name2elem.get(filename)
+                if filename not in sha256sums:
+                    msg = 'There is no sha256 sum for file %s.\n' \
+                          'This could be due to an outdated working copy.\n' \
+                          'Please update your working copy with osc update and\n' \
+                          'commit again afterwards.'
+                    print(msg % filename)
+                    return 1
+                fileelem.set('hash', 'sha256:%s' % sha256sums[filename])
+            sfilelist = self.__send_commitlog(msg, filelist)
         send = self.commit_get_missing(sfilelist)
         real_send = [i for i in real_send if not i in send]
         # abort after 3 tries
@@ -4629,6 +4648,18 @@ def dgst(file):
     f.close()
     return s.hexdigest()
 
+def sha256_dgst(file):
+
+    global BUFSIZE
+
+    f = open(file, 'rb')
+    s = hashlib.sha256()
+    while True:
+        buf = f.read(BUFSIZE)
+        if not buf: break
+        s.update(buf)
+    f.close()
+    return s.hexdigest()
 
 def binary(s):
     """return true if a string is binary data using diff's heuristic"""

tests/commit_fixtures/testAddedMissing_lfilelistwithSHA

@@ -0,0 +1 @@
+<directory><entry hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" md5="14758f1afd44c09b7992073ccf00b43d" name="bar" /><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /></directory>

tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA

@@ -0,0 +1,3 @@
+<directory error="missing" name="added_missing">
+  <entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="new"/>
+</directory>

tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum

@@ -0,0 +1,3 @@
+<directory error="missing" name="added_missing">
+  <entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"/>
+</directory>

tests/commit_fixtures/testSimple_lfilelistwithSHA

@@ -0,0 +1 @@
+<directory><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /><entry md5="17b9e9e1a032ed44e7a584dc6303ffa8" name="merge" /><entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="382588b92f5976de693f44c4d6df27b7" name="nochange" /></directory>

tests/commit_fixtures/testSimple_missingfilelistwithSHA

@@ -0,0 +1,3 @@
+<directory error="missing" name="simple">
+  <entry hash="missing" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
+</directory>

tests/commit_fixtures/testSimple_missingfilelistwithSHAsum

@@ -0,0 +1,3 @@
+<directory error="missing" name="simple">
+  <entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
+</directory>

tests/test_commit.py

@@ -25,7 +25,7 @@ def _get_fixtures_dir(self):
     @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
     @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
     @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
           exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@@ -48,7 +48,7 @@ def test_simple(self):
     @GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
     @POST('http://localhost/source/osctest/add?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAddfile_missingfilelist', expfile='testAddfile_lfilelist')
     @PUT('http://localhost/source/osctest/add/add?rev=repository',
          exp='added file\n', text=rev_dummy)
@@ -73,7 +73,7 @@ def test_addfile(self):
     @GET('http://localhost/source/osctest/delete?rev=latest', file='testDeletefile_filesremote')
     @POST('http://localhost/source/osctest/delete?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testDeletefile_cfilesremote', expfile='testDeletefile_lfilelist')
     def test_deletefile(self):
         """delete a file"""
@@ -120,7 +120,7 @@ def test_nochanges(self):
     @GET('http://localhost/source/osctest/multiple?rev=latest', file='testMultiple_filesremote')
     @POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testMultiple_missingfilelist', expfile='testMultiple_lfilelist')
     @PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
     @PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
@@ -149,7 +149,7 @@ def test_multiple(self):
     @GET('http://localhost/source/osctest/multiple?rev=latest', file='testPartial_filesremote')
     @POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testPartial_missingfilelist', expfile='testPartial_lfilelist')
     @PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
     @PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
@@ -176,7 +176,7 @@ def test_partial(self):
     @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
     @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
     @PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n',
         exception=IOError('test exception'), text=rev_dummy)
@@ -194,7 +194,7 @@ def test_interrupt(self):
     @GET('http://localhost/source/osctest/allstates?rev=latest', file='testPartial_filesremote')
     @POST('http://localhost/source/osctest/allstates?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAllStates_missingfilelist', expfile='testAllStates_lfilelist')
     @PUT('http://localhost/source/osctest/allstates/add?rev=repository', exp='added file\n', text=rev_dummy)
     @PUT('http://localhost/source/osctest/allstates/missing?rev=repository', exp='replaced\n', text=rev_dummy)
@@ -224,7 +224,7 @@ def test_allstates(self):
     @GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
     @POST('http://localhost/source/osctest/add?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAddfile_cfilesremote', expfile='testAddfile_lfilelist')
     def test_remoteexists(self):
         """file 'add' should be committed but already exists on the server"""
@@ -245,7 +245,7 @@ def test_remoteexists(self):
     @GET('http://localhost/source/osctest/branch?rev=latest', file='testExpand_filesremote')
     @POST('http://localhost/source/osctest/branch?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
+    @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&withvalidate=1&keeplink=1',
           file='testExpand_missingfilelist', expfile='testExpand_lfilelist')
     @PUT('http://localhost/source/osctest/branch/simple?rev=repository', exp='simple modified file.\n', text=rev_dummy)
     @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
@@ -277,7 +277,7 @@ def test_added_missing(self):
     @GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
     @POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAddedMissing_missingfilelist', expfile='testAddedMissing_lfilelist')
     @PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
     @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
@@ -296,7 +296,7 @@ def test_added_missing2(self):
     @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
     @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
     @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
           exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@@ -312,6 +312,52 @@ def test_commitfilelist_error(self):
         self.assertEqual(sys.stdout.getvalue(), exp)
         self._check_status(p, 'nochange', 'M')
 
-if __name__ == '__main__':
+    @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
+    @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
+          exp='', text='<services />')
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
+          file='testSimple_missingfilelistwithSHA', expfile='testSimple_lfilelist')
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+          file='testSimple_missingfilelistwithSHAsum', expfile='testSimple_lfilelistwithSHA')
+    @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
+          exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+          file='testSimple_cfilesremote', expfile='testSimple_lfilelistwithSHA')
+    def test_simple_sha256(self):
+        """a simple commit (only one modified file)"""
+        self._change_to_pkg('simple')
+        p = osc.core.Package('.')
+        p.commit()
+        exp = 'Sending    nochange\nTransmitting file data .\nCommitted revision 2.\n'
+        self.assertEqual(sys.stdout.getvalue(), exp)
+        self._check_digests('testSimple_cfilesremote')
+        self.assertTrue(os.path.exists('nochange'))
+        self.assertEqual(open('nochange', 'r').read(), open(os.path.join('.osc', 'nochange'), 'r').read())
+        self._check_status(p, 'nochange', ' ')
+        self._check_status(p, 'foo', ' ')
+        self._check_status(p, 'merge', ' ')
+
+    @GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
+    @POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
+          exp='', text='<services />')
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
+          file='testAddedMissing_missingfilelistwithSHA', expfile='testAddedMissing_lfilelist')
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+          file='testAddedMissing_missingfilelistwithSHAsum', expfile='testAddedMissing_lfilelistwithSHA')
+    @PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+          file='testAddedMissing_cfilesremote', expfile='testAddedMissing_lfilelistwithSHA')
+    def test_added_missing2_sha256(self):
+        """commit an added file, another added file missing (but it's not part of the commit)"""
+        self._change_to_pkg('added_missing')
+        p = osc.core.Package('.')
+        p.todo = ['bar']
+        p.commit()
+        exp = 'Sending    bar\nTransmitting file data .\nCommitted revision 2.\n'
+        self.assertEqual(sys.stdout.getvalue(), exp)
+        self._check_status(p, 'add', '!')
+        self._check_status(p, 'bar', ' ')
+
+if  __name__ == '__main__':
     import unittest
     unittest.main()