diff --git a/osc/core.py b/osc/core.py
index 6d02c6904..df499f27e 100644
--- a/osc/core.py
+++ b/osc/core.py
@@ -22,6 +22,7 @@
import socket
import errno
import shlex
+import hashlib
try:
from urllib.parse import urlsplit, urlunsplit, urlparse, quote_plus, urlencode, unquote
@@ -1393,7 +1394,7 @@ def commit_get_missing(filelist):
todo.append(n.get('name'))
return todo
- def __send_commitlog(self, msg, local_filelist):
+ def __send_commitlog(self, msg, local_filelist, validate=False):
"""send the commitlog and the local filelist to the server"""
query = {}
if self.islink() and self.isexpanded():
@@ -1405,6 +1406,8 @@ def __send_commitlog(self, msg, local_filelist):
query['linkrev'] = self.get_pulled_srcmd5()
if self.islinkrepair():
query['repairlink'] = '1'
+ if validate:
+ query['withvalidate'] = '1'
return self.commit_filelist(self.apiurl, self.prjname, self.name,
local_filelist, msg, **query)
@@ -1444,6 +1447,7 @@ def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch
todo_send = {}
todo_delete = []
real_send = []
+ sha256sums = {}
for filename in self.filenamelist + [i for i in self.to_be_added if not i in self.filenamelist]:
if filename.startswith('_service:') or filename.startswith('_service_'):
continue
@@ -1454,6 +1458,7 @@ def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch
elif filename in self.todo:
if st in ('A', 'R', 'M'):
todo_send[filename] = dgst(os.path.join(self.absdir, filename))
+ sha256sums[filename] = sha256_dgst(os.path.join(self.absdir, filename))
real_send.append(filename)
print(statfrmt('Sending', os.path.join(pathn, filename)))
elif st in (' ', '!', 'S'):
@@ -1486,7 +1491,21 @@ def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch
print('Transmitting file data', end=' ')
filelist = self.__generate_commitlist(todo_send)
- sfilelist = self.__send_commitlog(msg, filelist)
+ sfilelist = self.__send_commitlog(msg, filelist, validate=True)
+ if sfilelist.get('error') and sfilelist.findall('.//entry[@hash]'):
+ name2elem = dict([(e.get('name'), e) for e in filelist.findall('entry')])
+ for entry in sfilelist.findall('.//entry[@hash]'):
+ filename = entry.get('name')
+ fileelem = name2elem.get(filename)
+ if filename not in sha256sums:
+ msg = 'There is no sha256 sum for file %s.\n' \
+ 'This could be due to an outdated working copy.\n' \
+ 'Please update your working copy with osc update and\n' \
+ 'commit again afterwards.'
+ print(msg % filename)
+ return 1
+ fileelem.set('hash', 'sha256:%s' % sha256sums[filename])
+ sfilelist = self.__send_commitlog(msg, filelist)
send = self.commit_get_missing(sfilelist)
real_send = [i for i in real_send if not i in send]
# abort after 3 tries
@@ -4629,6 +4648,18 @@ def dgst(file):
f.close()
return s.hexdigest()
+def sha256_dgst(file):
+
+ global BUFSIZE
+
+ f = open(file, 'rb')
+ s = hashlib.sha256()
+ while True:
+ buf = f.read(BUFSIZE)
+ if not buf: break
+ s.update(buf)
+ f.close()
+ return s.hexdigest()
def binary(s):
"""return true if a string is binary data using diff's heuristic"""
diff --git a/tests/commit_fixtures/testAddedMissing_lfilelistwithSHA b/tests/commit_fixtures/testAddedMissing_lfilelistwithSHA
new file mode 100644
index 000000000..408146c67
--- /dev/null
+++ b/tests/commit_fixtures/testAddedMissing_lfilelistwithSHA
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA b/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA
new file mode 100644
index 000000000..4895b5ee4
--- /dev/null
+++ b/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA
@@ -0,0 +1,3 @@
+
+
+
diff --git a/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum b/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum
new file mode 100644
index 000000000..13a25531c
--- /dev/null
+++ b/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum
@@ -0,0 +1,3 @@
+
+
+
diff --git a/tests/commit_fixtures/testSimple_lfilelistwithSHA b/tests/commit_fixtures/testSimple_lfilelistwithSHA
new file mode 100644
index 000000000..aac9cf3c9
--- /dev/null
+++ b/tests/commit_fixtures/testSimple_lfilelistwithSHA
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/tests/commit_fixtures/testSimple_missingfilelistwithSHA b/tests/commit_fixtures/testSimple_missingfilelistwithSHA
new file mode 100644
index 000000000..1a7a99b96
--- /dev/null
+++ b/tests/commit_fixtures/testSimple_missingfilelistwithSHA
@@ -0,0 +1,3 @@
+
+
+
diff --git a/tests/commit_fixtures/testSimple_missingfilelistwithSHAsum b/tests/commit_fixtures/testSimple_missingfilelistwithSHAsum
new file mode 100644
index 000000000..077cd5168
--- /dev/null
+++ b/tests/commit_fixtures/testSimple_missingfilelistwithSHAsum
@@ -0,0 +1,3 @@
+
+
+
diff --git a/tests/test_commit.py b/tests/test_commit.py
index 02baa8558..349fc4b4f 100644
--- a/tests/test_commit.py
+++ b/tests/test_commit.py
@@ -25,7 +25,7 @@ def _get_fixtures_dir(self):
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@@ -48,7 +48,7 @@ def test_simple(self):
@GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
@POST('http://localhost/source/osctest/add?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddfile_missingfilelist', expfile='testAddfile_lfilelist')
@PUT('http://localhost/source/osctest/add/add?rev=repository',
exp='added file\n', text=rev_dummy)
@@ -73,7 +73,7 @@ def test_addfile(self):
@GET('http://localhost/source/osctest/delete?rev=latest', file='testDeletefile_filesremote')
@POST('http://localhost/source/osctest/delete?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testDeletefile_cfilesremote', expfile='testDeletefile_lfilelist')
def test_deletefile(self):
"""delete a file"""
@@ -120,7 +120,7 @@ def test_nochanges(self):
@GET('http://localhost/source/osctest/multiple?rev=latest', file='testMultiple_filesremote')
@POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testMultiple_missingfilelist', expfile='testMultiple_lfilelist')
@PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
@PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
@@ -149,7 +149,7 @@ def test_multiple(self):
@GET('http://localhost/source/osctest/multiple?rev=latest', file='testPartial_filesremote')
@POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testPartial_missingfilelist', expfile='testPartial_lfilelist')
@PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
@PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
@@ -176,7 +176,7 @@ def test_partial(self):
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n',
exception=IOError('test exception'), text=rev_dummy)
@@ -194,7 +194,7 @@ def test_interrupt(self):
@GET('http://localhost/source/osctest/allstates?rev=latest', file='testPartial_filesremote')
@POST('http://localhost/source/osctest/allstates?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAllStates_missingfilelist', expfile='testAllStates_lfilelist')
@PUT('http://localhost/source/osctest/allstates/add?rev=repository', exp='added file\n', text=rev_dummy)
@PUT('http://localhost/source/osctest/allstates/missing?rev=repository', exp='replaced\n', text=rev_dummy)
@@ -224,7 +224,7 @@ def test_allstates(self):
@GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
@POST('http://localhost/source/osctest/add?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddfile_cfilesremote', expfile='testAddfile_lfilelist')
def test_remoteexists(self):
"""file 'add' should be committed but already exists on the server"""
@@ -245,7 +245,7 @@ def test_remoteexists(self):
@GET('http://localhost/source/osctest/branch?rev=latest', file='testExpand_filesremote')
@POST('http://localhost/source/osctest/branch?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
+ @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&withvalidate=1&keeplink=1',
file='testExpand_missingfilelist', expfile='testExpand_lfilelist')
@PUT('http://localhost/source/osctest/branch/simple?rev=repository', exp='simple modified file.\n', text=rev_dummy)
@POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
@@ -277,7 +277,7 @@ def test_added_missing(self):
@GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
@POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddedMissing_missingfilelist', expfile='testAddedMissing_lfilelist')
@PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
@@ -296,7 +296,7 @@ def test_added_missing2(self):
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='')
- @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+ @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@@ -312,6 +312,52 @@ def test_commitfilelist_error(self):
self.assertEqual(sys.stdout.getvalue(), exp)
self._check_status(p, 'nochange', 'M')
-if __name__ == '__main__':
+ @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
+ @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
+ exp='', text='')
+ @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
+ file='testSimple_missingfilelistwithSHA', expfile='testSimple_lfilelist')
+ @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+ file='testSimple_missingfilelistwithSHAsum', expfile='testSimple_lfilelistwithSHA')
+ @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
+ exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
+ @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+ file='testSimple_cfilesremote', expfile='testSimple_lfilelistwithSHA')
+ def test_simple_sha256(self):
+ """a simple commit (only one modified file)"""
+ self._change_to_pkg('simple')
+ p = osc.core.Package('.')
+ p.commit()
+ exp = 'Sending nochange\nTransmitting file data .\nCommitted revision 2.\n'
+ self.assertEqual(sys.stdout.getvalue(), exp)
+ self._check_digests('testSimple_cfilesremote')
+ self.assertTrue(os.path.exists('nochange'))
+ self.assertEqual(open('nochange', 'r').read(), open(os.path.join('.osc', 'nochange'), 'r').read())
+ self._check_status(p, 'nochange', ' ')
+ self._check_status(p, 'foo', ' ')
+ self._check_status(p, 'merge', ' ')
+
+ @GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
+ @POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
+ exp='', text='')
+ @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
+ file='testAddedMissing_missingfilelistwithSHA', expfile='testAddedMissing_lfilelist')
+ @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+ file='testAddedMissing_missingfilelistwithSHAsum', expfile='testAddedMissing_lfilelistwithSHA')
+ @PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
+ @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+ file='testAddedMissing_cfilesremote', expfile='testAddedMissing_lfilelistwithSHA')
+ def test_added_missing2_sha256(self):
+ """commit an added file, another added file missing (but it's not part of the commit)"""
+ self._change_to_pkg('added_missing')
+ p = osc.core.Package('.')
+ p.todo = ['bar']
+ p.commit()
+ exp = 'Sending bar\nTransmitting file data .\nCommitted revision 2.\n'
+ self.assertEqual(sys.stdout.getvalue(), exp)
+ self._check_status(p, 'add', '!')
+ self._check_status(p, 'bar', ' ')
+
+if __name__ == '__main__':
import unittest
unittest.main()