[backend] [api] check the sha256 of file if needed

This will set $entry->{'hash'} = 'missing' if there is already a
file on the server with the same md5sum, but in another project
with the same package name.

The client(osc) will then calculate the sha256 for the file and return
it to the srcserver.

The sha256 sum of the uploaded file will be compared to the one from
the one already existing on the server.

added sha256 check in BSSrcrep::addfile

docs/api/api/api.txt

@@ -837,6 +837,7 @@ Parameters:
   onlyissues: used to limit to issues (for diff commands)
   setrelease: define a specific release tag when used with "release" command. Setting it to "-" strips
               the release string.
+  withvalidate: activate sha validation code
 
 
 

src/api/app/controllers/source_controller.rb

@@ -1369,7 +1369,7 @@ def package_command_commit
   # POST /source/<project>/<package>?cmd=commitfilelist
   def package_command_commitfilelist
     path = request.path_info
-    path += build_query_from_hash(params, [:cmd, :user, :comment, :rev, :linkrev, :keeplink, :repairlink])
+    path += build_query_from_hash(params, [:cmd, :user, :comment, :rev, :linkrev, :keeplink, :repairlink, :withvalidate])
     answer = pass_to_backend path
 
     @package.sources_changed(dir_xml: answer) if @package # except in case of _project package

src/backend/BSSrcrep.pm

@@ -23,6 +23,7 @@
 package BSSrcrep;
 
 use Digest::MD5 ();
+use Digest::SHA ();
 use Symbol;
 use BSSolv;
 
@@ -116,16 +117,33 @@ sub addfile {
     if (!rename($tmpfile, "$srcrep/$packid/$md5-$filename")) {
       mkdir_p("$srcrep/$packid");
       if (!rename($tmpfile, "$srcrep/$packid/$md5-$filename")) {
-	my $err = $!;
-	if (! -e "$srcrep/$packid/$md5-$filename") {
-	  $! = $err;
-	  die("rename $tmpfile $srcrep/$packid/$md5-$filename: $!\n");
-	}
+        my $err = $!;
+        if (! -e "$srcrep/$packid/$md5-$filename") {
+          $! = $err;
+          die("rename $tmpfile $srcrep/$packid/$md5-$filename: $!\n");
+        }
       }
     }
     adddeltastoreevent($projid, $packid, "$md5-$filename") if $filename =~ /\.obscpio$/s;
   } else {
+    # get the sha256 sum for the uploaded file
+    open(F, '<', $tmpfile) || die("$tmpfile: $!\n");
+    my $ctx = Digest::SHA->new(256);
+    $ctx->addfile(*F);
+    close F;
+    my $upload_sha = $ctx->hexdigest();
+    # get the sha256 sum for the already existing file
+    open(F, '<', "$srcrep/$packid/$md5-$filename") || die("$srcrep/$packid/$md5-$filename: $!\n");
+    $ctx = Digest::SHA->new(256);
+    $ctx->addfile(*F);
+    close F;
+    my $existing_sha = $ctx->hexdigest();
+    # if the sha sum is different, but the md5 and filename are the same someone might
+    # try to sneak in code.
     unlink($tmpfile);
+    if ($upload_sha ne $existing_sha) {
+      die("SHA missmatch for same md5sum in $packid for file $filename with sum $md5\n");
+    }
   }
   return $md5;
 }

src/backend/BSXML.pm

@@ -401,6 +401,7 @@ our $dir = [
      [[ 'entry' =>
 	    'name',
 	    'md5',
+            'hash',
 	    'size',
 	    'mtime',
 	    'error',

src/backend/bs_srcserver

@@ -34,6 +34,7 @@ use XML::Structured ':bytes';
 use POSIX;
 use Fcntl qw(:DEFAULT :flock);
 use Digest::MD5 ();
+use Digest::SHA ();
 use Data::Dumper;
 use Storable ();
 use Symbol;
@@ -2961,14 +2962,38 @@ sub sourcecommitfilelist {
   # make sure we know every file
   my @missing;
   my $files = {};
+  my $ofiles = {};
+  my $ofiles_expanded = {};
   my $orev = {'project' => $projid, 'package' => $packid};
+  if ($cgi->{'withvalidate'}) {
+    eval {
+      my $rev_old = getrev($projid, $packid);
+      $ofiles = BSRevision::lsrev($rev_old);
+      $ofiles_expanded = lsrev_expanded($rev_old);
+    };
+  }
   for my $entry (@{$fl->{'entry'} || []}) {
     BSVerify::verify_filename($entry->{'name'});
     BSVerify::verify_md5($entry->{'md5'});
     if (! -e BSRevision::revfilename($orev, $entry->{'name'}, $entry->{'md5'})) {
       push @missing, $entry;
     } else {
       die("duplicate file: $entry->{'name'}\n") if exists $files->{$entry->{'name'}};
+      if ($entry->{'hash'}) {
+        my $fd = gensym;
+        BSRevision::revopen($orev, $entry->{'name'}, $entry->{'md5'}, $fd);
+        my $sha256 = Digest::SHA->new(256);
+        my $hash_to_check = "sha256:" . $sha256->addfile($fd)->hexdigest;
+        if ($hash_to_check ne $entry->{'hash'}) {
+          die("SHA missmatch for same md5sum in $packid for file $entry->{'name'} with sum $entry->{'md5'}\n");
+        }
+      } elsif ($cgi->{'withvalidate'}) {
+        if ((!$ofiles->{$entry->{'name'}} || $ofiles->{$entry->{'name'}} ne $entry->{'md5'}) ||
+            (!$ofiles_expanded->{$entry->{'name'}} || $ofiles_expanded->{$entry->{'name'}} ne $entry->{'md5'})) {
+          $entry->{'hash'} = 'missing';
+          push @missing, $entry;
+        }
+      }
       $files->{$entry->{'name'}} = $entry->{'md5'};
     }
   }
@@ -6055,7 +6080,7 @@ my $dispatches = [
   'POST:/source/$project/$package cmd=linkdiff rev? linkrev? unified:bool? file:filename* filelimit:num? tarlimit:num? view:? withissues:bool? onlyissues:bool?' => \&linkdiff,
   'POST:/source/$project/$package cmd=servicediff rev? unified:bool? file:filename* filelimit:num? tarlimit:num? view:? withissues:bool? onlyissues:bool?' => \&servicediff,
   'POST:/source/$project/$package cmd=commit rev? user:? comment:? keeplink:bool? repairlink:bool? linkrev? setrev:bool? requestid:num? noservice:bool?' => \&sourcecommit,
-  'POST:/source/$project/$package cmd=commitfilelist rev? user:? comment:? keeplink:bool? repairlink:bool? linkrev? setrev:bool? requestid:num? time:num? version:? vrev:? noservice:bool? servicemark:?' => \&sourcecommitfilelist,
+  'POST:/source/$project/$package cmd=commitfilelist rev? user:? comment:? keeplink:bool? repairlink:bool? linkrev? setrev:bool? requestid:num? time:num? version:? vrev:? noservice:bool? servicemark:? withvalidate:?' => \&sourcecommitfilelist,
   'POST:/source/$project/$package cmd=copy rev? user:? comment:? orev:rev? oproject:project? opackage:package? expand:bool? keeplink:bool? repairlink:bool? linkrev? setrev:linkrev? olinkrev:linkrev? requestid:num? dontupdatesource:bool? noservice:bool? withvrev:bool? withacceptinfo:bool? makeoriginolder:bool? freezelink:bool? vrevbump:num? instantiate:bool?' => \&sourcecopy,
   'POST:/source/$project/$package cmd=collectbuildenv user:? comment:? orev:rev? oproject:project? opackage:package?' => \&sourcecollectbuildenv,
   'POST:/source/$project/$package cmd=branch rev? user:? comment:? orev:rev? oproject:project? opackage:package? olinkrev:linkrev? requestid:num? force:bool? keepcontent:bool? missingok:bool? noservice:bool? withacceptinfo:bool? time:num? extendvrev:bool?' => \&sourcebranch,