Skip to content
Permalink
Browse files

feat(db): Calculate the sha256 value of the uploading file and store …

…it in database
  • Loading branch information...
sandipbhuyan committed Jun 18, 2019
1 parent 6b70553 commit d3641939e358500ddc21eedca00b70bceb919fb5
@@ -133,7 +133,7 @@ function updateSHA256($dbManager, $tableName)
foreach ($rows as $row) {
$dbManager->execute($statement, [
$row["id"],
hash('sha256', file_get_contents(RepPath($row['id'],"files")))
hash_file('sha256', RepPath($row['id'],"files"))
]);
}
return count($rows);
@@ -193,19 +193,19 @@ function migrate_35_36($dbManager, $force = false)
// Updating the pfile findings
if ($totalPfile != 0)
{
echo "*** Updating the sha256 values of manual pfile findings ***\n";
echo "*** Updating the sha256 values of pfiles ***\n";
$countPfile = updateSHA256($dbManager, "pfile");
}
$dbManager->commit();
if($total != 0)
{
echo "*** Updated hash of $count/$total manual copyright/ecc/keyword/pfile findings ***\n";
echo "*** Updated hash of $count/$total manual copyright/ecc/keyword findings ***\n";
}
if ($totalPfile != 0)
{
echo "*** Updated sha256 of $countPfile/$totalPfile records of pfile findings ***\n";
echo "*** Updated sha256 of $countPfile/$totalPfile records of pfile ***\n";
}
} catch (Exception $e) {
echo "*** Something went wrong. Try running postinstall again! ***\n";
@@ -18,6 +18,10 @@
* \file
* \brief Contains all utility functions used by FOSSology
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "ununpack.h"
#include "externs.h"
#include "regex.h"
@@ -1131,8 +1135,8 @@ int DBInsertPfile (ContainerInfo *CI, char *Fuid)
/* Check if the pfile exists */
memset(SQL,'\0',MAXSQL);
snprintf(SQL,MAXSQL,"SELECT pfile_pk,pfile_mimetypefk FROM pfile "
"WHERE pfile_sha1 = '%.40s' AND pfile_md5 = '%.32s' AND pfile_size = '%s';",
Fuid,Fuid+41,Fuid+74);
"WHERE pfile_sha1 = '%.40s' AND pfile_md5 = '%.32s' AND pfile_sha256 = '%.64s' AND pfile_size = '%s';",
Fuid,Fuid+41,Fuid+74,Fuid+140);
result = PQexec(pgConn, SQL); /* SELECT */
if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(12);

@@ -1146,14 +1150,14 @@ int DBInsertPfile (ContainerInfo *CI, char *Fuid)
memset(SQL,'\0',MAXSQL);
if (CMD[CI->PI.Cmd].DBindex > 0)
{
snprintf(SQL,MAXSQL,"INSERT INTO pfile (pfile_sha1,pfile_md5,pfile_size,pfile_mimetypefk) "
"VALUES ('%.40s','%.32s','%s','%ld');",
Fuid,Fuid+41,Fuid+74,CMD[CI->PI.Cmd].DBindex);
snprintf(SQL,MAXSQL,"INSERT INTO pfile (pfile_sha1,pfile_md5,pfile_sha256,pfile_size,pfile_mimetypefk) "
"VALUES ('%.40s','%.32s','%.64s','%s','%ld');",
Fuid,Fuid+41,Fuid+74,Fuid+140,CMD[CI->PI.Cmd].DBindex);
}
else
{
snprintf(SQL,MAXSQL,"INSERT INTO pfile (pfile_sha1,pfile_md5,pfile_size) VALUES ('%.40s','%.32s','%s');",
Fuid,Fuid+41,Fuid+74);
snprintf(SQL,MAXSQL,"INSERT INTO pfile (pfile_sha1,pfile_md5,pfile_sha256,pfile_size) VALUES ('%.40s','%.32s','%.64s','%s');",
Fuid,Fuid+41,Fuid+74,Fuid+140);
}
result = PQexec(pgConn, SQL); /* INSERT INTO pfile */
// ignore duplicate constraint failure (23505), report others
@@ -1169,8 +1173,8 @@ int DBInsertPfile (ContainerInfo *CI, char *Fuid)
on currval(). */
memset(SQL,'\0',MAXSQL);
snprintf(SQL,MAXSQL,"SELECT pfile_pk,pfile_mimetypefk FROM pfile "
"WHERE pfile_sha1 = '%.40s' AND pfile_md5 = '%.32s' AND pfile_size = '%s';",
Fuid,Fuid+41,Fuid+74);
"WHERE pfile_sha1 = '%.40s' AND pfile_md5 = '%.32s' AND pfile_sha256 = '%.64s' AND pfile_size = '%s';",
Fuid,Fuid+41,Fuid+74,Fuid+140);
result = PQexec(pgConn, SQL); /* SELECT */
if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(14);
}
@@ -1403,17 +1407,25 @@ int AddToRepository (ContainerInfo *CI, char *Fuid, int Mask)
/* If we ever want to skip artifacts, use && !CI->Artifact */
if ((Fuid[0]!='\0') && UseRepository)
{
/* Translate the new Fuid into old Fuid */
char FuidNew[1024];
memset(FuidNew, '\0', sizeof(FuidNew));
// Copy the value till md5
strncpy(FuidNew, Fuid, 74);
// Copy the size of the file
strcat(FuidNew,Fuid+140);

/* put file in repository */
if (!fo_RepExist(REP_FILES,Fuid))
{
if (fo_RepImport(CI->Source,REP_FILES,Fuid,1) != 0)
if (fo_RepImport(CI->Source,REP_FILES,FuidNew,1) != 0)
{
LOG_ERROR("Failed to import '%s' as '%s' into the repository",CI->Source,Fuid);
LOG_ERROR("Failed to import '%s' as '%s' into the repository",CI->Source,FuidNew);
SafeExit(21);
}
}
if (Verbose) LOG_DEBUG("Repository[%s]: insert '%s' as '%s'",
REP_FILES,CI->Source,Fuid);
REP_FILES,CI->Source,FuidNew);
}

/* PERFORMANCE NOTE:
@@ -1557,19 +1569,36 @@ int DisplayContainerInfo (ContainerInfo *CI, int Cmd)
{
CksumFile *CF;
Cksum *Sum;
char SHA256[64], command[1000] = "sha256sum ";
strcat(command,CI->Source);
int retcode = -1;

CF = SumOpenFile(CI->Source);
FILE* file = popen(command, "r");
if (file != (FILE*) NULL)
{
fscanf(file, "%64s", SHA256);
retcode = WEXITSTATUS(pclose(file));
}
if (file == (FILE*) NULL || retcode != 0)
{
LOG_FATAL("Unable to calculate SHA256 of %s\n", CI->Source);
SafeExit(56);
}
if (CF)
{
Sum = SumComputeBuff(CF);
SumCloseFile(CF);

if (Sum)
{
for(i=0; i<20; i++) { sprintf(Fuid+0+i*2,"%02X",Sum->SHA1digest[i]); }
Fuid[40]='.';
for(i=0; i<16; i++) { sprintf(Fuid+41+i*2,"%02X",Sum->MD5digest[i]); }
Fuid[73]='.';
snprintf(Fuid+74,sizeof(Fuid)-74,"%Lu",(long long unsigned int)Sum->DataLen);
for(i=0; i<64; i++) { sprintf(Fuid+74+i,"%c",SHA256[i]); }
Fuid[139]='.';
snprintf(Fuid+140,sizeof(Fuid)-140,"%Lu",(long long unsigned int)Sum->DataLen);
if (ListOutFile) fprintf(ListOutFile,"fuid=\"%s\" ",Fuid);
free(Sum);
} /* if Sum */
@@ -1587,7 +1616,9 @@ int DisplayContainerInfo (ContainerInfo *CI, int Cmd)
Fuid[40]='.';
for(i=0; i<16; i++) { sprintf(Fuid+41+i*2,"%02X",Sum->MD5digest[i]); }
Fuid[73]='.';
snprintf(Fuid+74,sizeof(Fuid)-74,"%Lu",(long long unsigned int)Sum->DataLen);
for(i=0; i<64; i++) { sprintf(Fuid+74+i,"%c",SHA256[i]); }
Fuid[139]='.';
snprintf(Fuid+140,sizeof(Fuid)-140,"%Lu",(long long unsigned int)Sum->DataLen);
if (ListOutFile) fprintf(ListOutFile,"fuid=\"%s\" ",Fuid);
free(Sum);
}
@@ -150,7 +150,7 @@ void testDBInsertPfile()
ContainerInfo CITest = {"../testdata/test_1.orig.tar.gz", "./test-result/",
"test_1.orig.tar.gz", "test_1.orig.tar.gz.dir", 1, 1, 0, 0, Stat, PI, 0, 0, 0, 0, 0, 0};
CI = &CITest;
char *Fuid = "383A1791BA72A77F80698A90F22C1B7B04C59BEF.720B5CECCC4700FC90D628FCB45490E3.1312";
char *Fuid = "383A1791BA72A77F80698A90F22C1B7B04C59BEF.720B5CECCC4700FC90D628FCB45490E3.1aa248f65785e15aa9da4fa3701741d85653584544ab4003ef45e232a761a2f1.1312";
int result = DBInsertPfile(CI, Fuid);
CU_ASSERT_EQUAL(result, 1);
}
@@ -94,18 +94,34 @@ void DBLoadGold()
char SQL[MAXCMD];
long PfileKey;
char *Path;
char SHA256[64], command[1000] = "sha256sum ";
FILE *Fin;
int rc;
PGresult *result;
int retcode = -1;

LOG_VERBOSE0("Processing %s",GlobalTempFile);
Fin = fopen(GlobalTempFile,"rb");
// Calculate sha256 value
strcat(command,GlobalTempFile);
FILE* file = popen(command, "r");

if (!Fin)
{
LOG_FATAL("upload %ld Unable to open temp file %s from %s",
GlobalUploadKey,GlobalTempFile,GlobalURL);
SafeExit(1);
}
if (file != (FILE*) NULL)
{
fscanf(file, "%64s", SHA256);
retcode = WEXITSTATUS(pclose(file));
}
if (file == (FILE*) NULL || retcode != 0)
{
LOG_FATAL("Unable to calculate SHA256 of %s\n", GlobalTempFile);
SafeExit(2);
}
Sum = SumComputeFile(Fin);
fclose(Fin);
if ((int)ForceGroup > 0)
@@ -187,8 +203,8 @@ void DBLoadGold()
Len = Unique+41+33; /* 32 for md5 + 1 for '.' */
/* Set the pfile */
memset(SQL,'\0',MAXCMD);
snprintf(SQL,MAXCMD-1,"SELECT pfile_pk FROM pfile WHERE pfile_sha1 = '%.40s' AND pfile_md5 = '%.32s' AND pfile_size = %s;",
SHA1,MD5,Len);
snprintf(SQL,MAXCMD-1,"SELECT pfile_pk FROM pfile WHERE pfile_sha1 = '%.40s' AND pfile_md5 = '%.32s' AND pfile_sha256 = '%.64s' AND pfile_size = %s;",
SHA1,MD5,SHA256,Len);
result = PQexec(pgConn, SQL); /* SELECT */
if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(7);

@@ -197,8 +213,8 @@ void DBLoadGold()
{
/* Insert it */
memset(SQL,'\0',MAXCMD);
snprintf(SQL,MAXCMD-1,"INSERT INTO pfile (pfile_sha1, pfile_md5, pfile_size) VALUES ('%.40s','%.32s',%s)",
SHA1,MD5,Len);
snprintf(SQL,MAXCMD-1,"INSERT INTO pfile (pfile_sha1, pfile_md5, pfile_sha256, pfile_size) VALUES ('%.40s','%.32s','%.64s',%s)",
SHA1,MD5,SHA256,Len);
PQclear(result);
result = PQexec(pgConn, SQL);
if (fo_checkPQcommand(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(8);
@@ -327,6 +327,10 @@
<td align="center">{{ "MD5 Checksum"|trans }}</td>
<td align="right">{{ fileMd5|e }}</td>
</tr>
<tr>
<td align="center">{{ "SHA256 Checksum"|trans }}</td>
<td align="right">{{ fileSha256|e }}</td>
</tr>
<tr>
<td align="center">{{ "Repository ID"|trans }}</td>
<td align="right">{{ fileSha1|e }}.{{ fileMd5|e }}.{{ fileSize|e }}</td>
@@ -120,6 +120,7 @@ function ShowView($Upload, $Item, $ShowMenu=0)
$vars['sizeInMB'] = $bytesH;
$vars['fileSha1'] = $row['pfile_sha1'];
$vars['fileMd5'] = $row['pfile_md5'];
$vars['fileSha256'] = $row['pfile_sha256'];
$vars['fileSize'] = $row['pfile_size'];
$vars['filePfileId'] = $row['pfile_fk'];
}

0 comments on commit d364193

Please sign in to comment.
You can’t perform that action at this time.