Skip to content
Permalink
Browse files

- constify / more useful log message

- verify sdp when parsing (to early detect a truncated file)
- cleanup code / less pointers / fix filename parsing for tmp files
  • Loading branch information...
abma committed Apr 21, 2017
1 parent c78c235 commit 76af8b34a62ea183bb031bff2c014d7e4fc6a445
Showing with 62 additions and 70 deletions.
  1. +32 −42 src/Downloader/Rapid/Sdp.cpp
  2. +1 −1 src/Downloader/Rapid/Sdp.h
  3. +29 −27 src/FileSystem/FileSystem.cpp
@@ -33,13 +33,13 @@ CSdp::CSdp(const std::string& shortname, const std::string& md5,
, downloaded(false)
{
memset(cursize_buf, 0, LENGTH_SIZE);
std::string dir =
const std::string dir =
fileSystem->getSpringDir() + PATH_DELIMITER + "packages" + PATH_DELIMITER;
LOG_DEBUG("%s", dir.c_str());
if (!fileSystem->directoryExists(dir)) {
fileSystem->createSubdirs(dir);
}
sdpPath = dir + md5 + ".sdp";
LOG_DEBUG("%s", sdpPath.c_str());
}

CSdp::~CSdp()
@@ -67,9 +67,6 @@ bool createPoolDirs(const std::string& root)

bool CSdp::downloadSelf(IDownload* dl)
{
if (fileSystem->fileExists(sdpPath) && fileSystem->validateSDP(sdpPath)) { //.sdp is already downloaded
return true;
}

const std::string tmpFile = sdpPath + ".tmp";
IDownload tmpdl(tmpFile);
@@ -78,51 +75,47 @@ bool CSdp::downloadSelf(IDownload* dl)
LOG_ERROR("Couldn't download %s", (md5 + ".sdp").c_str());
return false;
}


if (!fileSystem->Rename(tmpFile, sdpPath)) {
LOG_ERROR("Couldn't rename %s to %s", tmpFile.c_str(), sdpPath.c_str());
return false;
}

return fileSystem->validateSDP(sdpPath);
return true;
}

bool CSdp::download(IDownload* dl)
{
if (downloaded) // allow download only once of the same sdp
return true;
m_download = dl;

if (!downloadSelf(dl))
return false;

std::list<FileData> tmpFiles;
files = &tmpFiles;

if (!fileSystem->parseSdp(sdpPath, *files))// parse downloaded file
return false;
if ((!fileSystem->fileExists(sdpPath)) || (!fileSystem->parseSdp(sdpPath, files))) {// parse downloaded file
if (!downloadSelf(dl))
return false;
fileSystem->parseSdp(sdpPath, files);
}

int i = 0;
int count = 0;
for (FileData& filedata: *files) { // check which file are available on local
for (FileData& filedata: files) { // check which file are available on local
// disk -> create list of files to download
HashMD5 fileMd5;
i++;
fileMd5.Set(filedata.md5, sizeof(filedata.md5));
std::string file;
fileSystem->getPoolFilename(fileMd5.toString(), file);
if (!fileSystem->fileExists(
file)) { // add non-existing files to download list
if (!fileSystem->fileExists(file)) { // add non-existing files to download list
count++;
filedata.download = true;
} else {
filedata.download = false;
}
if (i % 30 == 0) {
LOG_DEBUG("\r%d/%d checked", i, (int)files->size());
LOG_DEBUG("\r%d/%d checked", i, (int)files.size());
}
}
LOG_DEBUG("\r%d/%d need to download %d files", i, (int)files->size(),
LOG_DEBUG("\r%d/%d need to download %d files", i, (int)files.size(),
count);

std::string root = fileSystem->getSpringDir();
@@ -131,26 +124,23 @@ bool CSdp::download(IDownload* dl)
root += PATH_DELIMITER;
if (!createPoolDirs(root)) {
LOG_ERROR("Creating pool directories failed");
count = 0;
return false;
}
if (count > 0) {
downloaded = downloadStream();
if (!downloaded) {
LOG_ERROR("Couldn't download files for %s", md5.c_str());
fileSystem->removeFile(sdpPath);
return false;
}
LOG_DEBUG("Sucessfully downloaded %d files: %s %s", count,
shortname.c_str(), name.c_str());
} else {
LOG_DEBUG("Already downloaded: %s", shortname.c_str());
downloaded = true;
if (!downloadStream()) {
LOG_ERROR("Couldn't download files for %s", md5.c_str());
fileSystem->removeFile(sdpPath);
return false;
}
LOG_DEBUG("Sucessfully downloaded %d files: %s %s", count,
shortname.c_str(), name.c_str());

if (downloaded) {
dl->state = IDownload::STATE_FINISHED;
if (!fileSystem->validateSDP(sdpPath)) { //FIXME: in this call only the downloaded files should be checked
LOG_ERROR("Validation failed");
return false;
}
return downloaded;
downloaded = true;
dl->state = IDownload::STATE_FINISHED;
return true;
}

/**
@@ -167,7 +157,7 @@ static size_t write_streamed_data(const void* tmp, size_t size, size_t nmemb,
char buf[CURL_MAX_WRITE_SIZE];
memcpy(&buf, tmp, CURL_MAX_WRITE_SIZE);
if (!sdp->downloadInitialized) {
sdp->list_it = sdp->files->begin();
sdp->list_it = sdp->files.begin();
sdp->downloadInitialized = true;
sdp->file_handle = NULL;
sdp->file_name = "";
@@ -182,7 +172,7 @@ static size_t write_streamed_data(const void* tmp, size_t size, size_t nmemb,
while (!sdp->list_it->download) { // get file
++sdp->list_it;
}
assert(sdp->list_it != sdp->files->end());
assert(sdp->list_it != sdp->files.end());

FileData& fd = *(sdp->list_it);
HashMD5 fileMd5;
@@ -299,20 +289,20 @@ bool CSdp::downloadStream()

curl_easy_setopt(curlw.GetHandle(), CURLOPT_URL, downloadUrl.c_str());

const int buflen = (files->size() + 7) / 8;
const int buflen = (files.size() + 7) / 8;
std::vector<char> buf(buflen, 0);

int i = 0;
for (FileData& fd: *files) {
for (FileData& fd: files) {
if (fd.download) {
buf[i / 8] |= (1 << (i % 8));
}
i++;
}

int destlen = files->size() * 2;
int destlen = files.size() * 2;
std::vector<char> dest(destlen, 0);
LOG_DEBUG("%d %d %d", (int)files->size(), buflen, destlen);
LOG_DEBUG("%d %d %d", (int)files.size(), buflen, destlen);

gzip_str(&buf[0], buflen, &dest[0], &destlen);

@@ -61,7 +61,7 @@ class CSdp
IDownload* m_download;
bool downloadInitialized;
std::list<FileData>::iterator list_it;
std::list<FileData>* files; // list with all files of an sdp
std::list<FileData> files; // list with all files of an sdp
CFile* file_handle;
std::string file_name;

@@ -81,8 +81,14 @@ bool CFileSystem::fileIsValid(const FileData* mod,
return true;
}

bool CFileSystem::parseSdp(const std::string& filename,
std::list<FileData>& files)
std::string getMD5fromFilename(const std::string& path)
{
const size_t start = path.rfind(PATH_DELIMITER) + 1;
const size_t end = path.find(".", start);
return path.substr(start, end-start);
}

bool CFileSystem::parseSdp(const std::string& filename, std::list<FileData>& files)
{
char c_name[255];
unsigned char c_md5[16];
@@ -98,7 +104,10 @@ bool CFileSystem::parseSdp(const std::string& filename,
return false;
}
files.clear();
HashMD5 sdpmd5;
sdpmd5.Init();
while (true) {

if (!gzread(in, &length, 1)) {
if (gzeof(in)) {
break;
@@ -121,9 +130,25 @@ bool CFileSystem::parseSdp(const std::string& filename,
memcpy(fd.crc32, &c_crc32, 4);
fd.size = parse_int32(c_size);
files.push_back(fd);

HashMD5 nameMd5;
nameMd5.Init();
nameMd5.Update(fd.name.data(), fd.name.size());
nameMd5.Final();
assert(nameMd5.getSize() == 16);
assert(sizeof(fd.md5) == 16);
sdpmd5.Update((const char*)nameMd5.Data(), nameMd5.getSize());
sdpmd5.Update((const char*)&fd.md5[0], sizeof(fd.md5));
}
gzclose(in);
fclose(f);
sdpmd5.Final();
const std::string filehash = getMD5fromFilename(filename);
if (filehash != sdpmd5.toString()) {
LOG_ERROR("%s is invalid, deleted (%s vs %s)", filename.c_str(), filehash.c_str(), sdpmd5.toString().c_str());
removeFile(filename);
return false;
}
LOG_DEBUG("Parsed %s with %d files", filename.c_str(), (int)files.size());
return true;
}
@@ -491,15 +516,9 @@ bool CFileSystem::dumpSDP(const std::string& filename)
return true;
}

std::string getMD5fromFilename(const std::string& path)
{
const size_t start = path.rfind(PATH_DELIMITER) + 1;
const size_t end = path.rfind(".");
return path.substr(start, end-start);
}

bool CFileSystem::validateSDP(const std::string& sdpPath)
{
LOG_DEBUG("CFileSystem::validateSDP() ...");
if (!fileExists(sdpPath)){
LOG_ERROR("SDP file doesn't exist: %s", sdpPath.c_str());
return false;
@@ -516,17 +535,7 @@ bool CFileSystem::validateSDP(const std::string& sdpPath)
}

bool valid = true;
HashMD5 sdpmd5;
sdpmd5.Init();
for (FileData& fd : files) {
HashMD5 nameMd5;
nameMd5.Init();
nameMd5.Update(fd.name.data(), fd.name.size());
nameMd5.Final();
assert(nameMd5.getSize() == 16);
assert(sizeof(fd.md5) == 16);
sdpmd5.Update((const char*)nameMd5.Data(), nameMd5.getSize());
sdpmd5.Update((const char*)&fd.md5[0], sizeof(fd.md5));

std::string filePath;
HashMD5 fileMd5;
@@ -544,14 +553,7 @@ bool CFileSystem::validateSDP(const std::string& sdpPath)
}
}
}
sdpmd5.Final();
const std::string filehash = getMD5fromFilename(sdpPath);
if (filehash != sdpmd5.toString()) {
LOG_ERROR("%s is invalid, deleted (%s vs %s)", sdpPath.c_str(), filehash.c_str(), sdpmd5.toString().c_str());
removeFile(sdpPath);
return false;
}

LOG_DEBUG("CFileSystem::validateSDP() done");
return valid;
}

0 comments on commit 76af8b3

Please sign in to comment.
You can’t perform that action at this time.