Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Straighten out localization in the scraper. #1471

Merged
merged 2 commits into from May 26, 2019
Merged

Straighten out localization in the scraper. #1471

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
991046c
Straigten out localization in the scraper.
jamescowens May 26, 2019
b17721c
Removing unnecessary commented out localization code
jamescowens May 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 0 additions & 5 deletions src/init.cpp
View file
Open in desktop
Expand Up @@ -540,11 +540,6 @@ bool AppInit2(ThreadHandlerPtr threads)
if (!lock.try_lock())
return InitError(strprintf(_("Cannot obtain a lock on data directory %s. Gridcoin is probably already running."), strDataDir));

// Set the scraper file staging directory.
pathScraper = GetDataDir() / "Scraper";



#if !defined(WIN32)
if (fDaemon)
{
Expand Down
12 changes: 9 additions & 3 deletions src/net.cpp
View file
Open in desktop
Expand Up @@ -2253,9 +2253,15 @@ void StartNode(void* parg)
else
{
LogPrintf("Scraper disabled.");
LogPrintf("NN housekeeping thread enabled.");
if (!netThreads->createThread(ThreadNeuralNetwork,NULL,"NeuralNetwork"))
LogPrintf("Error: createThread(NeuralNetwork) failed");

// If the neural network is specified to be disabled, do not enable housekeeping thread.
if (!GetBoolArg("-disablenn", false))
{
LogPrintf("NN housekeeping thread enabled.");

if (!netThreads->createThread(ThreadNeuralNetwork,NULL,"NeuralNetwork"))
LogPrintf("Error: createThread(NeuralNetwork) failed");
}
}
}

Expand Down
54 changes: 24 additions & 30 deletions src/scraper/scraper.cpp
View file
Open in desktop
Expand Up @@ -163,7 +163,7 @@ class logger
LOCK(cs_log);

fs::path plogfile = pathDataDir / "scraper.log";
logfile.open(plogfile.c_str(), std::ios_base::out | std::ios_base::app);
logfile.open(plogfile, std::ios_base::out | std::ios_base::app);

if (!logfile.is_open())
LogPrintf("ERROR: Scraper: Logger: Failed to open logging file\n");
Expand Down Expand Up @@ -261,23 +261,23 @@ class logger

// Re-open log file.
fs::path plogfile = pathDataDir / "scraper.log";
logfile.open(plogfile.c_str(), std::ios_base::out | std::ios_base::app);
logfile.open(plogfile, std::ios_base::out | std::ios_base::app);

if (!logfile.is_open())
LogPrintf("ERROR: Scraper: Logger: Failed to open logging file\n");

PrevArchiveCheckDate = ArchiveCheckDate;
}

std::ifstream infile(pfile_temp.string().c_str(), std::ios_base::in | std::ios_base::binary);
std::ifstream infile(pfile_temp.string(), std::ios_base::in | std::ios_base::binary);

if (!infile)
{
LogPrintf("ERROR: logger: Failed to open archive log file for compression %s.", pfile_temp.string());
return false;
}

std::ofstream outgzfile(pfile_out.string().c_str(), std::ios_base::out | std::ios_base::binary);
std::ofstream outgzfile(pfile_out.string(), std::ios_base::out | std::ios_base::binary);

if (!outgzfile)
{
Expand Down Expand Up @@ -486,7 +486,7 @@ class userpass

fs::path plistfile = GetDataDir() / "userpass.dat";

userpassfile.open(plistfile.c_str(), std::ios_base::in);
userpassfile.open(plistfile, std::ios_base::in);

if (!userpassfile.is_open())
_log(logattribute::CRITICAL, "userpass_data", "Failed to open userpass file");
Expand Down Expand Up @@ -542,9 +542,9 @@ class authdata
authdata(const std::string& project)
{
std::string outfile = project + "_auth.dat";
fs::path poutfile = GetDataDir() / outfile.c_str();
fs::path poutfile = GetDataDir() / outfile;

oauthdata.open(poutfile.c_str(), std::ios_base::out | std::ios_base::app);
oauthdata.open(poutfile, std::ios_base::out | std::ios_base::app);

if (!oauthdata.is_open())
_log(logattribute::CRITICAL, "auth_data", "Failed to open auth data file");
Expand Down Expand Up @@ -707,9 +707,6 @@ void Scraper(bool bSingleShot)
if (pathDataDir.empty())
{
pathDataDir = GetDataDir();
// This is necessary to maintain compatibility with Windows.
pathDataDir.imbue(std::locale(std::locale(), new std::codecvt_utf8_utf16<wchar_t>()));

pathScraper = pathDataDir / "Scraper";
}

Expand Down Expand Up @@ -811,7 +808,7 @@ void Scraper(bool bSingleShot)
sbage = SuperblockAge();
_log(logattribute::INFO, "Scraper", "Superblock not needed. age=" + std::to_string(sbage));
_log(logattribute::INFO, "Scraper", "Sleeping for " + std::to_string(nScraperSleep / 1000) +" seconds");

MilliSleep(nScraperSleep);
}
}
Expand Down Expand Up @@ -957,9 +954,6 @@ void NeuralNetwork()
if (!fScraperActive && pathDataDir.empty())
{
pathDataDir = GetDataDir();
// This is necessary to maintain compatibility with Windows.
pathDataDir.imbue(std::locale(std::locale(), new std::codecvt_utf8_utf16<wchar_t>()));

pathScraper = pathDataDir / "Scraper";

// Initialize log singleton. Must be after the imbue.
Expand Down Expand Up @@ -1293,7 +1287,7 @@ bool DownloadProjectTeamFiles(const NN::WhitelistSnapshot& projectWhitelist)

std::string team_file_name = prjs.m_name + "-team.gz";

fs::path team_file = pathScraper / team_file_name.c_str();
fs::path team_file = pathScraper / team_file_name;

// Grab ETag of team file
Http http;
Expand Down Expand Up @@ -1344,7 +1338,7 @@ bool DownloadProjectTeamFiles(const NN::WhitelistSnapshot& projectWhitelist)
}

std::string chketagfile = prjs.m_name + "-" + sTeamETag + ".csv" + ".gz";
fs::path chkfile = pathScraper / chketagfile.c_str();
fs::path chkfile = pathScraper / chketagfile;

if (fs::exists(chkfile))
{
Expand Down Expand Up @@ -1387,7 +1381,7 @@ bool ProcessProjectTeamFile(const fs::path& file, const std::string& etag, std::
if (file.string().empty())
return false;

std::ifstream ingzfile(file.string().c_str(), std::ios_base::in | std::ios_base::binary);
std::ifstream ingzfile(file.string(), std::ios_base::in | std::ios_base::binary);

if (!ingzfile)
{
Expand Down Expand Up @@ -1529,7 +1523,7 @@ bool DownloadProjectRacFilesByCPID(const NN::WhitelistSnapshot& projectWhitelist

std::string rac_file_name = prjs.m_name + +"-user.gz";

fs::path rac_file = pathScraper / rac_file_name.c_str();
fs::path rac_file = pathScraper / rac_file_name;

// Grab ETag of rac file
Http http;
Expand Down Expand Up @@ -1580,7 +1574,7 @@ bool DownloadProjectRacFilesByCPID(const NN::WhitelistSnapshot& projectWhitelist
}

std::string chketagfile = prjs.m_name + "-" + sRacETag + ".csv" + ".gz";
fs::path chkfile = pathScraper / chketagfile.c_str();
fs::path chkfile = pathScraper / chketagfile;

if (fs::exists(chkfile))
{
Expand Down Expand Up @@ -1636,7 +1630,7 @@ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& fil
if (file.string().empty())
return false;

std::ifstream ingzfile(file.string().c_str(), std::ios_base::in | std::ios_base::binary);
std::ifstream ingzfile(file.string(), std::ios_base::in | std::ios_base::binary);

if (!ingzfile)
{
Expand Down Expand Up @@ -1772,7 +1766,7 @@ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& fil
try
{
size_t filea = fs::file_size(file);
fs::path temp = gzetagfile.c_str();
fs::path temp = gzetagfile;
size_t fileb = fs::file_size(temp);

_log(logattribute::INFO, "ProcessProjectRacFileByCPID", "Processing new rac file " + file.string() + "(" + std::to_string(filea) + " -> " + std::to_string(fileb) + ")");
Expand Down Expand Up @@ -1927,7 +1921,7 @@ uint256 GetmScraperFileManifestHash()

bool LoadBeaconList(const fs::path& file, BeaconMap& mBeaconMap)
{
std::ifstream ingzfile(file.string().c_str(), std::ios_base::in | std::ios_base::binary);
std::ifstream ingzfile(file.string(), std::ios_base::in | std::ios_base::binary);

if (!ingzfile)
{
Expand Down Expand Up @@ -1972,7 +1966,7 @@ bool LoadBeaconList(const fs::path& file, BeaconMap& mBeaconMap)

bool LoadTeamIDList(const fs::path& file)
{
std::ifstream ingzfile(file.string().c_str(), std::ios_base::in | std::ios_base::binary);
std::ifstream ingzfile(file.string(), std::ios_base::in | std::ios_base::binary);

if (!ingzfile)
{
Expand Down Expand Up @@ -2071,7 +2065,7 @@ bool StoreBeaconList(const fs::path& file)
if (fs::exists(file))
fs::remove(file);

std::ofstream outgzfile(file.string().c_str(), std::ios_base::out | std::ios_base::binary);
std::ofstream outgzfile(file.string(), std::ios_base::out | std::ios_base::binary);

if (!outgzfile)
{
Expand Down Expand Up @@ -2116,7 +2110,7 @@ bool StoreTeamIDList(const fs::path& file)
if (fs::exists(file))
fs::remove(file);

std::ofstream outgzfile(file.string().c_str(), std::ios_base::out | std::ios_base::binary);
std::ofstream outgzfile(file.string(), std::ios_base::out | std::ios_base::binary);

if (!outgzfile)
{
Expand Down Expand Up @@ -2236,7 +2230,7 @@ bool MarkScraperFileManifestEntryNonCurrent(ScraperFileManifestEntry& entry)

bool LoadScraperFileManifest(const fs::path& file)
{
std::ifstream ingzfile(file.string().c_str(), std::ios_base::in | std::ios_base::binary);
std::ifstream ingzfile(file.string(), std::ios_base::in | std::ios_base::binary);

if (!ingzfile)
{
Expand Down Expand Up @@ -2299,7 +2293,7 @@ bool StoreScraperFileManifest(const fs::path& file)
if (fs::exists(file))
fs::remove(file);

std::ofstream outgzfile(file.string().c_str(), std::ios_base::out | std::ios_base::binary);
std::ofstream outgzfile(file.string(), std::ios_base::out | std::ios_base::binary);

if (!outgzfile)
{
Expand Down Expand Up @@ -2357,7 +2351,7 @@ bool StoreStats(const fs::path& file, const ScraperStats& mScraperStats)
if (fs::exists(file))
fs::remove(file);

std::ofstream outgzfile(file.string().c_str(), std::ios_base::out | std::ios_base::binary);
std::ofstream outgzfile(file.string(), std::ios_base::out | std::ios_base::binary);

if (!outgzfile)
{
Expand Down Expand Up @@ -2435,7 +2429,7 @@ bool StoreStats(const fs::path& file, const ScraperStats& mScraperStats)

bool LoadProjectFileToStatsByCPID(const std::string& project, const fs::path& file, const double& projectmag, const BeaconMap& mBeaconMap, ScraperStats& mScraperStats)
{
std::ifstream ingzfile(file.string().c_str(), std::ios_base::in | std::ios_base::binary);
std::ifstream ingzfile(file.string(), std::ios_base::in | std::ios_base::binary);

if (!ingzfile)
{
Expand Down Expand Up @@ -2925,7 +2919,7 @@ bool ScraperSaveCScraperManifestToFiles(uint256 nManifestHash)

outputfilewpath = savepath / outputfile;

std::ofstream outfile(outputfilewpath.string().c_str(), std::ios_base::out | std::ios_base::binary);
std::ofstream outfile(outputfilewpath.string(), std::ios_base::out | std::ios_base::binary);

if (!outfile)
{
Expand Down