Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

pfc: add possibility to specify disk usage parameters in .. G, T units using XrdOuca2x::a2sz() #296

Merged
merged 7 commits into from
Oct 8, 2015
Merged

pfc: add possibility to specify disk usage parameters in .. G, T units using XrdOuca2x::a2sz() #296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a584f97
Work with absolute disk usage values instead of relative.
alja Sep 22, 2015
2eb2dce
Add option to set disk usage in absolute (k, g, or T) value.
alja Sep 22, 2015
3d105cc
Fix a2sz conversion check when parsing disk usage in absolute coordin…
alja Oct 7, 2015
0b79eb6
Return with error if pfc.diskusage lwm and hwm are zero
alja Oct 7, 2015
10846f2
Update default pfc.diskusage values.
alja Oct 7, 2015
1bbda06
Replace atof with strtod when convert relative diskusage limits.
alja Oct 8, 2015
33cffb8
Fix printf format in a debug message.
alja Oct 8, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/XrdFileCache/README
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ pfc.nread: number of in memory cached blocks reserved for read tasks
pfc.nprefetch: number of in memory cached blocks reserved for prefetch tasks

pfc.diskusage <lwm fraction> <hwm fraction>: high / low watermarks for disk cache
purge operation (default 0.7 and 0.9)
purge operation (default 0.9 and 0.95)

pfc.user <username>: username used by XrdOss plugin

Expand Down
90 changes: 59 additions & 31 deletions src/XrdFileCache/XrdFileCacheFactory.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,6 @@


using namespace XrdFileCache;
namespace {
static long long s_diskSpacePrecisionFactor = 10000000;
}

XrdVERSIONINFO(XrdOucGetCache, XrdFileCache);

Expand Down Expand Up @@ -212,6 +209,20 @@ bool Factory::Config(XrdSysLogger *logger, const char *config_filename, const ch
&XrdVERSIONINFOVAR(XrdOucGetCache));
if (!ofsCfg) return false;


if (ofsCfg->Load(XrdOfsConfigPI::theOssLib)) {
ofsCfg->Plugin(m_output_fs);
XrdOssCache_FS* ocfs = XrdOssCache::Find("public");
ocfs->Add(m_configuration.m_cache_dir.c_str());
}
else
{
clLog()->Error(XrdCl::AppMsg, "Factory::Config() Unable to create an OSS object");
m_output_fs = 0;
return false;
}


// Actual parsing of the config file.
bool retval = true;
char *var;
Expand Down Expand Up @@ -240,23 +251,19 @@ bool Factory::Config(XrdSysLogger *logger, const char *config_filename, const ch
}

Config.Close();


if (retval)
// sets default value for disk usage
if (m_configuration.m_diskUsageLWM < 0 || m_configuration.m_diskUsageHWM < 0)
{
if (ofsCfg->Load(XrdOfsConfigPI::theOssLib)) {
ofsCfg->Plugin(m_output_fs);
XrdOssCache_FS* ocfs = XrdOssCache::Find("public");
ocfs->Add(m_configuration.m_cache_dir.c_str());
}
else
{
clLog()->Error(XrdCl::AppMsg, "Factory::Config() Unable to create an OSS object");
retval = false;
m_output_fs = 0;
XrdOssVSInfo sP;
if (m_output_fs->StatVS(&sP, "public", 1) >= 0) {
m_configuration.m_diskUsageLWM = 0.90 * sP.Total;
m_configuration.m_diskUsageHWM = 0.95 * sP.Total;
clLog()->Debug(XrdCl::AppMsg, "Default disk usage [%lld, %lld]", m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM);
}
}


if (retval)
{
int loff = 0;
char buff[2048];
loff = snprintf(buff, sizeof(buff), "result\n"
Expand Down Expand Up @@ -310,17 +317,39 @@ bool Factory::ConfigParameters(std::string part, XrdOucStream& config )
}
else if ( part == "diskusage" )
{
const char* minV = config.GetWord();
if (minV) {
m_configuration.m_lwm = ::atof(minV);
const char* maxV = config.GetWord();
if (maxV) {
m_configuration.m_hwm = ::atof(maxV);
std::string minV = config.GetWord();
std::string maxV = config.GetWord();
if (!minV.empty() && !maxV.empty()) {
XrdOssVSInfo sP;
if (m_output_fs->StatVS(&sP, "public", 1) >= 0)
{
if (::isalpha(*(minV.rbegin())) && ::isalpha(*(minV.rbegin()))) {
if ( XrdOuca2x::a2sz(m_log, "Error getting disk usage low watermark", minV.c_str(), &m_configuration.m_diskUsageLWM, 0, sP.Total)
|| XrdOuca2x::a2sz(m_log, "Error getting disk usage high watermark", maxV.c_str(), &m_configuration.m_diskUsageHWM, 0, sP.Total))
{
return false;
}
}
else
{
char* eP;
errno = 0;
float lwmf = strtod(minV.c_str(), &eP);
if (errno || eP == minV.c_str()) {
m_log.Emsg("Factory::ConfigParameters() error parsing diskusage parameter ", minV.c_str());
return false;
}
float hwmf = strtod(maxV.c_str(), &eP);
if (errno || eP == maxV.c_str()) {
m_log.Emsg("Factory::ConfigParameters() error parsing diskusage parameter ", maxV.c_str());
return false;
}

m_configuration.m_diskUsageLWM = sP.Total * lwmf;
m_configuration.m_diskUsageHWM = sP.Total * hwmf;
}
}
}
else {
clLog()->Error(XrdCl::AppMsg, "Factory::ConfigParameters() pss.diskUsage min max value not specified");
}
}
else if ( part == "blocksize" )
{
Expand Down Expand Up @@ -488,12 +517,11 @@ void Factory::CacheDirCleanup()
}
else
{
float oc = 1 - float(sP.Free)/(sP.Total);
clLog()->Debug(XrdCl::AppMsg, "Factory::CacheDirCleanup() occupates disk space == %f", oc);
if (oc > m_configuration.m_hwm)
long long ausage = sP.Total - sP.Free;
clLog()->Debug(XrdCl::AppMsg, "Factory::CacheDirCleanup() occupates disk space == %lld", ausage);
if (ausage > m_configuration.m_diskUsageHWM)
{
long long bytesToRemoveLong = static_cast<long long> ((oc - m_configuration.m_lwm) * static_cast<float>(s_diskSpacePrecisionFactor));
bytesToRemove = (sP.Total * bytesToRemoveLong) / s_diskSpacePrecisionFactor;
bytesToRemove = ausage - m_configuration.m_diskUsageLWM;
clLog()->Info(XrdCl::AppMsg, "Factory::CacheDirCleanup() need space for %lld bytes", bytesToRemove);
}
}
Expand Down
8 changes: 4 additions & 4 deletions src/XrdFileCache/XrdFileCacheFactory.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ namespace XrdFileCache
{
Configuration() :
m_hdfsmode(false),
m_lwm(0.95),
m_hwm(0.9),
m_diskUsageLWM(-1),
m_diskUsageHWM(-1),
m_bufferSize(1024*1024),
m_NRamBuffersRead(8),
m_NRamBuffersPrefetch(1),
Expand All @@ -57,8 +57,8 @@ namespace XrdFileCache
std::string m_cache_dir; //!< path of disk cache
std::string m_username; //!< username passed to oss plugin

float m_lwm; //!< cache purge low water mark
float m_hwm; //!< cache purge high water mark
long long m_diskUsageLWM; //!< cache purge low water mark
long long m_diskUsageHWM; //!< cache purge high water mark

long long m_bufferSize; //!< prefetch buffer size, default 1MB
int m_NRamBuffersRead; //!< number of read in-memory cache blocks
Expand Down