Skip to content

Commit

Permalink
Update notes on Xcache monitroing statistics feature.
Browse files Browse the repository at this point in the history
  • Loading branch information
@abh3
abh3 committed Dec 2, 2019
1 parent 05fef42 commit de93530
Show file tree
Hide file tree
Showing 14 changed files with 216 additions and 45 deletions.
1 change: 1 addition & 0 deletions docs/PreReleaseNotes.txt
View file
Expand Up @@ -6,6 +6,7 @@ Prerelease Notes
================

+ **New Features**
* **[Xcache]** Provide monitoring summary statistics.
* **[All]** Add hardware assisted CRC32C checksum based on Mark Adler's code.
* **[All]** Implement new thread-safe strerror() replacement.
* **[Server]** Allow xrootd.fslib plugin to be generally stacked.
Expand Down
21 changes: 19 additions & 2 deletions src/XrdOuc/XrdOucCache.hh
View file
Expand Up @@ -525,10 +525,27 @@ virtual int Xeq(XeqCmd cmd, char *arg, int arglen)
XrdOucCacheStats Statistics;

//------------------------------------------------------------------------------
//! Constructor & Destructor
//! A 1-to-7 character cache type identifier (usually pfc or rmc).
//------------------------------------------------------------------------------

const char CacheType[8];

//------------------------------------------------------------------------------
//! Constructor
//!
//! @param ctype - A 1-to-7 character cache type identifier.
//------------------------------------------------------------------------------

XrdOucCache(const char *ctype)
: CacheType({'\0', '2', '3', '4', '5', '6', '7', '8'})
{strncpy(const_cast<char *>(CacheType), ctype,
sizeof(CacheType));
}

//------------------------------------------------------------------------------
//! Destructor
//------------------------------------------------------------------------------

XrdOucCache() {}
virtual ~XrdOucCache() {}
};

Expand Down
16 changes: 10 additions & 6 deletions src/XrdOuc/XrdOucCacheStats.hh
View file
Expand Up @@ -33,6 +33,7 @@
#include <stdint.h>
#include <string.h>

#include "XrdSys/XrdSysAtomics.hh"
#include "XrdSys/XrdSysPthread.hh"

/* The XrdOucCacheStats object holds statistics on cache usage. It is available
Expand Down Expand Up @@ -83,13 +84,13 @@ long long MemSize; // Maximum bytes that can be in memory
long long MemUsed; // Actual bytes that are allocated in memory
long long MemWriteQ; // Actual bytes that are in write queue

// Remote file information (supplied by the POSIX layer)
// File information (supplied by the POSIX layer)
//
long long RemoteOpens; // Number of remote files opened
long long RemoteClosed;// Number of remote files closed
long long OpenDefers; // Number of opens that were deferred
long long DeferOpens; // Number of deferes that were actually opened
} X; // This must be a POD type
long long OpenDefers; // Number of opens that were deferred
long long DeferOpens; // Number of defers that were actually opened
long long ClosDefers; // Number of closes that were deferred
long long ClosedLost; // Number of closed file objects that were lost
} X; // This must be a POD type

inline void Get(XrdOucCacheStats &D)
{sMutex.Lock();
Expand Down Expand Up @@ -126,6 +127,9 @@ inline void Set(XrdOucCacheStats &S)
inline void Add(long long &Dest, long long Val)
{sMutex.Lock(); Dest += Val; sMutex.UnLock();}

inline void Count(long long &Dest)
{AtomicBeg(sMutex); AtomicInc(Dest); AtomicEnd(sMutex);}

inline void Set(long long &Dest, long long Val)
{sMutex.Lock(); Dest = Val; sMutex.UnLock();}

Expand Down
2 changes: 1 addition & 1 deletion src/XrdPfc/XrdPfc.cc
View file
Expand Up @@ -168,7 +168,7 @@ bool Cache::Decide(XrdOucCacheIO* io)
}

Cache::Cache(XrdSysLogger *logger) :
XrdOucCache(),
XrdOucCache("pfc"),
m_log(logger, "XrdPfc_"),
m_trace(new XrdSysTrace("XrdPfc", logger)),
m_traceID("Manager"),
Expand Down
1 change: 1 addition & 0 deletions src/XrdPosix.cmake
View file
Expand Up @@ -26,6 +26,7 @@ add_library(
XrdPosix/XrdPosixObject.cc XrdPosix/XrdPosixObject.hh
XrdPosix/XrdPosixObjGuard.hh
XrdPosix/XrdPosixPrepIO.cc XrdPosix/XrdPosixPrepIO.hh
XrdPosix/XrdPosixStats.hh
XrdPosix/XrdPosixTrace.hh
XrdPosix/XrdPosixXrootd.cc XrdPosix/XrdPosixXrootd.hh
XrdPosix/XrdPosixXrootdPath.cc XrdPosix/XrdPosixXrootdPath.hh
Expand Down
93 changes: 92 additions & 1 deletion src/XrdPosix/XrdPosixConfig.cc
View file
Expand Up @@ -28,8 +28,10 @@
/* specific prior written permission of the institution or contributor. */
/******************************************************************************/

#include <algorithm>
#include <iostream>
#include <stdio.h>
#include <string>

#include "XrdCl/XrdClDefaultEnv.hh"

Expand All @@ -44,6 +46,7 @@
#include "XrdPosix/XrdPosixInfo.hh"
#include "XrdPosix/XrdPosixMap.hh"
#include "XrdPosix/XrdPosixPrepIO.hh"
#include "XrdPosix/XrdPosixStats.hh"
#include "XrdPosix/XrdPosixTrace.hh"
#include "XrdPosix/XrdPosixXrootd.hh"
#include "XrdPosix/XrdPosixXrootdPath.hh"
Expand All @@ -63,12 +66,13 @@ namespace XrdPosixGlobals
extern XrdScheduler *schedP;
extern XrdOucCache *theCache;
extern XrdOucName2Name *theN2N;
extern XrdCl::DirListFlags::Flags dlFlag;
extern XrdSysLogger *theLogger;
extern XrdSysError *eDest;
extern XrdPosixStats Stats;
extern XrdSysTrace Trace;
extern int ddInterval;
extern int ddMaxTries;
extern XrdCl::DirListFlags::Flags dlFlag;
extern bool oidsOK;
};

Expand Down Expand Up @@ -387,3 +391,90 @@ void XrdPosixConfig::setOids(bool isok)
{
XrdPosixGlobals::oidsOK = isok;
}

/******************************************************************************/
/* S t a t s C a c h e */
/******************************************************************************/

int XrdPosixConfig::Stats(const char *theID, char *buff, int blen)
{
static const char stats1[] = "<stats id=\"%s\">"
"<open>%lld<errs>%lld</errs></open>"
"<close>%lld<errs>%lld</errs></close>"
"</stats>";

static const char stats2[] = "<stats id=\"cache\" type=\"%s\">"
"<prerd><in>%lld</in><hits>%lld</hits><miss>%lld</miss></prerd>"
"<rd><in>%lld</in><out>%lld</out>"
"<hits>%lld></hits><miss>%lld</miss>"
"</rd>"
"<pass>%lld<cnt>%lld</cnt></pass>"
"<wr><out>%lld</out><updt>%lld</updt></wr>"
"<saved>%lld</saved><purge>%lld</purge>"
"<files><opened>%lld</opened><closed>%lld</closed><new>%lld</new>"
"<del>%lld</del><now>%lld</now><full>%lld</full>"
"</files>"
"<store><size>%lld</size><used>%lld</used>"
"<min>%lld</min><max>%lld</max>"
"</store>"
"<mem><size>%lld</size><used>%lld</used><wq>%lld</wq></mem>"
"<opcl><odefer>%lld</odefer><defero>%lld</defero>"
"<cdefer>%lld</cdefer><clost>%lld</clost>"
"</opcl>"
"</stats>";

// If the caller want the maximum length, then provide it.
//
if (!blen)
{size_t n;
int len1, digitsLL = strlen("9223372036854775807");
std::string fmt = stats1;
n = std::count(fmt.begin(), fmt.end(), '%');
len1 = fmt.size() + (digitsLL*n) - (n*3) + strlen(theID);
if (!XrdPosixGlobals::theCache) return len1;
fmt = stats2;
n = std::count(fmt.begin(), fmt.end(), '%');
return len1 + fmt.size() + (digitsLL*n) - (n*3) + 8;
}

// Get the standard statistics
//
XrdPosixStats Y;
XrdPosixGlobals::Stats.Get(Y);

// Format the line
//
int k = snprintf(buff, blen, stats1, theID,
Y.X.Opens, Y.X.OpenErrs, Y.X.Closes, Y.X.CloseErrs);

// If there is no cache then there nothing to return
//
if (!XrdPosixGlobals::theCache) return k;
buff += k; blen -= k;

// Get the statistics
//
XrdOucCacheStats Z;
XrdPosixGlobals::theCache->Statistics.Get(Z);

// Format the statisics into the supplied buffer
//
int n = snprintf(buff, blen, stats2, XrdPosixGlobals::theCache->CacheType,
Z.X.BytesPead, Z.X.HitsPR, Z.X.MissPR,
Z.X.BytesRead, Z.X.BytesGet, Z.X.Hits, Z.X.Miss,
Z.X.BytesPass, Z.X.Pass,
Z.X.BytesWrite, Z.X.BytesPut,
Z.X.BytesSaved, Z.X.BytesPurged,
Z.X.FilesOpened, Z.X.FilesClosed, Z.X.FilesCreated,
Z.X.FilesPurged, Z.X.FilesInCache, Z.X.FilesAreFull,
Z.X.DiskSize, Z.X.DiskUsed,
Z.X.DiskMin, Z.X.DiskMax,
Z.X.MemSize, Z.X.MemUsed, Z.X.MemWriteQ,
Z.X.OpenDefers, Z.X.DeferOpens,
Z.X.ClosDefers, Z.X.ClosedLost
);

// Return the right value
//
return (n < blen ? n+k : 0);
}
2 changes: 2 additions & 0 deletions src/XrdPosix/XrdPosixConfig.hh
View file
Expand Up @@ -55,6 +55,8 @@ static void SetEnv(const char *kword, int kval);

static void setOids(bool isok);

static int Stats(const char *theID, char *buff, int blen);

XrdPosixConfig() {}
~XrdPosixConfig() {}

Expand Down
52 changes: 35 additions & 17 deletions src/XrdPosix/XrdPosixFile.cc
View file
Expand Up @@ -42,6 +42,7 @@
#include "XrdPosix/XrdPosixFile.hh"
#include "XrdPosix/XrdPosixFileRH.hh"
#include "XrdPosix/XrdPosixPrepIO.hh"
#include "XrdPosix/XrdPosixStats.hh"
#include "XrdPosix/XrdPosixTrace.hh"
#include "XrdPosix/XrdPosixXrootdPath.hh"

Expand All @@ -57,9 +58,9 @@ namespace XrdPosixGlobals
extern XrdOucCache *theCache;
extern XrdOucName2Name *theN2N;
extern XrdSysError *eDest;
extern XrdPosixStats Stats;
extern int ddInterval;
extern int ddMaxTries;
int ddNumLost = 0;
};

namespace
Expand Down Expand Up @@ -122,13 +123,14 @@ XrdPosixFile::XrdPosixFile(bool &aOK, const char *path, XrdPosixCallBack *cbP,

XrdPosixFile::~XrdPosixFile()
{
// Detach the cache if it is attached
//??? We need to figure this out, shouldn't need to do this
// if (XCio != this) XCio->Detach();

// Close the remote connection
//
if (clFile.IsOpen()) {XrdCl::XRootDStatus status = clFile.Close();};
if (clFile.IsOpen())
{XrdPosixGlobals::Stats.Count(XrdPosixGlobals::Stats.X.Closes);
XrdCl::XRootDStatus status = clFile.Close();
if (!status.IsOK())
XrdPosixGlobals::Stats.Count(XrdPosixGlobals::Stats.X.CloseErrs);
}

// Get rid of defered open object
//
Expand Down Expand Up @@ -159,6 +161,7 @@ void* XrdPosixFile::DelayedDestroy(void* vpf)
const char *eTxt;
XrdPosixFile *fCurr, *fNext;
char buff[512], buff2[256];
static int ddNumLost = 0;
int ddCount, refNum;
bool doWait = false;

Expand All @@ -181,13 +184,12 @@ do{if (doWait)

// Do some debugging
//
DEBUG("DLY destory of "<<ddCount<<" objects; "<<XrdPosixGlobals::ddNumLost
<<" already lost.");
DEBUG("DLY destory of "<<ddCount<<" objects; "<<ddNumLost <<" already lost.");

// Try to delete all the files on the list. If we exceeded the try limit,
// remove the file from the list and let it sit forever.
//
int nowLost = XrdPosixGlobals::ddNumLost;
int nowLost = ddNumLost;
while((fCurr = fNext))
{fNext = fCurr->nextFile;
if (!(refNum = fCurr->Refs()))
Expand All @@ -199,7 +201,7 @@ do{if (doWait)
} else eTxt = 0;

if (fCurr->numTries > XrdPosixGlobals::ddMaxTries)
{XrdPosixGlobals::ddNumLost++; ddCount--;
{ddNumLost++; ddCount--;
if (!eTxt)
{snprintf(buff2, sizeof(buff2), "in use %d", refNum);
eTxt = buff2;
Expand All @@ -209,7 +211,7 @@ do{if (doWait)
Say->Emsg("DDestroy", buff, fCurr->Origin());
} else {
DMSG("DDestroy", eTxt <<" timeout closing " <<fCurr->Origin()
<<' ' <<XrdPosixGlobals::ddNumLost <<" objects lost");
<<' ' <<ddNumLost <<" objects lost");
}
fCurr->nextFile = ddLost;
ddLost = fCurr;
Expand All @@ -222,14 +224,17 @@ do{if (doWait)
ddMutex.UnLock();
}
}
if (Say && XrdPosixGlobals::ddNumLost - nowLost >= 3)
if (Say && ddNumLost - nowLost >= 3)
{snprintf(buff, sizeof(buff), "%d objects deferred and %d lost.",
ddCount, XrdPosixGlobals::ddNumLost);
ddCount, ddNumLost);
Say->Emsg("DDestroy", buff);
} else {
DEBUG("DLY destory end; "<<ddCount<<" objects deferred and "
<<XrdPosixGlobals::ddNumLost <<" lost.");
<<ddNumLost <<" lost.");
}
if (XrdPosixGlobals::theCache && ddNumLost != nowLost)
XrdPosixGlobals::theCache->Statistics.Set(
(XrdPosixGlobals::theCache->Statistics.X.ClosedLost), ddNumLost);
} while(true);

return 0;
Expand All @@ -243,6 +248,12 @@ void XrdPosixFile::DelayedDestroy(XrdPosixFile *fp)
int ddCount;
bool doPost;

// Count number of times this has happened (we should have a cache)
//
if (XrdPosixGlobals::theCache)
XrdPosixGlobals::theCache->Statistics.Count(
(XrdPosixGlobals::theCache->Statistics.X.ClosDefers));

// Place this file on the delayed delete list
//
ddMutex.Lock();
Expand Down Expand Up @@ -278,8 +289,11 @@ bool XrdPosixFile::Close(XrdCl::XRootDStatus &Status)
// from the file table at this point and should be unlocked.
//
if (clFile.IsOpen())
{Status = clFile.Close();
return Status.IsOK();
{XrdPosixGlobals::Stats.Count(XrdPosixGlobals::Stats.X.Closes);
Status = clFile.Close();
if (Status.IsOK()) return true;
XrdPosixGlobals::Stats.Count(XrdPosixGlobals::Stats.X.CloseErrs);
return false;
}
return true;
}
Expand All @@ -306,7 +320,11 @@ bool XrdPosixFile::Finalize(XrdCl::XRootDStatus *Status)
// Setup the cache if it is to be used
//
if (XrdPosixGlobals::theCache)
XCio = XrdPosixGlobals::theCache->Attach(ioP, cOpt);
{XCio = XrdPosixGlobals::theCache->Attach(ioP, cOpt);
if (ioP == (XrdOucCacheIO *)PrepIO)
XrdPosixGlobals::theCache->Statistics.Add(
(XrdPosixGlobals::theCache->Statistics.X.OpenDefers), 1LL);
}

return true;
}
Expand Down

0 comments on commit de93530

Please sign in to comment.