From d31439da4b432443f6030aef652e7dd783eb5d62 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Thu, 3 Mar 2016 05:43:04 -0800 Subject: [PATCH 001/171] [Posix] Stage 1: Implement async cache interface. --- src/XrdOuc/XrdOucCache2.hh | 304 +++++++++++++++++++++++++++++++++++++ src/XrdOuc/XrdOucIOVec.hh | 12 ++ 2 files changed, 316 insertions(+) create mode 100644 src/XrdOuc/XrdOucCache2.hh diff --git a/src/XrdOuc/XrdOucCache2.hh b/src/XrdOuc/XrdOucCache2.hh new file mode 100644 index 00000000000..0947d618c0c --- /dev/null +++ b/src/XrdOuc/XrdOucCache2.hh @@ -0,0 +1,304 @@ +#ifndef __XRDOUCCACHE2_HH__ +#define __XRDOUCCACHE2_HH__ +/******************************************************************************/ +/* */ +/* X r d O u c C a c h e 2 . h h */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "XrdOuc/XrdOucCache.hh" + +//----------------------------------------------------------------------------- +//! XrdOucCache2 +//! +//! This class is an extension of the original XrdOucCache class and provides +//! async I/O support, defered open and I/O decoupling. It is loaded as a +//! cache plugin by the POSIX package via the proxy directive +//! +//! pss.cache -2 \ +//! +//! Without the "-2" the original version is loaded. Implementation of a cache +//! is similar between versions (see XrdOucCache.hh for details). The only +//! difference is this version requires additional methods to be implemented +//! and uses an asynchrnous callback mechanism to return the results. +//----------------------------------------------------------------------------- + +/******************************************************************************/ +/* C l a s s X r d O u c C a c h e 2 C B */ +/******************************************************************************/ + +//----------------------------------------------------------------------------- +//! The XrdOucCache2CB defines a callback object that must be used to handle +//! asynchronous operations (e.g. I/O). +//----------------------------------------------------------------------------- + +class XrdOucCache2CB +{ +public: + +//------------------------------------------------------------------------------ +//! Handle result from a previous async operation. +//! +//! @param result is result from a previous operation. Successful results are +//! always values >= 0 while errors are negative values and are +//! always '-errno' indicate the reason for the error. +//------------------------------------------------------------------------------ +virtual +void Done(int result) = 0; + + XrdOucCache2CB() {} +virtual ~XrdOucCache2CB() {} +}; + +/******************************************************************************/ +/* C l a s s X r d O u c C a c h e I O 2 */ +/******************************************************************************/ + +//------------------------------------------------------------------------------ +//! The XrdOucCacheIO2 object is responsible for interacting with the original +//! data source/target. It can be used with or without a front-end cache. It +//! an extension of the XrdOucCacheIO class which defines base methods. +//------------------------------------------------------------------------------ + +class XrdOucCacheIO2 : public virtual XrdOucCacheIO +{ +public: + +//------------------------------------------------------------------------------ +//! Perform an asynchronous file open request. This method is meant to be +//! by the cache layer using the original XrdOucCacheIO2 object passed to +//! Attach() after the cache layer has defered an open request via Prepare(). +//! The file is opened using the original url, oflags, and mode. +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. Done() is passed +//! <0 - Open failed, value is -errno. +//! =0 - Open succeeded. +//------------------------------------------------------------------------------ + +virtual void Open(XrdOucCache2CB &iocb) {iocb.Done(-ENOSYS);} + +//------------------------------------------------------------------------------ +//! Perform an asynchronous read (defaults to synchrnous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. Done() is passed +//! < 0 - Read failed, value is -errno. +//! >=0 - Read succeeded, value is number of bytes read. +//! @param buff pointer to the buffer to receive the results. The buffer must +//! remain valid until the callback is invoked. +//! @param offs the offset into the file. +//! @param rlen the number of bytes to read. +//------------------------------------------------------------------------------ + +using XrdOucCacheIO::Read; + +virtual void Read (XrdOucCache2CB &iocb, char *buff, long long offs, int rlen) + {iocb.Done(Read(buff, offs, rlen));} + +//------------------------------------------------------------------------------ +//! Perform an asynchronous vector read (defaults to synchrnous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. Done() is passed +//! < 0 - ReadV failed, value is -errno. +//! >=0 - ReadV succeeded, value is number of bytes read. +//! @param readV pointer to a vector of read requests. +//! @param rnum the number of elements in the vector. +//------------------------------------------------------------------------------ + +using XrdOucCacheIO::ReadV; + +virtual void ReadV(XrdOucCache2CB &iocb, const XrdOucIOVec *readV, int rnum) + {iocb.Done(ReadV(readV, rnum));} + +//------------------------------------------------------------------------------ +//! Perform an asynchronous fsync() operation (defaults to synchronous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. Done() is passed +//! <0 - Sync failed, value is -errno. +//! =0 - Sync succeeded. +//------------------------------------------------------------------------------ + +using XrdOucCacheIO::Sync; + +virtual void Sync(XrdOucCache2CB &iocb) {iocb.Done(Sync());} + +//------------------------------------------------------------------------------ +//! Perform an asynchronous trunc() operation (defaults to synchrnous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. Done() is passsed +//! <0 - Trunc failed, value is -errno. +//! =0 - Trunc succeeded. +//! @param offs the size the file is to have. +//------------------------------------------------------------------------------ + +using XrdOucCacheIO::Trunc; + +virtual void Trunc(XrdOucCache2CB &iocb, long long offs) + {iocb.Done(Trunc(offs));} + +//------------------------------------------------------------------------------ +//! Perform an asynchronous write (defaults to synchronous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. +//! < 0 - Write failed, value is -errno. +//! >=0 - Write succeeded, value is number of bytes written. +//! @param buff pointer to the buffer holding the contents. The buffer must +//! remain valid until the callback is invoked. +//! @param offs the offset into the file. +//! @param wlen the number of bytes to write +//------------------------------------------------------------------------------ + +using XrdOucCacheIO::Write; + +virtual void Write(XrdOucCache2CB &iocb, char *buff, long long offs, int wlen) + {iocb.Done(Write(buff, offs, wlen));} + +//------------------------------------------------------------------------------ + +virtual ~XrdOucCacheIO2() {} // Always use Detach() instead of direct delete! +}; + +/******************************************************************************/ +/* C l a s s X r d O u c C a c h e 2 */ +/******************************************************************************/ + +//------------------------------------------------------------------------------ +//! The XrdOucCache2 class is used to define a version 2 cache. In version 2, +//! there can be only one such instance. the cache is associated with one or +//! more XrdOucCacheIO2 objects. Use the Attach() method in this class to +//! create such associations. +//------------------------------------------------------------------------------ + +class XrdOucCache2 : public virtual XrdOucCache +{ +public: + +//------------------------------------------------------------------------------ +//! Obtain a new XrdOucCacheIO2 object that fronts an existing XrdOucCacheIO2 +//! with this cache. Upon success a pointer to a new XrdOucCacheIO2 object is +//! returned and must be used to read and write data with the cache interposed. +//! Upon failure, the original XrdOucCacheIO2 object is returned with errno set. +//! You can continue using the object without any cache. +//! +//! @param ioP Pointer to the current CacheIO2 object used for I/O. +//! @param opts Cache options identical to those defined for XrdOucCache +//! Attach() method. +//! +//! @return Pointer to a new XrdOucCacheIO2 object (success) or the original +//! XrdOucCacheIO2 object (failure) with errno set. +//------------------------------------------------------------------------------ +virtual +XrdOucCacheIO2 *Attach(XrdOucCacheIO2 *ioP, int opts=0) = 0; + +virtual +XrdOucCacheIO *Attach(XrdOucCacheIO *ioP, int opts=0) + {errno = ENOSYS; return ioP;} + +//------------------------------------------------------------------------------ +//! Creates an instance of a version 1 cache. This method is no longer used so +//! we simply define a default for this method here for backward compatability. +//! +//! @return A pointer to an XrdOucCache2 object upon success or a nil pointer +//! with errno set upon failure. +//------------------------------------------------------------------------------ +virtual +XrdOucCache *Create(Parms &Params, XrdOucCacheIO::aprParms *aprP=0) + {return this;} + +//------------------------------------------------------------------------------ +//! Preapare the cache for a file open request. This method is called prior to +//! actually opening a file. This method is meant to allow defering an open +//! request or implementing the full I/O stack in the cache layer. +//! +//! @param url - Pointer to the url about to be opened. +//! @param oflags - Standard Unix open flags (see open(2)). +//! @param mode - Standard mode flags if file is being created. +//! +//! @return <0 Error has occurred, return value is -errno; fail open request. +//! =0 Continue with open() request. +//! >0 Defer open but treat the file as actually being open. Use the +//! XrdOucCacheIO2::Open() method to open the file at a later time. +//------------------------------------------------------------------------------ +virtual +int Prepare(const char *url, int oflags, mode_t mode) + {(void)url; (void)oflags; (void)mode; return 0;} + + XrdOucCache2() {} +virtual ~XrdOucCache2() {} +}; + +/******************************************************************************/ +/* C r e a t i n g C a c h e P l u g - I n s */ +/******************************************************************************/ + +//------------------------------------------------------------------------------ +//! You can create a V2 cache plug-in for those parts of the xrootd system that +//! allow a dynamically selectable cache implementation (e.g. the proxy server +//! plug-in supports cache plug-ins via the pss.cachelib). For the proxy server, +//! it preferentially looks for XrdOucGetCache2() and invokes this function if +//! is exists. Otherwise, it uses the XrdOucGetCache() function (old version 1). +//! +//! Your plug-in must exist in a shared library and have the following extern C +//! function defined whos parameters are: +//! +//! @param Logger Pointer to the logger object that should be used with an +//! instance of XrdSysError to direct messages to a log file. +//! If Logger is null, you should use cerr to output messages. +//! @param Config Pointer to the configuration file name from where you +//! should get additional information. If Config is null, there +//! is no configuration file is present. +//! @param Parms Pointer to any parameters specified after the shared library +//! path. If Parms is null, there are no parameters. +//! @return A usable, fully configured, instance of an XrdOucCache2 +//! object upon success and a null pointer otherwise. This +//! instance is used for all operations defined by methods in +//! XrdOucCache bas class as well as this class. +//! +//! extern "C" +//! { +//! XrdOucCache2 *XrdOucGetCache2(XrdSysLogger *Logger, // Where messages go +//! const char *Config, // Config file used +//! const char *Parms); // Optional parm string +//! } +#endif diff --git a/src/XrdOuc/XrdOucIOVec.hh b/src/XrdOuc/XrdOucIOVec.hh index 10afb78858c..337df2c91e7 100644 --- a/src/XrdOuc/XrdOucIOVec.hh +++ b/src/XrdOuc/XrdOucIOVec.hh @@ -44,4 +44,16 @@ struct XrdOucIOVec int info; // Available for arbitrary use char *data; // Location to read into. }; + +// Add backward compatible constructor to XrdOucIOVec +// +struct XrdOucIOVec2 : public XrdOucIOVec +{ + XrdOucIOVec2(char *buff, long long offs, int sz, int inf=0) + {XrdOucIOVec::offset = offs; + XrdOucIOVec::size = sz; + XrdOucIOVec::info = inf; + XrdOucIOVec::data = buff; + } +}; #endif From d8c1460d538cceae19eb635bd42e7c728b2b8a68 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Tue, 8 Mar 2016 07:00:04 -0800 Subject: [PATCH 002/171] Add Stat() and Update() to the Cache2 objects. --- src/XrdOuc/XrdOucCache2.hh | 96 ++++++++++++++------------------------ 1 file changed, 35 insertions(+), 61 deletions(-) diff --git a/src/XrdOuc/XrdOucCache2.hh b/src/XrdOuc/XrdOucCache2.hh index 0947d618c0c..4439e641c0d 100644 --- a/src/XrdOuc/XrdOucCache2.hh +++ b/src/XrdOuc/XrdOucCache2.hh @@ -46,33 +46,6 @@ //! difference is this version requires additional methods to be implemented //! and uses an asynchrnous callback mechanism to return the results. //----------------------------------------------------------------------------- - -/******************************************************************************/ -/* C l a s s X r d O u c C a c h e 2 C B */ -/******************************************************************************/ - -//----------------------------------------------------------------------------- -//! The XrdOucCache2CB defines a callback object that must be used to handle -//! asynchronous operations (e.g. I/O). -//----------------------------------------------------------------------------- - -class XrdOucCache2CB -{ -public: - -//------------------------------------------------------------------------------ -//! Handle result from a previous async operation. -//! -//! @param result is result from a previous operation. Successful results are -//! always values >= 0 while errors are negative values and are -//! always '-errno' indicate the reason for the error. -//------------------------------------------------------------------------------ -virtual -void Done(int result) = 0; - - XrdOucCache2CB() {} -virtual ~XrdOucCache2CB() {} -}; /******************************************************************************/ /* C l a s s X r d O u c C a c h e I O 2 */ @@ -88,22 +61,6 @@ class XrdOucCacheIO2 : public virtual XrdOucCacheIO { public: -//------------------------------------------------------------------------------ -//! Perform an asynchronous file open request. This method is meant to be -//! by the cache layer using the original XrdOucCacheIO2 object passed to -//! Attach() after the cache layer has defered an open request via Prepare(). -//! The file is opened using the original url, oflags, and mode. -//! -//! @param iocb reference to the callback object that receives the result. All -//! results are returned via this object's Done() method. If the -//! caller holds any locks they must be recursive locks as the -//! callback may occur on the calling thread. Done() is passed -//! <0 - Open failed, value is -errno. -//! =0 - Open succeeded. -//------------------------------------------------------------------------------ - -virtual void Open(XrdOucCache2CB &iocb) {iocb.Done(-ENOSYS);} - //------------------------------------------------------------------------------ //! Perform an asynchronous read (defaults to synchrnous). //! @@ -121,7 +78,7 @@ virtual void Open(XrdOucCache2CB &iocb) {iocb.Done(-ENOSYS);} using XrdOucCacheIO::Read; -virtual void Read (XrdOucCache2CB &iocb, char *buff, long long offs, int rlen) +virtual void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen) {iocb.Done(Read(buff, offs, rlen));} //------------------------------------------------------------------------------ @@ -139,7 +96,7 @@ virtual void Read (XrdOucCache2CB &iocb, char *buff, long long offs, int rlen) using XrdOucCacheIO::ReadV; -virtual void ReadV(XrdOucCache2CB &iocb, const XrdOucIOVec *readV, int rnum) +virtual void ReadV(XrdOucCacheIOCB &iocb, const XrdOucIOVec *readV, int rnum) {iocb.Done(ReadV(readV, rnum));} //------------------------------------------------------------------------------ @@ -155,24 +112,21 @@ virtual void ReadV(XrdOucCache2CB &iocb, const XrdOucIOVec *readV, int rnum) using XrdOucCacheIO::Sync; -virtual void Sync(XrdOucCache2CB &iocb) {iocb.Done(Sync());} +virtual void Sync(XrdOucCacheIOCB &iocb) {iocb.Done(Sync());} //------------------------------------------------------------------------------ -//! Perform an asynchronous trunc() operation (defaults to synchrnous). +//! Update the originally passed XrdOucCacheIO2 object with the object passed. +//! All future uses underlying XrdOucCacheIO2 object must now use this object. +//! Update() is called when Prepare() indicated that the file should not be +//! physically opened and a file method was invoked in the XrdOucCacheIO2 +//! passed to Attach(). When this occurs, the file is actually opened and +//! Update() called to replace the original XrdOucCacheIO2 object with one +//! that uses the newly opened file. //! -//! @param iocb reference to the callback object that receives the result. All -//! results are returned via this object's Done() method. If the -//! caller holds any locks they must be recursive locks as the -//! callback may occur on the calling thread. Done() is passsed -//! <0 - Trunc failed, value is -errno. -//! =0 - Trunc succeeded. -//! @param offs the size the file is to have. +//! @param iocp reference to the new XrdOucCacheIO2 object. //------------------------------------------------------------------------------ -using XrdOucCacheIO::Trunc; - -virtual void Trunc(XrdOucCache2CB &iocb, long long offs) - {iocb.Done(Trunc(offs));} +virtual void Update(XrdOucCacheIO2 &iocp) {} //------------------------------------------------------------------------------ //! Perform an asynchronous write (defaults to synchronous). @@ -191,7 +145,7 @@ virtual void Trunc(XrdOucCache2CB &iocb, long long offs) using XrdOucCacheIO::Write; -virtual void Write(XrdOucCache2CB &iocb, char *buff, long long offs, int wlen) +virtual void Write(XrdOucCacheIOCB &iocb, char *buff, long long offs, int wlen) {iocb.Done(Write(buff, offs, wlen));} //------------------------------------------------------------------------------ @@ -202,6 +156,8 @@ virtual ~XrdOucCacheIO2() {} // Always use Detach() instead of direct delete /******************************************************************************/ /* C l a s s X r d O u c C a c h e 2 */ /******************************************************************************/ + +struct stat; //------------------------------------------------------------------------------ //! The XrdOucCache2 class is used to define a version 2 cache. In version 2, @@ -219,7 +175,8 @@ public: //! with this cache. Upon success a pointer to a new XrdOucCacheIO2 object is //! returned and must be used to read and write data with the cache interposed. //! Upon failure, the original XrdOucCacheIO2 object is returned with errno set. -//! You can continue using the object without any cache. +//! You can continue using the object without any cache. The new cache should +//! use the methods in the passed CacheIO2 object to perform I/O operatios. //! //! @param ioP Pointer to the current CacheIO2 object used for I/O. //! @param opts Cache options identical to those defined for XrdOucCache @@ -239,7 +196,7 @@ XrdOucCacheIO *Attach(XrdOucCacheIO *ioP, int opts=0) //! Creates an instance of a version 1 cache. This method is no longer used so //! we simply define a default for this method here for backward compatability. //! -//! @return A pointer to an XrdOucCache2 object upon success or a nil pointer +//! @return A pointer to an XrdOucCache object upon success or a nil pointer //! with errno set upon failure. //------------------------------------------------------------------------------ virtual @@ -264,6 +221,23 @@ virtual int Prepare(const char *url, int oflags, mode_t mode) {(void)url; (void)oflags; (void)mode; return 0;} +//------------------------------------------------------------------------------ +//! Perform a stat() operation (defaults to passthrough). +//! +//! @param url pointer to the url whose stat information is wanted. +//! @param sbuff reference to the stat buffer to be filled in. Only fields +//! st_size, st_blocks, st_mtime (st_atime and st_ctime may be +//! set to st_mtime), st_ino, st_rdev, and st_mode need to be +//! set. All other fields should be set to zero. +//! +//! @return <0 - Stat failed, value is -errno. +//! =0 - Stat succeeded, sbuff holds stat information. +//! >0 - Stat could not be done, forward operation to next level. +//------------------------------------------------------------------------------ + +virtual int Stat(const char *url, struct stat &sbuff) + {(void)url; (void)sbuff; return 1;} + XrdOucCache2() {} virtual ~XrdOucCache2() {} }; From 6704ccabb65abe083e5906ec94946ac5491b962d Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 6 Apr 2015 13:50:36 -0700 Subject: [PATCH 003/171] Replace Prefetch class with File class to do async reads. --- src/XrdFileCache/XrdFileCache.cc | 10 +- src/XrdFileCache/XrdFileCache.hh | 15 +- src/XrdFileCache/XrdFileCacheFactory.cc | 2 +- src/XrdFileCache/XrdFileCacheFile.cc | 575 ++++++++++ src/XrdFileCache/XrdFileCacheFile.hh | 156 +++ src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 16 +- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 12 +- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 27 +- src/XrdFileCache/XrdFileCacheIOFileBlock.hh | 7 +- src/XrdFileCache/XrdFileCacheInfo.cc | 4 +- src/XrdFileCache/XrdFileCacheInfo.hh | 2 +- src/XrdFileCache/XrdFileCachePrefetch.cc | 1018 ------------------ src/XrdFileCache/XrdFileCachePrefetch.hh | 199 ---- 13 files changed, 763 insertions(+), 1280 deletions(-) create mode 100644 src/XrdFileCache/XrdFileCacheFile.cc create mode 100644 src/XrdFileCache/XrdFileCacheFile.hh delete mode 100644 src/XrdFileCache/XrdFileCachePrefetch.cc delete mode 100644 src/XrdFileCache/XrdFileCachePrefetch.hh diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 95be0e484f3..005aa4f065e 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -27,11 +27,10 @@ #include "XrdOuc/XrdOucEnv.hh" #include "XrdFileCache.hh" -#include "XrdFileCachePrefetch.hh" +#include "XrdFileCacheFile.hh" #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheIOFileBlock.hh" -#include "XrdFileCacheFactory.hh" -#include "XrdFileCachePrefetch.hh" +#include "XrdFileCacheFactory.hh"x XrdFileCache::Cache::WriteQ XrdFileCache::Cache::s_writeQ; @@ -68,7 +67,6 @@ XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) else cio = new IOEntireFile(*io, m_stats, *this); - cio->StartPrefetch(); return cio; } else @@ -117,7 +115,7 @@ Cache::HaveFreeWritingSlots() //______________________________________________________________________________ void -Cache::AddWriteTask(Prefetch* p, int ri, size_t s, bool fromRead) +Cache::AddWriteTask(File* p, int ri, size_t s, bool fromRead) { XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::AddWriteTask() wqsize = %d, bi=%d", s_writeQ.size, ri); s_writeQ.condVar.Lock(); @@ -131,7 +129,7 @@ Cache::AddWriteTask(Prefetch* p, int ri, size_t s, bool fromRead) } //______________________________________________________________________________ -void Cache::RemoveWriteQEntriesFor(Prefetch *p) +void Cache::RemoveWriteQEntriesFor(File *p) { s_writeQ.condVar.Lock(); std::list::iterator i = s_writeQ.queue.begin(); diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 4d48a4e0148..2db1faf5938 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -28,7 +28,7 @@ namespace XrdCl { class Log; } namespace XrdFileCache { -class Prefetch; +class File; } namespace XrdFileCache @@ -67,7 +67,7 @@ namespace XrdFileCache //--------------------------------------------------------------------- //! Add downloaded block in write queue. //--------------------------------------------------------------------- - static void AddWriteTask(Prefetch* p, int ramBlockidx, size_t size, bool fromRead); + static void AddWriteTask(File* p, int ramBlockidx, size_t size, bool fromRead); //--------------------------------------------------------------------- //! Check write queue size is not over limit. @@ -76,9 +76,9 @@ namespace XrdFileCache //--------------------------------------------------------------------- //! \brief Remove blocks from write queue which belong to given prefetch. - //! This method is used at the time of Prefetch destruction. + //! This method is used at the time of File destruction. //--------------------------------------------------------------------- - static void RemoveWriteQEntriesFor(Prefetch *p); + static void RemoveWriteQEntriesFor(File *p); //--------------------------------------------------------------------- //! Separate task which writes blocks from ram to disk. @@ -101,10 +101,10 @@ namespace XrdFileCache struct WriteTask { - Prefetch* prefetch; //!< object queued for writing + File* prefetch; //!< object queued for writing int ramBlockIdx; //!< in memory cache index size_t size; //!< write size -- block size except in case this is the end file block - WriteTask(Prefetch* p, int ri, size_t s):prefetch(p), ramBlockIdx(ri), size(s){} + WriteTask(File* p, int ri, size_t s):prefetch(p), ramBlockIdx(ri), size(s){} }; struct WriteQ @@ -124,7 +124,7 @@ namespace XrdFileCache //---------------------------------------------------------------------------- class IO : public XrdOucCacheIO { - friend class Prefetch; + friend class File; public: IO (XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache &cache) : @@ -146,7 +146,6 @@ namespace XrdFileCache virtual int Write(char *Buffer, long long Offset, int Length) { errno = ENOTSUP; return -1; } - virtual void StartPrefetch() {} protected: XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc index 93b63762395..3580b6344cb 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ b/src/XrdFileCache/XrdFileCacheFactory.cc @@ -42,7 +42,7 @@ #include "XrdFileCache.hh" #include "XrdFileCacheFactory.hh" -#include "XrdFileCachePrefetch.hh" +#include "XrdFileCacheFile.hh" using namespace XrdFileCache; diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc new file mode 100644 index 00000000000..e9c62c4fcbb --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -0,0 +1,575 @@ +#include "XrdFileCacheFile.hh" + +using namespace XrdFileCache; + +File::File(XrdOucCacheIO &io, std::string &path, long long off, long long size) : + m_input(io), + m_output(0), + m_infoFile(0), + m_temp_filename(path), + m_offset(off), + m_fileSize(size), + + m_block_cond(0), + m_num_reads(0) +{ + clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); +} + +File::~File() +{ + // see if we have to shut down + clLog()->Info(XrdCl::AppMsg, "File::~File() %p %s", (void*) this, m_input.Path()); + + clLog()->Info(XrdCl::AppMsg, "File::~File close data file %p",(void*)this ); + + if (m_output) + { + m_output->Close(); + delete m_output; + m_output = 0; + } + if (m_infoFile) + { + m_cfi.AppendIOStat(&m_stats, m_infoFile); + m_cfi.WriteHeader(m_infoFile); + + clLog()->Info(XrdCl::AppMsg, "File::~File close info file"); + + m_infoFile->Close(); + delete m_infoFile; + m_infoFile = 0; + } +} + +//============================================================================== + +bool File::Open() +{ + // clLog()->Debug(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_input.Path()); + + XrdOss &m_output_fs = *Factory::GetInstance().GetOss(); + // Create the data file itself. + XrdOucEnv myEnv; + m_output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), m_temp_filename.c_str(), 0600, myEnv, XRDOSS_mkpath); + m_output = m_output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); + if (m_output) + { + int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0600, myEnv); + if (res < 0) + { + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data-FD for %s %s", m_temp_filename.c_str(), m_input.Path()); + delete m_output; + m_output = 0; + + return false; + } + } + else + { + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data holder "); + return false; + } + + // Create the info file + std::string ifn = m_temp_filename + Info::m_infoExtension; + m_output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0600, myEnv, XRDOSS_mkpath); + m_infoFile = m_output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); + if (m_infoFile) + { + int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv); + if (res < 0) + { + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get info-FD %s %s", ifn.c_str(), m_input.Path()); + delete m_infoFile; + m_infoFile = 0; + return false; + } + } + else + { + return false; + } + + if (m_cfi.Read(m_infoFile) <= 0) + { + int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; + clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input.Path()); + m_cfi.ResizeBits(ss); + RecordDownloadInfo(); + } + else + { + clLog()->Debug(XrdCl::AppMsg, "Info file read from disk: %s", m_input.Path()); + } + + return true; +} + +//============================================================================== + +ssize_t File::ReadInBlocks(char *buff, off_t off, size_t size) +{ + long long off0 = off; + int idx_first = off0 / m_cfi.GetBufferSize(); + int idx_last = (off0 + size - 1)/ m_cfi.GetBufferSize(); + + size_t bytes_read = 0; + for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx) + { + int readBlockSize = size; + if (idx_first != idx_last) + { + if (blockIdx == idx_first) + { + readBlockSize = (blockIdx + 1) * m_cfi.GetBufferSize() - off0; + clLog()->Dump(XrdCl::AppMsg, "Read partially till the end of the block %", m_input.Path()); + } + else if (blockIdx == idx_last) + { + readBlockSize = (off0 + size) - blockIdx * m_cfi.GetBufferSize(); + clLog()->Dump(XrdCl::AppMsg, "Read partially from beginning of block %s", m_input.Path()); + } + else + { + readBlockSize = m_cfi.GetBufferSize(); + } + } + + if (readBlockSize > m_cfi.GetBufferSize()) { + clLog()->Error(XrdCl::AppMsg, "block size invalid"); + } + + int retvalBlock = -1; + // now do per block read at Read(buff, off, readBlockSize) + + m_downloadStatusMutex.Lock(); + bool dsl = m_cfi.TestBit(blockIdx); + m_downloadStatusMutex.UnLock(); + + if (dsl) + { + retvalBlock = m_output->Read(buff, off, readBlockSize); + m_stats.m_BytesDisk += retvalBlock; + clLog()->Dump(XrdCl::AppMsg, "File::ReadInBlocks [%d] disk = %d",blockIdx, retvalBlock); + } + else + { + if (ReadFromTask(blockIdx, buff, off, readBlockSize)) + { + retvalBlock = readBlockSize; // presume since ReadFromTask did not fail, could pass a refrence to ReadFromTask + m_stats.m_BytesRam += retvalBlock; + clLog()->Dump(XrdCl::AppMsg, "File::ReadInBlocks [%d] ram = %d", blockIdx, retvalBlock); + } + else + { + retvalBlock = m_input.Read(buff, off, readBlockSize); + clLog()->Dump(XrdCl::AppMsg, "File::ReadInBlocks [%d] client = %d", blockIdx, retvalBlock); + m_stats.m_BytesMissed += retvalBlock; + } + } + + if (retvalBlock > 0 ) + { + bytes_read += retvalBlock; + buff += retvalBlock; + off += retvalBlock; + if (readBlockSize != retvalBlock) + { + clLog()->Warning(XrdCl::AppMsg, "File::ReadInBlocks incomplete , missing = %d", readBlockSize-retvalBlock); + return bytes_read; + } + } + else + { + return bytes_read; + } + } + return bytes_read; +} + + +//============================================================================== +// Read and helpers +//============================================================================== + + + +namespace +{ + bool overlap(int blk, // block to query + long long blk_size, // + long long req_off, // offset of user request + int req_size, // size of user request + // output: + long_long &off, // offset in user buffer + long long &blk_off, // offset in block + long long &size) // size to copy + { + const long long beg = blk * blk_size; + const long long end = beg + blk_size; + const long long req_end = req_off + req_size; + + if (req_off < end && req_end > beg) + { + const long long ovlp_beg = std::max(beg, req_off); + const long long ovlp_end = std::min(end, req_end); + + off = ovlp_beg - req_off; + blk_off = ovlp_beg - beg; + size = ovlp_end - ovlp_beg; + + return true; + } + else + { + return false; + } + } +} + +//------------------------------------------------------------------------------ + +Block* File::RequestBlock(int i) +{ + // Must be called w/ block_map locked. + // Checks on size etc should be done before. + // + // Reference count is 0 so increase it in calling function if you want to + // catch the block while still in memory. + + XrdCl::XrdClFile &client = ((XrdPosixFile*)m_input).clFile; + + const long long BS = m_cfi.GetBufferSize(); + const int last_block = m_cfi.GetSizeInBits() - 1; + + long long off = i * BS; + long long this_bs = (i == last_block) ? m_input.FSize() - off : BS; + + Block *b = new Block(this, off, this_bs); + m_block_map[i] = b; + + client.Read(off, this_bs, b->get_buff(), new BlockRH(b)); +} + +//------------------------------------------------------------------------------ + +int File::RequestBlocksDirect(DirectRH *handler, std::list& blocks, + long long req_buf, long long req_off, long long req_size) +{ + XrdCl::XrdClFile &client = ((XrdPosixFile*)m_input).clFile; + + const long long BS = m_cfi.GetBufferSize(); + + // XXX Use readv to load more at the same time. + + long long total = 0; + + for (IntList_i ii = blocks ; ii != blocks.end(); ++ii) + { + // overlap and request + long_long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + + overlap(*ii, BS, req_off, req_size, off, blk_off, size); + + client.Read(*ii * BS + blk_off, size, req_buf + off, handler); + + total += size; + } + + return total; +} + +//------------------------------------------------------------------------------ + +int File::ReadBlocksFromDisk(std::list& blocks, + long long req_buf, long long req_off, long long req_size) +{ + const long long BS = m_cfi.GetBufferSize(); + + long long total = 0; + + // XXX Coalesce adjacent reads. + + for (IntList_i ii = blocks ; ii != blocks.end(); ++ii) + { + // overlap and read + long_long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + + overlap(*ii, BS, req_off, req_size, off, blk_off, size); + + long long rs = m_output.Read(req_buf + off, *ii * BS + blk_off, size); + + if (rs < 0) + return rs; + + total += rs; + } + + return total; +} + +//------------------------------------------------------------------------------ + +int File::Read(char* buff, long long offset, int size); +{ + XrdCl::XrdClFile &client = ((XrdPosixFile*)m_input).clFile; + + const long long BS = m_cfi.GetBufferSize(); + + // lock + // loop over reqired blocks: + // - if on disk, ok; + // - if in ram or incoming, inc ref-count + // - if not available, request and inc ref count + // before requesting the hell and more (esp. for sparse readv) assess if + // passing the req to client is actually better. + // unlock + + const int MaxBlocksForRead = 16; // Should be config var! Or even mutable for low client counts. + + m_block_cond.Lock(); + + // XXX Check for blocks to free? Later ... + + inc_num_readers(); + + const int idx_first = off / BS; + const int idx_last = (off + size - 1) / BS; + + BlockList_t blks_to_process, blks_processed; + IntList_t blks_on_disk, blks_direct; + + int bmap_size = m_block_map.size(); + + for (int i = idx_first; blockIdx <= idx_last; ++blockIdx) + { + BlockMap_i bi = m_block_map.find(i); + + // In RAM or incoming? + if (bi != m_block_map.end()) + { + // XXXX if failed before -- retry if timestamp sufficient or fail? + // XXXX Or just push it and handle errors in one place later? + + (*bi)->inc_ref_cont(); + blks_to_process.push_front(*bi); + } + // On disk? + else if (m_cfi.TestBit(i)) + { + blks_on_disk.push_back(i); + } + // Then we have to get it ... + else + { + // Is there room for one more RAM Block? + if (size < MaxBlocksForRead) + { + Block *b = RequestBlock(i); + b->inc_ref_cont(); + blks_to_process.push_back(b); + + ++size; + } + // Nope ... read this directly without caching. + else + { + blks_direct.push_back(i); + } + } + } + + m_block_cond.UnLock(); + + // First, send out any direct requests. + // XXX Could send them all out in a single vector read. + DirectRH *direct_handler = 0; + int direct_size = 0; + + if ( ! blks_direct.empty()) + { + direct_handler = new DirectRH(blks_direct.size()); + + direct_size = RequestBlocksDirect(direct_handler, blks_direct, *ii, buff, offset, size); + } + + long long bytes_read = 0; + + // Second, read blocks from disk. + int rc = ReadBlocksFromDisk(blks_on_disk, buff, offset, size); + if (rc >= 0) + { + bytes_read += rc; + } + else + { + bytes_read = rc; + } + + // Third, loop over blocks that are available or incoming + while ( ! blks_to_process.empty() && bytes_read >= 0) + { + BlockList_t finished; + + { + XrdSysConVarHelper _lck(m_block_cond); + + BlockList_i bi = blks_to_process.begin(); + while (bi != blks_to_process.end()) + { + if ((*bi)->is_finished()) + { + finished.push_back(*bi); + BlockList_i bj = bi++; + blks_to_process.erase(bj); + } + else + { + ++bi; + } + } + + if (finished.empty()) + { + m_block_cond.Wait(); + continue; + } + } + + bi = finished.begin(); + while (bi != finished.end()) + { + if ((*bi)->is_ok()) + { + // XXXX memcpy ! + } + else // it has failed ... krap up. + { + bytes_read = -1; + errno = (*bi)->errno; + break; + } + } + + std::copy(finished.begin(), finished.end(), std::back_inserter(blks_processed)); + } + + // Fourth, make sure all direct requests have arrived + if (m_direct_handler != 0) + { + XrdSysCondVarHelper _lck(direct_handler->m_cond); + + if (direct_handler->m_to_wait > 0) + { + direct_handler->m_cond.Wait(); + } + + if (direct_handler->m_errno == 0) + { + bytes_read += direct_size; + } + else + { + errno = direct_handler->m_errno; + bytes_read = -1; + } + + delete m_direct_handler; m_direct_handler = 0; + } + + // Last, stamp and release blocks, release file. + { + XrdSysConVarHelper _lck(m_block_cond); + + // XXXX stamp file + + // blks_to_process can be non-empty, if we're exiting with an error. + std::copy(blks_to_process.begin(), blks_to_process.end(), std::back_inserter(blks_processed)); + + for (BlockList_i bi = blks_processed.begin(); bi != blks_processed.end(); ++bi) + { + (*bi)->dec_ref_count(); + // XXXX stamp block + } + + dec_num_readers(); + } + + return bytes_read; +} + +//------------------------------------------------------------------------------ + +void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) +{ + m_block_cond.Lock(); + + if (status->IsOK()) + { + b->m_downloaded = true; + b->inc_ref_count(); + + // XXX Add to write queue (if needed) + // write_queue->QueueBlock(b); + } + else + { + // how long to keep? + // when to retry? + + XrdPosixMap::Result(*status); + + b->set_error_and_free(errno); + errno = 0; + + // ??? + b->inc_ref_count(); + } + + m_block_cond.Broadcast(); + + m_block_cond.UnLock(); +} + + + +//============================================================================== + +//============================================================================== + +void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response) +{ + m_block->m_file->ProcessBlockResponse(m_block, status); + + delete status; + delete response; + + delete this; +} + +//------------------------------------------------------------------------------ + +void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response) +{ + XrdSysCondVarHelper _lck(m_cond); + + --m_to_wait; + + if ( ! status->IsOK()) + { + XrdPosixMap::Result(*status); + m_errno = errno; + } + + if (m_to_wait == 0) + { + m_cond.Signal(); + } +} diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh new file mode 100644 index 00000000000..816f1cbd0ff --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -0,0 +1,156 @@ +#ifndef __XRDFILECACHE_FILE_HH__ +#define __XRDFILECACHE_FILE_HH__ + +#include "XrdCl/XrdClXRootDResponses.hh" + +#include +#include + + +namespace XrdFileCache +{ + class RefCounted + { + int m_refcnt; + + RefCounted() : m_refcnt(0) {} + }; + + class File; + + class Block + { + public: + std::vector m_buff; + long long m_offset; + File *m_file; + int m_refcnt; + int m_errno; + bool m_downloaded; + bool m_on_disk; + + Block(File *f, long long off, int size) : + m_offset(off), m_file(f), m_refcnt(0), + m_errno(0), m_downloaded(false), m_on_disk(false) + { + mbuff.resize(size); + } + + char* get_buff(long long pos = 0) const { return &m_buff[pos]; } + + bool is_finished() { return m_downloaded || m_errno != 0; } + bool is_ok() { return m_downloaded; } + bool is_failed() { return m_errno != 0; } + + void set_error_and_free(int err) + { + m_errno = err; + m_buff.resize(0); + } + }; + + class File + { + XrdOucCacheIO *m_input; //!< original data source + XrdOssDF *m_output; //!< file handle for data file on disk + XrdOssDF *m_infoFile; //!< file handle for data-info file on disk + Info m_cfi; //!< download status of file blocks and access statistics + + std::string m_temp_filename; //!< filename of data file on disk + long long m_offset; //!< offset of cached file for block-based operation + long long m_fileSize; //!< size of cached disk file for block-based operation + + typedef std::list IntList_t; + typedef IntList_t::iterator IntList_i; + + typedef std::list BlockList_t; + typedef BlockList_t::iterator BlockList_i; + + typedef std::map BlockMap_t; + typedef BlockMap_t::iterator BlockMap_i; + + + BlockMap_t m_block_map; + + XrdSysCondVar m_block_cond; + + int m_num_reads; + + // XrdSysCondVar m_stateCond; + + // XrdSysMutex m_downloadStatusMutex; //!< mutex locking access to m_cfi object + + // std::deque m_tasks_queue; //!< download queue + // XrdSysCondVar m_queueCond; + + Stats m_stats; //!< cache statistics, used in IO detach + + public: + + //------------------------------------------------------------------------ + //! Constructor. + //------------------------------------------------------------------------ + File(XrdOucCacheIO &io, std::string &path, + long long offset, long long fileSize); + + //------------------------------------------------------------------------ + //! Destructor. + //------------------------------------------------------------------------ + ~File(); + + //! Open file handle for data file and info file on local disk. + bool Open(); + + int Read(char* buff, off_t offset, size_t size); + + private: + Block* RequestBlock(int i); + + int RequestBlocksDirect(DirectRH *handler, IntList_t& blocks, + long long req_buf, long long req_off, long long req_size); + + int ReadBlocksFromDisk(IntList_t& blocks, + long long req_buf, long long req_off, long long req_size); + + + void ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status); + + int ReadInBlocks(char* buff, off_t offset, size_t size); + + + }; + + + // ================================================================ + + class BlockResponseHandler : public XrdCl::ResponseHandler + { + public: + Block *m_block; + + BlockResponseHandler(Block *b) : m_block(b) {} + + void HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response); + }; + + class DirectResponseHandler : public XrdCl::ResponseHandler + { + public: + XrdSysCondVar m_cond; + int m_to_wait; + int m_errno; + + DirectResponseHandler(int to_wait) : m_cond(0), m_to_wait(to_wait), m_errno(0) {} + + bool is_finished() { XrdSysCondVarHelper _lck(m_cond); return m_to_wait == 0; } + bool is_ok() { XrdSysCondVarHelper _lck(m_cond); return m_to_wait == 0 && m_errno == 0; } + bool is_failed() { XrdSysCondVarHelper _lck(m_cond); return m_errno != 0; } + + void HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response); + }; + +} + +#endif diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index ed009b61ec9..93075b0a5e1 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -29,13 +29,6 @@ using namespace XrdFileCache; -void *PrefetchRunner(void * prefetch_void) -{ - XrdFileCache::Prefetch *prefetch = static_cast(prefetch_void); - if (prefetch) - prefetch->Run(); - return NULL; -} //______________________________________________________________________________ @@ -48,7 +41,7 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache & c std::string fname; m_cache.getFilePathFromURL(io.Path(), fname); - m_prefetch = new Prefetch(io, fname, 0, io.FSize()); + m_prefetch = new File(io, fname, 0, io.FSize()); } @@ -60,13 +53,6 @@ bool IOEntireFile::ioActive() return m_prefetch->InitiateClose(); } -void IOEntireFile::StartPrefetch() -{ - pthread_t tid; - XrdSysThread::Run(&tid, PrefetchRunner, (void *)(m_prefetch), 0, "XrdFileCache Prefetcher"); -} - - XrdOucCacheIO *IOEntireFile::Detach() { m_statsGlobal.Add(m_prefetch->GetStats()); diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index 9917f174d8c..925c038f4b2 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -23,7 +23,7 @@ #include "XrdSys/XrdSysPthread.hh" #include "XrdFileCache.hh" #include "XrdFileCacheStats.hh" -#include "XrdFileCachePrefetch.hh" +#include "XrdFileCacheFile.hh" class XrdSysError; class XrdOssDF; @@ -50,7 +50,7 @@ namespace XrdFileCache ~IOEntireFile(); //--------------------------------------------------------------------- - //! Pass Read request to the corresponding Prefetch object. + //! Pass Read request to the corresponding File object. //! //! @param Buffer //! @param Offset @@ -61,7 +61,7 @@ namespace XrdFileCache virtual int Read(char *Buffer, long long Offset, int Length); //--------------------------------------------------------------------- - //! Pass ReadV request to the corresponding Prefetch object. + //! Pass ReadV request to the corresponding File object. //! //! @param readV //! @param n number of XrdOucIOVecs @@ -81,12 +81,8 @@ namespace XrdFileCache //! Called to check if destruction needs to be done in a separate task. virtual bool ioActive(); - protected: - //! Run prefetch outside constructor. - virtual void StartPrefetch(); - private: - Prefetch* m_prefetch; + File* m_prefetch; }; } diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index e2ef7ae55e4..9ad33320892 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -33,13 +33,6 @@ using namespace XrdFileCache; -void *PrefetchRunnerBl(void * prefetch_void) -{ - Prefetch *prefetch = static_cast(prefetch_void); - prefetch->Run(); - return NULL; -} - //______________________________________________________________________________ IOFileBlock::IOFileBlock(XrdOucCacheIO &io, XrdOucCacheStats &statsGlobal, Cache & cache) : IO(io, statsGlobal, cache) @@ -55,7 +48,7 @@ XrdOucCacheIO* IOFileBlock::Detach() XrdOucCacheIO * io = &m_io; - for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) + for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { m_statsGlobal.Add(it->second->GetStats()); delete it->second; @@ -91,7 +84,7 @@ void IOFileBlock::GetBlockSizeFromPath() } //______________________________________________________________________________ -Prefetch* IOFileBlock::newBlockPrefetcher(long long off, int blocksize, XrdOucCacheIO* io) +File* IOFileBlock::newBlockFileer(long long off, int blocksize, XrdOucCacheIO* io) { std::string fname; m_cache.getFilePathFromURL(io->Path(), fname); @@ -103,10 +96,8 @@ Prefetch* IOFileBlock::newBlockPrefetcher(long long off, int blocksize, XrdOucCa ss << &offExt[0]; fname = ss.str(); - clLog()->Debug(XrdCl::AppMsg, "FileBlock::FileBlock(), create XrdFileCachePrefetch. %s", m_io.Path()); - Prefetch* prefetch = new Prefetch(*io, fname, off, blocksize); - pthread_t tid; - XrdSysThread::Run(&tid, PrefetchRunnerBl, (void *)prefetch, 0, "BlockFile Prefetcher"); + clLog()->Debug(XrdCl::AppMsg, "FileBlock::FileBlock(), create XrdFileCacheFile. %s", m_io.Path()); + File* prefetch = new File(*io, fname, off, blocksize); return prefetch; } @@ -115,7 +106,7 @@ Prefetch* IOFileBlock::newBlockPrefetcher(long long off, int blocksize, XrdOucCa bool IOFileBlock::ioActive() { bool res = false; - for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { + for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { if (it->second->InitiateClose()) res = true; } @@ -146,9 +137,9 @@ int IOFileBlock::Read (char *buff, long long off, int size) for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx ) { // locate block - Prefetch* fb; + File* fb; m_mutex.Lock(); - std::map::iterator it = m_blocks.find(blockIdx); + std::map::iterator it = m_blocks.find(blockIdx); if ( it != m_blocks.end() ) { fb = it->second; @@ -164,8 +155,8 @@ int IOFileBlock::Read (char *buff, long long off, int size) clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() last block, change output file size to %lld \n %s", pbs, m_io.Path()); } - fb = newBlockPrefetcher(blockIdx*m_blocksize, pbs, &m_io); - m_blocks.insert(std::pair(blockIdx, (Prefetch*) fb)); + fb = newBlockFileer(blockIdx*m_blocksize, pbs, &m_io); + m_blocks.insert(std::pair(blockIdx, (File*) fb)); } m_mutex.UnLock(); diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index 1d0af73e6f6..944d171a93b 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -24,7 +24,7 @@ #include "XrdSys/XrdSysPthread.hh" #include "XrdFileCache.hh" -#include "XrdFileCachePrefetch.hh" +#include "XrdFileCacheFile.hh" class XrdSysError; class XrdOssDF; @@ -57,7 +57,7 @@ namespace XrdFileCache virtual XrdOucCacheIO *Detach(); //--------------------------------------------------------------------- - //! Pass Read request to the corresponding Prefetch object. + //! Pass Read request to the corresponding File object. //--------------------------------------------------------------------- virtual int Read(char *Buffer, long long Offset, int Length); @@ -67,11 +67,10 @@ namespace XrdFileCache private: long long m_blocksize; //!< size of file-block - std::map m_blocks; //!< map of created blocks + std::map m_blocks; //!< map of created blocks XrdSysMutex m_mutex; //!< map mutex void GetBlockSizeFromPath(); - Prefetch* newBlockPrefetcher(long long off, int blocksize, XrdOucCacheIO* io); }; } diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index 131c6161440..5db67079aa4 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -68,8 +68,8 @@ void Info::ResizeBits(int s) int Info::Read(XrdOssDF* fp) { - // does not need lock, called only in Prefetch::Open - // before Prefetch::Run() starts + // does not need lock, called only in File::Open + // before File::Run() starts int off = 0; off += fp->Read(&m_version, off, sizeof(int)); diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index 13f81f4e8de..f1915536f91 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -96,7 +96,7 @@ namespace XrdFileCache int Read(XrdOssDF* fp); //--------------------------------------------------------------------- - //! Write number of blocks and prefetch buffer size + //! Write number of blocks and read buffer size //--------------------------------------------------------------------- void WriteHeader(XrdOssDF* fp); diff --git a/src/XrdFileCache/XrdFileCachePrefetch.cc b/src/XrdFileCache/XrdFileCachePrefetch.cc deleted file mode 100644 index 9d4bed9ae35..00000000000 --- a/src/XrdFileCache/XrdFileCachePrefetch.cc +++ /dev/null @@ -1,1018 +0,0 @@ -//---------------------------------------------------------------------------------- -// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University -// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman -//---------------------------------------------------------------------------------- -// XRootD is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// XRootD is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with XRootD. If not, see . -//---------------------------------------------------------------------------------- - -#include -#include -#include - -#include "XrdCl/XrdClLog.hh" -#include "XrdCl/XrdClConstants.hh" -#include "XrdCl/XrdClFile.hh" -#include "XrdSys/XrdSysPthread.hh" -#include "XrdSys/XrdSysTimer.hh" -#include "XrdOss/XrdOss.hh" -#include "XrdOuc/XrdOucEnv.hh" -#include "Xrd/XrdScheduler.hh" - -#include "XrdSfs/XrdSfsInterface.hh" -#include "XrdPosix/XrdPosixFile.hh" - -#include "XrdFileCachePrefetch.hh" -#include "XrdFileCacheFactory.hh" -#include "XrdFileCache.hh" - -using namespace XrdFileCache; - -namespace XrdPosixGlobals -{ - extern XrdScheduler *schedP; -} - -namespace -{ - const int PREFETCH_MAX_ATTEMPTS = 10; - - class DiskSyncer : public XrdJob - { - private: - Prefetch *m_prefetch; - - public: - DiskSyncer(Prefetch *pref, const char *desc="") : - XrdJob(desc), - m_prefetch(pref) - {} - - void DoIt() - { - m_prefetch->Sync(); - } - }; -} - - -Prefetch::RAM::RAM():m_numBlocks(0),m_buffer(0), m_blockStates(0), m_writeMutex(0) -{ - m_numBlocks = Factory::GetInstance().RefConfiguration().m_NRamBuffersRead + Factory::GetInstance().RefConfiguration().m_NRamBuffersPrefetch; - m_buffer = (char*)malloc(m_numBlocks * Factory::GetInstance().RefConfiguration().m_bufferSize); - m_blockStates = new RAMBlock[m_numBlocks]; -} - -Prefetch::RAM::~RAM() -{ - free(m_buffer); - delete [] m_blockStates; -} - -Prefetch::Prefetch(XrdOucCacheIO &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : - m_output(NULL), - m_infoFile(NULL), - m_cfi(Factory::GetInstance().RefConfiguration().m_bufferSize), - m_input(inputIO), - m_temp_filename(disk_file_path), - m_offset(iOffset), - m_fileSize(iFileSize), - m_started(false), - m_failed(false), - m_stopping(false), - m_stopped(false), - m_stateCond(0), // We will explicitly lock the condition before use. - m_queueCond(0), - m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), - m_non_flushed_cnt(0), - m_in_sync(false) -{ - assert(m_fileSize > 0); - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Prefetch() %p %s", (void*)&m_input, lPath()); -} -//______________________________________________________________________________ - -bool Prefetch::InitiateClose() -{ - // Retruns true if delay is needed - - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Initiate close start", lPath()); - if (m_cfi.IsComplete()) return false; - - XrdSysCondVarHelper monitor(m_stateCond); - m_stopping = true; - if (m_started == false) - { - m_stopped = true; - return false; - } - - return true; -} - -//______________________________________________________________________________ -Prefetch::~Prefetch() -{ - clLog()->Debug(XrdCl::AppMsg, "Prefetch::~Prefetch() %p %s", (void*)this, lPath()); - - m_queueCond.Lock(); - m_queueCond.Signal(); - m_queueCond.UnLock(); - - Cache::RemoveWriteQEntriesFor(this); - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch() check write queues ...%s", lPath()); - while (true) - { - m_stateCond.Lock(); - bool isStopped = m_stopped; - m_stateCond.UnLock(); - - if (isStopped) - { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::~Prefetch sleep, waiting queues to empty begin %s", lPath()); - - bool writewait = false; - m_ram.m_writeMutex.Lock(); - for (int i = 0; i < m_ram.m_numBlocks;++i ) - { - if (m_ram.m_blockStates[i].refCount) - { - writewait = true; - break; - } - } - m_ram.m_writeMutex.UnLock(); - - // disk sync - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - - if (m_in_sync) writewait = true; - } - - if (!writewait) - break; - } - - XrdSysTimer::Wait(100); - } - clLog()->Debug(XrdCl::AppMsg, "Prefetch::~Prefetch finished with writing %s",lPath() ); - - bool do_sync = false; - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - if (m_non_flushed_cnt > 0) - { - do_sync = true; - m_in_sync = true; - - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch sync unflushed %d\n", m_non_flushed_cnt); - } - } - if (do_sync) - { - Sync(); - } - - if (m_output) - { - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch close data file %p",(void*)this , lPath()); - - m_output->Close(); - delete m_output; - m_output = NULL; - } - else - { - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch close data file -- not opened %p",(void*)this , lPath()); - } - if (m_infoFile) - { - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch close info file"); - - // write statistics in *cinfo file - AppendIOStatToFileInfo(); - - m_infoFile->Close(); - delete m_infoFile; - m_infoFile = NULL; - } - else - { - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch close info file -- not opened %p",(void*)this , lPath()); - } - - delete m_syncer; -} - -//______________________________________________________________________________ -const char* Prefetch::lPath() const -{ - return m_temp_filename.c_str(); -} - -//______________________________________________________________________________ - -bool Prefetch::Open() -{ - // clLog()->Debug(XrdCl::AppMsg, "Prefetch::Open() open file for disk cache %s", lPath()); - XrdOss &output_fs = *Factory::GetInstance().GetOss(); - // Create the data file itself. - XrdOucEnv myEnv; - output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), m_temp_filename.c_str(), 0644, myEnv, XRDOSS_mkpath); - m_output = output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); - if (m_output) - { - int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0644, myEnv); - if ( res < 0) - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::Open() can't open local file %s", m_temp_filename.c_str()); - delete m_output; - m_output = NULL; - return false; - } - } - else - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::Open() can't get data holder "); - return false; - } - - // Create the info file - std::string ifn = m_temp_filename + Info::m_infoExtension; - output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0644, myEnv, XRDOSS_mkpath); - m_infoFile = output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); - if (m_infoFile) - { - - int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0644, myEnv); - if ( res < 0 ) - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::Open() can't get info-FD %s %s", ifn.c_str(), lPath()); - delete m_output; - m_output = NULL; - delete m_infoFile; - m_infoFile = NULL; - - return false; - } - } - if (!m_infoFile) - { - return false; - } - if ( m_cfi.Read(m_infoFile) <= 0) - { - assert(m_fileSize > 0); - int ss = (m_fileSize -1)/m_cfi.GetBufferSize() + 1; - // clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, lPath()); - m_cfi.ResizeBits(ss); - m_cfi.WriteHeader(m_infoFile); - } - else - { - clLog()->Debug(XrdCl::AppMsg, "Info file already exists %s", lPath()); - // m_cfi.Print(); - } - - return true; -} - - -//_________________________________________________________________________________________________ -void -Prefetch::Run() -{ - { - XrdSysCondVarHelper monitor(m_stateCond); - - if (m_started) - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::Run() Already started for %s", lPath()); - m_stopped = true; - return; - } - - if (m_stopped) - { - return; - } - - if ( !Open()) - { - m_failed = true; - } - m_started = true; - // Broadcast to possible io-read waiting objects - m_stateCond.Broadcast(); - - if (m_failed) - { - m_stopped = true; - return; - } - } - assert(m_infoFile); - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Run() Starting loop over tasks for %s", lPath()); - - Task* task; - int numReadBlocks = 0; - while ((task = GetNextTask()) != 0) - { - DoTask(task); - - if (task->condVar) - { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Run() task %p condvar %p", task, task->condVar); - XrdSysCondVarHelper tmph(task->condVar); - task->condVar->Signal(); - } - - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Run() delete task %p condvar %p", task, task->condVar); - delete task; - - numReadBlocks++; - } // loop tasks - - - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Run() exits, download %s !", m_cfi.IsComplete() ? " completed " : "unfinished %s", lPath()); - - - m_cfi.CheckComplete(); - - m_stateCond.Lock(); - m_stopped = true; - m_stateCond.UnLock(); -} // end Run() - - -//______________________________________________________________________________ -Prefetch::Task* -Prefetch::CreateTaskForFirstUndownloadedBlock() -{ - // first check if there are enough write and ram resources - if (Cache::HaveFreeWritingSlots() == false) return 0; - - int nRP = 0; - for (int i =0 ; i < m_ram.m_numBlocks; ++i) { - if (m_ram.m_blockStates[i].fromRead == false && m_ram.m_blockStates[i].refCount > 0) nRP++; - } - if ( nRP >= Factory::GetInstance().RefConfiguration().m_NRamBuffersPrefetch ) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::CreateTaskForFirstUndownloadedBlock no resources %d %d, %s ", nRP, Factory::GetInstance().RefConfiguration().m_NRamBuffersPrefetch, lPath()); - return 0; - } - - Task *task = new Task; - Task &t = * task; - t.ramBlockIdx = -1; - int fileBlockIdx = -1; - for (int f = 0; f < m_cfi.GetSizeInBits(); ++f) - { - m_downloadStatusMutex.Lock(); - bool isdn = m_cfi.TestBit(f); - m_downloadStatusMutex.UnLock(); - - if (!isdn) - { - fileBlockIdx = f + m_offset/m_cfi.GetBufferSize(); - // get ram for the file block - m_ram.m_writeMutex.Lock(); - for (int r =0 ; r < m_ram.m_numBlocks; ++r) - { - - if (m_ram.m_blockStates[r].fileBlockIdx == fileBlockIdx) - { - // skip this task, the file block f is already downloaded - break; - } - - if (m_ram.m_blockStates[r].refCount == 0 ) - { - t.ramBlockIdx = r; - - assert(m_ram.m_blockStates[r].fileBlockIdx == -1); - m_ram.m_blockStates[r].refCount = 1; - m_ram.m_blockStates[r].fileBlockIdx = fileBlockIdx; - m_ram.m_blockStates[r].status = kReadWait; - break; - } - } - m_ram.m_writeMutex.UnLock(); - - break; - } - } - - if (t.ramBlockIdx >= 0) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::CreateTaskForFirstUndownloadedBlock success block %d %s ", fileBlockIdx, lPath()); - return task; - } - else if (fileBlockIdx == -1) { - m_cfi.CheckComplete(); - } - - delete task; - return 0; -} - -//_____________________________________________________________________________ -Prefetch::Task* -Prefetch::GetNextTask() -{ - while (true) - { - m_stateCond.Lock(); - bool doExit = m_stopping; - m_stateCond.UnLock(); - if (doExit) return 0; - - m_queueCond.Lock(); - - if ( ! m_tasks_queue.empty()) - { - // Exiting with queueMutex held !!! - break; - } - - // returns true on ETIMEDOUT - if ( ! m_queueCond.WaitMS(100)) - { - // Can be empty as result of a signal from destructor - if( ! m_tasks_queue.empty()) - // Exiting with queueMutex held !!! - break; - } - - m_queueCond.UnLock(); - - m_stateCond.Lock(); - doExit = m_stopping; - m_stateCond.UnLock(); - if (doExit) return 0; - - Task* t = CreateTaskForFirstUndownloadedBlock(); - if (t) - return t; - else if (m_cfi.IsComplete()) - return 0; - } - - Task *task = m_tasks_queue.front(); - m_tasks_queue.pop_front(); - - m_queueCond.UnLock(); - - assert(task->ramBlockIdx >=0); - clLog()->Info(XrdCl::AppMsg, "Prefetch::GetNextTask [%d] from queue %s", m_ram.m_blockStates[task->ramBlockIdx].fileBlockIdx, lPath()); - - return task; -} - -//______________________________________________________________________________ -void -Prefetch::DoTask(Task* task) -{ - // read block from client into buffer - int fileBlockIdx = m_ram.m_blockStates[task->ramBlockIdx].fileBlockIdx; - long long offset = fileBlockIdx * m_cfi.GetBufferSize(); - - long long rw_size = m_cfi.GetBufferSize(); - // fix size if this is the last file block - if ( offset + rw_size - m_offset > m_fileSize ) { - rw_size = m_fileSize + m_offset - offset; - assert (rw_size < m_cfi.GetBufferSize()); - } - int missing = rw_size; - int cnt = 0; - char* buff = m_ram.m_buffer; - buff += task->ramBlockIdx * m_cfi.GetBufferSize(); - while (missing) - { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::DoTask() for block f = %d r = %dsingal = %p %s", fileBlockIdx, task->ramBlockIdx, task->condVar, lPath()); - int retval = m_input.Read(buff, offset, missing); - if (retval < 0) - { - clLog()->Warning(XrdCl::AppMsg, "Prefetch::DoTask() failed for negative ret %d block %d %s", retval, fileBlockIdx , lPath()); - break; - } - - missing -= retval; - offset += retval; - buff += retval; - ++cnt; - if (cnt > PREFETCH_MAX_ATTEMPTS) - { - break; - } - } - - m_ram.m_writeMutex.Lock(); - if (missing) { - m_ram.m_blockStates[task->ramBlockIdx].status = kReadFailed; - m_ram.m_blockStates[task->ramBlockIdx].readErrno = errno; - } - else { - m_ram.m_blockStates[task->ramBlockIdx].status = kReadSuccess; - m_ram.m_blockStates[task->ramBlockIdx].readErrno = 0; - } - m_ram.m_writeMutex.Broadcast(); - m_ram.m_writeMutex.UnLock(); - - if (missing == 0) - { - // queue for ram to disk write - XrdSysCondVarHelper monitor(m_stateCond); - if (!m_stopping) { Cache::AddWriteTask(this, task->ramBlockIdx, rw_size, task->condVar ? true : false ); - } - else { - m_ram.m_blockStates[task->ramBlockIdx].refCount--; - } - - } - else - { - DecRamBlockRefCount(task->ramBlockIdx); - clLog()->Dump(XrdCl::AppMsg, "Prefetch::DoTask() incomplete read missing %d for block %d %s", missing, fileBlockIdx, lPath()); - } -} - -//_________________________________________________________________________________________________ -void -Prefetch::WriteBlockToDisk(int ramIdx, size_t size) -{ - // called from XrdFileCache::Cache when process queue - - int fileIdx = m_ram.m_blockStates[ramIdx].fileBlockIdx; - char* buff = m_ram.m_buffer; - buff += ramIdx*m_cfi.GetBufferSize(); - assert(ramIdx >=0 && ramIdx < m_ram.m_numBlocks); - int retval = 0; - - // write block buffer into disk file - long long offset = fileIdx * m_cfi.GetBufferSize() - m_offset; - int buffer_remaining = size; - int buffer_offset = 0; - int cnt = 0; - while ((buffer_remaining > 0) && // There is more to be written - (((retval = m_output->Write(buff, offset + buffer_offset, buffer_remaining)) != -1) - || (errno == EINTR))) // Write occurs without an error - { - buffer_remaining -= retval; - buff += retval; - cnt++; - - if (buffer_remaining) - { - clLog()->Warning(XrdCl::AppMsg, "Prefetch::WriteToDisk() reattempt[%d] writing missing %d for block %d %s", - cnt, buffer_remaining, fileIdx, lPath()); - } - if (cnt > PREFETCH_MAX_ATTEMPTS) - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::WriteToDisk() write failes too manny attempts %s", lPath()); - return; - } - } - - // set bit fetched - clLog()->Dump(XrdCl::AppMsg, "Prefetch::WriteToDisk() success set bit for block [%d] size [%d] %s", fileIdx, size, lPath()); - int pfIdx = fileIdx - m_offset/m_cfi.GetBufferSize(); - m_downloadStatusMutex.Lock(); - m_cfi.SetBitFetched(pfIdx); - m_downloadStatusMutex.UnLock(); - - - // set bit synced - bool schedule_sync = false; - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - - if (m_in_sync) - { - m_writes_during_sync.push_back(pfIdx); - } - else - { - m_cfi.SetBitWriteCalled(pfIdx); - ++m_non_flushed_cnt; - } - - if (m_non_flushed_cnt >= 100) - { - schedule_sync = true; - m_in_sync = true; - m_non_flushed_cnt = 0; - } - } - - if (schedule_sync) - { - XrdPosixGlobals::schedP->Schedule(m_syncer); - } -} - -//______________________________________________________________________________ -void Prefetch::Sync() -{ - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Sync %s", lPath()); - - m_output->Fsync(); - - m_cfi.WriteHeader(m_infoFile); - - int written_while_in_sync; - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - - for (std::vector::iterator i = m_writes_during_sync.begin(); i != m_writes_during_sync.end(); ++i) - { - m_cfi.SetBitWriteCalled(*i); - } - written_while_in_sync = m_non_flushed_cnt = (int) m_writes_during_sync.size(); - m_writes_during_sync.clear(); - - m_in_sync = false; - } - - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Sync %d blocks written during sync.", written_while_in_sync); - - m_infoFile->Fsync(); -} - -//______________________________________________________________________________ -void Prefetch::DecRamBlockRefCount(int ramIdx) -{ - clLog()->Dump(XrdCl::AppMsg, "Prefetch::DecRamBlockRefCount %d %d %s", m_ram.m_blockStates[ramIdx].fileBlockIdx, ramIdx,lPath() ); - - // mark ram block available - m_ram.m_writeMutex.Lock(); - assert(m_ram.m_blockStates[ramIdx].refCount); - assert(ramIdx >= 0 && ramIdx < m_ram.m_numBlocks); - - m_ram.m_blockStates[ramIdx].refCount --; - if (m_ram.m_blockStates[ramIdx].refCount == 0) { - m_ram.m_blockStates[ramIdx].fileBlockIdx = -1; - } - m_ram.m_writeMutex.UnLock(); -} - -//______________________________________________________________________________ -bool Prefetch::ReadFromTask(int iFileBlockIdx, char* iBuff, long long iOff, size_t iSize) -{ - // offs == offset inside the block, size read size in block - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadFromTask fileIdx= %d begin", iFileBlockIdx); - - m_stateCond.Lock(); - bool doExit = m_stopping; - m_stateCond.UnLock(); - if (doExit) return false; - - if (Cache::HaveFreeWritingSlots()) - { - int ramIdx = -1; - m_ram.m_writeMutex.Lock(); - - int nRR = 0; - for (int i =0 ; i < m_ram.m_numBlocks; ++i) { - if (m_ram.m_blockStates[i].fromRead && m_ram.m_blockStates[i].refCount > 0) nRR++; - } - - if (nRR < Factory::GetInstance().RefConfiguration().m_NRamBuffersRead) { - for (int i =0 ; i < m_ram.m_numBlocks; ++i) - { - if (m_ram.m_blockStates[i].refCount == 0) - { - assert(m_ram.m_blockStates[i].fileBlockIdx == -1); - ramIdx = i; - m_ram.m_blockStates[i].refCount = 1; - m_ram.m_blockStates[i].fileBlockIdx = iFileBlockIdx; - m_ram.m_blockStates[i].fromRead = true; - m_ram.m_blockStates[i].status = kReadWait; - break; - } - } - } - m_ram.m_writeMutex.UnLock(); - - if (ramIdx >= 0) - { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadFromTask, going to add task fileIdx=%d ", iFileBlockIdx); - XrdSysCondVar newTaskCond(0); - { - XrdSysCondVarHelper xx(newTaskCond); - - Task* task = new Task(ramIdx, &newTaskCond); - - m_queueCond.Lock(); - m_tasks_queue.push_front(task); - m_queueCond.Signal(); - m_queueCond.UnLock(); - - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadFromTask wait task %p confvar %p", task, task->condVar); - - newTaskCond.Wait(); - } - if (m_ram.m_blockStates[ramIdx].status == kReadSuccess) - { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadFromTask memcpy from RAM to IO::buffer fileIdx=%d ", iFileBlockIdx); - long long inBlockOff = iOff - iFileBlockIdx * m_cfi.GetBufferSize(); - char* srcBuff = m_ram.m_buffer + ramIdx*m_cfi.GetBufferSize(); - memcpy(iBuff, srcBuff + inBlockOff, iSize); - } - else - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::ReadFromTask client fileIdx=%d failed", iFileBlockIdx); - } - - return m_ram.m_blockStates[ramIdx].status == kReadSuccess; - } - else { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::ReadFromTask can't get free ram, not enough resources"); - return false; - } - } - else { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::ReadFromTask write queue full, not enough resources"); - return false; - } -} - -//______________________________________________________________________________ - -ssize_t Prefetch::ReadInBlocks(char *buff, off_t off, size_t size) -{ - long long off0 = off; - int idx_first = off0 / m_cfi.GetBufferSize(); - int idx_last = (off0 + size -1)/ m_cfi.GetBufferSize(); - - size_t bytes_read = 0; - for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx) - { - - int readBlockSize = size; - if (idx_first != idx_last) - { - if (blockIdx == idx_first) - { - readBlockSize = (blockIdx + 1) * m_cfi.GetBufferSize() - off0; - clLog()->Dump(XrdCl::AppMsg, "Read partially till the end of the block %s", lPath()); - } - else if (blockIdx == idx_last) - { - readBlockSize = (off0+size) - blockIdx*m_cfi.GetBufferSize(); - clLog()->Dump(XrdCl::AppMsg, "Read partially from beginning of block %s", lPath()); - } - else - { - readBlockSize = m_cfi.GetBufferSize(); - } - } - - if (readBlockSize > m_cfi.GetBufferSize()) { - clLog()->Error(XrdCl::AppMsg, "block size invalid"); - } - - int retvalBlock = -1; - // now do per block read at Read(buff, off, readBlockSize) - - m_downloadStatusMutex.Lock(); - bool dsl = m_cfi.TestBit(blockIdx - m_offset/m_cfi.GetBufferSize()); - m_downloadStatusMutex.UnLock(); - - if (dsl) - { - retvalBlock = m_output->Read(buff, off - m_offset, readBlockSize); - m_stats.m_BytesDisk += retvalBlock; - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks [%d] disk = %d",blockIdx, retvalBlock); - } - else - { - int RamIdx = -1; - m_ram.m_writeMutex.Lock(); - for (int ri = 0; ri < m_ram.m_numBlocks; ++ri ) - { - if (m_ram.m_blockStates[ri].fileBlockIdx == blockIdx) - { - RamIdx = ri; - m_ram.m_blockStates[ri].refCount++; - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks ram = %d file block = %d wait", RamIdx, blockIdx); - while (m_ram.m_blockStates[ri].status == kReadWait) - { - m_ram.m_writeMutex.Wait(); - } - break; - } - } - - m_ram.m_writeMutex.UnLock(); - - if (RamIdx >= 0 ) { - if ( m_ram.m_blockStates[RamIdx].status == kReadSuccess) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks ram = %d file block = %d", RamIdx, blockIdx); - int in_block_off = off - m_ram.m_blockStates[RamIdx].fileBlockIdx *m_cfi.GetBufferSize(); - char *rbuff = m_ram.m_buffer + RamIdx*m_cfi.GetBufferSize() + in_block_off; - memcpy(buff, rbuff, readBlockSize); - DecRamBlockRefCount(RamIdx); - retvalBlock = readBlockSize; - } - else { - errno = m_ram.m_blockStates[RamIdx].readErrno; - DecRamBlockRefCount(RamIdx); - return -1; - } - } - else - { - if (ReadFromTask(blockIdx, buff, off, readBlockSize)) - { - retvalBlock = readBlockSize; // presume since ReadFromTask did not fail, could pass a refrence to ReadFromTask - m_stats.m_BytesRam += retvalBlock; - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks [%d] fromTask = %d", blockIdx, blockIdx); - } - else - { - retvalBlock = m_input.Read(buff, off, readBlockSize); - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks [%d] client = %d", blockIdx, retvalBlock); - m_stats.m_BytesMissed += retvalBlock; - } - } - } - - if (retvalBlock > 0 ) - { - bytes_read += retvalBlock; - buff += retvalBlock; - off += retvalBlock; - if (readBlockSize != retvalBlock) - { - clLog()->Warning(XrdCl::AppMsg, "Prefetch::ReadInBlocks incomplete , missing = %d", readBlockSize-retvalBlock); - return bytes_read; - } - } - else - { - return bytes_read; - } - } - return bytes_read; -} - - -//______________________________________________________________________________ - -int Prefetch::ReadV (const XrdOucIOVec *readV, int n) -{ - { - XrdSysCondVarHelper monitor(m_stateCond); - - // AMT check if this can be done once during initalization - if (m_failed) return m_input.ReadV(readV, n); - - if ( ! m_started) - { - m_stateCond.Wait(); - if (m_failed) return 0; - } - } - - // check if read sizes are big enough to cache - - XrdCl::XRootDStatus Status; - XrdCl::ChunkList chunkVec; - XrdCl::VectorReadInfo *vrInfo = 0; - - std::vector cachedReads; - - int nbytes = 0; - for (int i=0; iDebug(XrdCl::AppMsg, "Prefetch::ReadV %d from cache ", i); - if (Read(readV[i].data, readV[i].offset, readV[i].size) < 0) - return -1; - } - else - { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::ReadV %d add back to client vector read ", i); - chunkVec.push_back(XrdCl::ChunkInfo((uint64_t)readV[i].offset, - (uint32_t)readV[i].size, - (void *)readV[i].data - )); - } - - } - if (!chunkVec.empty()) { - XrdCl::File& clFile = ((XrdPosixFile&)m_input).clFile; - Status = clFile.VectorRead(chunkVec, (void *)0, vrInfo); - delete vrInfo; - - if (!Status.IsOK()) - { - XrdPosixMap::Result(Status); - return -1; - } - } - return nbytes; -} -//______________________________________________________________________________ -ssize_t -Prefetch::Read(char *buff, off_t off, size_t size) -{ - { - XrdSysCondVarHelper monitor(m_stateCond); - - // AMT check if this can be done once during initalization - if (m_failed) return m_input.Read(buff, off, size); - - if ( ! m_started) - { - m_stateCond.Wait(); - if (m_failed) return 0; - } - } - - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Read() off = %lld size = %lld. %s", off, size, lPath()); - - bool fileComplete; - m_downloadStatusMutex.Lock(); - fileComplete = m_cfi.IsComplete(); - m_downloadStatusMutex.UnLock(); - - if (fileComplete) - { - int res = m_output->Read(buff, off - m_offset, size); - m_stats.m_BytesDisk += res; - return res; - } - else - { - return ReadInBlocks(buff, off, size); - } -} - - -//______________________________________________________________________________ -void Prefetch::AppendIOStatToFileInfo() -{ - // lock in case several IOs want to write in *cinfo file - m_downloadStatusMutex.Lock(); - if (m_infoFile) - { - Info::AStat as; - as.DetachTime = time(0); - as.BytesDisk = m_stats.m_BytesDisk; - as.BytesRam = m_stats.m_BytesRam; - as.BytesMissed = m_stats.m_BytesMissed; - m_cfi.AppendIOStat(as, (XrdOssDF*)m_infoFile); - } - else - { - clLog()->Warning(XrdCl::AppMsg, "Prefetch::AppendIOStatToFileInfo() info file not opened %s", lPath()); - } - m_downloadStatusMutex.UnLock(); -} - - - diff --git a/src/XrdFileCache/XrdFileCachePrefetch.hh b/src/XrdFileCache/XrdFileCachePrefetch.hh deleted file mode 100644 index f1cc467d761..00000000000 --- a/src/XrdFileCache/XrdFileCachePrefetch.hh +++ /dev/null @@ -1,199 +0,0 @@ -#ifndef __XRDFILECACHE_PREFETCH_HH__ -#define __XRDFILECACHE_PREFETCH_HH__ -//---------------------------------------------------------------------------------- -// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University -// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman -//---------------------------------------------------------------------------------- -// XRootD is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// XRootD is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with XRootD. If not, see . -//---------------------------------------------------------------------------------- - -#include -#include - -#include "XrdCl/XrdClDefaultEnv.hh" - -#include "XrdFileCacheInfo.hh" -#include "XrdFileCacheStats.hh" - -class XrdJob; -class XrdOucIOVec; - -namespace XrdCl -{ - class Log; -} - -namespace XrdFileCache -{ - //---------------------------------------------------------------------------- - //! Downloads data into a file on local disk and handles IO read requests. - //---------------------------------------------------------------------------- - class Prefetch - { - friend class IOEntireFile; - friend class IOFileBlock; - enum ReadRamState_t { kReadWait, kReadSuccess, kReadFailed}; - - struct Task; - public: - //------------------------------------------------------------------------ - //! Constructor. - //------------------------------------------------------------------------ - Prefetch(XrdOucCacheIO& inputFile, std::string& path, - long long offset, long long fileSize); - - //------------------------------------------------------------------------ - //! Destructor. - //------------------------------------------------------------------------ - ~Prefetch(); - - //--------------------------------------------------------------------- - //! Thread function for file prefetching. - //--------------------------------------------------------------------- - void Run(); - - //---------------------------------------------------------------------- - //! Reference to prefetch statistics. - //---------------------------------------------------------------------- - Stats& GetStats() { return m_stats; } - - //---------------------------------------------------------------------- - //! Write block to file on disk. Called from Cache. - //---------------------------------------------------------------------- - void WriteBlockToDisk(int ramIdx, size_t size); - - //---------------------------------------------------------------------- - //! Decrease block reference count. - //---------------------------------------------------------------------- - void DecRamBlockRefCount(int ramIdx); - - //---------------------------------------------------------------------- - //! \brief Initiate close. Return true if still IO active. - //! Used in XrdPosixXrootd::Close() - //---------------------------------------------------------------------- - bool InitiateClose(); - - //---------------------------------------------------------------------- - //! Sync file cache inf o and output data with disk - //---------------------------------------------------------------------- - void Sync(); - - - protected: - //! Read from disk, RAM, task, or client. - ssize_t Read(char * buff, off_t offset, size_t size); - - //! Vector read from disk if block is already downloaded, else ReadV from client. - int ReadV (const XrdOucIOVec *readV, int n); - - //! Write cache statistics in *cinfo file. - void AppendIOStatToFileInfo(); - - private: - //---------------------------------------------------------------------- - //! A prefetching task -- a file region that requires preferential treatment. - //---------------------------------------------------------------------- - struct Task - { - int ramBlockIdx; //!< idx in the in-memory buffer - XrdSysCondVar *condVar; //!< signal when complete - - Task(): ramBlockIdx(-1), condVar(0) {} - Task(int r, XrdSysCondVar *cv): - ramBlockIdx(r), condVar(cv) {} - ~Task() {} - }; - - struct RAMBlock { - int fileBlockIdx; //!< offset in output file - int refCount; //!< read and write reference count - bool fromRead; //!< is ram requested from prefetch or read - ReadRamState_t status; //!< read from client status - int readErrno; //!< posix error on read fail - - RAMBlock():fileBlockIdx(-1), refCount(0), fromRead(false), status(kReadWait) {} - }; - - struct RAM - { - int m_numBlocks; //!< number of in memory blocks - char* m_buffer; //!< buffer m_numBlocks x size_of_block - RAMBlock* m_blockStates; //!< referenced structure - XrdSysCondVar m_writeMutex; //!< write mutex - - RAM(); - ~RAM(); - }; - - //! Stop Run thread. - void CloseCleanly(); - - //! Get blocks to prefetch. - Task* GetNextTask(); - - //! Open file handle for data file and info file on local disk. - bool Open(); - - //! Short log alias. - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - - //! Split read in blocks. - ssize_t ReadInBlocks( char* buff, off_t offset, size_t size); - - //! Prefetch block. - Task* CreateTaskForFirstUndownloadedBlock(); - - //! Create task from read request and wait its completed. - bool ReadFromTask(int bIdx, char* buff, long long off, size_t size); - - //! Read from client into in memory cache, queue ram buffer for disk write. - void DoTask(Task* task); - - //! Log path - const char* lPath() const; - - RAM m_ram; //!< in memory cache - - XrdOssDF *m_output; //!< file handle for data file on disk - XrdOssDF *m_infoFile; //!< file handle for data-info file on disk - Info m_cfi; //!< download status of file blocks and access statistics - XrdOucCacheIO &m_input; //!< original data source - - std::string m_temp_filename; //!< filename of data file on disk - - long long m_offset; //!< offset of disk file for block-based operation - long long m_fileSize; //!< size of disk file for block-based operation - - bool m_started; //!< state of run thread - bool m_failed; //!< reading from original source or writing to disk has failed - bool m_stopping; //!< run thread should be stopped - bool m_stopped; //!< prefetch is stopped - XrdSysCondVar m_stateCond; //!< state condition variable - - XrdSysMutex m_downloadStatusMutex; //!< mutex locking access to m_cfi object - - std::deque m_tasks_queue; //!< download queue - XrdSysCondVar m_queueCond; //!< m_tasks_queue condition variable - - Stats m_stats; //!< cache statistics, used in IO detach - - // fsync - XrdSysMutex m_syncStatusMutex; //!< mutex locking fsync status - XrdJob *m_syncer; - std::vector m_writes_during_sync; - int m_non_flushed_cnt; - bool m_in_sync; - }; -} -#endif From 8dd2f98b6330cea005a13d72abc407174310a1ec Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 6 Apr 2015 14:10:16 -0700 Subject: [PATCH 004/171] Remove ReadInBlocks(). --- src/XrdFileCache/XrdFileCacheFile.cc | 82 ---------------------------- src/XrdFileCache/XrdFileCacheFile.hh | 10 ---- 2 files changed, 92 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index e9c62c4fcbb..a65457bc727 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -106,88 +106,6 @@ bool File::Open() return true; } -//============================================================================== - -ssize_t File::ReadInBlocks(char *buff, off_t off, size_t size) -{ - long long off0 = off; - int idx_first = off0 / m_cfi.GetBufferSize(); - int idx_last = (off0 + size - 1)/ m_cfi.GetBufferSize(); - - size_t bytes_read = 0; - for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx) - { - int readBlockSize = size; - if (idx_first != idx_last) - { - if (blockIdx == idx_first) - { - readBlockSize = (blockIdx + 1) * m_cfi.GetBufferSize() - off0; - clLog()->Dump(XrdCl::AppMsg, "Read partially till the end of the block %", m_input.Path()); - } - else if (blockIdx == idx_last) - { - readBlockSize = (off0 + size) - blockIdx * m_cfi.GetBufferSize(); - clLog()->Dump(XrdCl::AppMsg, "Read partially from beginning of block %s", m_input.Path()); - } - else - { - readBlockSize = m_cfi.GetBufferSize(); - } - } - - if (readBlockSize > m_cfi.GetBufferSize()) { - clLog()->Error(XrdCl::AppMsg, "block size invalid"); - } - - int retvalBlock = -1; - // now do per block read at Read(buff, off, readBlockSize) - - m_downloadStatusMutex.Lock(); - bool dsl = m_cfi.TestBit(blockIdx); - m_downloadStatusMutex.UnLock(); - - if (dsl) - { - retvalBlock = m_output->Read(buff, off, readBlockSize); - m_stats.m_BytesDisk += retvalBlock; - clLog()->Dump(XrdCl::AppMsg, "File::ReadInBlocks [%d] disk = %d",blockIdx, retvalBlock); - } - else - { - if (ReadFromTask(blockIdx, buff, off, readBlockSize)) - { - retvalBlock = readBlockSize; // presume since ReadFromTask did not fail, could pass a refrence to ReadFromTask - m_stats.m_BytesRam += retvalBlock; - clLog()->Dump(XrdCl::AppMsg, "File::ReadInBlocks [%d] ram = %d", blockIdx, retvalBlock); - } - else - { - retvalBlock = m_input.Read(buff, off, readBlockSize); - clLog()->Dump(XrdCl::AppMsg, "File::ReadInBlocks [%d] client = %d", blockIdx, retvalBlock); - m_stats.m_BytesMissed += retvalBlock; - } - } - - if (retvalBlock > 0 ) - { - bytes_read += retvalBlock; - buff += retvalBlock; - off += retvalBlock; - if (readBlockSize != retvalBlock) - { - clLog()->Warning(XrdCl::AppMsg, "File::ReadInBlocks incomplete , missing = %d", readBlockSize-retvalBlock); - return bytes_read; - } - } - else - { - return bytes_read; - } - } - return bytes_read; -} - //============================================================================== // Read and helpers diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 816f1cbd0ff..8692c50ca27 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -76,13 +76,6 @@ namespace XrdFileCache int m_num_reads; - // XrdSysCondVar m_stateCond; - - // XrdSysMutex m_downloadStatusMutex; //!< mutex locking access to m_cfi object - - // std::deque m_tasks_queue; //!< download queue - // XrdSysCondVar m_queueCond; - Stats m_stats; //!< cache statistics, used in IO detach public: @@ -115,9 +108,6 @@ namespace XrdFileCache void ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status); - int ReadInBlocks(char* buff, off_t offset, size_t size); - - }; From 80e40d04c272791507deac141edcdcf138129e95 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 15 Apr 2015 14:45:08 -0700 Subject: [PATCH 005/171] Compile success. --- src/XrdFileCache.cmake | 2 +- src/XrdFileCache/XrdFileCache.cc | 7 +- src/XrdFileCache/XrdFileCacheFile.cc | 223 +++++++++++++++----- src/XrdFileCache/XrdFileCacheFile.hh | 108 +++++++++- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 4 +- src/XrdFileCache/XrdFileCacheIOFileBlock.hh | 1 + 6 files changed, 275 insertions(+), 70 deletions(-) diff --git a/src/XrdFileCache.cmake b/src/XrdFileCache.cmake index 46ed47193c9..5622fbcaaa7 100644 --- a/src/XrdFileCache.cmake +++ b/src/XrdFileCache.cmake @@ -18,7 +18,7 @@ add_library( MODULE XrdFileCache/XrdFileCache.cc XrdFileCache/XrdFileCache.hh XrdFileCache/XrdFileCacheFactory.cc XrdFileCache/XrdFileCacheFactory.hh - XrdFileCache/XrdFileCachePrefetch.cc XrdFileCache/XrdFileCachePrefetch.hh + XrdFileCache/XrdFileCacheFile.cc XrdFileCache/XrdFileCacheFile.hh XrdFileCache/XrdFileCacheStats.hh XrdFileCache/XrdFileCacheInfo.cc XrdFileCache/XrdFileCacheInfo.hh XrdFileCache/XrdFileCacheIOEntireFile.cc XrdFileCache/XrdFileCacheIOEntireFile.hh diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 005aa4f065e..543c65e13e1 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -30,7 +30,7 @@ #include "XrdFileCacheFile.hh" #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheIOFileBlock.hh" -#include "XrdFileCacheFactory.hh"x +#include "XrdFileCacheFactory.hh" XrdFileCache::Cache::WriteQ XrdFileCache::Cache::s_writeQ; @@ -130,7 +130,7 @@ Cache::AddWriteTask(File* p, int ri, size_t s, bool fromRead) //______________________________________________________________________________ void Cache::RemoveWriteQEntriesFor(File *p) -{ +{/* s_writeQ.condVar.Lock(); std::list::iterator i = s_writeQ.queue.begin(); while (i != s_writeQ.queue.end()) @@ -148,12 +148,14 @@ void Cache::RemoveWriteQEntriesFor(File *p) } } s_writeQ.condVar.UnLock(); + */ } //______________________________________________________________________________ void Cache::ProcessWriteTasks() { + /* while (true) { s_writeQ.condVar.Lock(); @@ -169,4 +171,5 @@ Cache::ProcessWriteTasks() t.prefetch->WriteBlockToDisk(t.ramBlockIdx, t.size); t.prefetch->DecRamBlockRefCount(t.ramBlockIdx); } + */ } diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index a65457bc727..b25e03f8da9 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -1,17 +1,84 @@ +//---------------------------------------------------------------------------------- +// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University +// Author: Alja Mrak-Tadel, Matevz Tadel +//---------------------------------------------------------------------------------- +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +//---------------------------------------------------------------------------------- + + #include "XrdFileCacheFile.hh" + +#include +#include +#include +#include "XrdCl/XrdClLog.hh" +#include "XrdCl/XrdClConstants.hh" +#include "XrdCl/XrdClFile.hh" +#include "XrdSys/XrdSysPthread.hh" +#include "XrdSys/XrdSysTimer.hh" +#include "XrdOss/XrdOss.hh" +#include "XrdOuc/XrdOucEnv.hh" +#include "Xrd/XrdScheduler.hh" +#include "XrdSfs/XrdSfsInterface.hh" +#include "XrdPosix/XrdPosixFile.hh" +#include "XrdFileCacheFactory.hh" +#include "XrdFileCache.hh" + using namespace XrdFileCache; -File::File(XrdOucCacheIO &io, std::string &path, long long off, long long size) : - m_input(io), - m_output(0), - m_infoFile(0), - m_temp_filename(path), - m_offset(off), - m_fileSize(size), +namespace +{ +const int PREFETCH_MAX_ATTEMPTS = 10; +class DiskSyncer : public XrdJob +{ +private: + File *m_file; +public: + DiskSyncer(File *pref, const char *desc="") : + XrdJob(desc), + m_file(pref) + {} + void DoIt() + { + m_file->Sync(); + } +}; +} + + +File::File(XrdOucCacheIO &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : +m_input(inputIO), +m_output(NULL), +m_infoFile(NULL), - m_block_cond(0), - m_num_reads(0) +m_temp_filename(disk_file_path), +m_offset(iOffset), +m_fileSize(iFileSize), + +m_started(false), +m_failed(false), +m_stopping(false), +m_stopped(false), +m_stateCond(0), // We will explicitly lock the condition before use. + +m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), +m_non_flushed_cnt(0), +m_in_sync(false), + +m_block_cond(0) +// m_num_reads(0) { clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); } @@ -42,6 +109,43 @@ File::~File() } } +bool File::InitiateClose() +{ + // Retruns true if delay is needed + clLog()->Debug(XrdCl::AppMsg, "Prefetch::Initiate close start", lPath()); + if (m_cfi.IsComplete()) return false; + m_stateCond.Lock(); + if (m_started == false) return false; + m_stopping = true; + m_stateCond.UnLock(); + return true; +} + +//______________________________________________________________________________ + + +void File::Sync() +{ + clLog()->Dump(XrdCl::AppMsg, "Prefetch::Sync %s", lPath()); + m_output->Fsync(); + m_cfi.WriteHeader(m_infoFile); + int written_while_in_sync; + { + XrdSysMutexHelper _lck(&m_syncStatusMutex); + for (std::vector::iterator i = m_writes_during_sync.begin(); i != m_writes_during_sync.end(); ++i) + { + m_cfi.SetBitWriteCalled(*i); + } + written_while_in_sync = m_non_flushed_cnt = (int) m_writes_during_sync.size(); + m_writes_during_sync.clear(); + m_in_sync = false; + } + clLog()->Dump(XrdCl::AppMsg, "Prefetch::Sync %d blocks written during sync.", written_while_in_sync); + m_infoFile->Fsync(); +} + + + //============================================================================== bool File::Open() @@ -96,7 +200,7 @@ bool File::Open() int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input.Path()); m_cfi.ResizeBits(ss); - RecordDownloadInfo(); + m_cfi.WriteHeader(m_infoFile); } else { @@ -120,7 +224,7 @@ namespace long long req_off, // offset of user request int req_size, // size of user request // output: - long_long &off, // offset in user buffer + long long &off, // offset in user buffer long long &blk_off, // offset in block long long &size) // size to copy { @@ -156,7 +260,7 @@ Block* File::RequestBlock(int i) // Reference count is 0 so increase it in calling function if you want to // catch the block while still in memory. - XrdCl::XrdClFile &client = ((XrdPosixFile*)m_input).clFile; + XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; const long long BS = m_cfi.GetBufferSize(); const int last_block = m_cfi.GetSizeInBits() - 1; @@ -167,26 +271,29 @@ Block* File::RequestBlock(int i) Block *b = new Block(this, off, this_bs); m_block_map[i] = b; - client.Read(off, this_bs, b->get_buff(), new BlockRH(b)); + // AMT do I really have got cast to non-const ? + client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); + + return b; } //------------------------------------------------------------------------------ -int File::RequestBlocksDirect(DirectRH *handler, std::list& blocks, - long long req_buf, long long req_off, long long req_size) +int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, + char* req_buf, long long req_off, long long req_size) { - XrdCl::XrdClFile &client = ((XrdPosixFile*)m_input).clFile; + XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; const long long BS = m_cfi.GetBufferSize(); - // XXX Use readv to load more at the same time. + // XXX Use readv to load more at the same time. long long total = 0; - for (IntList_i ii = blocks ; ii != blocks.end(); ++ii) + for (IntList_i ii = blocks.begin() ; ii != blocks.end(); ++ii) { // overlap and request - long_long off; // offset in user buffer + long long off; // offset in user buffer long long blk_off; // offset in block long long size; // size to copy @@ -203,7 +310,7 @@ int File::RequestBlocksDirect(DirectRH *handler, std::list& blocks, //------------------------------------------------------------------------------ int File::ReadBlocksFromDisk(std::list& blocks, - long long req_buf, long long req_off, long long req_size) + char* req_buf, long long req_off, long long req_size) { const long long BS = m_cfi.GetBufferSize(); @@ -211,16 +318,16 @@ int File::ReadBlocksFromDisk(std::list& blocks, // XXX Coalesce adjacent reads. - for (IntList_i ii = blocks ; ii != blocks.end(); ++ii) + for (IntList_i ii = blocks.begin() ; ii != blocks.end(); ++ii) { // overlap and read - long_long off; // offset in user buffer + long long off; // offset in user buffer long long blk_off; // offset in block long long size; // size to copy overlap(*ii, BS, req_off, req_size, off, blk_off, size); - long long rs = m_output.Read(req_buf + off, *ii * BS + blk_off, size); + long long rs = m_output->Read(req_buf + off, *ii * BS + blk_off, size); if (rs < 0) return rs; @@ -233,10 +340,8 @@ int File::ReadBlocksFromDisk(std::list& blocks, //------------------------------------------------------------------------------ -int File::Read(char* buff, long long offset, int size); +int File::Read(char* buff, long long off, int size) { - XrdCl::XrdClFile &client = ((XrdPosixFile*)m_input).clFile; - const long long BS = m_cfi.GetBufferSize(); // lock @@ -248,13 +353,13 @@ int File::Read(char* buff, long long offset, int size); // passing the req to client is actually better. // unlock - const int MaxBlocksForRead = 16; // Should be config var! Or even mutable for low client counts. + const int MaxBlocksForRead = 16; // AMT Should be config var! Or even mutable for low client counts. m_block_cond.Lock(); // XXX Check for blocks to free? Later ... - inc_num_readers(); + // inc_num_readers(); const int idx_first = off / BS; const int idx_last = (off + size - 1) / BS; @@ -262,11 +367,9 @@ int File::Read(char* buff, long long offset, int size); BlockList_t blks_to_process, blks_processed; IntList_t blks_on_disk, blks_direct; - int bmap_size = m_block_map.size(); - - for (int i = idx_first; blockIdx <= idx_last; ++blockIdx) + for (int block_idx = idx_first; block_idx <= idx_last; ++block_idx) { - BlockMap_i bi = m_block_map.find(i); + BlockMap_i bi = m_block_map.find(block_idx); // In RAM or incoming? if (bi != m_block_map.end()) @@ -274,13 +377,13 @@ int File::Read(char* buff, long long offset, int size); // XXXX if failed before -- retry if timestamp sufficient or fail? // XXXX Or just push it and handle errors in one place later? - (*bi)->inc_ref_cont(); - blks_to_process.push_front(*bi); + bi->second->inc_ref_count(); + blks_to_process.push_front(bi->second); } // On disk? - else if (m_cfi.TestBit(i)) + else if (m_cfi.TestBit(block_idx)) { - blks_on_disk.push_back(i); + blks_on_disk.push_back(block_idx); } // Then we have to get it ... else @@ -288,45 +391,45 @@ int File::Read(char* buff, long long offset, int size); // Is there room for one more RAM Block? if (size < MaxBlocksForRead) { - Block *b = RequestBlock(i); - b->inc_ref_cont(); + Block *b = RequestBlock(block_idx); + b->inc_ref_count(); blks_to_process.push_back(b); - ++size; } // Nope ... read this directly without caching. else { - blks_direct.push_back(i); + blks_direct.push_back(block_idx); } } } m_block_cond.UnLock(); + long long bytes_read = 0; + // First, send out any direct requests. // XXX Could send them all out in a single vector read. - DirectRH *direct_handler = 0; + DirectResponseHandler *direct_handler = 0; int direct_size = 0; if ( ! blks_direct.empty()) { - direct_handler = new DirectRH(blks_direct.size()); + direct_handler = new DirectResponseHandler(blks_direct.size()); - direct_size = RequestBlocksDirect(direct_handler, blks_direct, *ii, buff, offset, size); + direct_size = RequestBlocksDirect(direct_handler, blks_direct, buff, off, size); + bytes_read += direct_size; // AMT added } - long long bytes_read = 0; - // Second, read blocks from disk. - int rc = ReadBlocksFromDisk(blks_on_disk, buff, offset, size); + int rc = ReadBlocksFromDisk(blks_on_disk, buff, off, size); if (rc >= 0) { bytes_read += rc; } else { - bytes_read = rc; + bytes_read = rc; // AMT ?? should there be an immediate return } // Third, loop over blocks that are available or incoming @@ -335,7 +438,7 @@ int File::Read(char* buff, long long offset, int size); BlockList_t finished; { - XrdSysConVarHelper _lck(m_block_cond); + XrdSysCondVarHelper _lck(m_block_cond); BlockList_i bi = blks_to_process.begin(); while (bi != blks_to_process.end()) @@ -359,7 +462,7 @@ int File::Read(char* buff, long long offset, int size); } } - bi = finished.begin(); + BlockList_i bi = finished.begin(); while (bi != finished.end()) { if ((*bi)->is_ok()) @@ -369,7 +472,7 @@ int File::Read(char* buff, long long offset, int size); else // it has failed ... krap up. { bytes_read = -1; - errno = (*bi)->errno; + errno = (*bi)->m_errno; break; } } @@ -378,7 +481,7 @@ int File::Read(char* buff, long long offset, int size); } // Fourth, make sure all direct requests have arrived - if (m_direct_handler != 0) + if (direct_handler != 0) { XrdSysCondVarHelper _lck(direct_handler->m_cond); @@ -397,12 +500,12 @@ int File::Read(char* buff, long long offset, int size); bytes_read = -1; } - delete m_direct_handler; m_direct_handler = 0; + delete direct_handler; } // Last, stamp and release blocks, release file. { - XrdSysConVarHelper _lck(m_block_cond); + XrdSysCondVarHelper _lck(m_block_cond); // XXXX stamp file @@ -415,7 +518,7 @@ int File::Read(char* buff, long long offset, int size); // XXXX stamp block } - dec_num_readers(); + // dec_num_readers(); } return bytes_read; @@ -491,3 +594,15 @@ void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, m_cond.Signal(); } } + + +int File::ReadV (const XrdOucIOVec *readV, int n) +{ + return 0; +} + +//______________________________________________________________________________ +const char* File::lPath() const +{ +return m_temp_filename.c_str(); +} diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 8692c50ca27..9528434a144 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -1,12 +1,47 @@ #ifndef __XRDFILECACHE_FILE_HH__ #define __XRDFILECACHE_FILE_HH__ +//---------------------------------------------------------------------------------- +// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University +// Author: Alja Mrak-Tadel, Matevz Tadel +//---------------------------------------------------------------------------------- +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +//---------------------------------------------------------------------------------- #include "XrdCl/XrdClXRootDResponses.hh" +#include "XrdCl/XrdClDefaultEnv.hh" + +#include "XrdFileCacheInfo.hh" +#include "XrdFileCacheStats.hh" #include #include +class XrdJob; +class XrdOucIOVec; +namespace XrdCl +{ +class Log; +} + +namespace XrdFileCache { + class BlockResponseHandler; + class DirectResponseHandler; +} + + + namespace XrdFileCache { class RefCounted @@ -33,10 +68,10 @@ namespace XrdFileCache m_offset(off), m_file(f), m_refcnt(0), m_errno(0), m_downloaded(false), m_on_disk(false) { - mbuff.resize(size); + m_buff.resize(size); } - char* get_buff(long long pos = 0) const { return &m_buff[pos]; } + const char* get_buff(long long pos = 0) const { return &m_buff[pos]; } bool is_finished() { return m_downloaded || m_errno != 0; } bool is_ok() { return m_downloaded; } @@ -47,11 +82,16 @@ namespace XrdFileCache m_errno = err; m_buff.resize(0); } + + void inc_ref_count() {m_refcnt++;} + void dec_ref_count() {m_refcnt--;} // AMT done under m_block_cond lock + }; class File { - XrdOucCacheIO *m_input; //!< original data source + private: + XrdOucCacheIO &m_input; //!< original data source XrdOssDF *m_output; //!< file handle for data file on disk XrdOssDF *m_infoFile; //!< file handle for data-info file on disk Info m_cfi; //!< download status of file blocks and access statistics @@ -60,8 +100,24 @@ namespace XrdFileCache long long m_offset; //!< offset of cached file for block-based operation long long m_fileSize; //!< size of cached disk file for block-based operation + bool m_started; //!< state of run thread + bool m_failed; //!< reading from original source or writing to disk has failed + bool m_stopping; //!< run thread should be stopped + bool m_stopped; //!< prefetch is stopped + XrdSysCondVar m_stateCond; //!< state condition variable + + XrdSysMutex m_downloadStatusMutex; //!< mutex locking access to m_cfi object + + // fsync + XrdSysMutex m_syncStatusMutex; //!< mutex locking fsync status + XrdJob *m_syncer; + std::vector m_writes_during_sync; + int m_non_flushed_cnt; + bool m_in_sync; + typedef std::list IntList_t; typedef IntList_t::iterator IntList_i; + // typedef IntList_t::const_iterator IntList_ci; typedef std::list BlockList_t; typedef BlockList_t::iterator BlockList_i; @@ -72,11 +128,11 @@ namespace XrdFileCache BlockMap_t m_block_map; - XrdSysCondVar m_block_cond; + XrdSysCondVar m_block_cond; - int m_num_reads; + // int m_num_reads; AMT don't know how this one should be used - Stats m_stats; //!< cache statistics, used in IO detach + Stats m_stats; //!< cache statistics, used in IO detach public: @@ -94,20 +150,50 @@ namespace XrdFileCache //! Open file handle for data file and info file on local disk. bool Open(); - int Read(char* buff, off_t offset, size_t size); + //! Vector read from disk if block is already downloaded, else ReadV from client. + int ReadV (const XrdOucIOVec *readV, int n); + + int Read(char* buff, long long offset, int size); + + //---------------------------------------------------------------------- + //! \brief Initiate close. Return true if still IO active. + //! Used in XrdPosixXrootd::Close() + //---------------------------------------------------------------------- + bool InitiateClose(); + + //---------------------------------------------------------------------- + //! Sync file cache inf o and output data with disk + //---------------------------------------------------------------------- + void Sync(); + + //---------------------------------------------------------------------- + //! Reference to prefetch statistics. + //---------------------------------------------------------------------- + Stats& GetStats() { return m_stats; } + + + void ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status); private: Block* RequestBlock(int i); - int RequestBlocksDirect(DirectRH *handler, IntList_t& blocks, - long long req_buf, long long req_off, long long req_size); + int RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, + char* buff, long long req_off, long long req_size); int ReadBlocksFromDisk(IntList_t& blocks, - long long req_buf, long long req_off, long long req_size); + char* req_buf, long long req_off, long long req_size); - void ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status); + + //! Short log alias. + XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } + + + //! Log path + const char* lPath() const; + + }; diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 9ad33320892..45d35b4f3af 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -84,7 +84,7 @@ void IOFileBlock::GetBlockSizeFromPath() } //______________________________________________________________________________ -File* IOFileBlock::newBlockFileer(long long off, int blocksize, XrdOucCacheIO* io) +File* IOFileBlock::newBlockFile(long long off, int blocksize, XrdOucCacheIO* io) { std::string fname; m_cache.getFilePathFromURL(io->Path(), fname); @@ -155,7 +155,7 @@ int IOFileBlock::Read (char *buff, long long off, int size) clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() last block, change output file size to %lld \n %s", pbs, m_io.Path()); } - fb = newBlockFileer(blockIdx*m_blocksize, pbs, &m_io); + fb = newBlockFile(blockIdx*m_blocksize, pbs, &m_io); m_blocks.insert(std::pair(blockIdx, (File*) fb)); } m_mutex.UnLock(); diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index 944d171a93b..c96f192f795 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -71,6 +71,7 @@ namespace XrdFileCache XrdSysMutex m_mutex; //!< map mutex void GetBlockSizeFromPath(); + File* newBlockFile(long long off, int blocksize, XrdOucCacheIO* io); }; } From 01c0f96c6ae091e36b9b5d2e2203dce5ca213ff0 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 20 Apr 2015 13:17:19 -0700 Subject: [PATCH 006/171] Implement write queue. --- src/XrdFileCache/XrdFileCache.cc | 29 ++- src/XrdFileCache/XrdFileCache.hh | 14 +- src/XrdFileCache/XrdFileCacheFile.cc | 283 +++++++++++++++++++++------ src/XrdFileCache/XrdFileCacheFile.hh | 9 +- 4 files changed, 240 insertions(+), 95 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 543c65e13e1..693d5bab4e8 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -27,7 +27,6 @@ #include "XrdOuc/XrdOucEnv.hh" #include "XrdFileCache.hh" -#include "XrdFileCacheFile.hh" #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheIOFileBlock.hh" #include "XrdFileCacheFactory.hh" @@ -115,30 +114,29 @@ Cache::HaveFreeWritingSlots() //______________________________________________________________________________ void -Cache::AddWriteTask(File* p, int ri, size_t s, bool fromRead) +Cache::AddWriteTask(Block* b, bool fromRead) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::AddWriteTask() wqsize = %d, bi=%d", s_writeQ.size, ri); + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::AddWriteTask() bOff=%ld", b->m_offset); s_writeQ.condVar.Lock(); if (fromRead) - s_writeQ.queue.push_back(WriteTask(p, ri, s)); + s_writeQ.queue.push_back(b); else - s_writeQ.queue.push_front(WriteTask(p, ri, s)); + s_writeQ.queue.push_front(b); // AMT should this not be the opposite s_writeQ.size++; s_writeQ.condVar.Signal(); s_writeQ.condVar.UnLock(); } //______________________________________________________________________________ -void Cache::RemoveWriteQEntriesFor(File *p) -{/* +void Cache::RemoveWriteQEntriesFor(File *iFile) +{ s_writeQ.condVar.Lock(); - std::list::iterator i = s_writeQ.queue.begin(); + std::list::iterator i = s_writeQ.queue.begin(); while (i != s_writeQ.queue.end()) { - if (i->prefetch == p) + if ((*i)->m_file == iFile) { - std::list::iterator j = i++; - j->prefetch->DecRamBlockRefCount(j->ramBlockIdx); + std::list::iterator j = i++; s_writeQ.queue.erase(j); --s_writeQ.size; } @@ -148,14 +146,12 @@ void Cache::RemoveWriteQEntriesFor(File *p) } } s_writeQ.condVar.UnLock(); - */ } //______________________________________________________________________________ void Cache::ProcessWriteTasks() { - /* while (true) { s_writeQ.condVar.Lock(); @@ -163,13 +159,12 @@ Cache::ProcessWriteTasks() { s_writeQ.condVar.Wait(); } - WriteTask t = s_writeQ.queue.front(); + Block* block = s_writeQ.queue.front(); // AMT should not be back ??? s_writeQ.queue.pop_front(); s_writeQ.size--; s_writeQ.condVar.UnLock(); - t.prefetch->WriteBlockToDisk(t.ramBlockIdx, t.size); - t.prefetch->DecRamBlockRefCount(t.ramBlockIdx); + block->m_file->WriteBlockToDisk(block); } - */ } + diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 2db1faf5938..5c9006b8f5f 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -23,6 +23,7 @@ #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOucCache.hh" #include "XrdCl/XrdClDefaultEnv.hh" +#include "XrdFileCacheFile.hh" namespace XrdCl { class Log; @@ -67,7 +68,7 @@ namespace XrdFileCache //--------------------------------------------------------------------- //! Add downloaded block in write queue. //--------------------------------------------------------------------- - static void AddWriteTask(File* p, int ramBlockidx, size_t size, bool fromRead); + static void AddWriteTask(Block* b, bool from_read); //--------------------------------------------------------------------- //! Check write queue size is not over limit. @@ -78,7 +79,7 @@ namespace XrdFileCache //! \brief Remove blocks from write queue which belong to given prefetch. //! This method is used at the time of File destruction. //--------------------------------------------------------------------- - static void RemoveWriteQEntriesFor(File *p); + static void RemoveWriteQEntriesFor(File *f); //--------------------------------------------------------------------- //! Separate task which writes blocks from ram to disk. @@ -99,20 +100,13 @@ namespace XrdFileCache unsigned int m_attached; //!< number of attached IO objects XrdOucCacheStats &m_stats; //!< global cache usage statistics - struct WriteTask - { - File* prefetch; //!< object queued for writing - int ramBlockIdx; //!< in memory cache index - size_t size; //!< write size -- block size except in case this is the end file block - WriteTask(File* p, int ri, size_t s):prefetch(p), ramBlockIdx(ri), size(s){} - }; struct WriteQ { WriteQ() : condVar(0), size(0) {} XrdSysCondVar condVar; //!< write list condVar size_t size; //!< cache size of a container - std::list queue; //!< container + std::list queue; //!< container }; static WriteQ s_writeQ; diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index b25e03f8da9..ee922f3bc58 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -33,11 +33,18 @@ #include "Xrd/XrdScheduler.hh" #include "XrdSfs/XrdSfsInterface.hh" #include "XrdPosix/XrdPosixFile.hh" +#include "XrdPosix/XrdPosix.hh" #include "XrdFileCacheFactory.hh" #include "XrdFileCache.hh" +#include "Xrd/XrdScheduler.hh" using namespace XrdFileCache; +namespace XrdPosixGlobals +{ + extern XrdScheduler *schedP; +} + namespace { const int PREFETCH_MAX_ATTEMPTS = 10; @@ -85,34 +92,61 @@ m_block_cond(0) File::~File() { - // see if we have to shut down - clLog()->Info(XrdCl::AppMsg, "File::~File() %p %s", (void*) this, m_input.Path()); - - clLog()->Info(XrdCl::AppMsg, "File::~File close data file %p",(void*)this ); + clLog()->Debug(XrdCl::AppMsg, "File::~File() %p %s", (void*)this, lPath()); + Cache::RemoveWriteQEntriesFor(this); + clLog()->Info(XrdCl::AppMsg, "File::~File() check write queues ...%s", lPath()); + // can I do anythong to stop waiting for asyc read callbacks ? + while (true) + { + m_stateCond.Lock(); + bool isStopped = m_stopped; + m_stateCond.UnLock(); + if (isStopped) + { + // TODO AMT: wait for map to clear + break; + } + XrdSysTimer::Wait(100); + } + clLog()->Debug(XrdCl::AppMsg, "File::~File finished with writing %s",lPath() ); + bool do_sync = false; + { + XrdSysMutexHelper _lck(&m_syncStatusMutex); + if (m_non_flushed_cnt > 0) + { + do_sync = true; + m_in_sync = true; + clLog()->Info(XrdCl::AppMsg, "File::~File sync unflushed %d\n", m_non_flushed_cnt); + } + } + if (do_sync) + { + Sync(); + } + // write statistics in *cinfo file + // AMT append IO stat --- look new interface in master branch + clLog()->Info(XrdCl::AppMsg, "File::~File close data file %p",(void*)this , lPath()); if (m_output) { m_output->Close(); delete m_output; - m_output = 0; + m_output = NULL; } if (m_infoFile) { - m_cfi.AppendIOStat(&m_stats, m_infoFile); - m_cfi.WriteHeader(m_infoFile); - clLog()->Info(XrdCl::AppMsg, "File::~File close info file"); - m_infoFile->Close(); delete m_infoFile; - m_infoFile = 0; + m_infoFile = NULL; } + delete m_syncer; } bool File::InitiateClose() { // Retruns true if delay is needed - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Initiate close start", lPath()); + clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start", lPath()); if (m_cfi.IsComplete()) return false; m_stateCond.Lock(); if (m_started == false) return false; @@ -124,27 +158,6 @@ bool File::InitiateClose() //______________________________________________________________________________ -void File::Sync() -{ - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Sync %s", lPath()); - m_output->Fsync(); - m_cfi.WriteHeader(m_infoFile); - int written_while_in_sync; - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - for (std::vector::iterator i = m_writes_during_sync.begin(); i != m_writes_during_sync.end(); ++i) - { - m_cfi.SetBitWriteCalled(*i); - } - written_while_in_sync = m_non_flushed_cnt = (int) m_writes_during_sync.size(); - m_writes_during_sync.clear(); - m_in_sync = false; - } - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Sync %d blocks written during sync.", written_while_in_sync); - m_infoFile->Fsync(); -} - - //============================================================================== @@ -268,7 +281,7 @@ Block* File::RequestBlock(int i) long long off = i * BS; long long this_bs = (i == last_block) ? m_input.FSize() - off : BS; - Block *b = new Block(this, off, this_bs); + Block *b = new Block(this, off, this_bs); // should block be reused to avoid recreation m_block_map[i] = b; // AMT do I really have got cast to non-const ? @@ -340,7 +353,7 @@ int File::ReadBlocksFromDisk(std::list& blocks, //------------------------------------------------------------------------------ -int File::Read(char* buff, long long off, int size) +int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) { const long long BS = m_cfi.GetBufferSize(); @@ -357,12 +370,13 @@ int File::Read(char* buff, long long off, int size) m_block_cond.Lock(); + size_t msize = m_block_map.size(); // XXX Check for blocks to free? Later ... // inc_num_readers(); - const int idx_first = off / BS; - const int idx_last = (off + size - 1) / BS; + const int idx_first = iUserOff / BS; + const int idx_last = (iUserOff + iUserSize - 1) / BS; BlockList_t blks_to_process, blks_processed; IntList_t blks_on_disk, blks_direct; @@ -377,7 +391,7 @@ int File::Read(char* buff, long long off, int size) // XXXX if failed before -- retry if timestamp sufficient or fail? // XXXX Or just push it and handle errors in one place later? - bi->second->inc_ref_count(); + inc_ref_count(bi->second); blks_to_process.push_front(bi->second); } // On disk? @@ -389,12 +403,12 @@ int File::Read(char* buff, long long off, int size) else { // Is there room for one more RAM Block? - if (size < MaxBlocksForRead) + if ( msize < MaxBlocksForRead) { Block *b = RequestBlock(block_idx); - b->inc_ref_count(); + inc_ref_count(b); blks_to_process.push_back(b); - ++size; + ++msize; } // Nope ... read this directly without caching. else @@ -411,18 +425,18 @@ int File::Read(char* buff, long long off, int size) // First, send out any direct requests. // XXX Could send them all out in a single vector read. DirectResponseHandler *direct_handler = 0; - int direct_size = 0; + int direct_size = 0; if ( ! blks_direct.empty()) { direct_handler = new DirectResponseHandler(blks_direct.size()); - direct_size = RequestBlocksDirect(direct_handler, blks_direct, buff, off, size); + direct_size = RequestBlocksDirect(direct_handler, blks_direct, iUserBuff, iUserOff, iUserSize); bytes_read += direct_size; // AMT added } // Second, read blocks from disk. - int rc = ReadBlocksFromDisk(blks_on_disk, buff, off, size); + int rc = ReadBlocksFromDisk(blks_on_disk, iUserBuff, iUserOff, iUserSize); if (rc >= 0) { bytes_read += rc; @@ -467,7 +481,13 @@ int File::Read(char* buff, long long off, int size) { if ((*bi)->is_ok()) { - // XXXX memcpy ! + // XXXX memcpy ! AMT ... + long long user_off; // offset in user buffer + long long off_in_block; // offset in block + long long size_to_copy; // size to copy + + overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); + memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); } else // it has failed ... krap up. { @@ -508,13 +528,14 @@ int File::Read(char* buff, long long off, int size) XrdSysCondVarHelper _lck(m_block_cond); // XXXX stamp file + // AMT ??? fetched status stampled in WriteDisk callback , what dies stamp mean ?? // blks_to_process can be non-empty, if we're exiting with an error. std::copy(blks_to_process.begin(), blks_to_process.end(), std::back_inserter(blks_processed)); for (BlockList_i bi = blks_processed.begin(); bi != blks_processed.end(); ++bi) { - (*bi)->dec_ref_count(); + dec_ref_count(*bi); // XXXX stamp block } @@ -526,39 +547,102 @@ int File::Read(char* buff, long long off, int size) //------------------------------------------------------------------------------ -void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) +void File::WriteBlockToDisk(Block* b) { - m_block_cond.Lock(); - - if (status->IsOK()) + int retval = 0; + // write block buffer into disk file + long long offset = b->m_offset - m_offset; + long long size = (b->m_offset + m_cfi.GetBufferSize()) > m_input.FSize() ? (m_input.FSize() - b->m_offset) : m_cfi.GetBufferSize(); + int buffer_remaining = size; + int buffer_offset = 0; + int cnt = 0; + const char* buff = &b->m_buff[0]; + while ((buffer_remaining > 0) && // There is more to be written + (((retval = m_output->Write(buff, offset + buffer_offset, buffer_remaining)) != -1) + || (errno == EINTR))) // Write occurs without an error { - b->m_downloaded = true; - b->inc_ref_count(); + buffer_remaining -= retval; + buff += retval; + cnt++; - // XXX Add to write queue (if needed) - // write_queue->QueueBlock(b); + if (buffer_remaining) + { + clLog()->Warning(XrdCl::AppMsg, "File::WriteToDisk() reattempt[%d] writing missing %ld for block %d %s", + cnt, buffer_remaining, b->m_offset, lPath()); + } + if (cnt > PREFETCH_MAX_ATTEMPTS) + { + clLog()->Error(XrdCl::AppMsg, "File::WriteToDisk() write failes too manny attempts %s", lPath()); + return; + } } - else + + // set bit fetched + clLog()->Dump(XrdCl::AppMsg, "File::WriteToDisk() success set bit for block [%ld] size [%d] %s", b->m_offset, size, lPath()); + int pfIdx = (b->m_offset - m_offset)/m_cfi.GetBufferSize(); + m_downloadStatusMutex.Lock(); + m_cfi.SetBitFetched(pfIdx); + m_downloadStatusMutex.UnLock(); + { - // how long to keep? - // when to retry? + XrdSysCondVarHelper _lck(m_block_cond); + dec_ref_count(b); + } - XrdPosixMap::Result(*status); + // set bit synced + bool schedule_sync = false; + { + XrdSysMutexHelper _lck(&m_syncStatusMutex); - b->set_error_and_free(errno); - errno = 0; + if (m_in_sync) + { + m_writes_during_sync.push_back(pfIdx); + } + else + { + m_cfi.SetBitWriteCalled(pfIdx); + ++m_non_flushed_cnt; + } - // ??? - b->inc_ref_count(); + if (m_non_flushed_cnt >= 100) + { + schedule_sync = true; + m_in_sync = true; + m_non_flushed_cnt = 0; + } } - m_block_cond.Broadcast(); + if (schedule_sync) + { + XrdPosixGlobals::schedP->Schedule(m_syncer); + } +} - m_block_cond.UnLock(); +//------------------------------------------------------------------------------ + +void File::Sync() +{ + clLog()->Dump(XrdCl::AppMsg, "File::Sync %s", lPath()); + m_output->Fsync(); + m_cfi.WriteHeader(m_infoFile); + int written_while_in_sync; + { + XrdSysMutexHelper _lck(&m_syncStatusMutex); + for (std::vector::iterator i = m_writes_during_sync.begin(); i != m_writes_during_sync.end(); ++i) + { + m_cfi.SetBitWriteCalled(*i); + } + written_while_in_sync = m_non_flushed_cnt = (int) m_writes_during_sync.size(); + m_writes_during_sync.clear(); + m_in_sync = false; + } + clLog()->Dump(XrdCl::AppMsg, "File::Sync %d blocks written during sync.", written_while_in_sync); + m_infoFile->Fsync(); } + //============================================================================== //============================================================================== @@ -596,6 +680,77 @@ void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, } + +//============================================================================== + +//============================================================================== + +//============================================================================== +void File::inc_ref_count(Block* b) +{ + b->m_refcnt++; +} + +void File::dec_ref_count(Block* b) +{ + // AMT this function could actually be member of File ... would be nicer + // called under block lock + b-> m_refcnt--; + if ( b->m_refcnt == 0 ) { + int i = b->m_offset/BufferSize(); + size_t ret = m_block_map.erase(i); + if (ret != 1) { + clLog()->Error(XrdCl::AppMsg, "File::OnBlockZeroRefCount did not erase %d from map.", i); + } + } +} + +//------------------------------------------------------------------------------ + +void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) +{ + m_block_cond.Lock(); + + if (status->IsOK()) + { + b->m_downloaded = true; + inc_ref_count(b); + + // XXX Add to write queue (if needed) + // AMT TODO check if state is stopped + XrdFileCache::Cache::AddWriteTask(b, true); + } + else + { + // how long to keep? + // when to retry? + + XrdPosixMap::Result(*status); + + b->set_error_and_free(errno); + errno = 0; + + // ??? + inc_ref_count(b); + } + + m_block_cond.Broadcast(); + + m_block_cond.UnLock(); +} + + + + long long File::BufferSize() { + return m_cfi.GetBufferSize(); + } + + +//============================================================================== + +//============================================================================== + + int File::ReadV (const XrdOucIOVec *readV, int n) { return 0; @@ -606,3 +761,5 @@ const char* File::lPath() const { return m_temp_filename.c_str(); } + + diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 9528434a144..5b713a2e0d9 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -82,10 +82,6 @@ namespace XrdFileCache m_errno = err; m_buff.resize(0); } - - void inc_ref_count() {m_refcnt++;} - void dec_ref_count() {m_refcnt--;} // AMT done under m_block_cond lock - }; class File @@ -173,6 +169,7 @@ namespace XrdFileCache void ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status); + void WriteBlockToDisk(Block* b); private: Block* RequestBlock(int i); @@ -184,7 +181,7 @@ namespace XrdFileCache char* req_buf, long long req_off, long long req_size); - + long long BufferSize(); //! Short log alias. XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } @@ -193,6 +190,8 @@ namespace XrdFileCache //! Log path const char* lPath() const; + void inc_ref_count(Block*); + void dec_ref_count(Block*); }; From 39adb0be1313433da3f5534fdf83f18960d359f7 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 22 Apr 2015 09:48:56 -0700 Subject: [PATCH 007/171] Fixes in destruction. --- src/XrdFileCache/XrdFileCacheFile.cc | 111 ++++++++++++++++----------- src/XrdFileCache/XrdFileCacheFile.hh | 3 +- 2 files changed, 69 insertions(+), 45 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index ee922f3bc58..d25cfafcc14 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -74,10 +74,8 @@ m_temp_filename(disk_file_path), m_offset(iOffset), m_fileSize(iFileSize), -m_started(false), m_failed(false), m_stopping(false), -m_stopped(false), m_stateCond(0), // We will explicitly lock the condition before use. m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), @@ -85,9 +83,9 @@ m_non_flushed_cnt(0), m_in_sync(false), m_block_cond(0) -// m_num_reads(0) { clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); + Open(); } File::~File() @@ -100,16 +98,20 @@ File::~File() while (true) { m_stateCond.Lock(); - bool isStopped = m_stopped; + bool isStopped = m_stopping; m_stateCond.UnLock(); if (isStopped) { - // TODO AMT: wait for map to clear - break; + printf("~FILE map size %ld \n", m_block_map.size()); + if ( m_block_map.empty()) + break; } XrdSysTimer::Wait(100); } clLog()->Debug(XrdCl::AppMsg, "File::~File finished with writing %s",lPath() ); + + + // Wait disk sync bool do_sync = false; { XrdSysMutexHelper _lck(&m_syncStatusMutex); @@ -147,11 +149,10 @@ bool File::InitiateClose() { // Retruns true if delay is needed clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start", lPath()); - if (m_cfi.IsComplete()) return false; m_stateCond.Lock(); - if (m_started == false) return false; m_stopping = true; m_stateCond.UnLock(); + if (m_cfi.IsComplete()) return false; // AMT maybe map size is here more meaningfull return true; } @@ -272,6 +273,7 @@ Block* File::RequestBlock(int i) // // Reference count is 0 so increase it in calling function if you want to // catch the block while still in memory. + clLog()->Dump(XrdCl::AppMsg, "Request block %d ", i); XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; @@ -295,6 +297,8 @@ Block* File::RequestBlock(int i) int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, char* req_buf, long long req_off, long long req_size) { + + clLog()->Dump(XrdCl::AppMsg, "RequestBlockDirect %ld ", blocks.size()); XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; const long long BS = m_cfi.GetBufferSize(); @@ -325,6 +329,8 @@ int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, int File::ReadBlocksFromDisk(std::list& blocks, char* req_buf, long long req_off, long long req_size) { + + clLog()->Dump(XrdCl::AppMsg, "ReadBlocksFromDisk %ld ", blocks.size()); const long long BS = m_cfi.GetBufferSize(); long long total = 0; @@ -427,7 +433,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) DirectResponseHandler *direct_handler = 0; int direct_size = 0; - if ( ! blks_direct.empty()) + if (!blks_direct.empty()) { direct_handler = new DirectResponseHandler(blks_direct.size()); @@ -436,45 +442,54 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } // Second, read blocks from disk. - int rc = ReadBlocksFromDisk(blks_on_disk, iUserBuff, iUserOff, iUserSize); - if (rc >= 0) - { - bytes_read += rc; + if (!blks_on_disk.empty()) { + int rc = ReadBlocksFromDisk(blks_on_disk, iUserBuff, iUserOff, iUserSize); + if (rc >= 0) + { + bytes_read += rc; + } + else + { + bytes_read = rc; // AMT ?? should there be an immediate return + } } - else - { - bytes_read = rc; // AMT ?? should there be an immediate return - } - // Third, loop over blocks that are available or incoming - while ( ! blks_to_process.empty() && bytes_read >= 0) + while ( ! blks_to_process.empty() && bytes_read >= 0) // AMT : do I need this loop ? { - BlockList_t finished; + BlockList_t finished; - { - XrdSysCondVarHelper _lck(m_block_cond); + { + XrdSysCondVarHelper _lck(m_block_cond); - BlockList_i bi = blks_to_process.begin(); - while (bi != blks_to_process.end()) - { - if ((*bi)->is_finished()) - { - finished.push_back(*bi); - BlockList_i bj = bi++; - blks_to_process.erase(bj); - } - else - { - ++bi; - } - } + BlockList_i bi = blks_to_process.begin(); + while (bi != blks_to_process.end()) + { + clLog()->Dump(XrdCl::AppMsg, "searcing for blocks finished"); + if ((*bi)->is_finished()) + { + clLog()->Dump(XrdCl::AppMsg, "found finished block"); + finished.push_back(*bi); + BlockList_i bj = bi++; + blks_to_process.erase(bj); + } + else + { + ++bi; + } + } - if (finished.empty()) - { - m_block_cond.Wait(); - continue; - } - } + if (finished.empty()) + { + + clLog()->Dump(XrdCl::AppMsg, "wait block begin"); + + m_block_cond.Wait(); + + clLog()->Dump(XrdCl::AppMsg, "wait block end"); + + continue; + } + } BlockList_i bi = finished.begin(); while (bi != finished.end()) @@ -486,23 +501,30 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) long long off_in_block; // offset in block long long size_to_copy; // size to copy + clLog()->Dump(XrdCl::AppMsg, "Block finished ok."); overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); + bytes_read += size_to_copy; } else // it has failed ... krap up. { + + clLog()->Error(XrdCl::AppMsg, "Block finished with eorror."); bytes_read = -1; errno = (*bi)->m_errno; break; } + ++bi; } std::copy(finished.begin(), finished.end(), std::back_inserter(blks_processed)); + finished.clear(); } // Fourth, make sure all direct requests have arrived if (direct_handler != 0) { + clLog()->Error(XrdCl::AppMsg, "Read() waiting for direct requests."); XrdSysCondVarHelper _lck(direct_handler->m_cond); if (direct_handler->m_to_wait > 0) @@ -572,7 +594,7 @@ void File::WriteBlockToDisk(Block* b) } if (cnt > PREFETCH_MAX_ATTEMPTS) { - clLog()->Error(XrdCl::AppMsg, "File::WriteToDisk() write failes too manny attempts %s", lPath()); + clLog()->Error(XrdCl::AppMsg, "File::WriteToDisk() write failed too manny attempts %s", lPath()); return; } } @@ -663,6 +685,7 @@ void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) { + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"Direct block response"); XrdSysCondVarHelper _lck(m_cond); --m_to_wait; @@ -709,6 +732,8 @@ void File::dec_ref_count(Block* b) void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { + clLog()->Dump(XrdCl::AppMsg, "File::ProcessBlockResponse %d ",(int)(b->m_offset/BufferSize())); + m_block_cond.Lock(); if (status->IsOK()) diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 5b713a2e0d9..d5b31aa5954 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -96,10 +96,9 @@ namespace XrdFileCache long long m_offset; //!< offset of cached file for block-based operation long long m_fileSize; //!< size of cached disk file for block-based operation - bool m_started; //!< state of run thread bool m_failed; //!< reading from original source or writing to disk has failed bool m_stopping; //!< run thread should be stopped - bool m_stopped; //!< prefetch is stopped + XrdSysCondVar m_stateCond; //!< state condition variable XrdSysMutex m_downloadStatusMutex; //!< mutex locking access to m_cfi object From 551e16aa129c5298dfc5c8f90e9590c079b88c0a Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 22 Apr 2015 09:49:20 -0700 Subject: [PATCH 008/171] Change variable names. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 19 ++++++++++--------- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 93075b0a5e1..00a7e06f90c 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -34,14 +34,14 @@ using namespace XrdFileCache; IOEntireFile::IOEntireFile(XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache & cache) : IO(io, stats, cache), - m_prefetch(0) + m_file(0) { clLog()->Info(XrdCl::AppMsg, "IO::IO() [%p] %s", this, m_io.Path()); std::string fname; m_cache.getFilePathFromURL(io.Path(), fname); - m_prefetch = new File(io, fname, 0, io.FSize()); + m_file = new File(io, fname, 0, io.FSize()); } @@ -50,17 +50,18 @@ IOEntireFile::~IOEntireFile() bool IOEntireFile::ioActive() { - return m_prefetch->InitiateClose(); + printf("called ioActive ...\n"); + return m_file->InitiateClose(); } XrdOucCacheIO *IOEntireFile::Detach() { - m_statsGlobal.Add(m_prefetch->GetStats()); + m_statsGlobal.Add(m_file->GetStats()); XrdOucCacheIO * io = &m_io; - delete m_prefetch; - m_prefetch = 0; + delete m_file; + m_file = 0; // This will delete us! m_cache.Detach(this); @@ -85,8 +86,8 @@ int IOEntireFile::Read (char *buff, long long off, int size) ssize_t bytes_read = 0; ssize_t retval = 0; - retval = m_prefetch->Read(buff, off, size); - clLog()->Debug(XrdCl::AppMsg, "IO::Read() read from prefetch retval = %d %s", retval, m_io.Path()); + retval = m_file->Read(buff, off, size); + clLog()->Debug(XrdCl::AppMsg, "IO::Read() read from File retval = %d %s", retval, m_io.Path()); if (retval > 0) { bytes_read += retval; @@ -114,5 +115,5 @@ int IOEntireFile::ReadV (const XrdOucIOVec *readV, int n) clLog()->Warning(XrdCl::AppMsg, "IO::ReadV(), get %d requests %s", n, m_io.Path()); - return m_prefetch->ReadV(readV, n); + return m_file->ReadV(readV, n); } diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index 925c038f4b2..5ae99e6972f 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -82,7 +82,7 @@ namespace XrdFileCache virtual bool ioActive(); private: - File* m_prefetch; + File* m_file; }; } From 0e792e4c5db1610debac21889fc93a5b48f2daa1 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 23 Apr 2015 10:53:11 -0700 Subject: [PATCH 009/171] Cleanup.e --- src/XrdFileCache/XrdFileCacheFile.cc | 129 +++++++++++++-------------- src/XrdFileCache/XrdFileCacheFile.hh | 6 +- 2 files changed, 63 insertions(+), 72 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index d25cfafcc14..5fe048a4ef7 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -127,7 +127,12 @@ File::~File() Sync(); } // write statistics in *cinfo file + // AMT append IO stat --- look new interface in master branch + // XXXX MT -- OK, what needs to be here? + AppendIOStatToFileInfo(); + // XXXX MT END + clLog()->Info(XrdCl::AppMsg, "File::~File close data file %p",(void*)this , lPath()); if (m_output) { @@ -152,7 +157,7 @@ bool File::InitiateClose() m_stateCond.Lock(); m_stopping = true; m_stateCond.UnLock(); - if (m_cfi.IsComplete()) return false; // AMT maybe map size is here more meaningfull + if (m_cfi.IsComplete()) return false; // AMT maybe map size is here more meaningfull, but might hold block state lock return true; } @@ -286,7 +291,6 @@ Block* File::RequestBlock(int i) Block *b = new Block(this, off, this_bs); // should block be reused to avoid recreation m_block_map[i] = b; - // AMT do I really have got cast to non-const ? client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); return b; @@ -379,8 +383,6 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) size_t msize = m_block_map.size(); // XXX Check for blocks to free? Later ... - // inc_num_readers(); - const int idx_first = iUserOff / BS; const int idx_last = (iUserOff + iUserSize - 1) / BS; @@ -496,7 +498,6 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) { if ((*bi)->is_ok()) { - // XXXX memcpy ! AMT ... long long user_off; // offset in user buffer long long off_in_block; // offset in block long long size_to_copy; // size to copy @@ -508,8 +509,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } else // it has failed ... krap up. { - - clLog()->Error(XrdCl::AppMsg, "Block finished with eorror."); + clLog()->Error(XrdCl::AppMsg, "Block finished with eorror."); bytes_read = -1; errno = (*bi)->m_errno; break; @@ -560,8 +560,6 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) dec_ref_count(*bi); // XXXX stamp block } - - // dec_num_readers(); } return bytes_read; @@ -663,52 +661,6 @@ void File::Sync() } - - -//============================================================================== - -//============================================================================== - -void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, - XrdCl::AnyObject *response) -{ - m_block->m_file->ProcessBlockResponse(m_block, status); - - delete status; - delete response; - - delete this; -} - -//------------------------------------------------------------------------------ - -void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, - XrdCl::AnyObject *response) -{ - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"Direct block response"); - XrdSysCondVarHelper _lck(m_cond); - - --m_to_wait; - - if ( ! status->IsOK()) - { - XrdPosixMap::Result(*status); - m_errno = errno; - } - - if (m_to_wait == 0) - { - m_cond.Signal(); - } -} - - - -//============================================================================== - -//============================================================================== - -//============================================================================== void File::inc_ref_count(Block* b) { b->m_refcnt++; @@ -739,15 +691,14 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) if (status->IsOK()) { b->m_downloaded = true; - inc_ref_count(b); - - // XXX Add to write queue (if needed) - // AMT TODO check if state is stopped - XrdFileCache::Cache::AddWriteTask(b, true); + if (!m_stopping) { // AMT theoretically this should be under state lock, but then are double locks + inc_ref_count(b); + XrdFileCache::Cache::AddWriteTask(b, true); + } } else { - // how long to keep? + // AMT how long to keep? // when to retry? XrdPosixMap::Result(*status); @@ -755,8 +706,8 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) b->set_error_and_free(errno); errno = 0; - // ??? - inc_ref_count(b); + // ??? AMT temprary commented out + // inc_ref_count(b); } m_block_cond.Broadcast(); @@ -770,10 +721,7 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) return m_cfi.GetBufferSize(); } - -//============================================================================== - -//============================================================================== +//______________________________________________________________________________ int File::ReadV (const XrdOucIOVec *readV, int n) @@ -788,3 +736,50 @@ return m_temp_filename.c_str(); } + + + +//============================================================================== + +//============================================================================== + +void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response) +{ + m_block->m_file->ProcessBlockResponse(m_block, status); + + delete status; + delete response; + + delete this; +} + +//------------------------------------------------------------------------------ + +void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response) +{ + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"Direct block response"); + XrdSysCondVarHelper _lck(m_cond); + + --m_to_wait; + + if ( ! status->IsOK()) + { + XrdPosixMap::Result(*status); + m_errno = errno; + } + + if (m_to_wait == 0) + { + m_cond.Signal(); + } +} + + + +//============================================================================== + +//============================================================================== + +//============================================================================== diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index d5b31aa5954..da636edc74b 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -62,11 +62,10 @@ namespace XrdFileCache int m_refcnt; int m_errno; bool m_downloaded; - bool m_on_disk; Block(File *f, long long off, int size) : m_offset(off), m_file(f), m_refcnt(0), - m_errno(0), m_downloaded(false), m_on_disk(false) + m_errno(0), m_downloaded(false) { m_buff.resize(size); } @@ -112,7 +111,6 @@ namespace XrdFileCache typedef std::list IntList_t; typedef IntList_t::iterator IntList_i; - // typedef IntList_t::const_iterator IntList_ci; typedef std::list BlockList_t; typedef BlockList_t::iterator BlockList_i; @@ -125,8 +123,6 @@ namespace XrdFileCache XrdSysCondVar m_block_cond; - // int m_num_reads; AMT don't know how this one should be used - Stats m_stats; //!< cache statistics, used in IO detach public: From a1e2c5697ef7ac500a77e0ded355f33d05b9df83 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 8 Jun 2015 15:27:39 -0700 Subject: [PATCH 010/171] Global limit of RAM resources. --- src/XrdFileCache/XrdFileCache.cc | 27 ++++++++++++++++++++++--- src/XrdFileCache/XrdFileCache.hh | 18 +++++++++-------- src/XrdFileCache/XrdFileCacheFactory.cc | 19 +++++++++-------- src/XrdFileCache/XrdFileCacheFactory.hh | 14 +++++++++---- src/XrdFileCache/XrdFileCacheFile.cc | 15 +++++++++----- 5 files changed, 63 insertions(+), 30 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 693d5bab4e8..1c69e2f9386 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -32,8 +32,6 @@ #include "XrdFileCacheFactory.hh" -XrdFileCache::Cache::WriteQ XrdFileCache::Cache::s_writeQ; - using namespace XrdFileCache; void *ProcessWriteTaskThread(void* c) { @@ -44,7 +42,8 @@ void *ProcessWriteTaskThread(void* c) Cache::Cache(XrdOucCacheStats & stats) : m_attached(0), - m_stats(stats) + m_stats(stats), + m_RAMblocks_used(0) { pthread_t tid; XrdSysThread::Run(&tid, ProcessWriteTaskThread, (void*)this, 0, "XrdFileCache WriteTasks "); @@ -103,6 +102,7 @@ void Cache::getFilePathFromURL(const char* iUrl, std::string &result) const result = Factory::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); } +// XXXX MT: is the following needed ??? //______________________________________________________________________________ bool Cache::HaveFreeWritingSlots() @@ -167,4 +167,25 @@ Cache::ProcessWriteTasks() block->m_file->WriteBlockToDisk(block); } } +//______________________________________________________________________________ + +bool +Cache::RequestRAMBlock() +{ + XrdSysMutexHelper lock(&m_RAMblock_mutex); + if ( m_RAMblocks_used > Factory::GetInstance().RefConfiguration().m_NRamBuffers ) + { + m_RAMblocks_used++; + return true; + } + + return false; +} + +void +Cache::RAMBlockReleased() +{ + XrdSysMutexHelper lock(&m_RAMblock_mutex); + m_RAMblocks_used--; +} diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 5c9006b8f5f..6d3c4e98292 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -68,23 +68,23 @@ namespace XrdFileCache //--------------------------------------------------------------------- //! Add downloaded block in write queue. //--------------------------------------------------------------------- - static void AddWriteTask(Block* b, bool from_read); + void AddWriteTask(Block* b, bool from_read); - //--------------------------------------------------------------------- - //! Check write queue size is not over limit. - //--------------------------------------------------------------------- - static bool HaveFreeWritingSlots(); //--------------------------------------------------------------------- //! \brief Remove blocks from write queue which belong to given prefetch. //! This method is used at the time of File destruction. //--------------------------------------------------------------------- - static void RemoveWriteQEntriesFor(File *f); + void RemoveWriteQEntriesFor(File *f); //--------------------------------------------------------------------- //! Separate task which writes blocks from ram to disk. //--------------------------------------------------------------------- - static void ProcessWriteTasks(); + void ProcessWriteTasks(); + + bool RequestRAMBlock(); + + void RAMBlockReleased(); private: //! Decrease attached count. Called from IO::Detach(). @@ -100,6 +100,8 @@ namespace XrdFileCache unsigned int m_attached; //!< number of attached IO objects XrdOucCacheStats &m_stats; //!< global cache usage statistics + XrdSysMutex m_RAMblock_mutex; //!< central lock for this class + int m_RAMblocks_used; struct WriteQ { @@ -109,7 +111,7 @@ namespace XrdFileCache std::list queue; //!< container }; - static WriteQ s_writeQ; + WriteQ s_writeQ; }; diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc index 3580b6344cb..88fc367f119 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ b/src/XrdFileCache/XrdFileCacheFactory.cc @@ -60,7 +60,8 @@ void *CacheDirCleanupThread(void* cache_void) Factory::Factory() - : m_log(0, "XrdFileCache_") + : m_log(0, "XrdFileCache_"), + m_cache(0) {} extern "C" @@ -97,7 +98,9 @@ Factory &Factory::GetInstance() XrdOucCache *Factory::Create(Parms & parms, XrdOucCacheIO::aprParms * prParms) { clLog()->Info(XrdCl::AppMsg, "Factory::Create() new cache object"); - return new Cache(m_stats); + assert(m_cache == 0); + m_cache = new Cache(m_stats); + return m_cache; } @@ -269,10 +272,10 @@ bool Factory::Config(XrdSysLogger *logger, const char *config_filename, const ch loff = snprintf(buff, sizeof(buff), "result\n" "\tpfc.cachedir %s\n" "\tpfc.blocksize %lld\n" - "\tpfc.nramread %d\n\tpfc.nramprefetch %d\n", + "\tpfc.nram %d\n\n", m_configuration.m_cache_dir.c_str() , m_configuration.m_bufferSize, - m_configuration.m_NRamBuffersRead, m_configuration.m_NRamBuffersPrefetch ); + m_configuration.m_NRamBuffers ); if (m_configuration.m_hdfsmode) { @@ -360,13 +363,9 @@ bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) return false; } } - else if (part == "nramread") + else if (part == "nram") { - m_configuration.m_NRamBuffersRead = ::atoi(config.GetWord()); - } - else if (part == "nramprefetch") - { - m_configuration.m_NRamBuffersPrefetch = ::atoi(config.GetWord()); + m_configuration.m_NRamBuffers = ::atoi(config.GetWord()); } else if ( part == "hdfsmode" ) { diff --git a/src/XrdFileCache/XrdFileCacheFactory.hh b/src/XrdFileCache/XrdFileCacheFactory.hh index 5bc679cbbd4..1c8d2ad1859 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.hh +++ b/src/XrdFileCache/XrdFileCacheFactory.hh @@ -37,6 +37,10 @@ namespace XrdCl class Log; } +namespace XrdFileCache { + class Cache; +} + namespace XrdFileCache { //---------------------------------------------------------------------------- @@ -49,8 +53,7 @@ namespace XrdFileCache m_diskUsageLWM(-1), m_diskUsageHWM(-1), m_bufferSize(1024*1024), - m_NRamBuffersRead(8), - m_NRamBuffersPrefetch(1), + m_NRamBuffers(8000), m_hdfsbsize(128*1024*1024) {} bool m_hdfsmode; //!< flag for enabling block-level operation @@ -61,8 +64,7 @@ namespace XrdFileCache long long m_diskUsageHWM; //!< cache purge high water mark long long m_bufferSize; //!< prefetch buffer size, default 1MB - int m_NRamBuffersRead; //!< number of read in-memory cache blocks - int m_NRamBuffersPrefetch; //!< number of prefetch in-memory cache blocks + int m_NRamBuffers; //!< number of total in-memory cache blocks long long m_hdfsbsize; //!< used with m_hdfsmode, default 128MB }; @@ -143,6 +145,8 @@ namespace XrdFileCache //--------------------------------------------------------------------- void CacheDirCleanup(); + + Cache* GetCache() { return m_cache; } private: bool ConfigParameters(std::string, XrdOucStream&); bool ConfigXeq(char *, XrdOucStream &); @@ -161,6 +165,8 @@ namespace XrdFileCache std::map m_filesInQueue; Configuration m_configuration; //!< configurable parameters + + Cache* m_cache; }; } diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 5fe048a4ef7..ca62cc72720 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -64,6 +64,10 @@ class DiskSyncer : public XrdJob }; } +namespace +{ + Cache* cache() {return Factory::GetInstance().GetCache();} +} File::File(XrdOucCacheIO &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : m_input(inputIO), @@ -91,7 +95,7 @@ m_block_cond(0) File::~File() { clLog()->Debug(XrdCl::AppMsg, "File::~File() %p %s", (void*)this, lPath()); - Cache::RemoveWriteQEntriesFor(this); + cache()->RemoveWriteQEntriesFor(this); clLog()->Info(XrdCl::AppMsg, "File::~File() check write queues ...%s", lPath()); // can I do anythong to stop waiting for asyc read callbacks ? @@ -376,8 +380,6 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // passing the req to client is actually better. // unlock - const int MaxBlocksForRead = 16; // AMT Should be config var! Or even mutable for low client counts. - m_block_cond.Lock(); size_t msize = m_block_map.size(); @@ -411,7 +413,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) else { // Is there room for one more RAM Block? - if ( msize < MaxBlocksForRead) + if ( cache()->RequestRAMBlock()) { Block *b = RequestBlock(block_idx); inc_ref_count(b); @@ -677,6 +679,9 @@ void File::dec_ref_count(Block* b) if (ret != 1) { clLog()->Error(XrdCl::AppMsg, "File::OnBlockZeroRefCount did not erase %d from map.", i); } + else { + cache()->RAMBlockReleased(); + } } } @@ -693,7 +698,7 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) b->m_downloaded = true; if (!m_stopping) { // AMT theoretically this should be under state lock, but then are double locks inc_ref_count(b); - XrdFileCache::Cache::AddWriteTask(b, true); + cache()->AddWriteTask(b, true); } } else From db6871b7c1e3414ce483159f398d1cbdfa35a760 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 9 Jun 2015 13:08:29 -0700 Subject: [PATCH 011/171] Use same lock for RAM and disk download status. Add Prefetch function(). --- src/XrdFileCache/XrdFileCacheFile.cc | 85 ++++++++++++++++++++++------ src/XrdFileCache/XrdFileCacheFile.hh | 13 +++-- 2 files changed, 74 insertions(+), 24 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index ca62cc72720..58fded002f2 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -85,8 +85,7 @@ m_stateCond(0), // We will explicitly lock the condition before use. m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), m_non_flushed_cnt(0), m_in_sync(false), - -m_block_cond(0) +m_downloadCond(0) { clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); Open(); @@ -275,7 +274,7 @@ namespace //------------------------------------------------------------------------------ -Block* File::RequestBlock(int i) +Block* File::RequestBlock(int i, bool prefetch) { // Must be called w/ block_map locked. // Checks on size etc should be done before. @@ -292,7 +291,7 @@ Block* File::RequestBlock(int i) long long off = i * BS; long long this_bs = (i == last_block) ? m_input.FSize() - off : BS; - Block *b = new Block(this, off, this_bs); // should block be reused to avoid recreation + Block *b = new Block(this, off, this_bs, prefetch); // should block be reused to avoid recreation m_block_map[i] = b; client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); @@ -380,7 +379,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // passing the req to client is actually better. // unlock - m_block_cond.Lock(); + m_downloadCond.Lock(); size_t msize = m_block_map.size(); // XXX Check for blocks to free? Later ... @@ -393,7 +392,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) for (int block_idx = idx_first; block_idx <= idx_last; ++block_idx) { - BlockMap_i bi = m_block_map.find(block_idx); + BlockMap_i bi = m_block_map.find(block_idx); // In RAM or incoming? if (bi != m_block_map.end()) @@ -415,7 +414,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Is there room for one more RAM Block? if ( cache()->RequestRAMBlock()) { - Block *b = RequestBlock(block_idx); + Block *b = RequestBlock(block_idx, false); inc_ref_count(b); blks_to_process.push_back(b); ++msize; @@ -428,7 +427,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } } - m_block_cond.UnLock(); + m_downloadCond.UnLock(); long long bytes_read = 0; @@ -463,7 +462,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) BlockList_t finished; { - XrdSysCondVarHelper _lck(m_block_cond); + XrdSysCondVarHelper _lck(m_downloadCond); BlockList_i bi = blks_to_process.begin(); while (bi != blks_to_process.end()) @@ -487,7 +486,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) clLog()->Dump(XrdCl::AppMsg, "wait block begin"); - m_block_cond.Wait(); + m_downloadCond.Wait(); clLog()->Dump(XrdCl::AppMsg, "wait block end"); @@ -549,7 +548,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Last, stamp and release blocks, release file. { - XrdSysCondVarHelper _lck(m_block_cond); + XrdSysCondVarHelper _lck(m_downloadCond); // XXXX stamp file // AMT ??? fetched status stampled in WriteDisk callback , what dies stamp mean ?? @@ -602,12 +601,14 @@ void File::WriteBlockToDisk(Block* b) // set bit fetched clLog()->Dump(XrdCl::AppMsg, "File::WriteToDisk() success set bit for block [%ld] size [%d] %s", b->m_offset, size, lPath()); int pfIdx = (b->m_offset - m_offset)/m_cfi.GetBufferSize(); - m_downloadStatusMutex.Lock(); + + m_downloadCond.Lock(); m_cfi.SetBitFetched(pfIdx); - m_downloadStatusMutex.UnLock(); + m_downloadCond.UnLock(); + { - XrdSysCondVarHelper _lck(m_block_cond); + XrdSysCondVarHelper _lck(m_downloadCond); dec_ref_count(b); } @@ -691,7 +692,7 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { clLog()->Dump(XrdCl::AppMsg, "File::ProcessBlockResponse %d ",(int)(b->m_offset/BufferSize())); - m_block_cond.Lock(); + m_downloadCond.Lock(); if (status->IsOK()) { @@ -715,9 +716,9 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) // inc_ref_count(b); } - m_block_cond.Broadcast(); + m_downloadCond.Broadcast(); - m_block_cond.UnLock(); + m_downloadCond.UnLock(); } @@ -740,8 +741,56 @@ const char* File::lPath() const return m_temp_filename.c_str(); } +// XXXX MT: is this needed ???? +//______________________________________________________________________________ +void File::AppendIOStatToFileInfo() +{ + // lock in case several IOs want to write in *cinfo file + if (m_infoFile) + { + Info::AStat as; + as.DetachTime = time(0); + as.BytesDisk = m_stats.m_BytesDisk; + as.BytesRam = m_stats.m_BytesRam; + as.BytesMissed = m_stats.m_BytesMissed; + m_cfi.AppendIOStat(as, (XrdOssDF*)m_infoFile); + } + else + { + clLog()->Warning(XrdCl::AppMsg, "Prefetch::AppendIOStatToFileInfo() info file not opened %s", lPath()); + } +} - +//______________________________________________________________________________ +void File::Prefetch() +{ + int block_idx = -1; + + XrdSysCondVarHelper _lck(m_downloadCond); + // AMT can this be sorted before calling Prefetch ?? + if (m_cfi.IsComplete()) return; + + // check index not on disk and not in RAM + for (int f = 0; f < m_cfi.GetSizeInBits(); ++f) + { + if (!m_cfi.TestBit(f)) + { + BlockMap_i bi = m_block_map.find(block_idx); + if (bi == m_block_map.end()) { + block_idx = f; + break; + } + } + } + + assert(block_idx >= 0); + + // decrease counter of globally available blocks, resources already checked in global thread + cache()->RequestRAMBlock(); + + Block *b = RequestBlock(block_idx, true); + inc_ref_count(b); +} //============================================================================== diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index da636edc74b..e797e1dc880 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -59,12 +59,13 @@ namespace XrdFileCache std::vector m_buff; long long m_offset; File *m_file; + bool m_prefetch; int m_refcnt; int m_errno; bool m_downloaded; - Block(File *f, long long off, int size) : - m_offset(off), m_file(f), m_refcnt(0), + Block(File *f, long long off, int size, bool m_prefetch) : + m_offset(off), m_file(f), m_prefetch(false), m_refcnt(0), m_errno(0), m_downloaded(false) { m_buff.resize(size); @@ -100,8 +101,6 @@ namespace XrdFileCache XrdSysCondVar m_stateCond; //!< state condition variable - XrdSysMutex m_downloadStatusMutex; //!< mutex locking access to m_cfi object - // fsync XrdSysMutex m_syncStatusMutex; //!< mutex locking fsync status XrdJob *m_syncer; @@ -121,7 +120,7 @@ namespace XrdFileCache BlockMap_t m_block_map; - XrdSysCondVar m_block_cond; + XrdSysCondVar m_downloadCond; Stats m_stats; //!< cache statistics, used in IO detach @@ -166,8 +165,10 @@ namespace XrdFileCache void ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status); void WriteBlockToDisk(Block* b); + void Prefetch(); + private: - Block* RequestBlock(int i); + Block* RequestBlock(int i, bool prefetch); int RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, char* buff, long long req_off, long long req_size); From 5c5ce15b39806695e1c4cd547ed95792bc52ad0f Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 9 Jun 2015 14:59:06 -0700 Subject: [PATCH 012/171] Add bit array for prefetch status. --- src/XrdFileCache/XrdFileCacheInfo.cc | 7 +++++- src/XrdFileCache/XrdFileCacheInfo.hh | 35 +++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index 5db67079aa4..45777823a25 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -49,18 +49,23 @@ Info::~Info() { if (m_buff_fetched) free(m_buff_fetched); if (m_buff_write_called) free(m_buff_write_called); + if (m_buff_prefetch) free(m_buff_prefetch); } //______________________________________________________________________________ -void Info::ResizeBits(int s) +void Info::ResizeBits(int s, bool prefetch_stat) { m_sizeInBits = s; m_buff_fetched = (unsigned char*)malloc(GetSizeInBytes()); m_buff_write_called = (unsigned char*)malloc(GetSizeInBytes()); memset(m_buff_fetched, 0, GetSizeInBytes()); memset(m_buff_write_called, 0, GetSizeInBytes()); + if (prefetch_stat) { + m_buff_prefetch = (unsigned char*)malloc(GetSizeInBytes()); + memset(m_buff_prefetch, 0, GetSizeInBytes()); + } } //______________________________________________________________________________ diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index f1915536f91..e2508216820 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -79,12 +79,19 @@ namespace XrdFileCache //--------------------------------------------------------------------- void SetBitWriteCalled(int i); + //! \brief Mark block as written from prefetch + //! + //! @param i block index + //--------------------------------------------------------------------- + void SetBitPrefetch(int i); + + //--------------------------------------------------------------------- //! \brief Reserve buffer for fileSize/bufferSize bytes //! //! @param n number of file blocks //--------------------------------------------------------------------- - void ResizeBits(int n); + void ResizeBits(int n, bool prefetch_stat = false); //--------------------------------------------------------------------- //! \brief Rea load content from cinfo file into this object @@ -140,6 +147,11 @@ namespace XrdFileCache //--------------------------------------------------------------------- bool TestBit(int i) const; + //--------------------------------------------------------------------- + //! Test if block at the given index is prefetched + //--------------------------------------------------------------------- + bool TestPrefetchBit(int i) const; + //--------------------------------------------------------------------- //! Get complete status //--------------------------------------------------------------------- @@ -186,6 +198,7 @@ namespace XrdFileCache int m_sizeInBits; //!< number of file blocks unsigned char *m_buff_fetched; //!< download state vector unsigned char *m_buff_write_called; //!< disk written state vector + unsigned char *m_buff_prefetch; //!< prefetch state vector int m_accessCnt; //!< number of written AStat structs bool m_complete; //!< cached }; @@ -199,6 +212,16 @@ namespace XrdFileCache return (m_buff_fetched[cn] & cfiBIT(off)) == cfiBIT(off); } + // AMT could have only one function to test bit and pass an argument, but would loose clarity + inline bool Info::TestPrefetchBit(int i) const + { + int cn = i/8; + assert(cn < GetSizeInBytes()); + + int off = i - cn*8; + return (m_buff_prefetch[cn] & cfiBIT(off)) == cfiBIT(off); + } + inline int Info::GetNDownloadedBlocks() const { @@ -260,6 +283,16 @@ namespace XrdFileCache m_buff_fetched[cn] |= cfiBIT(off); } + inline void Info::SetBitPrefetch(int i) + { + int cn = i/8; + assert(cn < GetSizeInBytes()); + + int off = i - cn*8; + m_buff_prefetch[cn] |= cfiBIT(off); + } + + inline long long Info::GetBufferSize() const { return m_bufferSize; From 22a27388f4d8fd93ae3065f275d65f213363d88d Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 9 Jun 2015 15:00:17 -0700 Subject: [PATCH 013/171] Add prefetch option in configuration. --- src/XrdFileCache/XrdFileCacheFactory.cc | 11 ++++++++--- src/XrdFileCache/XrdFileCacheFactory.hh | 3 +++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc index 88fc367f119..75c55f23c55 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ b/src/XrdFileCache/XrdFileCacheFactory.cc @@ -272,9 +272,11 @@ bool Factory::Config(XrdSysLogger *logger, const char *config_filename, const ch loff = snprintf(buff, sizeof(buff), "result\n" "\tpfc.cachedir %s\n" "\tpfc.blocksize %lld\n" + "\tpfc.prefetch %d\n" "\tpfc.nram %d\n\n", m_configuration.m_cache_dir.c_str() , - m_configuration.m_bufferSize, + m_configuration.m_bufferSize, + m_configuration.m_prefetch, // AMT not sure what parsing should be m_configuration.m_NRamBuffers ); if (m_configuration.m_hdfsmode) @@ -284,7 +286,6 @@ bool Factory::Config(XrdSysLogger *logger, const char *config_filename, const ch loff += snprintf(&buff[loff], strlen(buff2), "%s", buff2); } - char unameBuff[256]; if (m_configuration.m_username.empty()) { XrdOucUtils::UserName(getuid(), unameBuff, sizeof(unameBuff)); @@ -363,7 +364,11 @@ bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) return false; } } - else if (part == "nram") + else if (part == "prefetch" ) + { + m_configuration.m_prefetch = true; + } + else if (part == "nram" ) { m_configuration.m_NRamBuffers = ::atoi(config.GetWord()); } diff --git a/src/XrdFileCache/XrdFileCacheFactory.hh b/src/XrdFileCache/XrdFileCacheFactory.hh index 1c8d2ad1859..d4108efbbe0 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.hh +++ b/src/XrdFileCache/XrdFileCacheFactory.hh @@ -54,6 +54,7 @@ namespace XrdFileCache m_diskUsageHWM(-1), m_bufferSize(1024*1024), m_NRamBuffers(8000), + m_prefetch(false), m_hdfsbsize(128*1024*1024) {} bool m_hdfsmode; //!< flag for enabling block-level operation @@ -65,6 +66,8 @@ namespace XrdFileCache long long m_bufferSize; //!< prefetch buffer size, default 1MB int m_NRamBuffers; //!< number of total in-memory cache blocks + bool m_prefetch; //!< prefetch enable state + long long m_hdfsbsize; //!< used with m_hdfsmode, default 128MB }; From a908ec86267a0d46ee5a39a58e9646565f184b16 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 9 Jun 2015 15:00:48 -0700 Subject: [PATCH 014/171] Implement prefetch score. --- src/XrdFileCache/XrdFileCacheFile.cc | 38 ++++++++++++++++++++++++++-- src/XrdFileCache/XrdFileCacheFile.hh | 8 ++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 58fded002f2..daab517f0de 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -85,7 +85,9 @@ m_stateCond(0), // We will explicitly lock the condition before use. m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), m_non_flushed_cnt(0), m_in_sync(false), -m_downloadCond(0) +m_downloadCond(0), +m_prefetchReadCnt(0), +m_prefetchHitCnt(0) { clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); Open(); @@ -221,7 +223,7 @@ bool File::Open() { int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input.Path()); - m_cfi.ResizeBits(ss); + m_cfi.ResizeBits(ss, Factory::GetInstance().RefConfiguration().m_prefetch); m_cfi.WriteHeader(m_infoFile); } else @@ -359,6 +361,8 @@ int File::ReadBlocksFromDisk(std::list& blocks, return rs; total += rs; + + CheckPrefetchStatDisk(*ii); } return total; @@ -507,6 +511,8 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); bytes_read += size_to_copy; + + CheckPrefetchStatRAM(*bi); } else // it has failed ... krap up. { @@ -787,12 +793,40 @@ void File::Prefetch() // decrease counter of globally available blocks, resources already checked in global thread cache()->RequestRAMBlock(); + m_prefetchReadCnt++; Block *b = RequestBlock(block_idx, true); inc_ref_count(b); } +//______________________________________________________________________________ +void File::CheckPrefetchStatRAM(Block* b) +{ + if (Factory::GetInstance().RefConfiguration().m_prefetch) { + if (b->m_prefetch) + m_prefetchHitCnt++; + } +} + +//______________________________________________________________________________ +void File::CheckPrefetchStatDisk(int idx) +{ + if (Factory::GetInstance().RefConfiguration().m_prefetch) { + if (m_cfi.TestPrefetchBit(idx)) + m_prefetchHitCnt++; + } +} + +//______________________________________________________________________________ +float File::GetPrefetchScore() +{ + if (m_prefetchReadCnt) + return m_prefetchHitCnt/m_prefetchReadCnt; + + return 0; +} + //============================================================================== //============================================================================== diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index e797e1dc880..a549a73c301 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -124,6 +124,9 @@ namespace XrdFileCache Stats m_stats; //!< cache statistics, used in IO detach + int m_prefetchReadCnt; + int m_prefetchHitCnt; + public: //------------------------------------------------------------------------ @@ -167,6 +170,8 @@ namespace XrdFileCache void Prefetch(); + float GetPrefetchScore(); + private: Block* RequestBlock(int i, bool prefetch); @@ -179,6 +184,9 @@ namespace XrdFileCache long long BufferSize(); + void CheckPrefetchStatRAM(Block* b); + void CheckPrefetchStatDisk(int idx); + //! Short log alias. XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } From eeb4840c1fbc328a4d4e84739d885a44f5a7e597 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 24 Jun 2015 16:34:51 -0700 Subject: [PATCH 015/171] Implement prefetch. --- src/XrdFileCache/XrdFileCache.cc | 118 +++++++++++++++++++++++---- src/XrdFileCache/XrdFileCache.hh | 13 ++- src/XrdFileCache/XrdFileCacheFile.cc | 101 ++++++++++++++--------- src/XrdFileCache/XrdFileCacheFile.hh | 8 +- 4 files changed, 185 insertions(+), 55 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 1c69e2f9386..25664c6e32f 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -18,11 +18,13 @@ #include #include +#include #include #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClURL.hh" #include "XrdSys/XrdSysPthread.hh" +#include "XrdSys/XrdSysTimer.hh" #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucEnv.hh" @@ -33,6 +35,7 @@ using namespace XrdFileCache; + void *ProcessWriteTaskThread(void* c) { Cache *cache = static_cast(c); @@ -40,14 +43,26 @@ void *ProcessWriteTaskThread(void* c) return NULL; } -Cache::Cache(XrdOucCacheStats & stats) - : m_attached(0), +void *PrefetchThread(void* ptr) +{ + Cache* cache = static_cast(ptr); + cache->Prefetch(); + return NULL; +} +//______________________________________________________________________________ + + +Cache::Cache(XrdOucCacheStats & stats) : XrdOucCache(), m_stats(stats), m_RAMblocks_used(0) { - pthread_t tid; - XrdSysThread::Run(&tid, ProcessWriteTaskThread, (void*)this, 0, "XrdFileCache WriteTasks "); + pthread_t tid1; + XrdSysThread::Run(&tid1, ProcessWriteTaskThread, (void*)this, 0, "XrdFileCache WriteTasks "); + + pthread_t tid2; + XrdSysThread::Run(&tid2, PrefetchThread, (void*)this, 0, "XrdFileCache Prefetch "); } + //______________________________________________________________________________ XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) @@ -55,10 +70,6 @@ XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) if (Factory::GetInstance().Decide(io)) { clLog()->Info(XrdCl::AppMsg, "Cache::Attach() %s", io->Path()); - { - XrdSysMutexHelper lock(&m_io_mutex); - m_attached++; - } IO* cio; if (Factory::GetInstance().RefConfiguration().m_hdfsmode) cio = new IOFileBlock(*io, m_stats, *this); @@ -78,17 +89,13 @@ XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) int Cache::isAttached() { - XrdSysMutexHelper lock(&m_io_mutex); - return m_attached; + // virutal function of XrdOucCache, don't see it used in pfc or posix layer + return true; } void Cache::Detach(XrdOucCacheIO* io) { clLog()->Info(XrdCl::AppMsg, "Cache::Detach() %s", io->Path()); - { - XrdSysMutexHelper lock(&m_io_mutex); - m_attached--; - } delete io; } @@ -167,6 +174,7 @@ Cache::ProcessWriteTasks() block->m_file->WriteBlockToDisk(block); } } + //______________________________________________________________________________ bool @@ -178,7 +186,6 @@ Cache::RequestRAMBlock() m_RAMblocks_used++; return true; } - return false; } @@ -189,3 +196,84 @@ Cache::RAMBlockReleased() m_RAMblocks_used--; } + +//============================================================================== +//======================= PREFETCH =================================== +//============================================================================== + +namespace { +struct prefetch_less_than +{ + inline bool operator() (const File* struct1, const File* struct2) + { + return (struct1->GetPrefetchScore() < struct2->GetPrefetchScore()); + } +}myobject; +} +//______________________________________________________________________________ + +void +Cache::RegisterPrefetchFile(File* file) +{ + // called from File::Open() + + if (Factory::GetInstance().RefConfiguration().m_prefetch) + { + if (Factory::GetInstance().RefConfiguration().m_hdfsmode) { + XrdSysMutexHelper lock(&m_prefetch_mutex); + m_files.push_back(file); + } + else + { + // don't need to lock it becuse it File object is created in constructor of IOEntireFile + m_files.push_back(file); + } + } +} +//______________________________________________________________________________ + +void +Cache::DeRegisterPrefetchFile(File* file) +{ + // called from last line File::InitiateClose() + + XrdSysMutexHelper lock(&m_prefetch_mutex); + for (FileList::iterator it = m_files.begin(); it != m_files.end(); ++it) { + if (*it == file) { + m_files.erase(it); + break; + } + } +} +//______________________________________________________________________________ + +File* +Cache::GetNextFileToPrefetch() +{ + XrdSysMutexHelper lock(&m_prefetch_mutex); + if (m_files.empty()) + return 0; + + std::sort(m_files.begin(), m_files.end(), myobject); + File* f = m_files.back(); + f->MarkPrefetch(); + return f; +} + +//______________________________________________________________________________ + + +void +Cache::Prefetch() +{ + while (true) { + File* f = GetNextFileToPrefetch(); + if (f) { + f->Prefetch(); + } + else { + // wait for new file, AMT should I wait for the signal instead ??? + XrdSysTimer::Wait(10); + } + } +} diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 6d3c4e98292..9a2451e9cdd 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -86,6 +86,13 @@ namespace XrdFileCache void RAMBlockReleased(); + void RegisterPrefetchFile(File*); + void DeRegisterPrefetchFile(File*); + + File* GetNextFileToPrefetch(); + + void Prefetch(); + private: //! Decrease attached count. Called from IO::Detach(). void Detach(XrdOucCacheIO *); @@ -96,8 +103,7 @@ namespace XrdFileCache //! Short log alias. XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - XrdSysMutex m_io_mutex; //!< central lock for this class - unsigned int m_attached; //!< number of attached IO objects + XrdSysMutex m_prefetch_mutex; //!< central lock for this class XrdOucCacheStats &m_stats; //!< global cache usage statistics XrdSysMutex m_RAMblock_mutex; //!< central lock for this class @@ -113,6 +119,9 @@ namespace XrdFileCache WriteQ s_writeQ; + // prefetching + typedef std::vector FileList; + FileList m_files; }; //---------------------------------------------------------------------------- diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index daab517f0de..1e66f9efb33 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -87,7 +87,8 @@ m_non_flushed_cnt(0), m_in_sync(false), m_downloadCond(0), m_prefetchReadCnt(0), -m_prefetchHitCnt(0) +m_prefetchHitCnt(0), +m_prefetchCurrentCnt(0) { clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); Open(); @@ -104,14 +105,17 @@ File::~File() { m_stateCond.Lock(); bool isStopped = m_stopping; + bool isPrefetching = (m_prefetchCurrentCnt > 0); m_stateCond.UnLock(); - if (isStopped) + if ((isPrefetching == false) && isStopped) { - printf("~FILE map size %ld \n", m_block_map.size()); - if ( m_block_map.empty()) + m_downloadCond.Lock(); + bool blockMapEmpty = m_block_map.empty(); + m_downloadCond.UnLock(); + if ( blockMapEmpty) break; } - XrdSysTimer::Wait(100); + XrdSysTimer::Wait(10); } clLog()->Debug(XrdCl::AppMsg, "File::~File finished with writing %s",lPath() ); @@ -159,9 +163,12 @@ bool File::InitiateClose() { // Retruns true if delay is needed clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start", lPath()); + + cache()->DeRegisterPrefetchFile(this); + m_stateCond.Lock(); m_stopping = true; - m_stateCond.UnLock(); + m_stateCond.UnLock(); if (m_cfi.IsComplete()) return false; // AMT maybe map size is here more meaningfull, but might hold block state lock return true; } @@ -231,6 +238,8 @@ bool File::Open() clLog()->Debug(XrdCl::AppMsg, "Info file read from disk: %s", m_input.Path()); } + + cache()->RegisterPrefetchFile(this); return true; } @@ -770,33 +779,40 @@ void File::AppendIOStatToFileInfo() //______________________________________________________________________________ void File::Prefetch() { - int block_idx = -1; + bool stopping = false; + m_stateCond.Lock(); + stopping = m_stopping; + m_stateCond.UnLock(); + - XrdSysCondVarHelper _lck(m_downloadCond); - // AMT can this be sorted before calling Prefetch ?? - if (m_cfi.IsComplete()) return; + if (!stopping) { + XrdSysCondVarHelper _lck(m_downloadCond); + if (m_cfi.IsComplete() == false) + { + int block_idx = -1; + // check index not on disk and not in RAM + for (int f = 0; f < m_cfi.GetSizeInBits(); ++f) + { + if (!m_cfi.TestBit(f)) + { + BlockMap_i bi = m_block_map.find(block_idx); + if (bi == m_block_map.end()) { + block_idx = f; + break; + } + } + } - // check index not on disk and not in RAM - for (int f = 0; f < m_cfi.GetSizeInBits(); ++f) - { - if (!m_cfi.TestBit(f)) - { - BlockMap_i bi = m_block_map.find(block_idx); - if (bi == m_block_map.end()) { - block_idx = f; - break; + if (cache()->RequestRAMBlock()) { + m_prefetchReadCnt++; + + Block *b = RequestBlock(block_idx, true); + inc_ref_count(b); } } } - - assert(block_idx >= 0); - - // decrease counter of globally available blocks, resources already checked in global thread - cache()->RequestRAMBlock(); - m_prefetchReadCnt++; - - Block *b = RequestBlock(block_idx, true); - inc_ref_count(b); + + UnMarkPrefetch(); } @@ -819,16 +835,33 @@ void File::CheckPrefetchStatDisk(int idx) } //______________________________________________________________________________ -float File::GetPrefetchScore() +float File::GetPrefetchScore() const { if (m_prefetchReadCnt) return m_prefetchHitCnt/m_prefetchReadCnt; - return 0; + return 1; // AMT not sure if this should be 0.5 ... ???? } -//============================================================================== +//______________________________________________________________________________ +void File::MarkPrefetch() +{ + m_stateCond.Lock(); + m_prefetchCurrentCnt++; + m_stateCond.UnLock(); + +} +//______________________________________________________________________________ +void File::UnMarkPrefetch() +{ + m_stateCond.Lock(); + m_prefetchCurrentCnt--; + m_stateCond.UnLock(); +} + +//============================================================================== +//================== RESPONSE HANDLER ================================== //============================================================================== void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, @@ -865,9 +898,3 @@ void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, } - -//============================================================================== - -//============================================================================== - -//============================================================================== diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index a549a73c301..8656db34a30 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -126,6 +126,8 @@ namespace XrdFileCache int m_prefetchReadCnt; int m_prefetchHitCnt; + int m_prefetchCurrentCnt; + // AMT should I cache prefetch score for optimization of Cache::getNextFileToPrefetch() ??? public: @@ -170,7 +172,9 @@ namespace XrdFileCache void Prefetch(); - float GetPrefetchScore(); + float GetPrefetchScore() const; + + void MarkPrefetch(); private: Block* RequestBlock(int i, bool prefetch); @@ -187,6 +191,8 @@ namespace XrdFileCache void CheckPrefetchStatRAM(Block* b); void CheckPrefetchStatDisk(int idx); + void UnMarkPrefetch(); + //! Short log alias. XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } From a8975780df0ffd49b689daebf9945a03e80faf12 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 24 Jun 2015 16:41:21 -0700 Subject: [PATCH 016/171] Cache prefetch score. --- src/XrdFileCache/XrdFileCacheFile.cc | 12 ++++++------ src/XrdFileCache/XrdFileCacheFile.hh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 1e66f9efb33..82220abebf2 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -88,6 +88,7 @@ m_in_sync(false), m_downloadCond(0), m_prefetchReadCnt(0), m_prefetchHitCnt(0), +m_prefetchScore(1), m_prefetchCurrentCnt(0) { clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); @@ -805,7 +806,7 @@ void File::Prefetch() if (cache()->RequestRAMBlock()) { m_prefetchReadCnt++; - + m_prefetchScore = m_prefetchHitCnt/m_prefetchReadCnt; Block *b = RequestBlock(block_idx, true); inc_ref_count(b); } @@ -820,8 +821,10 @@ void File::Prefetch() void File::CheckPrefetchStatRAM(Block* b) { if (Factory::GetInstance().RefConfiguration().m_prefetch) { - if (b->m_prefetch) + if (b->m_prefetch) { m_prefetchHitCnt++; + m_prefetchScore = m_prefetchHitCnt/m_prefetchReadCnt; + } } } @@ -837,10 +840,7 @@ void File::CheckPrefetchStatDisk(int idx) //______________________________________________________________________________ float File::GetPrefetchScore() const { - if (m_prefetchReadCnt) - return m_prefetchHitCnt/m_prefetchReadCnt; - - return 1; // AMT not sure if this should be 0.5 ... ???? + return m_prefetchScore; } //______________________________________________________________________________ diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 8656db34a30..975ec2eef54 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -126,8 +126,8 @@ namespace XrdFileCache int m_prefetchReadCnt; int m_prefetchHitCnt; + int m_prefetchScore; //cached int m_prefetchCurrentCnt; - // AMT should I cache prefetch score for optimization of Cache::getNextFileToPrefetch() ??? public: From 49ef88d10fc53dc15fe1b487fcea27ae05d913cb Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 24 Jun 2015 16:42:48 -0700 Subject: [PATCH 017/171] Remove oboslete member. --- src/XrdFileCache/XrdFileCacheFile.cc | 2 -- src/XrdFileCache/XrdFileCacheFile.hh | 1 - 2 files changed, 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 82220abebf2..436c1813bcf 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -77,8 +77,6 @@ m_infoFile(NULL), m_temp_filename(disk_file_path), m_offset(iOffset), m_fileSize(iFileSize), - -m_failed(false), m_stopping(false), m_stateCond(0), // We will explicitly lock the condition before use. diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 975ec2eef54..583e572d865 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -96,7 +96,6 @@ namespace XrdFileCache long long m_offset; //!< offset of cached file for block-based operation long long m_fileSize; //!< size of cached disk file for block-based operation - bool m_failed; //!< reading from original source or writing to disk has failed bool m_stopping; //!< run thread should be stopped XrdSysCondVar m_stateCond; //!< state condition variable From e4c57c6aba354342698a65440280ed04b4d2c9b2 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 24 Jun 2015 17:01:44 -0700 Subject: [PATCH 018/171] Move base IO class to separate file. --- src/XrdFileCache/XrdFileCacheIO.hh | 45 ++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 src/XrdFileCache/XrdFileCacheIO.hh diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh new file mode 100644 index 00000000000..df2b0fd18c8 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -0,0 +1,45 @@ +#ifndef __XRDFILECACHE_CACHE_IO_HH__ +#define __XRDFILECACHE_CACHE_IO_HH__ + +#include "XrdFileCache.hh" +#include "XrdOuc/XrdOucCache.hh" +#include "XrdCl/XrdClDefaultEnv.hh" + +namespace XrdFileCache +{ + //---------------------------------------------------------------------------- + //! Base cache-io class that implements XrdOucCacheIO abstract methods. + //---------------------------------------------------------------------------- + class IO : public XrdOucCacheIO + { + public: + IO (XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache &cache) : + m_io(io), m_statsGlobal(stats), m_cache(cache) {} + + //! Original data source. + virtual XrdOucCacheIO *Base() { return &m_io; } + + //! Original data source URL. + virtual long long FSize() { return m_io.FSize(); } + + //! Original data source URL. + virtual const char *Path() { return m_io.Path(); } + + virtual int Sync() { return 0; } + + virtual int Trunc(long long Offset) { errno = ENOTSUP; return -1; } + + virtual int Write(char *Buffer, long long Offset, int Length) + { errno = ENOTSUP; return -1; } + + + protected: + XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } + + XrdOucCacheIO &m_io; //!< original data source + XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics + Cache &m_cache; //!< reference to Cache needed in detach + }; +} + +#endif From a0c5e5cf2802a6c9790d35409f46b3d085bc50d5 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 24 Jun 2015 17:01:50 -0700 Subject: [PATCH 019/171] Move base IO class to separate file. --- src/XrdFileCache.cmake | 1 + src/XrdFileCache/XrdFileCache.cc | 4 +- src/XrdFileCache/XrdFileCache.hh | 45 ++------------------ src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 7 ++- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 1 + src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 5 ++- src/XrdFileCache/XrdFileCacheIOFileBlock.hh | 3 +- 7 files changed, 13 insertions(+), 53 deletions(-) diff --git a/src/XrdFileCache.cmake b/src/XrdFileCache.cmake index 5622fbcaaa7..ebd5ca029b9 100644 --- a/src/XrdFileCache.cmake +++ b/src/XrdFileCache.cmake @@ -21,6 +21,7 @@ add_library( XrdFileCache/XrdFileCacheFile.cc XrdFileCache/XrdFileCacheFile.hh XrdFileCache/XrdFileCacheStats.hh XrdFileCache/XrdFileCacheInfo.cc XrdFileCache/XrdFileCacheInfo.hh + XrdFileCache/XrdFileCacheIO.hh XrdFileCache/XrdFileCacheIOEntireFile.cc XrdFileCache/XrdFileCacheIOEntireFile.hh XrdFileCache/XrdFileCacheIOFileBlock.cc XrdFileCache/XrdFileCacheIOFileBlock.hh XrdFileCache/XrdFileCacheDecision.hh) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 25664c6e32f..6e808c9b448 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -100,9 +100,8 @@ void Cache::Detach(XrdOucCacheIO* io) delete io; } +// XXXX MT: is the following needed ??? //______________________________________________________________________________ - - void Cache::getFilePathFromURL(const char* iUrl, std::string &result) const { XrdCl::URL url(iUrl); @@ -118,7 +117,6 @@ Cache::HaveFreeWritingSlots() return s_writeQ.size < maxWriteWaits; } - //______________________________________________________________________________ void Cache::AddWriteTask(Block* b, bool fromRead) diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 9a2451e9cdd..7ab75e92e22 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -30,6 +30,7 @@ namespace XrdCl { } namespace XrdFileCache { class File; +class IO; } namespace XrdFileCache @@ -39,9 +40,6 @@ namespace XrdFileCache //---------------------------------------------------------------------------- class Cache : public XrdOucCache { - friend class IOEntireFile; - friend class IOFileBlock; - public: //--------------------------------------------------------------------- //! Constructor @@ -93,12 +91,10 @@ namespace XrdFileCache void Prefetch(); - private: //! Decrease attached count. Called from IO::Detach(). void Detach(XrdOucCacheIO *); - //! Transfor URL to path on local disk. - void getFilePathFromURL(const char* url, std::string& res) const; + private: //! Short log alias. XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } @@ -114,7 +110,7 @@ namespace XrdFileCache WriteQ() : condVar(0), size(0) {} XrdSysCondVar condVar; //!< write list condVar size_t size; //!< cache size of a container - std::list queue; //!< container + std::list queue; //!< container }; WriteQ s_writeQ; @@ -124,41 +120,6 @@ namespace XrdFileCache FileList m_files; }; - //---------------------------------------------------------------------------- - //! Base cache-io class that implements XrdOucCacheIO abstract methods. - //---------------------------------------------------------------------------- - class IO : public XrdOucCacheIO - { - friend class File; - - public: - IO (XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache &cache) : - m_io(io), m_statsGlobal(stats), m_cache(cache) {} - - //! Original data source. - virtual XrdOucCacheIO *Base() { return &m_io; } - - //! Original data source URL. - virtual long long FSize() { return m_io.FSize(); } - - //! Original data source URL. - virtual const char *Path() { return m_io.Path(); } - - virtual int Sync() { return 0; } - - virtual int Trunc(long long Offset) { errno = ENOTSUP; return -1; } - - virtual int Write(char *Buffer, long long Offset, int Length) - { errno = ENOTSUP; return -1; } - - - protected: - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - - XrdOucCacheIO &m_io; //!< original data source - XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics - Cache &m_cache; //!< reference to Cache needed in detach - }; } #endif diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 00a7e06f90c..2a6c9d66e8c 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -37,12 +37,11 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache & c m_file(0) { clLog()->Info(XrdCl::AppMsg, "IO::IO() [%p] %s", this, m_io.Path()); - - std::string fname; - m_cache.getFilePathFromURL(io.Path(), fname); + + XrdCl::URL url(io.Path()); + std::string fname = Factory::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); m_file = new File(io, fname, 0, io.FSize()); - } IOEntireFile::~IOEntireFile() diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index 5ae99e6972f..c9019bd4db2 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -21,6 +21,7 @@ #include #include "XrdSys/XrdSysPthread.hh" +#include "XrdFileCacheIO.hh" #include "XrdFileCache.hh" #include "XrdFileCacheStats.hh" #include "XrdFileCacheFile.hh" diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 45d35b4f3af..e09a31a9a86 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -86,8 +86,9 @@ void IOFileBlock::GetBlockSizeFromPath() //______________________________________________________________________________ File* IOFileBlock::newBlockFile(long long off, int blocksize, XrdOucCacheIO* io) { - std::string fname; - m_cache.getFilePathFromURL(io->Path(), fname); + XrdCl::URL url(io->Path()); + std::string fname = Factory::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); + std::stringstream ss; ss << fname; char offExt[64]; diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index c96f192f795..b11cf139ab6 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -23,8 +23,7 @@ #include "XrdOuc/XrdOucCache.hh" #include "XrdSys/XrdSysPthread.hh" -#include "XrdFileCache.hh" -#include "XrdFileCacheFile.hh" +#include "XrdFileCacheIO.hh" class XrdSysError; class XrdOssDF; From 9703af00f545113dcf52e081a6cd48d0a80456b9 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 25 Jun 2015 15:29:01 -0700 Subject: [PATCH 020/171] Add missing initialization. --- src/XrdFileCache/XrdFileCacheInfo.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index 45777823a25..9323e5748d7 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -39,7 +39,8 @@ using namespace XrdFileCache; Info::Info(long long iBufferSize) : m_version(0), m_bufferSize(iBufferSize), - m_sizeInBits(0), m_buff_fetched(0), m_buff_write_called(0), + m_sizeInBits(0), + m_buff_fetched(0), m_buff_write_called(0), m_buff_prefetch(0), m_accessCnt(0), m_complete(false) { From a1fab5a9333731b77ad5704111f3c370f3ad1d00 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 26 Jun 2015 15:33:25 -0700 Subject: [PATCH 021/171] Add dump log messages. --- src/XrdFileCache/XrdFileCache.cc | 12 +++- src/XrdFileCache/XrdFileCacheFactory.cc | 3 + src/XrdFileCache/XrdFileCacheFile.cc | 63 +++++++++++++------- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 2 +- src/XrdFileCache/XrdFileCacheInfo.cc | 13 +++- src/XrdFileCache/XrdFileCacheInfo.hh | 2 +- 6 files changed, 65 insertions(+), 30 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 6e808c9b448..b2b678240c0 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -179,7 +179,7 @@ bool Cache::RequestRAMBlock() { XrdSysMutexHelper lock(&m_RAMblock_mutex); - if ( m_RAMblocks_used > Factory::GetInstance().RefConfiguration().m_NRamBuffers ) + if ( m_RAMblocks_used < Factory::GetInstance().RefConfiguration().m_NRamBuffers ) { m_RAMblocks_used++; return true; @@ -249,8 +249,10 @@ File* Cache::GetNextFileToPrefetch() { XrdSysMutexHelper lock(&m_prefetch_mutex); - if (m_files.empty()) + if (m_files.empty()) { + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::GetNextFileToPrefetch ... no open files"); return 0; + } std::sort(m_files.begin(), m_files.end(), myobject); File* f = m_files.back(); @@ -264,14 +266,18 @@ Cache::GetNextFileToPrefetch() void Cache::Prefetch() { + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch thread start"); + while (true) { File* f = GetNextFileToPrefetch(); + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch got file (%p)", f); + if (f) { f->Prefetch(); } else { // wait for new file, AMT should I wait for the signal instead ??? - XrdSysTimer::Wait(10); + XrdSysTimer::Wait(1); } } } diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc index 75c55f23c55..68611c5ddc3 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ b/src/XrdFileCache/XrdFileCacheFactory.cc @@ -310,6 +310,7 @@ bool Factory::Config(XrdSysLogger *logger, const char *config_filename, const ch bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) { + printf("part %s \n", part.c_str()); XrdSysError err(0, ""); if ( part == "user" ) { @@ -366,7 +367,9 @@ bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) } else if (part == "prefetch" ) { + printf("prefetch enabled !!!!\n"); m_configuration.m_prefetch = true; + config.GetWord(); } else if (part == "nram" ) { diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 436c1813bcf..2bba3dbaa59 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -225,7 +225,7 @@ bool File::Open() return false; } - if (m_cfi.Read(m_infoFile) <= 0) + if (m_cfi.Read(m_infoFile, Factory::GetInstance().RefConfiguration().m_prefetch) <= 0) { int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input.Path()); @@ -291,7 +291,7 @@ Block* File::RequestBlock(int i, bool prefetch) // // Reference count is 0 so increase it in calling function if you want to // catch the block while still in memory. - clLog()->Dump(XrdCl::AppMsg, "Request block %d ", i); + clLog()->Debug(XrdCl::AppMsg, "RequestBlock() %d pOn=(%d)", i, prefetch); XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; @@ -306,6 +306,7 @@ Block* File::RequestBlock(int i, bool prefetch) client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); + clLog()->Debug(XrdCl::AppMsg, "RequestBlock() %d END", i); return b; } @@ -347,7 +348,7 @@ int File::ReadBlocksFromDisk(std::list& blocks, char* req_buf, long long req_off, long long req_size) { - clLog()->Dump(XrdCl::AppMsg, "ReadBlocksFromDisk %ld ", blocks.size()); + clLog()->Dump(XrdCl::AppMsg, "File::ReadBlocksFromDisk %ld ", blocks.size()); const long long BS = m_cfi.GetBufferSize(); long long total = 0; @@ -380,6 +381,8 @@ int File::ReadBlocksFromDisk(std::list& blocks, int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) { + clLog()->Dump(XrdCl::AppMsg, "File::Read() begin "); + const long long BS = m_cfi.GetBufferSize(); // lock @@ -426,6 +429,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Is there room for one more RAM Block? if ( cache()->RequestRAMBlock()) { + clLog()->Debug(XrdCl::AppMsg, "File::Read() request block to fetch %d", block_idx); Block *b = RequestBlock(block_idx, false); inc_ref_count(b); blks_to_process.push_back(b); @@ -434,6 +438,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Nope ... read this directly without caching. else { + clLog()->Debug(XrdCl::AppMsg, "File::Read() direct block %d", block_idx); blks_direct.push_back(block_idx); } } @@ -479,10 +484,10 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) BlockList_i bi = blks_to_process.begin(); while (bi != blks_to_process.end()) { - clLog()->Dump(XrdCl::AppMsg, "searcing for blocks finished"); + clLog()->Dump(XrdCl::AppMsg, "File::Read() searcing for blocks finished"); if ((*bi)->is_finished()) { - clLog()->Dump(XrdCl::AppMsg, "found finished block"); + clLog()->Dump(XrdCl::AppMsg, "File::Read() found finished block"); finished.push_back(*bi); BlockList_i bj = bi++; blks_to_process.erase(bj); @@ -496,11 +501,11 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) if (finished.empty()) { - clLog()->Dump(XrdCl::AppMsg, "wait block begin"); + clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block begin"); m_downloadCond.Wait(); - clLog()->Dump(XrdCl::AppMsg, "wait block end"); + clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block end"); continue; } @@ -515,7 +520,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) long long off_in_block; // offset in block long long size_to_copy; // size to copy - clLog()->Dump(XrdCl::AppMsg, "Block finished ok."); + clLog()->Dump(XrdCl::AppMsg, "File::Read() Block finished ok."); overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); bytes_read += size_to_copy; @@ -524,7 +529,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } else // it has failed ... krap up. { - clLog()->Error(XrdCl::AppMsg, "Block finished with eorror."); + clLog()->Error(XrdCl::AppMsg, "File::Read() Block finished with eorror."); bytes_read = -1; errno = (*bi)->m_errno; break; @@ -539,7 +544,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Fourth, make sure all direct requests have arrived if (direct_handler != 0) { - clLog()->Error(XrdCl::AppMsg, "Read() waiting for direct requests."); + clLog()->Error(XrdCl::AppMsg, "File::Read() waiting for direct requests."); XrdSysCondVarHelper _lck(direct_handler->m_cond); if (direct_handler->m_to_wait > 0) @@ -785,29 +790,40 @@ void File::Prefetch() if (!stopping) { + + clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch enter to check download status \n"); XrdSysCondVarHelper _lck(m_downloadCond); - if (m_cfi.IsComplete() == false) + clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch enter to check download status BEGIN \n"); + if (m_cfi.IsComplete() == false && m_block_map.size() < 1600) { - int block_idx = -1; + clLog()->Debug(XrdCl::AppMsg, "Prefetch::Prefetch begin, block size %ld", m_block_map.size()); + // check index not on disk and not in RAM - for (int f = 0; f < m_cfi.GetSizeInBits(); ++f) + bool found = false; + for (int f=0; f < m_cfi.GetSizeInBits(); ++f) { + clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch test bit %d", f); if (!m_cfi.TestBit(f)) { - BlockMap_i bi = m_block_map.find(block_idx); + BlockMap_i bi = m_block_map.find(f); if (bi == m_block_map.end()) { - block_idx = f; + clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch take block %d", f); + Block *b = RequestBlock(f, true); + inc_ref_count(b); + m_prefetchReadCnt++; + m_prefetchScore = m_prefetchHitCnt/m_prefetchReadCnt; + found = true; break; } } } - - if (cache()->RequestRAMBlock()) { - m_prefetchReadCnt++; - m_prefetchScore = m_prefetchHitCnt/m_prefetchReadCnt; - Block *b = RequestBlock(block_idx, true); - inc_ref_count(b); + if (!found) { + clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch no free blcok found "); + m_cfi.CheckComplete(); + if (m_cfi.IsComplete() == false) + clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch This shoulf not happedn !!!"); } + clLog()->Debug(XrdCl::AppMsg, "Prefetch::Prefetch end"); } } @@ -844,6 +860,8 @@ float File::GetPrefetchScore() const //______________________________________________________________________________ void File::MarkPrefetch() { + + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"File::MarkPrefetch()"); m_stateCond.Lock(); m_prefetchCurrentCnt++; m_stateCond.UnLock(); @@ -865,6 +883,7 @@ void File::UnMarkPrefetch() void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) { + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"BlockResponseHandler::HandleResponse()"); m_block->m_file->ProcessBlockResponse(m_block, status); delete status; @@ -878,7 +897,7 @@ void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"Direct block response"); + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"DirectResponseHandler::HandleRespons()"); XrdSysCondVarHelper _lck(m_cond); --m_to_wait; diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 2a6c9d66e8c..7844f29d484 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -49,7 +49,7 @@ IOEntireFile::~IOEntireFile() bool IOEntireFile::ioActive() { - printf("called ioActive ...\n"); + printf("called ioActive ...\n"); return m_file->InitiateClose(); } diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index 9323e5748d7..b08d8cbc51b 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -56,14 +56,14 @@ Info::~Info() //______________________________________________________________________________ -void Info::ResizeBits(int s, bool prefetch_stat) +void Info::ResizeBits(int s, bool init_prefetch_buff) { m_sizeInBits = s; m_buff_fetched = (unsigned char*)malloc(GetSizeInBytes()); m_buff_write_called = (unsigned char*)malloc(GetSizeInBytes()); memset(m_buff_fetched, 0, GetSizeInBytes()); memset(m_buff_write_called, 0, GetSizeInBytes()); - if (prefetch_stat) { + if (init_prefetch_buff) { m_buff_prefetch = (unsigned char*)malloc(GetSizeInBytes()); memset(m_buff_prefetch, 0, GetSizeInBytes()); } @@ -72,7 +72,7 @@ void Info::ResizeBits(int s, bool prefetch_stat) //______________________________________________________________________________ -int Info::Read(XrdOssDF* fp) +int Info::Read(XrdOssDF* fp, bool init_prefetch_buff ) { // does not need lock, called only in File::Open // before File::Run() starts @@ -95,6 +95,13 @@ int Info::Read(XrdOssDF* fp) off += fp->Read(&m_accessCnt, off, sizeof(int)); clLog()->Dump(XrdCl::AppMsg, "Info:::Read() complete %d access_cnt %d", m_complete, m_accessCnt); + + + if (init_prefetch_buff) { + m_buff_prefetch = (unsigned char*)malloc(GetSizeInBytes()); + memset(m_buff_prefetch, 0, GetSizeInBytes()); + } + return off; } diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index e2508216820..adcbeb6a54d 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -100,7 +100,7 @@ namespace XrdFileCache //! //! @return number of bytes read //--------------------------------------------------------------------- - int Read(XrdOssDF* fp); + int Read(XrdOssDF* fp, bool init_prefetch = false); //--------------------------------------------------------------------- //! Write number of blocks and read buffer size From 3fd83e94fac88f24f63bf8af00a92dd1edff9d2a Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 30 Jun 2015 12:20:54 -0700 Subject: [PATCH 022/171] Don't increase ref count to prefetching blocks. --- src/XrdFileCache/XrdFileCacheFile.cc | 29 ++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 2bba3dbaa59..1a5b0d294c0 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -95,24 +95,37 @@ m_prefetchCurrentCnt(0) File::~File() { - clLog()->Debug(XrdCl::AppMsg, "File::~File() %p %s", (void*)this, lPath()); - cache()->RemoveWriteQEntriesFor(this); + clLog()->Debug(XrdCl::AppMsg, "File::~File() %p %s", (void*)this, lPath()); + /* + cache()->RemoveWriteQEntriesFor(this); + m_downloadCond.Lock(); + for ( BlockMap_t::iterator it = m_block_map.begin(); it != m_block_map.end(); ++it) + { + if ((it)->second->m_downloaded || (it)->second->m_errno ) + dec_ref_count(it->second); + } + m_downloadCond.UnLock(); + */ clLog()->Info(XrdCl::AppMsg, "File::~File() check write queues ...%s", lPath()); + - // can I do anythong to stop waiting for asyc read callbacks ? while (true) { m_stateCond.Lock(); bool isStopped = m_stopping; bool isPrefetching = (m_prefetchCurrentCnt > 0); m_stateCond.UnLock(); + + clLog()->Info(XrdCl::AppMsg, "File::~File() stopped %d isPrefetching %d ...%s", isStopped, isPrefetching, lPath()); if ((isPrefetching == false) && isStopped) { m_downloadCond.Lock(); bool blockMapEmpty = m_block_map.empty(); + int blocksize = (int)m_block_map.size(); m_downloadCond.UnLock(); if ( blockMapEmpty) break; + clLog()->Info(XrdCl::AppMsg, "File::~File() block size %d ", blocksize); } XrdSysTimer::Wait(10); } @@ -123,7 +136,7 @@ File::~File() bool do_sync = false; { XrdSysMutexHelper _lck(&m_syncStatusMutex); - if (m_non_flushed_cnt > 0) + if (m_non_flushed_cnt > 0 || !m_writes_during_sync.empty()) { do_sync = true; m_in_sync = true; @@ -693,7 +706,7 @@ void File::dec_ref_count(Block* b) // AMT this function could actually be member of File ... would be nicer // called under block lock b-> m_refcnt--; - if ( b->m_refcnt == 0 ) { + if ( b->m_refcnt == 0 || ( b->m_prefetch && b->m_refcnt ==-1)) { int i = b->m_offset/BufferSize(); size_t ret = m_block_map.erase(i); if (ret != 1) { @@ -794,7 +807,7 @@ void File::Prefetch() clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch enter to check download status \n"); XrdSysCondVarHelper _lck(m_downloadCond); clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch enter to check download status BEGIN \n"); - if (m_cfi.IsComplete() == false && m_block_map.size() < 1600) + if (m_cfi.IsComplete() == false && m_block_map.size() < 100) { clLog()->Debug(XrdCl::AppMsg, "Prefetch::Prefetch begin, block size %ld", m_block_map.size()); @@ -808,8 +821,8 @@ void File::Prefetch() BlockMap_i bi = m_block_map.find(f); if (bi == m_block_map.end()) { clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch take block %d", f); - Block *b = RequestBlock(f, true); - inc_ref_count(b); + RequestBlock(f, true); + /// inc_ref_count(b); AMT don't increase it, there is no-one to annulate it 0 m_prefetchReadCnt++; m_prefetchScore = m_prefetchHitCnt/m_prefetchReadCnt; found = true; From a3134e594dd6a3c6086d095678ae08fa6565278c Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Sun, 5 Jul 2015 08:40:34 -0700 Subject: [PATCH 023/171] Add missing initialization. --- src/XrdFileCache/XrdFileCacheFile.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 583e572d865..911938a01c1 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -65,7 +65,7 @@ namespace XrdFileCache bool m_downloaded; Block(File *f, long long off, int size, bool m_prefetch) : - m_offset(off), m_file(f), m_prefetch(false), m_refcnt(0), + m_offset(off), m_file(f), m_prefetch(m_prefetch), m_refcnt(0), m_errno(0), m_downloaded(false) { m_buff.resize(size); From cc8cdcffc2676161189f2d22a0ea7e4569884414 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Sun, 5 Jul 2015 09:47:41 -0700 Subject: [PATCH 024/171] Remove prefetching block from map in ProcessBlockResponse if it does not have a ref count. --- src/XrdFileCache/XrdFileCacheFile.cc | 49 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 1a5b0d294c0..6f35c4f28ba 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -93,31 +93,29 @@ m_prefetchCurrentCnt(0) Open(); } + File::~File() { - clLog()->Debug(XrdCl::AppMsg, "File::~File() %p %s", (void*)this, lPath()); - /* - cache()->RemoveWriteQEntriesFor(this); - m_downloadCond.Lock(); - for ( BlockMap_t::iterator it = m_block_map.begin(); it != m_block_map.end(); ++it) - { - if ((it)->second->m_downloaded || (it)->second->m_errno ) - dec_ref_count(it->second); - } - m_downloadCond.UnLock(); - */ + clLog()->Debug(XrdCl::AppMsg, "File::~File() %p %s", (void*)this, lPath()); + + // assert if in debug mode + if (XrdCl::DefaultEnv::GetLog()->GetLevel() >= XrdCl::Log::DebugMsg ) { + m_stateCond.Lock(); + assert (m_stopping == true); + m_stateCond.UnLock(); + } + + cache()->RemoveWriteQEntriesFor(this); + clLog()->Info(XrdCl::AppMsg, "File::~File() check write queues ...%s", lPath()); - while (true) { m_stateCond.Lock(); - bool isStopped = m_stopping; bool isPrefetching = (m_prefetchCurrentCnt > 0); m_stateCond.UnLock(); - clLog()->Info(XrdCl::AppMsg, "File::~File() stopped %d isPrefetching %d ...%s", isStopped, isPrefetching, lPath()); - if ((isPrefetching == false) && isStopped) + if (isPrefetching == false) { m_downloadCond.Lock(); bool blockMapEmpty = m_block_map.empty(); @@ -695,18 +693,22 @@ void File::Sync() m_infoFile->Fsync(); } +//______________________________________________________________________________ void File::inc_ref_count(Block* b) { + // Method always called under lock b->m_refcnt++; } +//______________________________________________________________________________ + + void File::dec_ref_count(Block* b) { - // AMT this function could actually be member of File ... would be nicer - // called under block lock + // Method always called under lock b-> m_refcnt--; - if ( b->m_refcnt == 0 || ( b->m_prefetch && b->m_refcnt ==-1)) { + if ( b->m_refcnt == 0) { int i = b->m_offset/BufferSize(); size_t ret = m_block_map.erase(i); if (ret != 1) { @@ -722,23 +724,29 @@ void File::dec_ref_count(Block* b) void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { - clLog()->Dump(XrdCl::AppMsg, "File::ProcessBlockResponse %d ",(int)(b->m_offset/BufferSize())); + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d ",(int)(b->m_offset/BufferSize())); m_downloadCond.Lock(); if (status->IsOK()) { b->m_downloaded = true; + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d finidhed %d",(int)(b->m_offset/BufferSize()), b->is_finished()); if (!m_stopping) { // AMT theoretically this should be under state lock, but then are double locks inc_ref_count(b); cache()->AddWriteTask(b, true); } + else { + if (b->m_prefetch && ( b->m_refcnt == 0)) { + m_block_map.erase((int)(b->m_offset/BufferSize())); + } + } } else { // AMT how long to keep? // when to retry? - + clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %d error ",(int)(b->m_offset/BufferSize())); XrdPosixMap::Result(*status); b->set_error_and_free(errno); @@ -809,7 +817,6 @@ void File::Prefetch() clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch enter to check download status BEGIN \n"); if (m_cfi.IsComplete() == false && m_block_map.size() < 100) { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Prefetch begin, block size %ld", m_block_map.size()); // check index not on disk and not in RAM bool found = false; From f4e8a0bf9214e39145e4f75293675062f416605e Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Sun, 5 Jul 2015 10:06:19 -0700 Subject: [PATCH 025/171] Removed failed prefetched block in ProcessRequest(). --- src/XrdFileCache/XrdFileCacheFile.cc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 6f35c4f28ba..0d447de80cd 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -736,11 +736,6 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) inc_ref_count(b); cache()->AddWriteTask(b, true); } - else { - if (b->m_prefetch && ( b->m_refcnt == 0)) { - m_block_map.erase((int)(b->m_offset/BufferSize())); - } - } } else { @@ -752,10 +747,18 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) b->set_error_and_free(errno); errno = 0; - // ??? AMT temprary commented out + // ??? AMT temprary commented out -- throw away failed attempts // inc_ref_count(b); } + + // case when there is a prefetch that is failed or prefetch that stopped has just been initated + if (b->m_refcnt == 0) { + assert(b->m_prefetch); + m_block_map.erase((int)(b->m_offset/BufferSize())); + } + + m_downloadCond.Broadcast(); m_downloadCond.UnLock(); From 041129d4790a1e603fa1e1044f6bd417d71dcc85 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Sun, 5 Jul 2015 10:12:58 -0700 Subject: [PATCH 026/171] return immediately if read from disk failes in Read(). --- src/XrdFileCache/XrdFileCacheFile.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 0d447de80cd..b0e1554b3bb 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -481,7 +481,9 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } else { - bytes_read = rc; // AMT ?? should there be an immediate return + bytes_read = rc; // AMT ?? should there be an immediate return + clLog()->Error(XrdCl::AppMsg, "File::Read() failed to read from disk."); + return rc; } } // Third, loop over blocks that are available or incoming From b411cc2d2a94e1296047c600728f1876fd2a79f7 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Sun, 5 Jul 2015 10:22:22 -0700 Subject: [PATCH 027/171] Write statistics in info file. --- src/XrdFileCache/XrdFileCacheFile.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index b0e1554b3bb..d279771f73a 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -407,7 +407,6 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) m_downloadCond.Lock(); - size_t msize = m_block_map.size(); // XXX Check for blocks to free? Later ... const int idx_first = iUserOff / BS; @@ -428,11 +427,13 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) inc_ref_count(bi->second); blks_to_process.push_front(bi->second); + m_stats.m_BytesRam++; } // On disk? else if (m_cfi.TestBit(block_idx)) { blks_on_disk.push_back(block_idx); + m_stats.m_BytesDisk++; } // Then we have to get it ... else @@ -444,13 +445,14 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) Block *b = RequestBlock(block_idx, false); inc_ref_count(b); blks_to_process.push_back(b); - ++msize; + m_stats.m_BytesRam++; } // Nope ... read this directly without caching. else { clLog()->Debug(XrdCl::AppMsg, "File::Read() direct block %d", block_idx); blks_direct.push_back(block_idx); + m_stats.m_BytesMissed++; } } } From 94e44162269f2d8c3fe534f4c57d786118c3e992 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Sun, 5 Jul 2015 10:28:57 -0700 Subject: [PATCH 028/171] Save score as float. --- src/XrdFileCache/XrdFileCacheFile.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 911938a01c1..0505f2565f4 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -125,7 +125,7 @@ namespace XrdFileCache int m_prefetchReadCnt; int m_prefetchHitCnt; - int m_prefetchScore; //cached + float m_prefetchScore; //cached int m_prefetchCurrentCnt; public: From 2c135f211a838bb9fec0e0a07682ccd0fafd4bac Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Sun, 5 Jul 2015 10:29:32 -0700 Subject: [PATCH 029/171] Print info message about prefetch score at the end of destructor. --- src/XrdFileCache/XrdFileCacheFile.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index d279771f73a..b2bbd832639 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -167,6 +167,12 @@ File::~File() m_infoFile = NULL; } delete m_syncer; + + + + + // print just for curiosity + clLog()->Debug(XrdCl::AppMsg, "File::~File() ended, prefetch score ...%d/%d=%.2f", m_prefetchReadCnt, m_prefetchHitCnt, m_prefetchScore); } bool File::InitiateClose() From 60516e3915e5f5854e8c0286252fae49f63060af Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jul 2015 13:18:28 -0700 Subject: [PATCH 030/171] Reserver ram resources in prefetching. --- src/XrdFileCache/XrdFileCache.cc | 39 +++++++++++++++------------- src/XrdFileCache/XrdFileCacheFile.cc | 7 ++++- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index b2b678240c0..1a3ad8bf9fe 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -217,15 +217,8 @@ Cache::RegisterPrefetchFile(File* file) if (Factory::GetInstance().RefConfiguration().m_prefetch) { - if (Factory::GetInstance().RefConfiguration().m_hdfsmode) { XrdSysMutexHelper lock(&m_prefetch_mutex); m_files.push_back(file); - } - else - { - // don't need to lock it becuse it File object is created in constructor of IOEntireFile - m_files.push_back(file); - } } } //______________________________________________________________________________ @@ -266,18 +259,28 @@ Cache::GetNextFileToPrefetch() void Cache::Prefetch() { + const static int limitRAM= Factory::GetInstance().RefConfiguration().m_NRamBuffers * 0.7; + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch thread start"); - - while (true) { - File* f = GetNextFileToPrefetch(); - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch got file (%p)", f); - if (f) { - f->Prefetch(); - } - else { - // wait for new file, AMT should I wait for the signal instead ??? - XrdSysTimer::Wait(1); + while (true) { + bool doPrefetch = false; + m_RAMblock_mutex.Lock(); + if (m_RAMblocks_used < limitRAM) + doPrefetch = true; + m_RAMblock_mutex.UnLock(); + + if (doPrefetch) { + File* f = GetNextFileToPrefetch(); + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch got file (%p)", f); + if (f) { + f->Prefetch(); + continue; + } } - } + + // wait for new file or more resources, AMT should I wait for the signal instead ??? + XrdSysTimer::Wait(1); + + } } diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index b2bbd832639..9be06a3f979 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -720,6 +720,7 @@ void File::dec_ref_count(Block* b) b-> m_refcnt--; if ( b->m_refcnt == 0) { int i = b->m_offset/BufferSize(); + delete m_block_map[i]; size_t ret = m_block_map.erase(i); if (ret != 1) { clLog()->Error(XrdCl::AppMsg, "File::OnBlockZeroRefCount did not erase %d from map.", i); @@ -765,7 +766,10 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) // case when there is a prefetch that is failed or prefetch that stopped has just been initated if (b->m_refcnt == 0) { assert(b->m_prefetch); - m_block_map.erase((int)(b->m_offset/BufferSize())); + cache()->RAMBlockReleased(); + int bidx = (int)(b->m_offset/BufferSize()); + delete m_block_map[bidx]; + m_block_map.erase(bidx); } @@ -841,6 +845,7 @@ void File::Prefetch() BlockMap_i bi = m_block_map.find(f); if (bi == m_block_map.end()) { clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch take block %d", f); + cache()->RequestRAMBlock(); RequestBlock(f, true); /// inc_ref_count(b); AMT don't increase it, there is no-one to annulate it 0 m_prefetchReadCnt++; From f29ea20f6c189abebb880ea346963bf0358eceae Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 17 Jul 2015 11:34:02 -0700 Subject: [PATCH 031/171] Fix class name in log messages. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 7844f29d484..1c20cd88917 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -69,7 +69,7 @@ XrdOucCacheIO *IOEntireFile::Detach() int IOEntireFile::Read (char *buff, long long off, int size) { - clLog()->Debug(XrdCl::AppMsg, "IO::Read() [%p] %lld@%d %s", this, off, size, m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() [%p] %lld@%d %s", this, off, size, m_io.Path()); // protect from reads over the file size if (off >= m_io.FSize()) @@ -86,19 +86,23 @@ int IOEntireFile::Read (char *buff, long long off, int size) ssize_t retval = 0; retval = m_file->Read(buff, off, size); - clLog()->Debug(XrdCl::AppMsg, "IO::Read() read from File retval = %d %s", retval, m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() read from File retval = %d %s", retval, m_io.Path()); if (retval > 0) { bytes_read += retval; buff += retval; size -= retval; + // XXXX MT: the logick here is strange, see versions in + // alja/pfc-async-prefetch and in master. + // Also, what if retval == 0? + if ((size > 0)) - clLog()->Debug(XrdCl::AppMsg, "IO::Read() missed %d bytes %s", size, m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() missed %d bytes %s", size, m_io.Path()); } if (retval < 0) { - clLog()->Error(XrdCl::AppMsg, "IO::Read(), origin bytes read %d %s", retval, m_io.Path()); + clLog()->Error(XrdCl::AppMsg, "IOEntireFile::Read(), origin bytes read %d %s", retval, m_io.Path()); } return (retval < 0) ? retval : bytes_read; @@ -111,7 +115,7 @@ int IOEntireFile::Read (char *buff, long long off, int size) */ int IOEntireFile::ReadV (const XrdOucIOVec *readV, int n) { - clLog()->Warning(XrdCl::AppMsg, "IO::ReadV(), get %d requests %s", n, m_io.Path()); + clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::ReadV(), get %d requests %s", n, m_io.Path()); return m_file->ReadV(readV, n); From e6fc4569008c099b896186de15e2ae73a3617960 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 17 Jul 2015 11:35:23 -0700 Subject: [PATCH 032/171] Check write queue size. --- src/XrdFileCache/XrdFileCache.cc | 18 +++++------------- src/XrdFileCache/XrdFileCache.hh | 4 ++++ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 1a3ad8bf9fe..0e5d3343208 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -99,24 +99,15 @@ void Cache::Detach(XrdOucCacheIO* io) delete io; } - -// XXXX MT: is the following needed ??? -//______________________________________________________________________________ -void Cache::getFilePathFromURL(const char* iUrl, std::string &result) const -{ - XrdCl::URL url(iUrl); - result = Factory::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); -} - -// XXXX MT: is the following needed ??? //______________________________________________________________________________ bool Cache::HaveFreeWritingSlots() { + + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::HaveFreeWritingSlots() %ld", s_writeQ.size); const static size_t maxWriteWaits=500; return s_writeQ.size < maxWriteWaits; } - //______________________________________________________________________________ void Cache::AddWriteTask(Block* b, bool fromRead) @@ -266,7 +257,7 @@ Cache::Prefetch() while (true) { bool doPrefetch = false; m_RAMblock_mutex.Lock(); - if (m_RAMblocks_used < limitRAM) + if (m_RAMblocks_used < limitRAM && HaveFreeWritingSlots()) doPrefetch = true; m_RAMblock_mutex.UnLock(); @@ -275,12 +266,13 @@ Cache::Prefetch() XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch got file (%p)", f); if (f) { f->Prefetch(); + XrdSysTimer::Wait(1); continue; } } // wait for new file or more resources, AMT should I wait for the signal instead ??? - XrdSysTimer::Wait(1); + XrdSysTimer::Wait(10); } } diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 7ab75e92e22..ff703a2a08b 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -68,6 +68,10 @@ namespace XrdFileCache //--------------------------------------------------------------------- void AddWriteTask(Block* b, bool from_read); + //--------------------------------------------------------------------- + //! Check write queue size is not over limit. + //--------------------------------------------------------------------- + bool HaveFreeWritingSlots(); //--------------------------------------------------------------------- //! \brief Remove blocks from write queue which belong to given prefetch. From a38088b3424aca04c17f887f60976c2332f092a7 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 17 Jul 2015 11:36:05 -0700 Subject: [PATCH 033/171] Change log messages. --- src/XrdFileCache/XrdFileCacheFile.cc | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 9be06a3f979..de768d7648d 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -105,7 +105,7 @@ File::~File() m_stateCond.UnLock(); } - cache()->RemoveWriteQEntriesFor(this); + // cache()->RemoveWriteQEntriesFor(this); clLog()->Info(XrdCl::AppMsg, "File::~File() check write queues ...%s", lPath()); @@ -814,7 +814,7 @@ void File::AppendIOStatToFileInfo() } else { - clLog()->Warning(XrdCl::AppMsg, "Prefetch::AppendIOStatToFileInfo() info file not opened %s", lPath()); + clLog()->Warning(XrdCl::AppMsg, "File::AppendIOStatToFileInfo() info file not opened %s", lPath()); } } @@ -829,22 +829,22 @@ void File::Prefetch() if (!stopping) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch enter to check download status \n"); + // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status \n"); XrdSysCondVarHelper _lck(m_downloadCond); - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch enter to check download status BEGIN \n"); - if (m_cfi.IsComplete() == false && m_block_map.size() < 100) + // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN \n"); + if (m_cfi.IsComplete() == false && m_block_map.size() < 1) { // check index not on disk and not in RAM bool found = false; for (int f=0; f < m_cfi.GetSizeInBits(); ++f) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch test bit %d", f); + // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch test bit %d", f); if (!m_cfi.TestBit(f)) { BlockMap_i bi = m_block_map.find(f); if (bi == m_block_map.end()) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch take block %d", f); + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch take block %d", f); cache()->RequestRAMBlock(); RequestBlock(f, true); /// inc_ref_count(b); AMT don't increase it, there is no-one to annulate it 0 @@ -856,12 +856,12 @@ void File::Prefetch() } } if (!found) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch no free blcok found "); + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch no free blcok found "); m_cfi.CheckComplete(); if (m_cfi.IsComplete() == false) - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Prefetch This shoulf not happedn !!!"); + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch This shoulf not happedn !!!"); } - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Prefetch end"); + clLog()->Debug(XrdCl::AppMsg, "File::Prefetch end"); } } @@ -898,8 +898,6 @@ float File::GetPrefetchScore() const //______________________________________________________________________________ void File::MarkPrefetch() { - - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"File::MarkPrefetch()"); m_stateCond.Lock(); m_prefetchCurrentCnt++; m_stateCond.UnLock(); From 6b56beaecf88f57976201a0dee25b5a533e509a3 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 17 Jul 2015 14:47:50 -0700 Subject: [PATCH 034/171] Fix spelling in log. --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index de768d7648d..645bda852e8 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -742,7 +742,7 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) if (status->IsOK()) { b->m_downloaded = true; - clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d finidhed %d",(int)(b->m_offset/BufferSize()), b->is_finished()); + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d finished %d",(int)(b->m_offset/BufferSize()), b->is_finished()); if (!m_stopping) { // AMT theoretically this should be under state lock, but then are double locks inc_ref_count(b); cache()->AddWriteTask(b, true); From dc56300893e7750af1bf02b912f6d7b2801a40e2 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 20 Jul 2015 11:16:47 -0700 Subject: [PATCH 035/171] Add log message in Cache::HaveFreeWritingSlots. --- src/XrdFileCache/XrdFileCache.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 0e5d3343208..328f635046e 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -103,10 +103,14 @@ void Cache::Detach(XrdOucCacheIO* io) bool Cache::HaveFreeWritingSlots() { - - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::HaveFreeWritingSlots() %ld", s_writeQ.size); const static size_t maxWriteWaits=500; - return s_writeQ.size < maxWriteWaits; + if ( s_writeQ.size < maxWriteWaits) { + return true; + } + else { + XrdCl::DefaultEnv::GetLog()->Debug(XrdCl::AppMsg, "Cache::HaveFreeWritingSlots() negative", s_writeQ.size); + return false; + } } //______________________________________________________________________________ void @@ -212,6 +216,7 @@ Cache::RegisterPrefetchFile(File* file) m_files.push_back(file); } } + //______________________________________________________________________________ void @@ -263,7 +268,6 @@ Cache::Prefetch() if (doPrefetch) { File* f = GetNextFileToPrefetch(); - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch got file (%p)", f); if (f) { f->Prefetch(); XrdSysTimer::Wait(1); @@ -273,6 +277,5 @@ Cache::Prefetch() // wait for new file or more resources, AMT should I wait for the signal instead ??? XrdSysTimer::Wait(10); - } } From cec6a5afc808825f6aff9f107771868640aa37d2 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 20 Jul 2015 11:17:35 -0700 Subject: [PATCH 036/171] Changes in log messages. --- src/XrdFileCache/XrdFileCacheFile.cc | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 645bda852e8..57721988ff0 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -96,7 +96,7 @@ m_prefetchCurrentCnt(0) File::~File() { - clLog()->Debug(XrdCl::AppMsg, "File::~File() %p %s", (void*)this, lPath()); + clLog()->Debug(XrdCl::AppMsg, "File::~File() enter %p %s", (void*)this, lPath()); // assert if in debug mode if (XrdCl::DefaultEnv::GetLog()->GetLevel() >= XrdCl::Log::DebugMsg ) { @@ -172,7 +172,7 @@ File::~File() // print just for curiosity - clLog()->Debug(XrdCl::AppMsg, "File::~File() ended, prefetch score ...%d/%d=%.2f", m_prefetchReadCnt, m_prefetchHitCnt, m_prefetchScore); + clLog()->Debug(XrdCl::AppMsg, "File::~File() ended, prefetch score ...%d/%d=%.2f", m_prefetchHitCnt, m_prefetchReadCnt, m_prefetchScore); } bool File::InitiateClose() @@ -708,6 +708,7 @@ void File::Sync() void File::inc_ref_count(Block* b) { // Method always called under lock + clLog()->Error(XrdCl::AppMsg, "File::inc_ref_count %d %s ",b->m_refcnt, lPath()); b->m_refcnt++; } @@ -717,7 +718,10 @@ void File::inc_ref_count(Block* b) void File::dec_ref_count(Block* b) { // Method always called under lock + clLog()->Error(XrdCl::AppMsg, "File::dec_ref_count %d %s ",b->m_refcnt, lPath()); b-> m_refcnt--; + assert(b->m_refcnt >= 0); + if ( b->m_refcnt == 0) { int i = b->m_offset/BufferSize(); delete m_block_map[i]; @@ -735,14 +739,14 @@ void File::dec_ref_count(Block* b) void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { - clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d ",(int)(b->m_offset/BufferSize())); + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d %s",(int)(b->m_offset/BufferSize()), lPath()); m_downloadCond.Lock(); if (status->IsOK()) { b->m_downloaded = true; - clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d finished %d",(int)(b->m_offset/BufferSize()), b->is_finished()); + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d finished %d %s",(int)(b->m_offset/BufferSize()), b->is_finished(), lPath()); if (!m_stopping) { // AMT theoretically this should be under state lock, but then are double locks inc_ref_count(b); cache()->AddWriteTask(b, true); @@ -752,7 +756,7 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { // AMT how long to keep? // when to retry? - clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %d error ",(int)(b->m_offset/BufferSize())); + clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %d error %s",(int)(b->m_offset/BufferSize()), lPath()); XrdPosixMap::Result(*status); b->set_error_and_free(errno); @@ -831,8 +835,8 @@ void File::Prefetch() // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status \n"); XrdSysCondVarHelper _lck(m_downloadCond); - // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN \n"); - if (m_cfi.IsComplete() == false && m_block_map.size() < 1) + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN \n"); + if (m_cfi.IsComplete() == false && m_block_map.size() < 3) { // check index not on disk and not in RAM From 04c654b655ef7e34dcc31ccc483862fd7a63746b Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 20 Jul 2015 14:22:23 -0700 Subject: [PATCH 037/171] Unregister file from cache prefetch when it is complete. --- src/XrdFileCache/XrdFileCacheFile.cc | 30 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 57721988ff0..bd862032911 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -398,8 +398,6 @@ int File::ReadBlocksFromDisk(std::list& blocks, int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() begin "); - const long long BS = m_cfi.GetBufferSize(); // lock @@ -432,6 +430,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // XXXX Or just push it and handle errors in one place later? inc_ref_count(bi->second); + clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count for existing %d %s", block_idx, lPath()); blks_to_process.push_front(bi->second); m_stats.m_BytesRam++; } @@ -448,6 +447,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) if ( cache()->RequestRAMBlock()) { clLog()->Debug(XrdCl::AppMsg, "File::Read() request block to fetch %d", block_idx); + clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count new %d %s", block_idx, lPath()); Block *b = RequestBlock(block_idx, false); inc_ref_count(b); blks_to_process.push_back(b); @@ -598,6 +598,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) for (BlockList_i bi = blks_processed.begin(); bi != blks_processed.end(); ++bi) { + clLog()->Dump(XrdCl::AppMsg, "File::Read() dec_ref_count %d %s", ((*bi)->m_offset/BufferSize()), lPath()); dec_ref_count(*bi); // XXXX stamp block } @@ -649,6 +650,7 @@ void File::WriteBlockToDisk(Block* b) { XrdSysCondVarHelper _lck(m_downloadCond); + clLog()->Dump(XrdCl::AppMsg, "File::WriteToDisk() dec_ref_count %d %s", pfIdx, lPath()); dec_ref_count(b); } @@ -718,7 +720,7 @@ void File::inc_ref_count(Block* b) void File::dec_ref_count(Block* b) { // Method always called under lock - clLog()->Error(XrdCl::AppMsg, "File::dec_ref_count %d %s ",b->m_refcnt, lPath()); + // clLog()->Error(XrdCl::AppMsg, "File::dec_ref_count %d %s ",b->m_refcnt, lPath()); b-> m_refcnt--; assert(b->m_refcnt >= 0); @@ -739,7 +741,7 @@ void File::dec_ref_count(Block* b) void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { - clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d %s",(int)(b->m_offset/BufferSize()), lPath()); + // clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d %s",(int)(b->m_offset/BufferSize()), lPath()); m_downloadCond.Lock(); @@ -748,6 +750,7 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) b->m_downloaded = true; clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d finished %d %s",(int)(b->m_offset/BufferSize()), b->is_finished(), lPath()); if (!m_stopping) { // AMT theoretically this should be under state lock, but then are double locks + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse inc_ref_count %d %s\n", (int)(b->m_offset/BufferSize()), lPath()); inc_ref_count(b); cache()->AddWriteTask(b, true); } @@ -835,7 +838,7 @@ void File::Prefetch() // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status \n"); XrdSysCondVarHelper _lck(m_downloadCond); - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN \n"); + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN %s \n", lPath()); if (m_cfi.IsComplete() == false && m_block_map.size() < 3) { @@ -851,7 +854,6 @@ void File::Prefetch() clLog()->Dump(XrdCl::AppMsg, "File::Prefetch take block %d", f); cache()->RequestRAMBlock(); RequestBlock(f, true); - /// inc_ref_count(b); AMT don't increase it, there is no-one to annulate it 0 m_prefetchReadCnt++; m_prefetchScore = m_prefetchHitCnt/m_prefetchReadCnt; found = true; @@ -862,11 +864,21 @@ void File::Prefetch() if (!found) { clLog()->Dump(XrdCl::AppMsg, "File::Prefetch no free blcok found "); m_cfi.CheckComplete(); - if (m_cfi.IsComplete() == false) - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch This shoulf not happedn !!!"); + // assert (m_cfi.IsComplete()); + // it is possible all missing blocks are in map but downlaoded status is still not complete + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch -- unlikely to happen ... file seem to be complete %s", lPath()); + // remove block from map + cache()->DeRegisterPrefetchFile(this); } clLog()->Debug(XrdCl::AppMsg, "File::Prefetch end"); } + else if (m_block_map.size() >= 3) { + clLog()->Dump(XrdCl::AppMsg,"skip prefetch %s ", lPath()); + for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it ) + { + clLog()->Dump(XrdCl::AppMsg, "block idx = %d, ref_count = %d, prefetch=%d [%s]", it->first, it->second->m_refcnt, it->second->m_prefetch, lPath()); + } + } } UnMarkPrefetch(); @@ -924,7 +936,7 @@ void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) { XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"BlockResponseHandler::HandleResponse()"); - m_block->m_file->ProcessBlockResponse(m_block, status); + m_block->m_file->ProcessBlockResponse(m_block, status); delete status; delete response; From 133d5eaf97c91c7578ffe36c238bd1162fc626d6 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 20 Jul 2015 14:24:00 -0700 Subject: [PATCH 038/171] Temporary change -- Randomize prefetch file list. --- src/XrdFileCache/XrdFileCache.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 328f635046e..29475a52dbb 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -193,7 +193,7 @@ Cache::RAMBlockReleased() //============================================================================== //======================= PREFETCH =================================== //============================================================================== - +/* namespace { struct prefetch_less_than { @@ -202,7 +202,7 @@ struct prefetch_less_than return (struct1->GetPrefetchScore() < struct2->GetPrefetchScore()); } }myobject; -} +}*/ //______________________________________________________________________________ void @@ -243,7 +243,8 @@ Cache::GetNextFileToPrefetch() return 0; } - std::sort(m_files.begin(), m_files.end(), myobject); + // std::sort(m_files.begin(), m_files.end(), myobject); + std::random_shuffle(m_files.begin(), m_files.end()); File* f = m_files.back(); f->MarkPrefetch(); return f; @@ -270,7 +271,7 @@ Cache::Prefetch() File* f = GetNextFileToPrefetch(); if (f) { f->Prefetch(); - XrdSysTimer::Wait(1); + // XrdSysTimer::Wait(1); continue; } } From 7f16d3795ba45fc12b0e9cb8b7f0f34955838bdd Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 21 Jul 2015 15:14:47 -0700 Subject: [PATCH 039/171] Keep failed blocks in map. --- src/XrdFileCache/XrdFileCacheFile.cc | 54 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index bd862032911..72f46baee00 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -90,7 +90,9 @@ m_prefetchScore(1), m_prefetchCurrentCnt(0) { clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); - Open(); + if (!Open()) { + clLog()->Error(XrdCl::AppMsg, "File::File() Open failed %s !!!", m_input.Path()); + } } @@ -105,7 +107,7 @@ File::~File() m_stateCond.UnLock(); } - // cache()->RemoveWriteQEntriesFor(this); + cache()->RemoveWriteQEntriesFor(this); clLog()->Info(XrdCl::AppMsg, "File::~File() check write queues ...%s", lPath()); @@ -118,6 +120,15 @@ File::~File() if (isPrefetching == false) { m_downloadCond.Lock(); + // remove failed blocks + for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end();) { + if (it->second->is_finished() && it->second->m_refcnt == 1) // refcounf more than 1 if used by Read() + m_block_map.erase(it); + else + ++it; + } + + bool blockMapEmpty = m_block_map.empty(); int blocksize = (int)m_block_map.size(); m_downloadCond.UnLock(); @@ -197,7 +208,7 @@ bool File::InitiateClose() bool File::Open() { - // clLog()->Debug(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_input.Path()); + clLog()->Dump(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_input.Path()); XrdOss &m_output_fs = *Factory::GetInstance().GetOss(); // Create the data file itself. @@ -320,10 +331,10 @@ Block* File::RequestBlock(int i, bool prefetch) Block *b = new Block(this, off, this_bs, prefetch); // should block be reused to avoid recreation m_block_map[i] = b; + clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() %p idx=%d pOn=(%d) %s", (void*)b, i, prefetch, lPath()); client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); - clLog()->Debug(XrdCl::AppMsg, "RequestBlock() %d END", i); return b; } @@ -446,8 +457,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Is there room for one more RAM Block? if ( cache()->RequestRAMBlock()) { - clLog()->Debug(XrdCl::AppMsg, "File::Read() request block to fetch %d", block_idx); - clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count new %d %s", block_idx, lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count new %d %s", block_idx, lPath()); Block *b = RequestBlock(block_idx, false); inc_ref_count(b); blks_to_process.push_back(b); @@ -550,7 +560,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } else // it has failed ... krap up. { - clLog()->Error(XrdCl::AppMsg, "File::Read() Block finished with eorror."); + clLog()->Error(XrdCl::AppMsg, "File::Read() Block finished with error."); bytes_read = -1; errno = (*bi)->m_errno; break; @@ -590,8 +600,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) { XrdSysCondVarHelper _lck(m_downloadCond); - // XXXX stamp file - // AMT ??? fetched status stampled in WriteDisk callback , what dies stamp mean ?? + // XXXX stamp file ??? // blks_to_process can be non-empty, if we're exiting with an error. std::copy(blks_to_process.begin(), blks_to_process.end(), std::back_inserter(blks_processed)); @@ -650,7 +659,7 @@ void File::WriteBlockToDisk(Block* b) { XrdSysCondVarHelper _lck(m_downloadCond); - clLog()->Dump(XrdCl::AppMsg, "File::WriteToDisk() dec_ref_count %d %s", pfIdx, lPath()); + // clLog()->Dump(XrdCl::AppMsg, "File::WriteToDisk() dec_ref_count %d %s", pfIdx, lPath()); dec_ref_count(b); } @@ -669,7 +678,7 @@ void File::WriteBlockToDisk(Block* b) ++m_non_flushed_cnt; } - if (m_non_flushed_cnt >= 100) + if (m_non_flushed_cnt >= 100 && (m_cfi.IsComplete() && m_non_flushed_cnt > 0)) { schedule_sync = true; m_in_sync = true; @@ -710,7 +719,7 @@ void File::Sync() void File::inc_ref_count(Block* b) { // Method always called under lock - clLog()->Error(XrdCl::AppMsg, "File::inc_ref_count %d %s ",b->m_refcnt, lPath()); + // clLog()->Dump(XrdCl::AppMsg, "File::inc_ref_count %d %s ",b->m_refcnt, lPath()); b->m_refcnt++; } @@ -720,12 +729,12 @@ void File::inc_ref_count(Block* b) void File::dec_ref_count(Block* b) { // Method always called under lock - // clLog()->Error(XrdCl::AppMsg, "File::dec_ref_count %d %s ",b->m_refcnt, lPath()); b-> m_refcnt--; assert(b->m_refcnt >= 0); if ( b->m_refcnt == 0) { int i = b->m_offset/BufferSize(); + clLog()->Error(XrdCl::AppMsg, "File::dec_ref_count erase block (%p) %d %s ", (void*)b, i, lPath()); delete m_block_map[i]; size_t ret = m_block_map.erase(i); if (ret != 1) { @@ -741,7 +750,7 @@ void File::dec_ref_count(Block* b) void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { - // clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d %s",(int)(b->m_offset/BufferSize()), lPath()); + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d %s",(int)(b->m_offset/BufferSize()), lPath()); m_downloadCond.Lock(); @@ -761,25 +770,14 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) // when to retry? clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %d error %s",(int)(b->m_offset/BufferSize()), lPath()); XrdPosixMap::Result(*status); - + // AMT could notfiy global cache we dont need RAM for that block b->set_error_and_free(errno); errno = 0; - // ??? AMT temprary commented out -- throw away failed attempts - // inc_ref_count(b); + // ??? AMT how long to keep + inc_ref_count(b); } - - // case when there is a prefetch that is failed or prefetch that stopped has just been initated - if (b->m_refcnt == 0) { - assert(b->m_prefetch); - cache()->RAMBlockReleased(); - int bidx = (int)(b->m_offset/BufferSize()); - delete m_block_map[bidx]; - m_block_map.erase(bidx); - } - - m_downloadCond.Broadcast(); m_downloadCond.UnLock(); From 4d1ea0d166d0512c8f97d3937ef8307656a608e8 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 24 Jul 2015 11:47:26 -0700 Subject: [PATCH 040/171] Fix valgrind error in constructor. --- src/XrdFileCache/XrdFileCacheFile.cc | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 72f46baee00..07cd1e9cf61 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -121,11 +121,20 @@ File::~File() { m_downloadCond.Lock(); // remove failed blocks - for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end();) { - if (it->second->is_finished() && it->second->m_refcnt == 1) // refcounf more than 1 if used by Read() - m_block_map.erase(it); - else - ++it; + //for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end();) { + + BlockMap_i itr = m_block_map.begin(); + while (itr != m_block_map.end()) { + if (itr->second->is_finished() && itr->second->m_refcnt == 1) { + // refcounf more than 1 if used by Read() + BlockMap_i toErase = itr; + ++itr; + m_block_map.erase(toErase); + // Relase RAM AMT !!! + } + else { + ++itr; + } } @@ -734,7 +743,7 @@ void File::dec_ref_count(Block* b) if ( b->m_refcnt == 0) { int i = b->m_offset/BufferSize(); - clLog()->Error(XrdCl::AppMsg, "File::dec_ref_count erase block (%p) %d %s ", (void*)b, i, lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::dec_ref_count erase block (%p) %d %s ", (void*)b, i, lPath()); delete m_block_map[i]; size_t ret = m_block_map.erase(i); if (ret != 1) { From fe6b6be375ea338d809028fb5afdce6a73f4c89d Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 24 Jul 2015 14:34:25 -0700 Subject: [PATCH 041/171] Handle case where File::Read() does not wait blocks to be processed. --- src/XrdFileCache/XrdFileCache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 29475a52dbb..7765f743ab0 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -239,7 +239,7 @@ Cache::GetNextFileToPrefetch() { XrdSysMutexHelper lock(&m_prefetch_mutex); if (m_files.empty()) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::GetNextFileToPrefetch ... no open files"); + // XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::GetNextFileToPrefetch ... no open files"); return 0; } From 176f81e2bbcb14b679048fe847a7a8bd6e7d8cc1 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 24 Jul 2015 14:47:01 -0700 Subject: [PATCH 042/171] Handle case where File::Read() does not wait blocks to be processed. --- src/XrdFileCache/XrdFileCacheFile.cc | 42 +++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 07cd1e9cf61..4379a61e6d0 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -130,7 +130,7 @@ File::~File() BlockMap_i toErase = itr; ++itr; m_block_map.erase(toErase); - // Relase RAM AMT !!! + cache()->RAMBlockReleased(); } else { ++itr; @@ -340,7 +340,7 @@ Block* File::RequestBlock(int i, bool prefetch) Block *b = new Block(this, off, this_bs, prefetch); // should block be reused to avoid recreation m_block_map[i] = b; - clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() %p idx=%d pOn=(%d) %s", (void*)b, i, prefetch, lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); @@ -450,9 +450,9 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // XXXX Or just push it and handle errors in one place later? inc_ref_count(bi->second); - clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count for existing %d %s", block_idx, lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count for existing b=%p %d %s", (void*)bi->second, block_idx, lPath()); blks_to_process.push_front(bi->second); - m_stats.m_BytesRam++; + m_stats.m_BytesRam++; // AMT what if block fails } // On disk? else if (m_cfi.TestBit(block_idx)) @@ -508,13 +508,14 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } else { - bytes_read = rc; // AMT ?? should there be an immediate return + bytes_read = rc; clLog()->Error(XrdCl::AppMsg, "File::Read() failed to read from disk."); - return rc; + // AMT commented line below should not be an immediate return, can have block refcount increased and map increased + // return rc; } } // Third, loop over blocks that are available or incoming - while ( ! blks_to_process.empty() && bytes_read >= 0) // AMT : do I need this loop ? + while ( (! blks_to_process.empty()) && (bytes_read >= 0)) { BlockList_t finished; @@ -524,10 +525,10 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) BlockList_i bi = blks_to_process.begin(); while (bi != blks_to_process.end()) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() searcing for blocks finished"); + // clLog()->Dump(XrdCl::AppMsg, "File::Read() searcing for block %p finished", (void*)(*bi)); if ((*bi)->is_finished()) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() found finished block"); + clLog()->Dump(XrdCl::AppMsg, "File::Read() found finished block %p", (void*)(*bi)); finished.push_back(*bi); BlockList_i bj = bi++; blks_to_process.erase(bj); @@ -560,7 +561,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) long long off_in_block; // offset in block long long size_to_copy; // size to copy - clLog()->Dump(XrdCl::AppMsg, "File::Read() Block finished ok."); + // clLog()->Dump(XrdCl::AppMsg, "File::Read() Block finished ok."); overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); bytes_read += size_to_copy; @@ -584,7 +585,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Fourth, make sure all direct requests have arrived if (direct_handler != 0) { - clLog()->Error(XrdCl::AppMsg, "File::Read() waiting for direct requests."); + clLog()->Debug(XrdCl::AppMsg, "File::Read() waiting for direct requests."); XrdSysCondVarHelper _lck(direct_handler->m_cond); if (direct_handler->m_to_wait > 0) @@ -616,7 +617,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) for (BlockList_i bi = blks_processed.begin(); bi != blks_processed.end(); ++bi) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() dec_ref_count %d %s", ((*bi)->m_offset/BufferSize()), lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::Read() dec_ref_count b=%p, %d %s", (void*)(*bi), ((*bi)->m_offset/BufferSize()), lPath()); dec_ref_count(*bi); // XXXX stamp block } @@ -728,8 +729,8 @@ void File::Sync() void File::inc_ref_count(Block* b) { // Method always called under lock - // clLog()->Dump(XrdCl::AppMsg, "File::inc_ref_count %d %s ",b->m_refcnt, lPath()); b->m_refcnt++; + clLog()->Dump(XrdCl::AppMsg, "File::inc_ref_count b=%p, %d %s ",(void*)b, b->m_refcnt, lPath()); } //______________________________________________________________________________ @@ -741,11 +742,13 @@ void File::dec_ref_count(Block* b) b-> m_refcnt--; assert(b->m_refcnt >= 0); - if ( b->m_refcnt == 0) { + //AMT ... this is ugly, ... File::Read() can decrease ref count before waiting to be , prefetch starts with refcnt 0 + if ( b->m_refcnt == 0 && b->is_finished()) { int i = b->m_offset/BufferSize(); clLog()->Dump(XrdCl::AppMsg, "File::dec_ref_count erase block (%p) %d %s ", (void*)b, i, lPath()); delete m_block_map[i]; size_t ret = m_block_map.erase(i); + // AMT free ram counter if (ret != 1) { clLog()->Error(XrdCl::AppMsg, "File::OnBlockZeroRefCount did not erase %d from map.", i); } @@ -759,10 +762,10 @@ void File::dec_ref_count(Block* b) void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { - clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d %s",(int)(b->m_offset/BufferSize()), lPath()); m_downloadCond.Lock(); + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %p, %d %s",(void*)b,(int)(b->m_offset/BufferSize()), lPath()); if (status->IsOK()) { b->m_downloaded = true; @@ -777,7 +780,7 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { // AMT how long to keep? // when to retry? - clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %d error %s",(int)(b->m_offset/BufferSize()), lPath()); + clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %p %d error %s",(void*)b,(int)(b->m_offset/BufferSize()), lPath()); XrdPosixMap::Result(*status); // AMT could notfiy global cache we dont need RAM for that block b->set_error_and_free(errno); @@ -877,15 +880,15 @@ void File::Prefetch() // remove block from map cache()->DeRegisterPrefetchFile(this); } - clLog()->Debug(XrdCl::AppMsg, "File::Prefetch end"); - } + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch end"); + } /* else if (m_block_map.size() >= 3) { clLog()->Dump(XrdCl::AppMsg,"skip prefetch %s ", lPath()); for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it ) { clLog()->Dump(XrdCl::AppMsg, "block idx = %d, ref_count = %d, prefetch=%d [%s]", it->first, it->second->m_refcnt, it->second->m_prefetch, lPath()); } - } + }*/ } UnMarkPrefetch(); @@ -943,6 +946,7 @@ void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) { XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"BlockResponseHandler::HandleResponse()"); + m_block->m_file->ProcessBlockResponse(m_block, status); delete status; From 652ba12164667230215e97ac5815fa9672bbc420 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 29 Jul 2015 12:05:35 -0700 Subject: [PATCH 043/171] Add util function to release block. --- src/XrdFileCache/XrdFileCacheFile.cc | 43 +++++++++++++++++----------- src/XrdFileCache/XrdFileCacheFile.hh | 1 + 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 4379a61e6d0..e5f2de92c20 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -125,12 +125,12 @@ File::~File() BlockMap_i itr = m_block_map.begin(); while (itr != m_block_map.end()) { - if (itr->second->is_finished() && itr->second->m_refcnt == 1) { - // refcounf more than 1 if used by Read() + bool dropWqOrError = itr->second->is_finished() && itr->second->m_refcnt == 1; // refcounf more than 1 if used by Read() + bool prefetchWCancel = itr->second->m_prefetch && itr->second->m_refcnt == 0 && itr->second->m_downloaded; + if (dropWqOrError || prefetchWCancel) { BlockMap_i toErase = itr; ++itr; - m_block_map.erase(toErase); - cache()->RAMBlockReleased(); + free_block(toErase->second); } else { ++itr; @@ -143,7 +143,12 @@ File::~File() m_downloadCond.UnLock(); if ( blockMapEmpty) break; - clLog()->Info(XrdCl::AppMsg, "File::~File() block size %d ", blocksize); + clLog()->Info(XrdCl::AppMsg, "File::~File() mapsize %d %s", blocksize,lPath()); + for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it) { + Block* b = it->second; + clLog()->Debug(XrdCl::AppMsg, "File::~File() block idx=%d p=%d rcnt=%d dwnd=%d %s", + b->m_offset/m_cfi.GetBufferSize(), b->m_prefetch, b->m_refcnt, b->m_downloaded, lPath()); + } } XrdSysTimer::Wait(10); } @@ -744,20 +749,24 @@ void File::dec_ref_count(Block* b) //AMT ... this is ugly, ... File::Read() can decrease ref count before waiting to be , prefetch starts with refcnt 0 if ( b->m_refcnt == 0 && b->is_finished()) { - int i = b->m_offset/BufferSize(); - clLog()->Dump(XrdCl::AppMsg, "File::dec_ref_count erase block (%p) %d %s ", (void*)b, i, lPath()); - delete m_block_map[i]; - size_t ret = m_block_map.erase(i); - // AMT free ram counter - if (ret != 1) { - clLog()->Error(XrdCl::AppMsg, "File::OnBlockZeroRefCount did not erase %d from map.", i); - } - else { - cache()->RAMBlockReleased(); - } + free_block(b); } } +void File::free_block(Block* b) +{ + int i = b->m_offset/BufferSize(); + clLog()->Dump(XrdCl::AppMsg, "File::free_block block (%p) %d %s ", (void*)b, i, lPath()); + delete m_block_map[i]; + size_t ret = m_block_map.erase(i); + if (ret != 1) { + clLog()->Error(XrdCl::AppMsg, "File::OnBlockZeroRefCount did not erase %d from map.", i); + } + else { + cache()->RAMBlockReleased(); + } +} + //------------------------------------------------------------------------------ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) @@ -848,7 +857,7 @@ void File::Prefetch() // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status \n"); XrdSysCondVarHelper _lck(m_downloadCond); - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN %s \n", lPath()); + // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN %s \n", lPath()); if (m_cfi.IsComplete() == false && m_block_map.size() < 3) { diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 0505f2565f4..4ab4f91af0a 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -201,6 +201,7 @@ namespace XrdFileCache void inc_ref_count(Block*); void dec_ref_count(Block*); + void free_block(Block*); }; From d9b414b878c1b929cae99b2013745ea73aa58a18 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 5 Aug 2015 15:07:14 -0700 Subject: [PATCH 044/171] Report client error codes in log. --- src/XrdFileCache/XrdFileCacheFile.cc | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index e5f2de92c20..f28a5aaec57 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -344,12 +344,18 @@ Block* File::RequestBlock(int i, bool prefetch) long long this_bs = (i == last_block) ? m_input.FSize() - off : BS; Block *b = new Block(this, off, this_bs, prefetch); // should block be reused to avoid recreation - m_block_map[i] = b; - clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); - client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); + XrdCl::XRootDStatus status = client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); + if (status.IsOK()) { + clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); + m_block_map[i] = b; + return b; + } + else { - return b; + clLog()->Error(XrdCl::AppMsg, "File::RequestBlock() error %d, this = %p, b=%p, this idx=%d pOn=(%d) %s", status.code, (void*)this, (void*)b, i, prefetch, lPath()); + return 0; + } } //------------------------------------------------------------------------------ @@ -376,7 +382,12 @@ int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, overlap(*ii, BS, req_off, req_size, off, blk_off, size); - client.Read(*ii * BS + blk_off, size, req_buf + off, handler); + XrdCl::Status status = client.Read(*ii * BS + blk_off, size, req_buf + off, handler); + if (!status.IsOK()) + { + clLog()->Error(XrdCl::AppMsg, "File::RequestBlocksDirect error %s\n", lPath()); + return total; + } total += size; } @@ -473,6 +484,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) { clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count new %d %s", block_idx, lPath()); Block *b = RequestBlock(block_idx, false); + break; //AMT ??? inc_ref_count(b); blks_to_process.push_back(b); m_stats.m_BytesRam++; @@ -789,7 +801,7 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { // AMT how long to keep? // when to retry? - clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %p %d error %s",(void*)b,(int)(b->m_offset/BufferSize()), lPath()); + clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %p %d error=%d, [%s] %s",(void*)b,(int)(b->m_offset/BufferSize()), status->code, status->GetErrorMessage().c_str(), lPath()); XrdPosixMap::Result(*status); // AMT could notfiy global cache we dont need RAM for that block b->set_error_and_free(errno); @@ -870,7 +882,7 @@ void File::Prefetch() { BlockMap_i bi = m_block_map.find(f); if (bi == m_block_map.end()) { - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch take block %d", f); + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch take block %d %s", f, lPath()); cache()->RequestRAMBlock(); RequestBlock(f, true); m_prefetchReadCnt++; From 993d77044767c9309d3ef02765d28251dac4de50 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 6 Aug 2015 11:46:35 -0700 Subject: [PATCH 045/171] Add path info in File::InitiateClose log. --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index f28a5aaec57..f0c7ce9c86f 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -203,7 +203,7 @@ File::~File() bool File::InitiateClose() { // Retruns true if delay is needed - clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start", lPath()); + clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start %s", lPath()); cache()->DeRegisterPrefetchFile(this); From 86d94151fabdb5c6bf8d0ad0f22e4581e93b0192 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 10 Aug 2015 09:40:27 -0700 Subject: [PATCH 046/171] Add asserts. --- src/XrdFileCache/XrdFileCacheFile.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index f0c7ce9c86f..c27c7bed2d4 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -466,7 +466,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // XXXX Or just push it and handle errors in one place later? inc_ref_count(bi->second); - clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count for existing b=%p %d %s", (void*)bi->second, block_idx, lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count for existing block %p %d %s", (void*)bi->second, block_idx, lPath()); blks_to_process.push_front(bi->second); m_stats.m_BytesRam++; // AMT what if block fails } @@ -484,7 +484,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) { clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count new %d %s", block_idx, lPath()); Block *b = RequestBlock(block_idx, false); - break; //AMT ??? + assert(b); inc_ref_count(b); blks_to_process.push_back(b); m_stats.m_BytesRam++; @@ -680,6 +680,7 @@ void File::WriteBlockToDisk(Block* b) int pfIdx = (b->m_offset - m_offset)/m_cfi.GetBufferSize(); m_downloadCond.Lock(); + assert((m_cfi.TestBit(pfIdx) == false) && "Block not yet fetched."); m_cfi.SetBitFetched(pfIdx); m_downloadCond.UnLock(); From 98525169d5c7c8a04d1caebc8e0a308b544fd929 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 10 Aug 2015 11:40:06 -0700 Subject: [PATCH 047/171] Check free writing slots before requesting ram block in File::Read(). --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index c27c7bed2d4..1e5a9d02336 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -480,7 +480,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) else { // Is there room for one more RAM Block? - if ( cache()->RequestRAMBlock()) + if ( cache()->HaveFreeWritingSlots() && cache()->RequestRAMBlock()) { clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count new %d %s", block_idx, lPath()); Block *b = RequestBlock(block_idx, false); From ca271c8b052d5c9b510c471fd2ec2e1c6f7922b2 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 10 Aug 2015 11:42:05 -0700 Subject: [PATCH 048/171] Increase severity of message if bytes were missed. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 1c20cd88917..dc4ad208e52 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -96,14 +96,16 @@ int IOEntireFile::Read (char *buff, long long off, int size) // XXXX MT: the logick here is strange, see versions in // alja/pfc-async-prefetch and in master. // Also, what if retval == 0? - - if ((size > 0)) - clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() missed %d bytes %s", size, m_io.Path()); - } + } if (retval < 0) { clLog()->Error(XrdCl::AppMsg, "IOEntireFile::Read(), origin bytes read %d %s", retval, m_io.Path()); } + else if ((size > 0)) + { + clLog()->Error(XrdCl::AppMsg, "IOEntireFile::Read() missed %d bytes %s", size, m_io.Path()); + bytes_read += retval; + } return (retval < 0) ? retval : bytes_read; } From 47328dc72b57da61cc5a8a44364d14e6e851a469 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 11 Aug 2015 16:20:11 -0700 Subject: [PATCH 049/171] Fix double read size from direct reads. --- src/XrdFileCache/XrdFileCacheFile.cc | 65 ++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 1e5a9d02336..a5bc4120e49 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClFile.hh" @@ -315,6 +316,7 @@ namespace blk_off = ovlp_beg - beg; size = ovlp_end - ovlp_beg; + assert(size <= blk_size); return true; } else @@ -363,8 +365,6 @@ Block* File::RequestBlock(int i, bool prefetch) int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, char* req_buf, long long req_off, long long req_size) { - - clLog()->Dump(XrdCl::AppMsg, "RequestBlockDirect %ld ", blocks.size()); XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; const long long BS = m_cfi.GetBufferSize(); @@ -386,9 +386,12 @@ int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, if (!status.IsOK()) { clLog()->Error(XrdCl::AppMsg, "File::RequestBlocksDirect error %s\n", lPath()); - return total; + //return total; + return -1; // AMT all reads should be canceled in this case + } + else { + clLog()->Dump(XrdCl::AppMsg, "RequestBlockDirect success %d %ld %s", *ii, size, lPath()); } - total += size; } @@ -401,7 +404,7 @@ int File::ReadBlocksFromDisk(std::list& blocks, char* req_buf, long long req_off, long long req_size) { - clLog()->Dump(XrdCl::AppMsg, "File::ReadBlocksFromDisk %ld ", blocks.size()); + clLog()->Dump(XrdCl::AppMsg, "File::ReadBlocksFromDisk %ld %s", blocks.size(), lPath()); const long long BS = m_cfi.GetBufferSize(); long long total = 0; @@ -418,12 +421,24 @@ int File::ReadBlocksFromDisk(std::list& blocks, overlap(*ii, BS, req_off, req_size, off, blk_off, size); long long rs = m_output->Read(req_buf + off, *ii * BS + blk_off, size); + clLog()->Dump(XrdCl::AppMsg, "File::ReadBlocksFromDisk block %d size %d %s", *ii, size, lPath()); + - if (rs < 0) + if (rs < 0) { + clLog()->Error(XrdCl::AppMsg, "File::ReadBlocksFromDisk neg retval %ld (%ld@%d) %s", rs, *ii * BS + blk_off, lPath()); return rs; + } + + + // AMT I think we should exit in this case too + if (rs !=size) { + clLog()->Error(XrdCl::AppMsg, "File::ReadBlocksFromDisk incomplete %ld (%ld@%d) %s", rs, *ii * BS + blk_off, lPath()); + return -1; + } total += rs; + CheckPrefetchStatDisk(*ii); } @@ -457,6 +472,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) for (int block_idx = idx_first; block_idx <= idx_last; ++block_idx) { + clLog()->Dump(XrdCl::AppMsg, "--- File::Read() idx %d %s \n", block_idx, lPath()); BlockMap_i bi = m_block_map.find(block_idx); // In RAM or incoming? @@ -466,13 +482,14 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // XXXX Or just push it and handle errors in one place later? inc_ref_count(bi->second); - clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count for existing block %p %d %s", (void*)bi->second, block_idx, lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p inc_ref_count for existing block %p %d %s", (void*)iUserBuff, (void*)bi->second, block_idx, lPath()); blks_to_process.push_front(bi->second); m_stats.m_BytesRam++; // AMT what if block fails } // On disk? else if (m_cfi.TestBit(block_idx)) { + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p read from disk %d %s", (void*)iUserBuff, block_idx, lPath()); blks_on_disk.push_back(block_idx); m_stats.m_BytesDisk++; } @@ -482,7 +499,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Is there room for one more RAM Block? if ( cache()->HaveFreeWritingSlots() && cache()->RequestRAMBlock()) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() inc_ref_count new %d %s", block_idx, lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p inc_ref_count new %d %s", (void*)iUserBuff, block_idx, lPath()); Block *b = RequestBlock(block_idx, false); assert(b); inc_ref_count(b); @@ -492,7 +509,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Nope ... read this directly without caching. else { - clLog()->Debug(XrdCl::AppMsg, "File::Read() direct block %d", block_idx); + clLog()->Debug(XrdCl::AppMsg, "File::Read() direct block %d %s", block_idx, lPath()); blks_direct.push_back(block_idx); m_stats.m_BytesMissed++; } @@ -513,12 +530,13 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) direct_handler = new DirectResponseHandler(blks_direct.size()); direct_size = RequestBlocksDirect(direct_handler, blks_direct, iUserBuff, iUserOff, iUserSize); - bytes_read += direct_size; // AMT added + clLog()->Dump(XrdCl::AppMsg, "File::Read() direct read %d. %s", direct_size, lPath()); } // Second, read blocks from disk. - if (!blks_on_disk.empty()) { + if ((!blks_on_disk.empty()) && (bytes_read >= 0)) { int rc = ReadBlocksFromDisk(blks_on_disk, iUserBuff, iUserOff, iUserSize); + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from disk %d. %s", (void*)iUserBuff, rc, lPath()); if (rc >= 0) { bytes_read += rc; @@ -526,11 +544,12 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) else { bytes_read = rc; - clLog()->Error(XrdCl::AppMsg, "File::Read() failed to read from disk."); + clLog()->Error(XrdCl::AppMsg, "File::Read() failed to read from disk. %s", lPath()); // AMT commented line below should not be an immediate return, can have block refcount increased and map increased // return rc; } } + // Third, loop over blocks that are available or incoming while ( (! blks_to_process.empty()) && (bytes_read >= 0)) { @@ -545,7 +564,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // clLog()->Dump(XrdCl::AppMsg, "File::Read() searcing for block %p finished", (void*)(*bi)); if ((*bi)->is_finished()) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() found finished block %p", (void*)(*bi)); + clLog()->Dump(XrdCl::AppMsg, "File::Read() found finished block %p %s", (void*)(*bi), lPath()); finished.push_back(*bi); BlockList_i bj = bi++; blks_to_process.erase(bj); @@ -559,16 +578,18 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) if (finished.empty()) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block begin"); + clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block begin %s", lPath()); m_downloadCond.Wait(); - clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block end"); + clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block end %s", lPath()); continue; } } + clLog()->Dump(XrdCl::AppMsg, "File::Read() bytes read before processing blocks %d %s\n", bytes_read, lPath()); + BlockList_i bi = finished.begin(); while (bi != finished.end()) { @@ -580,6 +601,8 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // clLog()->Dump(XrdCl::AppMsg, "File::Read() Block finished ok."); overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); + + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from finished block %d , size %d end %s", (void*)iUserBuff, (*bi)->m_offset/BS, size_to_copy, lPath()); memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); bytes_read += size_to_copy; @@ -587,7 +610,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } else // it has failed ... krap up. { - clLog()->Error(XrdCl::AppMsg, "File::Read() Block finished with error."); + clLog()->Error(XrdCl::AppMsg, "File::Read() Block finished with error %s.", lPath()); bytes_read = -1; errno = (*bi)->m_errno; break; @@ -599,10 +622,12 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) finished.clear(); } + clLog()->Dump(XrdCl::AppMsg, "File::Read() bytes read after processing blocks %d %s\n", bytes_read, lPath()); + // Fourth, make sure all direct requests have arrived - if (direct_handler != 0) + if ((direct_handler != 0) && (bytes_read >= 0 )) { - clLog()->Debug(XrdCl::AppMsg, "File::Read() waiting for direct requests."); + clLog()->Debug(XrdCl::AppMsg, "File::Read() waiting for direct requests %s.", lPath()); XrdSysCondVarHelper _lck(direct_handler->m_cond); if (direct_handler->m_to_wait > 0) @@ -622,12 +647,14 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) delete direct_handler; } + clLog()->Debug(XrdCl::AppMsg, "File::Read() before assert %s.", lPath()); + assert(iUserSize >= bytes_read); // Last, stamp and release blocks, release file. { XrdSysCondVarHelper _lck(m_downloadCond); - // XXXX stamp file ??? + // AMT what is stamp block ??? // blks_to_process can be non-empty, if we're exiting with an error. std::copy(blks_to_process.begin(), blks_to_process.end(), std::back_inserter(blks_processed)); From 28c5f2c0b77f0cf86ab140d2586bfd3d4384ebca Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 13 Aug 2015 10:06:18 -0700 Subject: [PATCH 050/171] Fix error in de-registering blocks from write queue at the distruction time. This change solves bug #61. --- src/XrdFileCache/XrdFileCache.cc | 4 ++++ src/XrdFileCache/XrdFileCacheFile.cc | 18 +++++++++++++----- src/XrdFileCache/XrdFileCacheFile.hh | 8 +++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 7765f743ab0..09d092ec5aa 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -136,7 +136,10 @@ void Cache::RemoveWriteQEntriesFor(File *iFile) { if ((*i)->m_file == iFile) { + + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Remove entries for %p path %s", (void*)(*i), iFile->lPath()); std::list::iterator j = i++; + iFile->BlockRemovedFromWriteQ(*j); s_writeQ.queue.erase(j); --s_writeQ.size; } @@ -162,6 +165,7 @@ Cache::ProcessWriteTasks() Block* block = s_writeQ.queue.front(); // AMT should not be back ??? s_writeQ.queue.pop_front(); s_writeQ.size--; + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::ProcessWriteTasks for %p path %s", (void*)(block), block->m_file->lPath()); s_writeQ.condVar.UnLock(); block->m_file->WriteBlockToDisk(block); diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index a5bc4120e49..223526b3eab 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -96,6 +96,13 @@ m_prefetchCurrentCnt(0) } } +void File::BlockRemovedFromWriteQ(Block* b) +{ + m_downloadCond.Lock(); + dec_ref_count(b); + clLog()->Dump(XrdCl::AppMsg, "File::BlockRemovedFromWriteQ() check write queues %p %d...%s", (void*)b, b->m_offset/m_cfi.GetBufferSize(), lPath()); + m_downloadCond.UnLock(); +} File::~File() { @@ -122,13 +129,9 @@ File::~File() { m_downloadCond.Lock(); // remove failed blocks - //for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end();) { - BlockMap_i itr = m_block_map.begin(); while (itr != m_block_map.end()) { - bool dropWqOrError = itr->second->is_finished() && itr->second->m_refcnt == 1; // refcounf more than 1 if used by Read() - bool prefetchWCancel = itr->second->m_prefetch && itr->second->m_refcnt == 0 && itr->second->m_downloaded; - if (dropWqOrError || prefetchWCancel) { + if (itr->second->is_failed() && itr->second->m_refcnt == 1) { BlockMap_i toErase = itr; ++itr; free_block(toErase->second); @@ -824,6 +827,11 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) inc_ref_count(b); cache()->AddWriteTask(b, true); } + else { + // there is no refcount +/- to remove dropped prefetched blocks on destruction + if (b->m_prefetch && (b->m_refcnt == 0)) + free_block(b); + } } else { diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 4ab4f91af0a..58cd0626e48 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -141,6 +141,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ ~File(); + void BlockRemovedFromWriteQ(Block*); //! Open file handle for data file and info file on local disk. bool Open(); @@ -175,6 +176,10 @@ namespace XrdFileCache void MarkPrefetch(); + + + //! Log path + const char* lPath() const; private: Block* RequestBlock(int i, bool prefetch); @@ -196,9 +201,6 @@ namespace XrdFileCache XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - //! Log path - const char* lPath() const; - void inc_ref_count(Block*); void dec_ref_count(Block*); void free_block(Block*); From 8840bc80a023004b47f9f4ab09ae232316780d53 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 13 Aug 2015 12:08:43 -0700 Subject: [PATCH 051/171] Fix evaluation of prefetch score. --- src/XrdFileCache/XrdFileCacheFile.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 223526b3eab..bad57321fef 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -922,7 +922,7 @@ void File::Prefetch() cache()->RequestRAMBlock(); RequestBlock(f, true); m_prefetchReadCnt++; - m_prefetchScore = m_prefetchHitCnt/m_prefetchReadCnt; + m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; found = true; break; } @@ -958,7 +958,7 @@ void File::CheckPrefetchStatRAM(Block* b) if (Factory::GetInstance().RefConfiguration().m_prefetch) { if (b->m_prefetch) { m_prefetchHitCnt++; - m_prefetchScore = m_prefetchHitCnt/m_prefetchReadCnt; + m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; } } } From 8358a75e34e71e48d2cbfa65ea3bd0c8d6e808c8 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 20 Aug 2015 14:54:47 -0700 Subject: [PATCH 052/171] Handle failed async requests. --- src/XrdFileCache/XrdFileCacheFile.cc | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index bad57321fef..762406454da 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -463,6 +463,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // passing the req to client is actually better. // unlock + bool preProcOK = true; m_downloadCond.Lock(); // XXX Check for blocks to free? Later ... @@ -504,7 +505,9 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) { clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p inc_ref_count new %d %s", (void*)iUserBuff, block_idx, lPath()); Block *b = RequestBlock(block_idx, false); - assert(b); + // assert(b); + preProcOK = false; + break; inc_ref_count(b); blks_to_process.push_back(b); m_stats.m_BytesRam++; @@ -516,11 +519,23 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) blks_direct.push_back(block_idx); m_stats.m_BytesMissed++; } - } + } + } + m_downloadCond.UnLock(); + + + + if (!preProcOK) { + for (BlockList_i i = blks_to_process.begin(); i!= blks_to_process.end(); ++i ) + dec_ref_count(*i); + return -1; // AMT ??? + } + + long long bytes_read = 0; // First, send out any direct requests. @@ -533,6 +548,13 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) direct_handler = new DirectResponseHandler(blks_direct.size()); direct_size = RequestBlocksDirect(direct_handler, blks_direct, iUserBuff, iUserOff, iUserSize); + // failed to send direct client request + if (direct_size < 0) { + for (BlockList_i i = blks_to_process.begin(); i!= blks_to_process.end(); ++i ) + dec_ref_count(*i); + delete direct_handler; + return -1; // AMT ??? + } clLog()->Dump(XrdCl::AppMsg, "File::Read() direct read %d. %s", direct_size, lPath()); } From 7eefc7b096864eb5dd3361cccb1cb3bac913fb27 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 21 Aug 2015 10:45:18 -0700 Subject: [PATCH 053/171] Fix error from the previous commit. --- src/XrdFileCache/XrdFileCacheFile.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 762406454da..084073f4b4f 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -506,8 +506,10 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p inc_ref_count new %d %s", (void*)iUserBuff, block_idx, lPath()); Block *b = RequestBlock(block_idx, false); // assert(b); + if (!b) { preProcOK = false; break; + } inc_ref_count(b); blks_to_process.push_back(b); m_stats.m_BytesRam++; From c0f53946c44f1842d61c0befa60cf977ad9a14ba Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 25 Aug 2015 12:06:50 -0700 Subject: [PATCH 054/171] Set errno in RequestBlock() and RequestBlocksDirect() if failed. --- src/XrdFileCache/XrdFileCacheFile.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 084073f4b4f..fdc91f3cebf 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -357,8 +357,8 @@ Block* File::RequestBlock(int i, bool prefetch) return b; } else { - clLog()->Error(XrdCl::AppMsg, "File::RequestBlock() error %d, this = %p, b=%p, this idx=%d pOn=(%d) %s", status.code, (void*)this, (void*)b, i, prefetch, lPath()); + XrdPosixMap::Result(status); return 0; } } @@ -389,7 +389,7 @@ int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, if (!status.IsOK()) { clLog()->Error(XrdCl::AppMsg, "File::RequestBlocksDirect error %s\n", lPath()); - //return total; + XrdPosixMap::Result(status); return -1; // AMT all reads should be canceled in this case } else { From d0df2df60e2600450115e96f285b9f223a52acaa Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 3 Sep 2015 13:19:45 -0700 Subject: [PATCH 055/171] Wait for signal in Prefetch() function when there is no open file --- src/XrdFileCache/XrdFileCache.cc | 37 ++++++++++++++++---------------- src/XrdFileCache/XrdFileCache.hh | 2 +- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 09d092ec5aa..b30ec394544 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -53,6 +53,7 @@ void *PrefetchThread(void* ptr) Cache::Cache(XrdOucCacheStats & stats) : XrdOucCache(), + m_prefetch_condVar(0), m_stats(stats), m_RAMblocks_used(0) { @@ -214,11 +215,16 @@ Cache::RegisterPrefetchFile(File* file) { // called from File::Open() - if (Factory::GetInstance().RefConfiguration().m_prefetch) - { - XrdSysMutexHelper lock(&m_prefetch_mutex); - m_files.push_back(file); - } + if (Factory::GetInstance().RefConfiguration().m_prefetch) + { + + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file BEGIN"); + m_prefetch_condVar.Lock(); + m_files.push_back(file); + m_prefetch_condVar.Signal(); + m_prefetch_condVar.UnLock(); + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file End"); + } } //______________________________________________________________________________ @@ -228,29 +234,30 @@ Cache::DeRegisterPrefetchFile(File* file) { // called from last line File::InitiateClose() - XrdSysMutexHelper lock(&m_prefetch_mutex); + m_prefetch_condVar.Lock(); for (FileList::iterator it = m_files.begin(); it != m_files.end(); ++it) { if (*it == file) { m_files.erase(it); break; } } + m_prefetch_condVar.UnLock(); } //______________________________________________________________________________ File* Cache::GetNextFileToPrefetch() { - XrdSysMutexHelper lock(&m_prefetch_mutex); + m_prefetch_condVar.Lock(); if (m_files.empty()) { - // XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::GetNextFileToPrefetch ... no open files"); - return 0; + m_prefetch_condVar.Wait(); } // std::sort(m_files.begin(), m_files.end(), myobject); std::random_shuffle(m_files.begin(), m_files.end()); File* f = m_files.back(); f->MarkPrefetch(); + m_prefetch_condVar.UnLock(); return f; } @@ -264,7 +271,7 @@ Cache::Prefetch() XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch thread start"); - while (true) { + while (true) { bool doPrefetch = false; m_RAMblock_mutex.Lock(); if (m_RAMblocks_used < limitRAM && HaveFreeWritingSlots()) @@ -273,14 +280,8 @@ Cache::Prefetch() if (doPrefetch) { File* f = GetNextFileToPrefetch(); - if (f) { - f->Prefetch(); - // XrdSysTimer::Wait(1); - continue; - } + f->Prefetch(); + XrdSysTimer::Wait(5); // ??? what is here a reasonable value } - - // wait for new file or more resources, AMT should I wait for the signal instead ??? - XrdSysTimer::Wait(10); } } diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index ff703a2a08b..af616ec0c75 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -103,7 +103,7 @@ namespace XrdFileCache //! Short log alias. XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - XrdSysMutex m_prefetch_mutex; //!< central lock for this class + XrdSysCondVar m_prefetch_condVar; //!< central lock for this class XrdOucCacheStats &m_stats; //!< global cache usage statistics XrdSysMutex m_RAMblock_mutex; //!< central lock for this class From 81a6006aa8e308ffa61343db94c27a911d4680c3 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 9 Sep 2015 14:13:12 -0700 Subject: [PATCH 056/171] in Prefetch wait 5ms if ram limit has been reached. --- src/XrdFileCache/XrdFileCache.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index b30ec394544..c2f6bad21d7 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -104,12 +104,12 @@ void Cache::Detach(XrdOucCacheIO* io) bool Cache::HaveFreeWritingSlots() { - const static size_t maxWriteWaits=500; + const static size_t maxWriteWaits=100000; if ( s_writeQ.size < maxWriteWaits) { return true; } else { - XrdCl::DefaultEnv::GetLog()->Debug(XrdCl::AppMsg, "Cache::HaveFreeWritingSlots() negative", s_writeQ.size); + XrdCl::DefaultEnv::GetLog()->Info(XrdCl::AppMsg, "Cache::HaveFreeWritingSlots() negative", s_writeQ.size); return false; } } @@ -272,16 +272,16 @@ Cache::Prefetch() XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch thread start"); while (true) { - bool doPrefetch = false; m_RAMblock_mutex.Lock(); - if (m_RAMblocks_used < limitRAM && HaveFreeWritingSlots()) - doPrefetch = true; + bool doPrefetch = (m_RAMblocks_used < limitRAM && HaveFreeWritingSlots()); m_RAMblock_mutex.UnLock(); if (doPrefetch) { File* f = GetNextFileToPrefetch(); f->Prefetch(); - XrdSysTimer::Wait(5); // ??? what is here a reasonable value + } + else { + XrdSysTimer::Wait(5); } } } From 13c6e23d20980f4c5a2c2acd0ace34543eb89a18 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 11 Sep 2015 15:57:29 -0700 Subject: [PATCH 057/171] Optimize random file access in GetNextFileToPrefetch(). --- src/XrdFileCache/XrdFileCache.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index c2f6bad21d7..39329047cce 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -254,6 +254,11 @@ Cache::GetNextFileToPrefetch() } // std::sort(m_files.begin(), m_files.end(), myobject); + + size_t l = m_files.size(); + int idx = rand() % l; + File* f = m_files[idx]; + std::random_shuffle(m_files.begin(), m_files.end()); File* f = m_files.back(); f->MarkPrefetch(); From 4af1628f77fe56d49bf5978b1f8d72a86a1145ed Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 14 Sep 2015 11:59:52 -0700 Subject: [PATCH 058/171] Optimize detection of file available for prefetching. --- src/XrdFileCache/XrdFileCache.cc | 3 - src/XrdFileCache/XrdFileCacheFile.cc | 87 +++++++++++++--------------- src/XrdFileCache/XrdFileCacheFile.hh | 4 ++ 3 files changed, 45 insertions(+), 49 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 39329047cce..a103b032176 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -258,9 +258,6 @@ Cache::GetNextFileToPrefetch() size_t l = m_files.size(); int idx = rand() % l; File* f = m_files[idx]; - - std::random_shuffle(m_files.begin(), m_files.end()); - File* f = m_files.back(); f->MarkPrefetch(); m_prefetch_condVar.UnLock(); return f; diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index fdc91f3cebf..bd786965df0 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -49,6 +49,8 @@ namespace XrdPosixGlobals namespace { const int PREFETCH_MAX_ATTEMPTS = 10; +const size_t PREFETCH_MAX_BLOCKS=5; + class DiskSyncer : public XrdJob { private: @@ -85,6 +87,7 @@ m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), m_non_flushed_cnt(0), m_in_sync(false), m_downloadCond(0), +m_prefetchState(kOn), m_prefetchReadCnt(0), m_prefetchHitCnt(0), m_prefetchScore(1), @@ -213,7 +216,8 @@ bool File::InitiateClose() m_stateCond.Lock(); m_stopping = true; - m_stateCond.UnLock(); + m_stateCond.UnLock(); + m_prefetchState = kCanceled; if (m_cfi.IsComplete()) return false; // AMT maybe map size is here more meaningfull, but might hold block state lock return true; } @@ -354,6 +358,10 @@ Block* File::RequestBlock(int i, bool prefetch) if (status.IsOK()) { clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); m_block_map[i] = b; + + if (m_prefetchState == kOn && m_block_map.size() > PREFETCH_MAX_BLOCKS) { + m_prefetchState = kHold; + } return b; } else { @@ -807,7 +815,6 @@ void File::inc_ref_count(Block* b) //______________________________________________________________________________ - void File::dec_ref_count(Block* b) { // Method always called under lock @@ -832,6 +839,9 @@ void File::free_block(Block* b) else { cache()->RAMBlockReleased(); } + + if (m_prefetchState == kHold && m_block_map.size() < PREFETCH_MAX_BLOCKS) + m_prefetchState = kOn; } //------------------------------------------------------------------------------ @@ -919,57 +929,42 @@ void File::AppendIOStatToFileInfo() //______________________________________________________________________________ void File::Prefetch() { - bool stopping = false; - m_stateCond.Lock(); - stopping = m_stopping; - m_stateCond.UnLock(); - - if (!stopping) { + if (m_prefetchState == kOn) { - // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status \n"); + // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status \n"); XrdSysCondVarHelper _lck(m_downloadCond); // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN %s \n", lPath()); - if (m_cfi.IsComplete() == false && m_block_map.size() < 3) - { - // check index not on disk and not in RAM - bool found = false; - for (int f=0; f < m_cfi.GetSizeInBits(); ++f) - { - // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch test bit %d", f); - if (!m_cfi.TestBit(f)) - { - BlockMap_i bi = m_block_map.find(f); - if (bi == m_block_map.end()) { - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch take block %d %s", f, lPath()); - cache()->RequestRAMBlock(); - RequestBlock(f, true); - m_prefetchReadCnt++; - m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; - found = true; - break; - } + // check index not on disk and not in RAM + bool found = false; + for (int f=0; f < m_cfi.GetSizeInBits(); ++f) + { + // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch test bit %d", f); + if (!m_cfi.TestBit(f)) + { + BlockMap_i bi = m_block_map.find(f); + if (bi == m_block_map.end()) { + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch take block %d %s", f, lPath()); + cache()->RequestRAMBlock(); + RequestBlock(f, true); + m_prefetchReadCnt++; + m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; + found = true; + break; } } - if (!found) { - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch no free blcok found "); - m_cfi.CheckComplete(); - // assert (m_cfi.IsComplete()); - // it is possible all missing blocks are in map but downlaoded status is still not complete - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch -- unlikely to happen ... file seem to be complete %s", lPath()); - // remove block from map - cache()->DeRegisterPrefetchFile(this); - } - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch end"); - } /* - else if (m_block_map.size() >= 3) { - clLog()->Dump(XrdCl::AppMsg,"skip prefetch %s ", lPath()); - for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it ) - { - clLog()->Dump(XrdCl::AppMsg, "block idx = %d, ref_count = %d, prefetch=%d [%s]", it->first, it->second->m_refcnt, it->second->m_prefetch, lPath()); - } - }*/ + } + if (!found) { + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch no free blcok found "); + m_cfi.CheckComplete(); + // assert (m_cfi.IsComplete()); + // it is possible all missing blocks are in map but downlaoded status is still not complete + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch -- unlikely to happen ... file seem to be complete %s", lPath()); + // remove block from map + cache()->DeRegisterPrefetchFile(this); + } + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch end"); } UnMarkPrefetch(); diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 58cd0626e48..3719bd64560 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -87,6 +87,8 @@ namespace XrdFileCache class File { private: + enum PrefetchState_e { kOn, kHold, kCanceled }; + XrdOucCacheIO &m_input; //!< original data source XrdOssDF *m_output; //!< file handle for data file on disk XrdOssDF *m_infoFile; //!< file handle for data-info file on disk @@ -123,6 +125,8 @@ namespace XrdFileCache Stats m_stats; //!< cache statistics, used in IO detach + PrefetchState_e m_prefetchState; + int m_prefetchReadCnt; int m_prefetchHitCnt; float m_prefetchScore; //cached From 4dbe4693453d3233e0edbea3619a6d6abb78db60 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 14 Sep 2015 12:53:42 -0700 Subject: [PATCH 059/171] DeRegister/Register files in case when they are ready for prefetching. --- src/XrdFileCache/XrdFileCacheFile.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index bd786965df0..53edffa52c9 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -49,7 +49,7 @@ namespace XrdPosixGlobals namespace { const int PREFETCH_MAX_ATTEMPTS = 10; -const size_t PREFETCH_MAX_BLOCKS=5; +const size_t PREFETCH_MAX_BLOCKS=10; class DiskSyncer : public XrdJob { @@ -361,6 +361,7 @@ Block* File::RequestBlock(int i, bool prefetch) if (m_prefetchState == kOn && m_block_map.size() > PREFETCH_MAX_BLOCKS) { m_prefetchState = kHold; + cache()->DeRegisterPrefetchFile(this); } return b; } @@ -840,8 +841,10 @@ void File::free_block(Block* b) cache()->RAMBlockReleased(); } - if (m_prefetchState == kHold && m_block_map.size() < PREFETCH_MAX_BLOCKS) + if (m_prefetchState == kHold && m_block_map.size() < PREFETCH_MAX_BLOCKS) { m_prefetchState = kOn; + cache()->RegisterPrefetchFile(this); + } } //------------------------------------------------------------------------------ From c824c79e5d68c890c8659a77ee31e11ecda2cd73 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 25 Sep 2015 12:23:36 -0700 Subject: [PATCH 060/171] First implementation of File::ReadV(). --- src/XrdFileCache.cmake | 5 +- src/XrdFileCache/XrdFileCacheFile.cc | 16 +- src/XrdFileCache/XrdFileCacheFile.hh | 20 ++ src/XrdFileCache/XrdFileCacheVRead.cc | 305 ++++++++++++++++++++++++++ 4 files changed, 332 insertions(+), 14 deletions(-) create mode 100644 src/XrdFileCache/XrdFileCacheVRead.cc diff --git a/src/XrdFileCache.cmake b/src/XrdFileCache.cmake index ebd5ca029b9..1104286c44d 100644 --- a/src/XrdFileCache.cmake +++ b/src/XrdFileCache.cmake @@ -17,8 +17,9 @@ add_library( ${LIB_XRD_FILECACHE} MODULE XrdFileCache/XrdFileCache.cc XrdFileCache/XrdFileCache.hh - XrdFileCache/XrdFileCacheFactory.cc XrdFileCache/XrdFileCacheFactory.hh - XrdFileCache/XrdFileCacheFile.cc XrdFileCache/XrdFileCacheFile.hh +XrdFileCache/XrdFileCacheFactory.cc XrdFileCache/XrdFileCacheFactory.hh +XrdFileCache/XrdFileCacheFile.cc XrdFileCache/XrdFileCacheFile.hh +XrdFileCache/XrdFileCacheVRead.cc XrdFileCache/XrdFileCacheStats.hh XrdFileCache/XrdFileCacheInfo.cc XrdFileCache/XrdFileCacheInfo.hh XrdFileCache/XrdFileCacheIO.hh diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 53edffa52c9..4ee4cf9ed2e 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -299,9 +299,9 @@ bool File::Open() -namespace -{ - bool overlap(int blk, // block to query +//namespace +//{ +bool File::overlap(int blk, // block to query long long blk_size, // long long req_off, // offset of user request int req_size, // size of user request @@ -331,7 +331,7 @@ namespace return false; } } -} +//} //------------------------------------------------------------------------------ @@ -896,14 +896,6 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) } //______________________________________________________________________________ - - -int File::ReadV (const XrdOucIOVec *readV, int n) -{ - return 0; -} - -//______________________________________________________________________________ const char* File::lPath() const { return m_temp_filename.c_str(); diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 3719bd64560..afdab32b0f0 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -38,6 +38,11 @@ class Log; namespace XrdFileCache { class BlockResponseHandler; class DirectResponseHandler; + + struct ReadVBlockListRAM; + struct ReadVChunkListRAM; + struct ReadVBlockListDisk; + struct ReadVChunkListDisk; } @@ -185,6 +190,16 @@ namespace XrdFileCache //! Log path const char* lPath() const; private: + + bool overlap(int blk, // block to query + long long blk_size, // + long long req_off, // offset of user request + int req_size, // size of user request + // output: + long long &off, // offset in user buffer + long long &blk_off, // offset in block + long long &size); + // Read Block* RequestBlock(int i, bool prefetch); int RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, @@ -193,7 +208,12 @@ namespace XrdFileCache int ReadBlocksFromDisk(IntList_t& blocks, char* req_buf, long long req_off, long long req_size); + // VRead + bool VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& blks_to_process, ReadVBlockListDisk& blks_on_disk, XrdCl::ChunkList& chunkVec); + int VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blks_on_disk); + int VReadProcessBlocks(const XrdOucIOVec *readV, int n, std::vector& blks_to_process, std::vector& blks_rocessed); + long long BufferSize(); void CheckPrefetchStatRAM(Block* b); diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc new file mode 100644 index 00000000000..93a890c7e83 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -0,0 +1,305 @@ + +#include "XrdFileCacheFile.hh" +#include "XrdFileCacheFactory.hh" +#include "XrdFileCache.hh" + +#include "XrdFileCacheInfo.hh" +#include "XrdFileCacheStats.hh" + +#include "XrdOss/XrdOss.hh" +#include "XrdCl/XrdClDefaultEnv.hh" +#include "XrdCl/XrdClFile.hh" +#include "XrdCl/XrdClXRootDResponses.hh" +#include "XrdPosix/XrdPosixFile.hh" + +namespace XrdFileCache { + + // a list of IOVec chuncks that match a given block index + // first element is block index, the following vector elements are chunk readv indicies + struct ReadVChunkListDisk { + ReadVChunkListDisk(int i) : block_idx(i) {}; + std::vector arr; + int block_idx; + }; + + struct ReadVChunkListRAM { + ReadVChunkListRAM(Block*b, std::vector * iarr) : block(b), arr(iarr) { }; + std::vector * arr; + Block* block; + }; + + // RAM + struct ReadVBlockListRAM { + std::vector bv; + bool AddEntry(Block* block, int chunkIdx) + { + for ( std::vector::iterator i = bv.begin(); i != bv.end(); ++i) + { + if (i->block == block) { + i->arr->push_back(chunkIdx); + return false; + } + } + bv.push_back(ReadVChunkListRAM(block, new std::vector)); + bv.back().arr->push_back(chunkIdx); + return true; + } + }; + + // Disk + struct ReadVBlockListDisk { + std::vector bv; + void AddEntry(int blockIdx, int chunkIdx) + { + for ( std::vector::iterator i = bv.begin(); i != bv.end(); ++i) + { + if (i->block_idx == blockIdx) { + i->arr.push_back(chunkIdx); + return; + } + } + bv.push_back(XrdFileCache::ReadVChunkListDisk(blockIdx)); + bv.back().arr.push_back(chunkIdx); + } + }; +} + +using namespace XrdFileCache; + +//------------------------------------- + +int File::ReadV (const XrdOucIOVec *readV, int n) +{ + int bytesRead = 0; + + ReadVBlockListRAM blocks_to_process; + std::vector blks_processed; + ReadVBlockListDisk blocks_on_disk; + XrdCl::ChunkList chunkVec; + DirectResponseHandler* direct_handler = 0; + if (!VReadPreProcess(readV, n, blocks_to_process, blocks_on_disk, chunkVec)) + bytesRead = -1; + + + // issue a client read + + if (bytesRead) { + if (!chunkVec.empty()) { + direct_handler = new DirectResponseHandler(1); + XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; + // TODO check interface in the client file + XrdCl::XRootDStatus vrStatus = client.VectorRead(chunkVec, (void*) 0, direct_handler); + if (!vrStatus.IsOK()) { + bytesRead = -1; + } + } + } + + // disk read + if (bytesRead) { + int dr = VReadFromDisk(readV, n, blocks_on_disk); + if (dr < 0) + bytesRead = dr; + else + bytesRead += dr; + } + + // read from cached blocks + if (bytesRead >=0) { + int br = VReadProcessBlocks(readV, n, blocks_to_process.bv, blks_processed); + if (br < 0) + bytesRead = br; + else + bytesRead += br; + } + + + // check direct requests have arrived, get bytes read from read handle + if (bytesRead >=0 && direct_handler != 0) + { + XrdSysCondVarHelper _lck(direct_handler->m_cond); + if (direct_handler->m_to_wait == 1) + { + direct_handler->m_cond.Wait(); + } + + if (direct_handler->m_errno == 0) + { + for (XrdCl::ChunkList::iterator i = chunkVec.begin(); i != chunkVec.end(); ++i) + bytesRead += i->length; + } + else + { + errno = direct_handler->m_errno; + bytesRead = -1; + } + } + + + { + XrdSysCondVarHelper _lck(m_downloadCond); + + // decrease ref count on the remaining blocks + // this happens in case read process has been broke due to previous errors + for (std::vector::iterator i = blocks_to_process.bv.begin(); i != blocks_to_process.bv.end(); ++i) + dec_ref_count(i->block); + + for (std::vector::iterator i = blks_processed.begin(); i != blks_processed.end(); ++i) + dec_ref_count(i->block); + } + + // remove objects on heap + delete direct_handler; + for (std::vector::iterator i = blocks_to_process.bv.begin(); i != blocks_to_process.bv.end(); ++i) + delete i->arr; + for (std::vector::iterator i = blks_processed.begin(); i != blks_processed.end(); ++i) + delete i->arr; + + return bytesRead; +} + +//______________________________________________________________________________ + + +bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& blocks_to_process, ReadVBlockListDisk& blocks_on_disk, XrdCl::ChunkList& chunkVec) +{ + XrdSysCondVarHelper _lck(m_downloadCond); + for (int iov_idx=0; iov_idxsecond, iov_idx)) + inc_ref_count(bi->second); + } + else if (m_cfi.TestBit(block_idx)) + { + blocks_on_disk.AddEntry(block_idx, iov_idx); + } + else { + if ( Factory::GetInstance().GetCache()->HaveFreeWritingSlots() && Factory::GetInstance().GetCache()->RequestRAMBlock()) + { + Block *b = RequestBlock(block_idx, false); + if (!b) return false; + blocks_to_process.AddEntry(b, iov_idx); + inc_ref_count(b); + } + else { + long long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + const long long BS = m_cfi.GetBufferSize(); + overlap(block_idx, BS, readV[iov_idx].offset, readV[iov_idx].size, off, blk_off, size); + chunkVec.push_back(XrdCl::ChunkInfo( BS*block_idx + blk_off,size, readV[iov_idx].data+off)); + } + } + } + } + + return true; +} + + +//----------------------------------------------------------------------------------- + +int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blocks_on_disk) +{ + int bytes_read = 0; + for (std::vector::iterator bit = blocks_on_disk.bv.begin(); bit <= blocks_on_disk.bv.end(); bit++ ) + { + int blockIdx = bit->block_idx; + for (std::vector::iterator chunkIt = bit->arr.begin(); chunkIt != bit->arr.end(); ++bit) + { + int chunkIdx = *chunkIt; + + long long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + + + overlap(blockIdx, m_cfi.GetBufferSize(), readV[chunkIdx].offset, readV[chunkIdx].size, off, blk_off, size); + int rs = m_output->Read(readV[chunkIdx].data + readV[chunkIdx].offset+ off, blockIdx*m_cfi.GetBufferSize() + blk_off , size); + if (rs >=0 ) { + bytes_read += rs; + } + else { + // ofs read shoul set the errno + return -1; + } + } + } + + return bytes_read; +} + +//______________________________________________________________________________ + + +int File::VReadProcessBlocks(const XrdOucIOVec *readV, int n, + std::vector& blocks_to_process, std::vector& blocks_processed) +{ + int bytes_read = 0; + while ( (! blocks_to_process.empty()) && (bytes_read >= 0)) + { + std::vector finished; + { + XrdSysCondVarHelper _lck(m_downloadCond); + std::vector::iterator bi = blocks_to_process.begin(); + while (bi != blocks_to_process.end()) + { + if (bi->block->is_finished()) + { + finished.push_back(ReadVChunkListRAM(bi->block, bi->arr)); + std::vector::iterator bj = bi++; + blocks_to_process.erase(bj); + } + else + { + ++bi; + } + } + + if (finished.empty()) + { + m_downloadCond.Wait(); + continue; + } + } + + + std::vector::iterator bi = finished.begin(); + while (bi != finished.end()) + { + if (bi->block->is_ok()) { + + for ( std::vector::iterator chunkIt = bi->arr->begin(); chunkIt < bi->arr->end(); ++chunkIt ) { + long long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + + int block_idx =bi->block->m_offset/m_cfi.GetBufferSize(); + overlap(block_idx, m_cfi.GetBufferSize(), readV[*chunkIt].offset, readV[*chunkIt].size, off, blk_off, size); + memcpy(readV[*chunkIt].data + off, &(bi->block->m_buff[blk_off]), size); + bytes_read += size; + } + } + else { + bytes_read = -1; + errno = bi->block->m_errno; + break; + } + + ++bi; + } + + // add finished to processed list + std::copy(finished.begin(), finished.end(), std::back_inserter(blocks_processed)); + finished.clear(); + } + + return bytes_read; +} From f31c829ff165b3967283129902ccaa2a6fd7813e Mon Sep 17 00:00:00 2001 From: alja Date: Mon, 12 Oct 2015 12:18:01 -0700 Subject: [PATCH 061/171] Fix compilation errror. --- src/XrdFileCache/XrdFileCacheVRead.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index 93a890c7e83..573b96b36ad 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -24,8 +24,8 @@ namespace XrdFileCache { struct ReadVChunkListRAM { ReadVChunkListRAM(Block*b, std::vector * iarr) : block(b), arr(iarr) { }; - std::vector * arr; Block* block; + std::vector * arr; }; // RAM From 62443be72e91a19ca061d7ace2d90ed80dcadec8 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 20 Oct 2015 14:30:48 -0700 Subject: [PATCH 062/171] Import vread code from async-vread-baq branch. --- src/XrdFileCache.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/XrdFileCache.cmake b/src/XrdFileCache.cmake index 1104286c44d..f241adc7fec 100644 --- a/src/XrdFileCache.cmake +++ b/src/XrdFileCache.cmake @@ -17,9 +17,9 @@ add_library( ${LIB_XRD_FILECACHE} MODULE XrdFileCache/XrdFileCache.cc XrdFileCache/XrdFileCache.hh -XrdFileCache/XrdFileCacheFactory.cc XrdFileCache/XrdFileCacheFactory.hh -XrdFileCache/XrdFileCacheFile.cc XrdFileCache/XrdFileCacheFile.hh -XrdFileCache/XrdFileCacheVRead.cc + XrdFileCache/XrdFileCacheFactory.cc XrdFileCache/XrdFileCacheFactory.hh + XrdFileCache/XrdFileCacheFile.cc XrdFileCache/XrdFileCacheFile.hh + XrdFileCache/XrdFileCacheVRead.cc XrdFileCache/XrdFileCacheStats.hh XrdFileCache/XrdFileCacheInfo.cc XrdFileCache/XrdFileCacheInfo.hh XrdFileCache/XrdFileCacheIO.hh From 7dbb8ffd639b36aeb1cf870d60d708671e542c57 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 20 Oct 2015 14:48:42 -0700 Subject: [PATCH 063/171] Add debug print. --- src/XrdFileCache/XrdFileCacheVRead.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index 573b96b36ad..4c24b44bee5 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -70,6 +70,8 @@ using namespace XrdFileCache; int File::ReadV (const XrdOucIOVec *readV, int n) { + clLog()->Debug(XrdCl::AppMsg, "ReadV for %d chunks", n); + int bytesRead = 0; ReadVBlockListRAM blocks_to_process; @@ -170,15 +172,19 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b const int blck_idx_last = (readV[iov_idx].offset + readV[iov_idx].size - 1) / m_cfi.GetBufferSize(); for (int block_idx = blck_idx_first; block_idx <= blck_idx_last; ++block_idx) { + + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess chunk %lld@lld", readV[iov_idx].size, readV[iov_idx].offset); BlockMap_i bi = m_block_map.find(block_idx); if (bi != m_block_map.end()) { if ( blocks_to_process.AddEntry(bi->second, iov_idx)) inc_ref_count(bi->second); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d in map", block_idx); } else if (m_cfi.TestBit(block_idx)) { blocks_on_disk.AddEntry(block_idx, iov_idx); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d on disk", block_idx); } else { if ( Factory::GetInstance().GetCache()->HaveFreeWritingSlots() && Factory::GetInstance().GetCache()->RequestRAMBlock()) @@ -195,6 +201,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b const long long BS = m_cfi.GetBufferSize(); overlap(block_idx, BS, readV[iov_idx].offset, readV[iov_idx].size, off, blk_off, size); chunkVec.push_back(XrdCl::ChunkInfo( BS*block_idx + blk_off,size, readV[iov_idx].data+off)); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess direct read %d", block_idx); } } } @@ -221,6 +228,7 @@ int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blo long long size; // size to copy + clLog()->Debug(XrdCl::AppMsg, "VreadFromDisk block=%d chunk=%d", blockIdx, chunkIdx); overlap(blockIdx, m_cfi.GetBufferSize(), readV[chunkIdx].offset, readV[chunkIdx].size, off, blk_off, size); int rs = m_output->Read(readV[chunkIdx].data + readV[chunkIdx].offset+ off, blockIdx*m_cfi.GetBufferSize() + blk_off , size); if (rs >=0 ) { From 60267b5e89ea5c60d1213b2a286c57d3c8b292d1 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 21 Oct 2015 10:10:19 -0700 Subject: [PATCH 064/171] Parse prefetch entry correctly. --- src/XrdFileCache/XrdFileCacheFactory.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc index 68611c5ddc3..999a752111f 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ b/src/XrdFileCache/XrdFileCacheFactory.cc @@ -367,9 +367,14 @@ bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) } else if (part == "prefetch" ) { - printf("prefetch enabled !!!!\n"); - m_configuration.m_prefetch = true; - config.GetWord(); + int p = ::atoi(config.GetWord()); + if (p != 0) { + printf("prefetch enabled !!!!\n"); + m_configuration.m_prefetch = true; + } + else { + m_configuration.m_prefetch = false; + } } else if (part == "nram" ) { From cc78ca0c51e02dc7884c93e2b760db06313f92a1 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 21 Oct 2015 10:19:13 -0700 Subject: [PATCH 065/171] Fix bug in VReadProcessBlocks -- removing vector while iterating. --- src/XrdFileCache/XrdFileCacheVRead.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index 4c24b44bee5..8be7759004d 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -18,8 +18,8 @@ namespace XrdFileCache { // first element is block index, the following vector elements are chunk readv indicies struct ReadVChunkListDisk { ReadVChunkListDisk(int i) : block_idx(i) {}; - std::vector arr; int block_idx; + std::vector arr; }; struct ReadVChunkListRAM { @@ -173,7 +173,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b for (int block_idx = blck_idx_first; block_idx <= blck_idx_last; ++block_idx) { - clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess chunk %lld@lld", readV[iov_idx].size, readV[iov_idx].offset); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess chunk %lld@%lld", readV[iov_idx].size, readV[iov_idx].offset); BlockMap_i bi = m_block_map.find(block_idx); if (bi != m_block_map.end()) { @@ -192,6 +192,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b Block *b = RequestBlock(block_idx, false); if (!b) return false; blocks_to_process.AddEntry(b, iov_idx); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess requst block %d", block_idx); inc_ref_count(b); } else { @@ -262,8 +263,8 @@ int File::VReadProcessBlocks(const XrdOucIOVec *readV, int n, if (bi->block->is_finished()) { finished.push_back(ReadVChunkListRAM(bi->block, bi->arr)); - std::vector::iterator bj = bi++; - blocks_to_process.erase(bj); + // std::vector::iterator bj = bi++; + blocks_to_process.erase(bi); } else { From 6b12ae03d407b8c291ec38988b54f222cf5376b4 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 21 Oct 2015 11:26:18 -0700 Subject: [PATCH 066/171] Fix iteration in VReadFromDisk. --- src/XrdFileCache/XrdFileCacheVRead.cc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index 8be7759004d..4bf02914e7d 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -85,7 +85,7 @@ int File::ReadV (const XrdOucIOVec *readV, int n) // issue a client read - if (bytesRead) { + if (bytesRead >= 0) { if (!chunkVec.empty()) { direct_handler = new DirectResponseHandler(1); XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; @@ -98,7 +98,7 @@ int File::ReadV (const XrdOucIOVec *readV, int n) } // disk read - if (bytesRead) { + if (bytesRead >= 0) { int dr = VReadFromDisk(readV, n, blocks_on_disk); if (dr < 0) bytesRead = dr; @@ -157,6 +157,8 @@ int File::ReadV (const XrdOucIOVec *readV, int n) for (std::vector::iterator i = blks_processed.begin(); i != blks_processed.end(); ++i) delete i->arr; + + clLog()->Debug(XrdCl::AppMsg, "VRead exit, total = %d", bytesRead); return bytesRead; } @@ -192,7 +194,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b Block *b = RequestBlock(block_idx, false); if (!b) return false; blocks_to_process.AddEntry(b, iov_idx); - clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess requst block %d", block_idx); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess request block %d", block_idx); inc_ref_count(b); } else { @@ -217,10 +219,10 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blocks_on_disk) { int bytes_read = 0; - for (std::vector::iterator bit = blocks_on_disk.bv.begin(); bit <= blocks_on_disk.bv.end(); bit++ ) + for (std::vector::iterator bit = blocks_on_disk.bv.begin(); bit != blocks_on_disk.bv.end(); ++bit ) { int blockIdx = bit->block_idx; - for (std::vector::iterator chunkIt = bit->arr.begin(); chunkIt != bit->arr.end(); ++bit) + for (std::vector::iterator chunkIt = bit->arr.begin(); chunkIt != bit->arr.end(); ++chunkIt) { int chunkIdx = *chunkIt; @@ -229,9 +231,9 @@ int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blo long long size; // size to copy - clLog()->Debug(XrdCl::AppMsg, "VreadFromDisk block=%d chunk=%d", blockIdx, chunkIdx); + clLog()->Debug(XrdCl::AppMsg, "VReadFromDisk block=%d chunk=%d", blockIdx, chunkIdx); overlap(blockIdx, m_cfi.GetBufferSize(), readV[chunkIdx].offset, readV[chunkIdx].size, off, blk_off, size); - int rs = m_output->Read(readV[chunkIdx].data + readV[chunkIdx].offset+ off, blockIdx*m_cfi.GetBufferSize() + blk_off , size); + int rs = m_output->Read(readV[chunkIdx].data + readV[chunkIdx].offset + off, blockIdx*m_cfi.GetBufferSize() + blk_off, size); if (rs >=0 ) { bytes_read += rs; } @@ -310,5 +312,6 @@ int File::VReadProcessBlocks(const XrdOucIOVec *readV, int n, finished.clear(); } + clLog()->Info(XrdCl::AppMsg, "VReadProcessBlocks total read == %d", bytes_read); return bytes_read; } From 673e7e55774650f145aaeb7164be0b884004fd69 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 21 Oct 2015 14:32:41 -0700 Subject: [PATCH 067/171] Fix offset in disk reads --- src/XrdFileCache/XrdFileCacheVRead.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index 4bf02914e7d..cbfdb3ae471 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -186,7 +186,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b else if (m_cfi.TestBit(block_idx)) { blocks_on_disk.AddEntry(block_idx, iov_idx); - clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d on disk", block_idx); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d , chunk idx = %d on disk", block_idx,iov_idx ); } else { if ( Factory::GetInstance().GetCache()->HaveFreeWritingSlots() && Factory::GetInstance().GetCache()->RequestRAMBlock()) @@ -194,7 +194,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b Block *b = RequestBlock(block_idx, false); if (!b) return false; blocks_to_process.AddEntry(b, iov_idx); - clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess request block %d", block_idx); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess request block %d", block_idx); inc_ref_count(b); } else { @@ -233,12 +233,16 @@ int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blo clLog()->Debug(XrdCl::AppMsg, "VReadFromDisk block=%d chunk=%d", blockIdx, chunkIdx); overlap(blockIdx, m_cfi.GetBufferSize(), readV[chunkIdx].offset, readV[chunkIdx].size, off, blk_off, size); - int rs = m_output->Read(readV[chunkIdx].data + readV[chunkIdx].offset + off, blockIdx*m_cfi.GetBufferSize() + blk_off, size); + + int rs = m_output->Read(readV[chunkIdx].data + off, blockIdx*m_cfi.GetBufferSize() + blk_off, size); if (rs >=0 ) { bytes_read += rs; } else { // ofs read shoul set the errno + + clLog()->Error(XrdCl::AppMsg, "VReadFromDisk FAILED block=%d chunk=%d off=%lld, blk_off=%lld, size=%lld, chunfoff =%lld", blockIdx, chunkIdx, off, blk_off, size,readV[chunkIdx].offset ); + return -1; } } From f8848353d7389156577ab101270d04fa3b7ad3ad Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Tue, 16 Feb 2016 14:52:50 -0800 Subject: [PATCH 068/171] Review open issues, make it compile. --- src/XrdFileCache/XrdFileCacheFactory.cc | 65 +++++----- src/XrdFileCache/XrdFileCacheFile.cc | 123 +++++++++---------- src/XrdFileCache/XrdFileCacheFile.hh | 30 ++--- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 16 +-- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 2 - src/XrdFileCache/XrdFileCacheInfo.hh | 5 +- 6 files changed, 107 insertions(+), 134 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc index 999a752111f..c2ce1f07803 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ b/src/XrdFileCache/XrdFileCacheFactory.cc @@ -413,52 +413,51 @@ bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) } //______________________________________________________________________________ -//namespace { -class FPurgeState +namespace { -public: - struct FS + class FPurgeState { - std::string path; - long long nByte; + public: + struct FS + { + std::string path; + long long nByte; - FS(const char* p, long long n) : path(p), nByte(n) {} - }; + FS(const char* p, long long n) : path(p), nByte(n) {} + }; - typedef std::multimap map_t; - typedef map_t::iterator map_i; + typedef std::multimap map_t; + typedef map_t::iterator map_i; - FPurgeState(long long iNByteReq) : nByteReq(iNByteReq), nByteAccum(0) {} + FPurgeState(long long iNByteReq) : nByteReq(iNByteReq), nByteAccum(0) {} - map_t fmap; + map_t fmap; - void checkFile (time_t iTime, const char* iPath, long long iNByte) - { - if (nByteAccum < nByteReq || iTime < fmap.rbegin()->first) + void checkFile (time_t iTime, const char* iPath, long long iNByte) { - fmap.insert(std::pair (iTime, FS(iPath, iNByte))); - nByteAccum += iNByte; - - // remove newest files from map if necessary - while (nByteAccum > nByteReq) + if (nByteAccum < nByteReq || iTime < fmap.rbegin()->first) { - time_t nt = fmap.begin()->first; - std::pair ret = fmap.equal_range(nt); - for (map_i it2 = ret.first; it2 != ret.second; ++it2) - nByteAccum -= it2->second.nByte; - fmap.erase(ret.first, ret.second); + fmap.insert(std::pair (iTime, FS(iPath, iNByte))); + nByteAccum += iNByte; + + // remove newest files from map if necessary + while (nByteAccum > nByteReq) + { + time_t nt = fmap.begin()->first; + std::pair ret = fmap.equal_range(nt); + for (map_i it2 = ret.first; it2 != ret.second; ++it2) + nByteAccum -= it2->second.nByte; + fmap.erase(ret.first, ret.second); + } } } - } - -private: - long long nByteReq; - long long nByteAccum; -}; - -//} + private: + long long nByteReq; + long long nByteAccum; + }; +} void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& purgeState) { diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 4ee4cf9ed2e..a24f21d6365 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -76,7 +76,7 @@ File::File(XrdOucCacheIO &inputIO, std::string& disk_file_path, long long iOffse m_input(inputIO), m_output(NULL), m_infoFile(NULL), - +m_cfi(Factory::GetInstance().RefConfiguration().m_bufferSize), m_temp_filename(disk_file_path), m_offset(iOffset), m_fileSize(iFileSize), @@ -178,11 +178,7 @@ File::~File() Sync(); } // write statistics in *cinfo file - - // AMT append IO stat --- look new interface in master branch - // XXXX MT -- OK, what needs to be here? AppendIOStatToFileInfo(); - // XXXX MT END clLog()->Info(XrdCl::AppMsg, "File::~File close data file %p",(void*)this , lPath()); if (m_output) @@ -485,7 +481,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) for (int block_idx = idx_first; block_idx <= idx_last; ++block_idx) { - clLog()->Dump(XrdCl::AppMsg, "--- File::Read() idx %d %s \n", block_idx, lPath()); + clLog()->Dump(XrdCl::AppMsg, "--- File::Read() idx %d %s \n", block_idx, lPath()); BlockMap_i bi = m_block_map.find(block_idx); // In RAM or incoming? @@ -516,8 +512,8 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) Block *b = RequestBlock(block_idx, false); // assert(b); if (!b) { - preProcOK = false; - break; + preProcOK = false; + break; } inc_ref_count(b); blks_to_process.push_back(b); @@ -534,19 +530,14 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } - m_downloadCond.UnLock(); - - - - if (!preProcOK) { + if (!preProcOK) { for (BlockList_i i = blks_to_process.begin(); i!= blks_to_process.end(); ++i ) dec_ref_count(*i); return -1; // AMT ??? } - long long bytes_read = 0; // First, send out any direct requests. @@ -571,78 +562,78 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Second, read blocks from disk. if ((!blks_on_disk.empty()) && (bytes_read >= 0)) { - int rc = ReadBlocksFromDisk(blks_on_disk, iUserBuff, iUserOff, iUserSize); - clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from disk %d. %s", (void*)iUserBuff, rc, lPath()); - if (rc >= 0) - { - bytes_read += rc; - } - else - { - bytes_read = rc; - clLog()->Error(XrdCl::AppMsg, "File::Read() failed to read from disk. %s", lPath()); - // AMT commented line below should not be an immediate return, can have block refcount increased and map increased - // return rc; - } + int rc = ReadBlocksFromDisk(blks_on_disk, iUserBuff, iUserOff, iUserSize); + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from disk %d. %s", (void*)iUserBuff, rc, lPath()); + if (rc >= 0) + { + bytes_read += rc; + } + else + { + bytes_read = rc; + clLog()->Error(XrdCl::AppMsg, "File::Read() failed to read from disk. %s", lPath()); + // AMT commented line below should not be an immediate return, can have block refcount increased and map increased + // return rc; + } } // Third, loop over blocks that are available or incoming while ( (! blks_to_process.empty()) && (bytes_read >= 0)) { - BlockList_t finished; + BlockList_t finished; - { - XrdSysCondVarHelper _lck(m_downloadCond); + { + XrdSysCondVarHelper _lck(m_downloadCond); - BlockList_i bi = blks_to_process.begin(); - while (bi != blks_to_process.end()) - { - // clLog()->Dump(XrdCl::AppMsg, "File::Read() searcing for block %p finished", (void*)(*bi)); - if ((*bi)->is_finished()) - { - clLog()->Dump(XrdCl::AppMsg, "File::Read() found finished block %p %s", (void*)(*bi), lPath()); - finished.push_back(*bi); - BlockList_i bj = bi++; - blks_to_process.erase(bj); - } - else - { - ++bi; - } - } + BlockList_i bi = blks_to_process.begin(); + while (bi != blks_to_process.end()) + { + // clLog()->Dump(XrdCl::AppMsg, "File::Read() searcing for block %p finished", (void*)(*bi)); + if ((*bi)->is_finished()) + { + clLog()->Dump(XrdCl::AppMsg, "File::Read() found finished block %p %s", (void*)(*bi), lPath()); + finished.push_back(*bi); + BlockList_i bj = bi++; + blks_to_process.erase(bj); + } + else + { + ++bi; + } + } - if (finished.empty()) - { + if (finished.empty()) + { - clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block begin %s", lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block begin %s", lPath()); - m_downloadCond.Wait(); + m_downloadCond.Wait(); - clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block end %s", lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block end %s", lPath()); - continue; - } - } + continue; + } + } - clLog()->Dump(XrdCl::AppMsg, "File::Read() bytes read before processing blocks %d %s\n", bytes_read, lPath()); + clLog()->Dump(XrdCl::AppMsg, "File::Read() bytes read before processing blocks %d %s\n", bytes_read, lPath()); BlockList_i bi = finished.begin(); while (bi != finished.end()) { if ((*bi)->is_ok()) { - long long user_off; // offset in user buffer - long long off_in_block; // offset in block - long long size_to_copy; // size to copy + long long user_off; // offset in user buffer + long long off_in_block; // offset in block + long long size_to_copy; // size to copy - // clLog()->Dump(XrdCl::AppMsg, "File::Read() Block finished ok."); - overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); + // clLog()->Dump(XrdCl::AppMsg, "File::Read() Block finished ok."); + overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); - clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from finished block %d , size %d end %s", (void*)iUserBuff, (*bi)->m_offset/BS, size_to_copy, lPath()); - memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); - bytes_read += size_to_copy; + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from finished block %d , size %d end %s", (void*)iUserBuff, (*bi)->m_offset/BS, size_to_copy, lPath()); + memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); + bytes_read += size_to_copy; - CheckPrefetchStatRAM(*bi); + CheckPrefetchStatRAM(*bi); } else // it has failed ... krap up. { @@ -683,7 +674,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) delete direct_handler; } - clLog()->Debug(XrdCl::AppMsg, "File::Read() before assert %s.", lPath()); + clLog()->Debug(XrdCl::AppMsg, "File::Read() before assert %s.", lPath()); assert(iUserSize >= bytes_read); // Last, stamp and release blocks, release file. @@ -747,7 +738,6 @@ void File::WriteBlockToDisk(Block* b) m_cfi.SetBitFetched(pfIdx); m_downloadCond.UnLock(); - { XrdSysCondVarHelper _lck(m_downloadCond); // clLog()->Dump(XrdCl::AppMsg, "File::WriteToDisk() dec_ref_count %d %s", pfIdx, lPath()); @@ -901,7 +891,6 @@ const char* File::lPath() const return m_temp_filename.c_str(); } -// XXXX MT: is this needed ???? //______________________________________________________________________________ void File::AppendIOStatToFileInfo() { diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index afdab32b0f0..88bda3a0cfd 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -138,7 +138,6 @@ namespace XrdFileCache int m_prefetchCurrentCnt; public: - //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ @@ -175,7 +174,6 @@ namespace XrdFileCache //---------------------------------------------------------------------- Stats& GetStats() { return m_stats; } - void ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status); void WriteBlockToDisk(Block* b); @@ -185,20 +183,18 @@ namespace XrdFileCache void MarkPrefetch(); - - //! Log path const char* lPath() const; + private: - bool overlap(int blk, // block to query - long long blk_size, // - long long req_off, // offset of user request - int req_size, // size of user request - // output: - long long &off, // offset in user buffer - long long &blk_off, // offset in block - long long &size); + long long blk_size, // + long long req_off, // offset of user request + int req_size, // size of user request + // output: + long long &off, // offset in user buffer + long long &blk_off, // offset in block + long long &size); // Read Block* RequestBlock(int i, bool prefetch); @@ -210,11 +206,12 @@ namespace XrdFileCache // VRead bool VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& blks_to_process, ReadVBlockListDisk& blks_on_disk, XrdCl::ChunkList& chunkVec); - int VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blks_on_disk); - int VReadProcessBlocks(const XrdOucIOVec *readV, int n, std::vector& blks_to_process, std::vector& blks_rocessed); + int VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blks_on_disk); + int VReadProcessBlocks(const XrdOucIOVec *readV, int n, std::vector& blks_to_process, std::vector& blks_rocessed); + - - long long BufferSize(); + long long BufferSize(); + void AppendIOStatToFileInfo(); void CheckPrefetchStatRAM(Block* b); void CheckPrefetchStatDisk(int idx); @@ -224,7 +221,6 @@ namespace XrdFileCache //! Short log alias. XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - void inc_ref_count(Block*); void dec_ref_count(Block*); void free_block(Block*); diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index dc4ad208e52..e7728db67e2 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -49,7 +49,6 @@ IOEntireFile::~IOEntireFile() bool IOEntireFile::ioActive() { - printf("called ioActive ...\n"); return m_file->InitiateClose(); } @@ -87,31 +86,24 @@ int IOEntireFile::Read (char *buff, long long off, int size) retval = m_file->Read(buff, off, size); clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() read from File retval = %d %s", retval, m_io.Path()); - if (retval > 0) + if (retval >= 0) { bytes_read += retval; buff += retval; size -= retval; - // XXXX MT: the logick here is strange, see versions in - // alja/pfc-async-prefetch and in master. - // Also, what if retval == 0? + if (size > 0) + clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::Read() missed %d bytes %s", size, m_io.Path()); } - if (retval < 0) + else { clLog()->Error(XrdCl::AppMsg, "IOEntireFile::Read(), origin bytes read %d %s", retval, m_io.Path()); } - else if ((size > 0)) - { - clLog()->Error(XrdCl::AppMsg, "IOEntireFile::Read() missed %d bytes %s", size, m_io.Path()); - bytes_read += retval; - } return (retval < 0) ? retval : bytes_read; } - /* * Perform a readv from the cache */ diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index e09a31a9a86..1b63bf17c11 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -209,5 +209,3 @@ int IOFileBlock::Read (char *buff, long long off, int size) return bytes_read; } - - diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index adcbeb6a54d..e4378710d94 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -100,12 +100,12 @@ namespace XrdFileCache //! //! @return number of bytes read //--------------------------------------------------------------------- - int Read(XrdOssDF* fp, bool init_prefetch = false); + int Read(XrdOssDF* fp, bool init_prefetch = false); //--------------------------------------------------------------------- //! Write number of blocks and read buffer size //--------------------------------------------------------------------- - void WriteHeader(XrdOssDF* fp); + void WriteHeader(XrdOssDF* fp); //--------------------------------------------------------------------- //! Append access time, and cache statistics @@ -222,7 +222,6 @@ namespace XrdFileCache return (m_buff_prefetch[cn] & cfiBIT(off)) == cfiBIT(off); } - inline int Info::GetNDownloadedBlocks() const { int cntd = 0; From f8b27a37fab911feda3b829f0161ed26ee172d1d Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Thu, 18 Feb 2016 22:47:33 -0800 Subject: [PATCH 069/171] Whitespace changes. --- src/XrdFileCache/XrdFileCacheFactory.cc | 12 +++--------- src/XrdFileCache/XrdFileCacheFile.cc | 9 ++------- src/XrdFileCache/XrdFileCacheFile.hh | 8 ++++---- src/XrdFileCache/XrdFileCachePrint.cc | 5 +++-- 4 files changed, 12 insertions(+), 22 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc index c2ce1f07803..ec8c68f76c8 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ b/src/XrdFileCache/XrdFileCacheFactory.cc @@ -132,7 +132,6 @@ bool Factory::xdlib(XrdOucStream &Config) const char* params; params = (val[0]) ? Config.GetWord() : 0; - XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "decisionlib", libp.c_str()); @@ -153,16 +152,16 @@ bool Factory::xdlib(XrdOucStream &Config) clLog()->Info(XrdCl::AppMsg, "Factory::Config() successfully created decision lib from %s", libp.c_str()); return true; } -//______________________________________________________________________________ +//______________________________________________________________________________ bool Factory::Decide(XrdOucCacheIO* io) { - if(!m_decisionpoints.empty()) + if (!m_decisionpoints.empty()) { std::string filename = io->Path(); std::vector::const_iterator it; - for ( it = m_decisionpoints.begin(); it != m_decisionpoints.end(); ++it) + for (it = m_decisionpoints.begin(); it != m_decisionpoints.end(); ++it) { XrdFileCache::Decision *d = *it; if (!d) continue; @@ -176,11 +175,8 @@ bool Factory::Decide(XrdOucCacheIO* io) return true; } - - //______________________________________________________________________________ - bool Factory::Config(XrdSysLogger *logger, const char *config_filename, const char *parameters) { m_log.logger(logger); @@ -538,7 +534,6 @@ void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& } } - void Factory::CacheDirCleanup() { // check state every sleep seconds @@ -615,4 +610,3 @@ void Factory::CacheDirCleanup() sleep(sleept); } } - diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index a24f21d6365..5785d0467f5 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -143,7 +143,6 @@ File::~File() ++itr; } } - bool blockMapEmpty = m_block_map.empty(); int blocksize = (int)m_block_map.size(); @@ -196,9 +195,6 @@ File::~File() } delete m_syncer; - - - // print just for curiosity clLog()->Debug(XrdCl::AppMsg, "File::~File() ended, prefetch score ...%d/%d=%.2f", m_prefetchHitCnt, m_prefetchReadCnt, m_prefetchScore); } @@ -913,9 +909,8 @@ void File::AppendIOStatToFileInfo() //______________________________________________________________________________ void File::Prefetch() { - - if (m_prefetchState == kOn) { - + if (m_prefetchState == kOn) + { // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status \n"); XrdSysCondVarHelper _lck(m_downloadCond); // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN %s \n", lPath()); diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 88bda3a0cfd..36b1d8a7e4f 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -27,15 +27,16 @@ #include #include - class XrdJob; class XrdOucIOVec; + namespace XrdCl { -class Log; + class Log; } -namespace XrdFileCache { +namespace XrdFileCache +{ class BlockResponseHandler; class DirectResponseHandler; @@ -46,7 +47,6 @@ namespace XrdFileCache { } - namespace XrdFileCache { class RefCounted diff --git a/src/XrdFileCache/XrdFileCachePrint.cc b/src/XrdFileCache/XrdFileCachePrint.cc index a72da409b3c..a87fb5956d5 100644 --- a/src/XrdFileCache/XrdFileCachePrint.cc +++ b/src/XrdFileCache/XrdFileCachePrint.cc @@ -71,8 +71,9 @@ void Print::printFile(const std::string& path) int cntd = 0; for (int i = 0; i < cfi.GetSizeInBits(); ++i) if (cfi.TestBit(i)) cntd++; - - printf("version == %d, bufferSize %lld nBlocks %d nDownlaoded %d %s\n",cfi.GetVersion(), cfi.GetBufferSize(), cfi.GetSizeInBits() , cntd, (cfi.GetSizeInBits() == cntd) ? " complete" :""); + printf("version == %d, bufferSize %lld nBlocks %d nDownloaded %d %s\n", + cfi.GetVersion(), cfi.GetBufferSize(), cfi.GetSizeInBits(), cntd, + (cfi.GetSizeInBits() == cntd) ? "complete" : ""); if (m_verbose) { printf("printing %d blocks: \n", cfi.GetSizeInBits()); From 2a1427b307c80b873da98a11be27272e29cfeaae Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Thu, 18 Feb 2016 23:27:25 -0800 Subject: [PATCH 070/171] pfc.prefetch now takes int argument: max number of blocks to prefetch per file. --- src/XrdFileCache/XrdFileCacheFactory.cc | 14 ++++---- src/XrdFileCache/XrdFileCacheFactory.hh | 6 ++-- src/XrdFileCache/XrdFileCacheFile.cc | 43 +++++++++++++------------ src/XrdFileCache/XrdFileCacheFile.hh | 2 +- 4 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc index ec8c68f76c8..5449e7a2183 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ b/src/XrdFileCache/XrdFileCacheFactory.cc @@ -312,11 +312,11 @@ bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) { m_configuration.m_username = config.GetWord(); } - else if ( part == "cachedir" ) + else if ( part == "cachedir" ) { m_configuration.m_cache_dir = config.GetWord(); } - else if ( part == "diskusage" ) + else if ( part == "diskusage" ) { std::string minV = config.GetWord(); std::string maxV = config.GetWord(); @@ -364,12 +364,12 @@ bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) else if (part == "prefetch" ) { int p = ::atoi(config.GetWord()); - if (p != 0) { - printf("prefetch enabled !!!!\n"); + if (p > 0) { + printf("prefetch enabled, max blocks per file=%d\n", p); m_configuration.m_prefetch = true; - } - else { - m_configuration.m_prefetch = false; + m_configuration.m_prefetch_max_blocks = p; + } else { + m_configuration.m_prefetch = false; } } else if (part == "nram" ) diff --git a/src/XrdFileCache/XrdFileCacheFactory.hh b/src/XrdFileCache/XrdFileCacheFactory.hh index d4108efbbe0..b2c30f783ff 100644 --- a/src/XrdFileCache/XrdFileCacheFactory.hh +++ b/src/XrdFileCache/XrdFileCacheFactory.hh @@ -55,6 +55,7 @@ namespace XrdFileCache m_bufferSize(1024*1024), m_NRamBuffers(8000), m_prefetch(false), + m_prefetch_max_blocks(10), m_hdfsbsize(128*1024*1024) {} bool m_hdfsmode; //!< flag for enabling block-level operation @@ -65,8 +66,9 @@ namespace XrdFileCache long long m_diskUsageHWM; //!< cache purge high water mark long long m_bufferSize; //!< prefetch buffer size, default 1MB - int m_NRamBuffers; //!< number of total in-memory cache blocks - bool m_prefetch; //!< prefetch enable state + int m_NRamBuffers; //!< number of total in-memory cache blocks + bool m_prefetch; //!< prefetch enable state + size_t m_prefetch_max_blocks;//!< maximum number of blocks to prefetch per file long long m_hdfsbsize; //!< used with m_hdfsmode, default 128MB }; diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 5785d0467f5..41fc3170133 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -48,28 +48,27 @@ namespace XrdPosixGlobals namespace { -const int PREFETCH_MAX_ATTEMPTS = 10; -const size_t PREFETCH_MAX_BLOCKS=10; + const int PREFETCH_MAX_ATTEMPTS = 10; -class DiskSyncer : public XrdJob -{ -private: - File *m_file; -public: - DiskSyncer(File *pref, const char *desc="") : - XrdJob(desc), - m_file(pref) - {} - void DoIt() + class DiskSyncer : public XrdJob { - m_file->Sync(); - } -}; + private: + File *m_file; + public: + DiskSyncer(File *pref, const char *desc="") : + XrdJob(desc), + m_file(pref) + {} + void DoIt() + { + m_file->Sync(); + } + }; } namespace { - Cache* cache() {return Factory::GetInstance().GetCache();} + Cache* cache() { return Factory::GetInstance().GetCache(); } } File::File(XrdOucCacheIO &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : @@ -351,7 +350,8 @@ Block* File::RequestBlock(int i, bool prefetch) clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); m_block_map[i] = b; - if (m_prefetchState == kOn && m_block_map.size() > PREFETCH_MAX_BLOCKS) { + if (m_prefetchState == kOn && m_block_map.size() > Factory::GetInstance().RefConfiguration().m_prefetch_max_blocks) + { m_prefetchState = kHold; cache()->DeRegisterPrefetchFile(this); } @@ -820,14 +820,17 @@ void File::free_block(Block* b) clLog()->Dump(XrdCl::AppMsg, "File::free_block block (%p) %d %s ", (void*)b, i, lPath()); delete m_block_map[i]; size_t ret = m_block_map.erase(i); - if (ret != 1) { + if (ret != 1) + { clLog()->Error(XrdCl::AppMsg, "File::OnBlockZeroRefCount did not erase %d from map.", i); } - else { + else + { cache()->RAMBlockReleased(); } - if (m_prefetchState == kHold && m_block_map.size() < PREFETCH_MAX_BLOCKS) { + if (m_prefetchState == kHold && m_block_map.size() < Factory::GetInstance().RefConfiguration().m_prefetch_max_blocks) + { m_prefetchState = kOn; cache()->RegisterPrefetchFile(this); } diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 36b1d8a7e4f..1a98651cc45 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -111,7 +111,7 @@ namespace XrdFileCache XrdSysMutex m_syncStatusMutex; //!< mutex locking fsync status XrdJob *m_syncer; std::vector m_writes_during_sync; - int m_non_flushed_cnt; + int m_non_flushed_cnt; bool m_in_sync; typedef std::list IntList_t; From c00a774c2c370679facd91486c2c76aab84c6c53 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 9 Mar 2016 15:55:07 -0800 Subject: [PATCH 071/171] Merge Factory class into Cache class. This makes OucCache::Create() potentially obsolete. --- src/XrdFileCache/XrdFileCacheConfiguration.cc | 299 +++++++++ src/XrdFileCache/XrdFileCacheFactory.cc | 612 ------------------ src/XrdFileCache/XrdFileCacheFactory.hh | 178 ----- src/XrdFileCache/XrdFileCachePurge.cc | 207 ++++++ 4 files changed, 506 insertions(+), 790 deletions(-) create mode 100644 src/XrdFileCache/XrdFileCacheConfiguration.cc delete mode 100644 src/XrdFileCache/XrdFileCacheFactory.cc delete mode 100644 src/XrdFileCache/XrdFileCacheFactory.hh create mode 100644 src/XrdFileCache/XrdFileCachePurge.cc diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc new file mode 100644 index 00000000000..7f45d454a83 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -0,0 +1,299 @@ +#include "XrdFileCache.hh" + +#include "XrdOss/XrdOss.hh" +#include "XrdOss/XrdOssCache.hh" + +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucUtils.hh" +#include "XrdOuc/XrdOucStream.hh" +#include "XrdOuc/XrdOucPinLoader.hh" +#include "XrdOuc/XrdOuca2x.hh" + +#include "XrdOfs/XrdOfsConfigPI.hh" +#include "XrdVersion.hh" + +#include +using namespace XrdFileCache; + +XrdVERSIONINFO(XrdOucGetCache, XrdFileCache); +/* Function: xdlib + + Purpose: To parse the directive: decisionlib [] + + the path of the decision library to be used. + optional parameters to be passed. + + + Output: true upon success or false upon failure. + */ +bool Cache::xdlib(XrdOucStream &Config) +{ + const char* val; + + std::string libp; + if (!(val = Config.GetWord()) || !val[0]) + { + clLog()->Info(XrdCl::AppMsg, " Cache::Config() decisionlib not specified; always caching files"); + return true; + } + else + { + libp = val; + } + + const char* params; + params = (val[0]) ? Config.GetWord() : 0; + + XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "decisionlib", + libp.c_str()); + + Decision *(*ep)(XrdSysError&); + ep = (Decision *(*)(XrdSysError&))myLib->Resolve("XrdFileCacheGetDecision"); + if (!ep) {myLib->Unload(true); return false;} + + Decision * d = ep(m_log); + if (!d) + { + clLog()->Error(XrdCl::AppMsg, "Cache::Config() decisionlib was not able to create a decision object"); + return false; + } + if (params) + d->ConfigDecision(params); + + m_decisionpoints.push_back(d); + clLog()->Info(XrdCl::AppMsg, "Cache::Config() successfully created decision lib from %s", libp.c_str()); + return true; +} + +//______________________________________________________________________________ + +bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char *parameters) +{ + m_log.logger(logger); + + const char * cache_env; + if (!(cache_env = getenv("XRDPOSIX_CACHE")) || !*cache_env) + XrdOucEnv::Export("XRDPOSIX_CACHE", "mode=s&optwr=0"); + + XrdOucEnv myEnv; + XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); + + if (!config_filename || !*config_filename) + { + clLog()->Warning(XrdCl::AppMsg, "Cache::Config() configuration file not specified."); + return false; + } + + int fd; + if ( (fd = open(config_filename, O_RDONLY, 0)) < 0) + { + clLog()->Error(XrdCl::AppMsg, "Cache::Config() can't open configuration file %s", config_filename); + return false; + } + + Config.Attach(fd); + + // Obtain plugin configurator + XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(config_filename,&Config,&m_log, + &XrdVERSIONINFOVAR(XrdOucGetCache)); + if (!ofsCfg) return false; + + + if (ofsCfg->Load(XrdOfsConfigPI::theOssLib)) { + ofsCfg->Plugin(m_output_fs); + XrdOssCache_FS* ocfs = XrdOssCache::Find("public"); + ocfs->Add(m_configuration.m_cache_dir.c_str()); + } + else + { + clLog()->Error(XrdCl::AppMsg, "Cache::Config() Unable to create an OSS object"); + m_output_fs = 0; + return false; + } + + + // Actual parsing of the config file. + bool retval = true; + char *var; + while((var = Config.GetMyFirstWord())) + { + if (!strcmp(var,"pfc.osslib")) + { + ofsCfg->Parse(XrdOfsConfigPI::theOssLib); + } + else if (!strcmp(var,"pfc.decisionlib")) + { + xdlib(Config); + } + else if (!strncmp(var,"pfc.", 4)) + { + retval = ConfigParameters(std::string(var+4), Config); + } + + if (!retval) + { + retval = false; + clLog()->Error(XrdCl::AppMsg, "Cache::Config() error in parsing"); + break; + } + + } + + Config.Close(); + // sets default value for disk usage + if (m_configuration.m_diskUsageLWM < 0 || m_configuration.m_diskUsageHWM < 0) + { + XrdOssVSInfo sP; + if (m_output_fs->StatVS(&sP, "public", 1) >= 0) { + m_configuration.m_diskUsageLWM = static_cast(0.90 * sP.Total + 0.5); + m_configuration.m_diskUsageHWM = static_cast(0.95 * sP.Total + 0.5); + clLog()->Debug(XrdCl::AppMsg, "Default disk usage [%lld, %lld]", m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM); + } + } + + if (retval) + { + int loff = 0; + char buff[2048]; + loff = snprintf(buff, sizeof(buff), "result\n" + "\tpfc.cachedir %s\n" + "\tpfc.blocksize %lld\n" + "\tpfc.prefetch %d\n" + "\tpfc.nram %d\n\n", + m_configuration.m_cache_dir.c_str() , + m_configuration.m_bufferSize, + m_configuration.m_prefetch, // AMT not sure what parsing should be + m_configuration.m_NRamBuffers ); + + if (m_configuration.m_hdfsmode) + { + char buff2[512]; + snprintf(buff2, sizeof(buff2), "\tpfc.hdfsmode hdfsbsize %lld\n", m_configuration.m_hdfsbsize); + loff += snprintf(&buff[loff], strlen(buff2), "%s", buff2); + } + + char unameBuff[256]; + if (m_configuration.m_username.empty()) { + XrdOucUtils::UserName(getuid(), unameBuff, sizeof(unameBuff)); + m_configuration.m_username = unameBuff; + } + else { + snprintf(unameBuff, sizeof(unameBuff), "\tpfc.user %s \n", m_configuration.m_username.c_str()); + loff += snprintf(&buff[loff], strlen(unameBuff), "%s", unameBuff); + } + + m_log.Emsg("Config", buff); + } + + m_log.Emsg("Config", "Configuration = ", retval ? "Success" : "Fail"); + + if (ofsCfg) delete ofsCfg; + return retval; +} + +//______________________________________________________________________________ + + +bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) +{ + printf("part %s \n", part.c_str()); + XrdSysError err(0, ""); + if ( part == "user" ) + { + m_configuration.m_username = config.GetWord(); + } + else if ( part == "cachedir" ) + { + m_configuration.m_cache_dir = config.GetWord(); + } + else if ( part == "diskusage" ) + { + std::string minV = config.GetWord(); + std::string maxV = config.GetWord(); + if (!minV.empty() && !maxV.empty()) { + XrdOssVSInfo sP; + if (m_output_fs->StatVS(&sP, "public", 1) >= 0) + { + if (::isalpha(*(minV.rbegin())) && ::isalpha(*(minV.rbegin()))) { + if ( XrdOuca2x::a2sz(m_log, "Error getting disk usage low watermark", minV.c_str(), &m_configuration.m_diskUsageLWM, 0, sP.Total) + || XrdOuca2x::a2sz(m_log, "Error getting disk usage high watermark", maxV.c_str(), &m_configuration.m_diskUsageHWM, 0, sP.Total)) + { + return false; + } + } + else + { + char* eP; + errno = 0; + float lwmf = strtod(minV.c_str(), &eP); + if (errno || eP == minV.c_str()) { + m_log.Emsg("Cache::ConfigParameters() error parsing diskusage parameter ", minV.c_str()); + return false; + } + float hwmf = strtod(maxV.c_str(), &eP); + if (errno || eP == maxV.c_str()) { + m_log.Emsg("Cache::ConfigParameters() error parsing diskusage parameter ", maxV.c_str()); + return false; + } + + m_configuration.m_diskUsageLWM = static_cast(sP.Total * lwmf + 0.5); + m_configuration.m_diskUsageHWM = static_cast(sP.Total * hwmf + 0.5); + } + } + } + } + else if ( part == "blocksize" ) + { + long long minBSize = 64 * 1024; + long long maxBSize = 16 * 1024 * 1024; + if ( XrdOuca2x::a2sz(m_log, "get block size", config.GetWord(), &m_configuration.m_bufferSize, minBSize, maxBSize)) + { + return false; + } + } + else if (part == "prefetch" ) + { + int p = ::atoi(config.GetWord()); + if (p > 0) { + printf("prefetch enabled, max blocks per file=%d\n", p); + m_configuration.m_prefetch = true; + m_configuration.m_prefetch_max_blocks = p; + } else { + m_configuration.m_prefetch = false; + } + } + else if (part == "nram" ) + { + m_configuration.m_NRamBuffers = ::atoi(config.GetWord()); + } + else if ( part == "hdfsmode" ) + { + m_configuration.m_hdfsmode = true; + + const char* params = config.GetWord(); + if (params) { + if (!strncmp("hdfsbsize", params, 9)) { + long long minBlSize = 128 * 1024; + long long maxBlSize = 1024 * 1024 * 1024; + params = config.GetWord(); + if ( XrdOuca2x::a2sz(m_log, "Error getting file fragment size", params, &m_configuration.m_hdfsbsize, minBlSize, maxBlSize)) + { + return false; + } + } + else { + m_log.Emsg("Config", "Error setting the fragment size parameter name"); + return false; + } + } + } + else + { + m_log.Emsg("Cache::ConfigParameters() unmatched pfc parameter", part.c_str()); + return false; + } + + assert ( config.GetWord() == 0 && "Cache::ConfigParameters() lost argument"); + + return true; +} diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc deleted file mode 100644 index 5449e7a2183..00000000000 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ /dev/null @@ -1,612 +0,0 @@ -//---------------------------------------------------------------------------------- -// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University -// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman -//---------------------------------------------------------------------------------- -// XRootD is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// XRootD is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with XRootD. If not, see . -//---------------------------------------------------------------------------------- - -#include -#include -#include -#include - - -#include - - -#include "XrdSys/XrdSysPthread.hh" -#include "XrdSys/XrdSysError.hh" -#include "XrdSys/XrdSysLogger.hh" -#include "XrdOuc/XrdOucEnv.hh" -#include "XrdOuc/XrdOucUtils.hh" -#include "XrdOss/XrdOssCache.hh" -#include "XrdOuc/XrdOucPinLoader.hh" -#include "XrdOuc/XrdOucStream.hh" -#include "XrdOuc/XrdOuca2x.hh" -#include "XrdOfs/XrdOfsConfigPI.hh" -#include "XrdOss/XrdOss.hh" -#include "XrdVersion.hh" -#include "XrdPosix/XrdPosixXrootd.hh" -#include "XrdCl/XrdClConstants.hh" - -#include "XrdFileCache.hh" -#include "XrdFileCacheFactory.hh" -#include "XrdFileCacheFile.hh" - - -using namespace XrdFileCache; - -XrdVERSIONINFO(XrdOucGetCache, XrdFileCache); - - -Factory * Factory::m_factory = NULL; - -void *CacheDirCleanupThread(void* cache_void) -{ - Factory::GetInstance().CacheDirCleanup(); - return NULL; -} - - -Factory::Factory() - : m_log(0, "XrdFileCache_"), - m_cache(0) -{} - -extern "C" -{ -XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, - const char *config_filename, - const char *parameters) -{ - XrdSysError err(0, ""); - err.logger(logger); - err.Emsg("Retrieve", "Retrieving a caching proxy factory."); - Factory &factory = Factory::GetInstance(); - if (!factory.Config(logger, config_filename, parameters)) - { - err.Emsg("Retrieve", "Error - unable to create a factory."); - return NULL; - } - err.Emsg("Retrieve", "Success - returning a factory."); - - - pthread_t tid; - XrdSysThread::Run(&tid, CacheDirCleanupThread, NULL, 0, "XrdFileCache CacheDirCleanup"); - return &factory; -} -} - -Factory &Factory::GetInstance() -{ - if (m_factory == NULL) - m_factory = new Factory(); - return *m_factory; -} - -XrdOucCache *Factory::Create(Parms & parms, XrdOucCacheIO::aprParms * prParms) -{ - clLog()->Info(XrdCl::AppMsg, "Factory::Create() new cache object"); - assert(m_cache == 0); - m_cache = new Cache(m_stats); - return m_cache; -} - - -/* Function: xdlib - - Purpose: To parse the directive: decisionlib [] - - the path of the decision library to be used. - optional parameters to be passed. - - - Output: true upon success or false upon failure. - */ -bool Factory::xdlib(XrdOucStream &Config) -{ - const char* val; - - std::string libp; - if (!(val = Config.GetWord()) || !val[0]) - { - clLog()->Info(XrdCl::AppMsg, " Factory::Config() decisionlib not specified; always caching files"); - return true; - } - else - { - libp = val; - } - - const char* params; - params = (val[0]) ? Config.GetWord() : 0; - - XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "decisionlib", - libp.c_str()); - - Decision *(*ep)(XrdSysError&); - ep = (Decision *(*)(XrdSysError&))myLib->Resolve("XrdFileCacheGetDecision"); - if (!ep) {myLib->Unload(true); return false;} - - Decision * d = ep(m_log); - if (!d) - { - clLog()->Error(XrdCl::AppMsg, "Factory::Config() decisionlib was not able to create a decision object"); - return false; - } - if (params) - d->ConfigDecision(params); - - m_decisionpoints.push_back(d); - clLog()->Info(XrdCl::AppMsg, "Factory::Config() successfully created decision lib from %s", libp.c_str()); - return true; -} - -//______________________________________________________________________________ - -bool Factory::Decide(XrdOucCacheIO* io) -{ - if (!m_decisionpoints.empty()) - { - std::string filename = io->Path(); - std::vector::const_iterator it; - for (it = m_decisionpoints.begin(); it != m_decisionpoints.end(); ++it) - { - XrdFileCache::Decision *d = *it; - if (!d) continue; - if (!d->Decide(filename, *m_output_fs)) - { - return false; - } - } - } - - return true; -} - -//______________________________________________________________________________ - -bool Factory::Config(XrdSysLogger *logger, const char *config_filename, const char *parameters) -{ - m_log.logger(logger); - - const char * cache_env; - if (!(cache_env = getenv("XRDPOSIX_CACHE")) || !*cache_env) - XrdOucEnv::Export("XRDPOSIX_CACHE", "mode=s&optwr=0"); - - XrdOucEnv myEnv; - XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); - - if (!config_filename || !*config_filename) - { - clLog()->Warning(XrdCl::AppMsg, "Factory::Config() configuration file not specified."); - return false; - } - - int fd; - if ( (fd = open(config_filename, O_RDONLY, 0)) < 0) - { - clLog()->Error(XrdCl::AppMsg, "Factory::Config() can't open configuration file %s", config_filename); - return false; - } - - Config.Attach(fd); - - // Obtain plugin configurator - XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(config_filename,&Config,&m_log, - &XrdVERSIONINFOVAR(XrdOucGetCache)); - if (!ofsCfg) return false; - - - if (ofsCfg->Load(XrdOfsConfigPI::theOssLib)) { - ofsCfg->Plugin(m_output_fs); - XrdOssCache_FS* ocfs = XrdOssCache::Find("public"); - ocfs->Add(m_configuration.m_cache_dir.c_str()); - } - else - { - clLog()->Error(XrdCl::AppMsg, "Factory::Config() Unable to create an OSS object"); - m_output_fs = 0; - return false; - } - - - // Actual parsing of the config file. - bool retval = true; - char *var; - while((var = Config.GetMyFirstWord())) - { - if (!strcmp(var,"pfc.osslib")) - { - ofsCfg->Parse(XrdOfsConfigPI::theOssLib); - } - else if (!strcmp(var,"pfc.decisionlib")) - { - xdlib(Config); - } - else if (!strncmp(var,"pfc.", 4)) - { - retval = ConfigParameters(std::string(var+4), Config); - } - - if (!retval) - { - retval = false; - clLog()->Error(XrdCl::AppMsg, "Factory::Config() error in parsing"); - break; - } - - } - - Config.Close(); - // sets default value for disk usage - if (m_configuration.m_diskUsageLWM < 0 || m_configuration.m_diskUsageHWM < 0) - { - XrdOssVSInfo sP; - if (m_output_fs->StatVS(&sP, "public", 1) >= 0) { - m_configuration.m_diskUsageLWM = static_cast(0.90 * sP.Total + 0.5); - m_configuration.m_diskUsageHWM = static_cast(0.95 * sP.Total + 0.5); - clLog()->Debug(XrdCl::AppMsg, "Default disk usage [%lld, %lld]", m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM); - } - } - - if (retval) - { - int loff = 0; - char buff[2048]; - loff = snprintf(buff, sizeof(buff), "result\n" - "\tpfc.cachedir %s\n" - "\tpfc.blocksize %lld\n" - "\tpfc.prefetch %d\n" - "\tpfc.nram %d\n\n", - m_configuration.m_cache_dir.c_str() , - m_configuration.m_bufferSize, - m_configuration.m_prefetch, // AMT not sure what parsing should be - m_configuration.m_NRamBuffers ); - - if (m_configuration.m_hdfsmode) - { - char buff2[512]; - snprintf(buff2, sizeof(buff2), "\tpfc.hdfsmode hdfsbsize %lld\n", m_configuration.m_hdfsbsize); - loff += snprintf(&buff[loff], strlen(buff2), "%s", buff2); - } - - char unameBuff[256]; - if (m_configuration.m_username.empty()) { - XrdOucUtils::UserName(getuid(), unameBuff, sizeof(unameBuff)); - m_configuration.m_username = unameBuff; - } - else { - snprintf(unameBuff, sizeof(unameBuff), "\tpfc.user %s \n", m_configuration.m_username.c_str()); - loff += snprintf(&buff[loff], strlen(unameBuff), "%s", unameBuff); - } - - m_log.Emsg("Config", buff); - } - - m_log.Emsg("Config", "Configuration = ", retval ? "Success" : "Fail"); - - if (ofsCfg) delete ofsCfg; - return retval; -} - -//______________________________________________________________________________ - - -bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) -{ - printf("part %s \n", part.c_str()); - XrdSysError err(0, ""); - if ( part == "user" ) - { - m_configuration.m_username = config.GetWord(); - } - else if ( part == "cachedir" ) - { - m_configuration.m_cache_dir = config.GetWord(); - } - else if ( part == "diskusage" ) - { - std::string minV = config.GetWord(); - std::string maxV = config.GetWord(); - if (!minV.empty() && !maxV.empty()) { - XrdOssVSInfo sP; - if (m_output_fs->StatVS(&sP, "public", 1) >= 0) - { - if (::isalpha(*(minV.rbegin())) && ::isalpha(*(minV.rbegin()))) { - if ( XrdOuca2x::a2sz(m_log, "Error getting disk usage low watermark", minV.c_str(), &m_configuration.m_diskUsageLWM, 0, sP.Total) - || XrdOuca2x::a2sz(m_log, "Error getting disk usage high watermark", maxV.c_str(), &m_configuration.m_diskUsageHWM, 0, sP.Total)) - { - return false; - } - } - else - { - char* eP; - errno = 0; - float lwmf = strtod(minV.c_str(), &eP); - if (errno || eP == minV.c_str()) { - m_log.Emsg("Factory::ConfigParameters() error parsing diskusage parameter ", minV.c_str()); - return false; - } - float hwmf = strtod(maxV.c_str(), &eP); - if (errno || eP == maxV.c_str()) { - m_log.Emsg("Factory::ConfigParameters() error parsing diskusage parameter ", maxV.c_str()); - return false; - } - - m_configuration.m_diskUsageLWM = static_cast(sP.Total * lwmf + 0.5); - m_configuration.m_diskUsageHWM = static_cast(sP.Total * hwmf + 0.5); - } - } - } - } - else if ( part == "blocksize" ) - { - long long minBSize = 64 * 1024; - long long maxBSize = 16 * 1024 * 1024; - if ( XrdOuca2x::a2sz(m_log, "get block size", config.GetWord(), &m_configuration.m_bufferSize, minBSize, maxBSize)) - { - return false; - } - } - else if (part == "prefetch" ) - { - int p = ::atoi(config.GetWord()); - if (p > 0) { - printf("prefetch enabled, max blocks per file=%d\n", p); - m_configuration.m_prefetch = true; - m_configuration.m_prefetch_max_blocks = p; - } else { - m_configuration.m_prefetch = false; - } - } - else if (part == "nram" ) - { - m_configuration.m_NRamBuffers = ::atoi(config.GetWord()); - } - else if ( part == "hdfsmode" ) - { - m_configuration.m_hdfsmode = true; - - const char* params = config.GetWord(); - if (params) { - if (!strncmp("hdfsbsize", params, 9)) { - long long minBlSize = 128 * 1024; - long long maxBlSize = 1024 * 1024 * 1024; - params = config.GetWord(); - if ( XrdOuca2x::a2sz(m_log, "Error getting file fragment size", params, &m_configuration.m_hdfsbsize, minBlSize, maxBlSize)) - { - return false; - } - } - else { - m_log.Emsg("Config", "Error setting the fragment size parameter name"); - return false; - } - } - } - else - { - m_log.Emsg("Factory::ConfigParameters() unmatched pfc parameter", part.c_str()); - return false; - } - - assert ( config.GetWord() == 0 && "Factory::ConfigParameters() lost argument"); - - return true; -} - -//______________________________________________________________________________ - -namespace -{ - class FPurgeState - { - public: - struct FS - { - std::string path; - long long nByte; - - FS(const char* p, long long n) : path(p), nByte(n) {} - }; - - typedef std::multimap map_t; - typedef map_t::iterator map_i; - - FPurgeState(long long iNByteReq) : nByteReq(iNByteReq), nByteAccum(0) {} - - map_t fmap; - - void checkFile (time_t iTime, const char* iPath, long long iNByte) - { - if (nByteAccum < nByteReq || iTime < fmap.rbegin()->first) - { - fmap.insert(std::pair (iTime, FS(iPath, iNByte))); - nByteAccum += iNByte; - - // remove newest files from map if necessary - while (nByteAccum > nByteReq) - { - time_t nt = fmap.begin()->first; - std::pair ret = fmap.equal_range(nt); - for (map_i it2 = ret.first; it2 != ret.second; ++it2) - nByteAccum -= it2->second.nByte; - fmap.erase(ret.first, ret.second); - } - } - } - - private: - long long nByteReq; - long long nByteAccum; - }; -} - -void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& purgeState) -{ - char buff[256]; - XrdOucEnv env; - int rdr; - const size_t InfoExtLen = strlen(XrdFileCache::Info::m_infoExtension); // cached var - XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); - - Factory& factory = Factory::GetInstance(); - while ((rdr = iOssDF->Readdir(&buff[0], 256)) >= 0) - { - // printf("readdir [%s]\n", buff); - std::string np = path + "/" + std::string(buff); - size_t fname_len = strlen(&buff[0]); - if (fname_len == 0) - { - // std::cout << "Finish read dir.[" << np <<"] Break loop \n"; - break; - } - - if (strncmp("..", &buff[0], 2) && strncmp(".", &buff[0], 1)) - { - XrdOssDF* dh = factory.GetOss()->newDir(factory.RefConfiguration().m_username.c_str()); - XrdOssDF* fh = factory.GetOss()->newFile(factory.RefConfiguration().m_username.c_str()); - - if (fname_len > InfoExtLen && strncmp(&buff[fname_len - InfoExtLen ], XrdFileCache::Info::m_infoExtension, InfoExtLen) == 0) - { - // XXXX MT - shouldn't we also check if it is currently opened? - - fh->Open(np.c_str(), O_RDONLY, 0600, env); - Info cinfo(factory.RefConfiguration().m_bufferSize); - time_t accessTime; - cinfo.Read(fh); - if (cinfo.GetLatestDetachTime(accessTime, fh)) - { - log->Debug(XrdCl::AppMsg, "FillFileMapRecurse() checking %s accessTime %d ", buff, (int)accessTime); - purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); - } - else - { - // cinfo file does not contain any known accesses, use stat.mtime instead. - - log->Info(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s, trying stat.\n", np.c_str()); - - XrdOss* oss = Factory::GetInstance().GetOss(); - struct stat fstat; - - if (oss->Stat(np.c_str(), &fstat) == XrdOssOK) - { - accessTime = fstat.st_mtime; - log->Info(XrdCl::AppMsg, "FillFileMapRecurse() determined access time for %s via stat: %lld\n", - np.c_str(), accessTime); - - purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); - } - else - { - // This really shouldn't happen ... but if it does remove cinfo and the data file right away. - - log->Warning(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s. Purging directly.\n", - np.c_str()); - - oss->Unlink(np.c_str()); - np = np.substr(0, np.size() - strlen(XrdFileCache::Info::m_infoExtension)); - oss->Unlink(np.c_str()); - } - } - } - else if (dh->Opendir(np.c_str(), env) >= 0) - { - FillFileMapRecurse(dh, np, purgeState); - } - - delete dh; dh = 0; - delete fh; fh = 0; - } - } -} - -void Factory::CacheDirCleanup() -{ - // check state every sleep seconds - const static int sleept = 300; - struct stat fstat; - XrdOucEnv env; - - XrdOss* oss = Factory::GetInstance().GetOss(); - XrdOssVSInfo sP; - - while (1) - { - // get amount of space to erase - long long bytesToRemove = 0; - if (oss->StatVS(&sP, "public", 1) < 0) - { - clLog()->Error(XrdCl::AppMsg, "Factory::CacheDirCleanup() can't get statvs for dir [%s] \n", m_configuration.m_cache_dir.c_str()); - exit(1); - } - else - { - long long ausage = sP.Total - sP.Free; - clLog()->Info(XrdCl::AppMsg, "Factory::CacheDirCleanup() occupates disk space == %lld", ausage); - if (ausage > m_configuration.m_diskUsageHWM) - { - bytesToRemove = ausage - m_configuration.m_diskUsageLWM; - clLog()->Info(XrdCl::AppMsg, "Factory::CacheDirCleanup() need space for %lld bytes", bytesToRemove); - } - } - - if (bytesToRemove > 0) - { - // make a sorted map of file patch by access time - XrdOssDF* dh = oss->newDir(m_configuration.m_username.c_str()); - if (dh->Opendir(m_configuration.m_cache_dir.c_str(), env) >= 0) - { - FPurgeState purgeState(bytesToRemove * 5 / 4); // prepare 20% more volume than required - - FillFileMapRecurse(dh, m_configuration.m_cache_dir, purgeState); - - // loop over map and remove files with highest value of access time - for (FPurgeState::map_i it = purgeState.fmap.begin(); it != purgeState.fmap.end(); ++it) - { - // XXXX MT - shouldn't we re-check if the file is currently opened? - - std::string path = it->second.path; - // remove info file - if (oss->Stat(path.c_str(), &fstat) == XrdOssOK) - { - bytesToRemove -= fstat.st_size; - oss->Unlink(path.c_str()); - clLog()->Info(XrdCl::AppMsg, "Factory::CacheDirCleanup() removed %s size %lld", - path.c_str(), fstat.st_size); - } - - // remove data file - path = path.substr(0, path.size() - strlen(XrdFileCache::Info::m_infoExtension)); - if (oss->Stat(path.c_str(), &fstat) == XrdOssOK) - { - bytesToRemove -= it->second.nByte; - oss->Unlink(path.c_str()); - clLog()->Info(XrdCl::AppMsg, "Factory::CacheDirCleanup() removed %s bytes %lld, stat_size %lld", - path.c_str(), it->second.nByte, fstat.st_size); - } - - if (bytesToRemove <= 0) - break; - } - } - dh->Close(); - delete dh; dh =0; - } - - sleep(sleept); - } -} diff --git a/src/XrdFileCache/XrdFileCacheFactory.hh b/src/XrdFileCache/XrdFileCacheFactory.hh deleted file mode 100644 index b2c30f783ff..00000000000 --- a/src/XrdFileCache/XrdFileCacheFactory.hh +++ /dev/null @@ -1,178 +0,0 @@ -#ifndef __XRDFILECACHE_FACTORY_HH__ -#define __XRDFILECACHE_FACTORY_HH__ -//---------------------------------------------------------------------------------- -// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University -// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman -//---------------------------------------------------------------------------------- -// XRootD is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// XRootD is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with XRootD. If not, see . -//---------------------------------------------------------------------------------- - -#include -#include -#include - -#include "XrdSys/XrdSysPthread.hh" -#include "XrdOuc/XrdOucCache.hh" - -#include "XrdCl/XrdClDefaultEnv.hh" -#include "XrdVersion.hh" -#include "XrdFileCacheDecision.hh" - -class XrdOucStream; -class XrdSysError; - -namespace XrdCl -{ - class Log; -} - -namespace XrdFileCache { - class Cache; -} - -namespace XrdFileCache -{ - //---------------------------------------------------------------------------- - //! Contains parameters configurable from the xrootd config file. - //---------------------------------------------------------------------------- - struct Configuration - { - Configuration() : - m_hdfsmode(false), - m_diskUsageLWM(-1), - m_diskUsageHWM(-1), - m_bufferSize(1024*1024), - m_NRamBuffers(8000), - m_prefetch(false), - m_prefetch_max_blocks(10), - m_hdfsbsize(128*1024*1024) {} - - bool m_hdfsmode; //!< flag for enabling block-level operation - std::string m_cache_dir; //!< path of disk cache - std::string m_username; //!< username passed to oss plugin - - long long m_diskUsageLWM; //!< cache purge low water mark - long long m_diskUsageHWM; //!< cache purge high water mark - - long long m_bufferSize; //!< prefetch buffer size, default 1MB - int m_NRamBuffers; //!< number of total in-memory cache blocks - bool m_prefetch; //!< prefetch enable state - size_t m_prefetch_max_blocks;//!< maximum number of blocks to prefetch per file - - long long m_hdfsbsize; //!< used with m_hdfsmode, default 128MB - }; - - - //---------------------------------------------------------------------------- - //! Instantiates Cache and Decision plugins. Parses configuration file. - //---------------------------------------------------------------------------- - class Factory : public XrdOucCache - { - public: - //-------------------------------------------------------------------------- - //! Constructor - //-------------------------------------------------------------------------- - Factory(); - - //--------------------------------------------------------------------- - //! \brief Unused abstract method. This method is implemented in the - //! the Cache class. - //--------------------------------------------------------------------- - virtual XrdOucCacheIO *Attach(XrdOucCacheIO *, int Options=0) { return NULL; } - - //--------------------------------------------------------------------- - //! \brief Unused abstract method. This information is available in - //! the Cache class. - //--------------------------------------------------------------------- - virtual int isAttached() { return false; } - - //--------------------------------------------------------------------- - //! Creates XrdFileCache::Cache object - //--------------------------------------------------------------------- - virtual XrdOucCache* Create(Parms &, XrdOucCacheIO::aprParms *aprP); - - XrdOss* GetOss() const { return m_output_fs; } - - //--------------------------------------------------------------------- - //! Getter for xrootd logger - //--------------------------------------------------------------------- - XrdSysError& GetSysError() { return m_log; } - - //-------------------------------------------------------------------- - //! \brief Makes decision if the original XrdOucCacheIO should be cached. - //! - //! @param & URL of file - //! - //! @return decision if IO object will be cached. - //-------------------------------------------------------------------- - bool Decide(XrdOucCacheIO*); - - //------------------------------------------------------------------------ - //! Reference XrdFileCache configuration - //------------------------------------------------------------------------ - const Configuration& RefConfiguration() const { return m_configuration; } - - - //--------------------------------------------------------------------- - //! \brief Parse configuration file - //! - //! @param logger xrootd logger - //! @param config_filename path to configuration file - //! @param parameters optional parameters to be passed - //! - //! @return parse status - //--------------------------------------------------------------------- - bool Config(XrdSysLogger *logger, const char *config_filename, const char *parameters); - - //--------------------------------------------------------------------- - //! Singleton access. - //--------------------------------------------------------------------- - static Factory &GetInstance(); - - //--------------------------------------------------------------------- - //! Version check. - //--------------------------------------------------------------------- - static bool VCheck(XrdVersionInfo &urVersion) { return true; } - - //--------------------------------------------------------------------- - //! Thread function running disk cache purge periodically. - //--------------------------------------------------------------------- - void CacheDirCleanup(); - - - Cache* GetCache() { return m_cache; } - private: - bool ConfigParameters(std::string, XrdOucStream&); - bool ConfigXeq(char *, XrdOucStream &); - bool xdlib(XrdOucStream &); - - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - - static Factory *m_factory; //!< this object - - XrdSysError m_log; //!< XrdFileCache namespace logger - XrdOucCacheStats m_stats; //!< passed to cache, currently not used - XrdOss *m_output_fs; //!< disk cache file system - - std::vector m_decisionpoints; //!< decision plugins - - std::map m_filesInQueue; - - Configuration m_configuration; //!< configurable parameters - - Cache* m_cache; - }; -} - -#endif diff --git a/src/XrdFileCache/XrdFileCachePurge.cc b/src/XrdFileCache/XrdFileCachePurge.cc new file mode 100644 index 00000000000..19ed5a3dd3d --- /dev/null +++ b/src/XrdFileCache/XrdFileCachePurge.cc @@ -0,0 +1,207 @@ +#include "XrdFileCache.hh" + +using namespace XrdFileCache; + +#include +#include "XrdOuc/XrdOucEnv.hh" + +namespace +{ + class FPurgeState + { + public: + struct FS + { + std::string path; + long long nByte; + + FS(const char* p, long long n) : path(p), nByte(n) {} + }; + + typedef std::multimap map_t; + typedef map_t::iterator map_i; + + FPurgeState(long long iNByteReq) : nByteReq(iNByteReq), nByteAccum(0) {} + + map_t fmap; + + void checkFile (time_t iTime, const char* iPath, long long iNByte) + { + if (nByteAccum < nByteReq || iTime < fmap.rbegin()->first) + { + fmap.insert(std::pair (iTime, FS(iPath, iNByte))); + nByteAccum += iNByte; + + // remove newest files from map if necessary + while (nByteAccum > nByteReq) + { + time_t nt = fmap.begin()->first; + std::pair ret = fmap.equal_range(nt); + for (map_i it2 = ret.first; it2 != ret.second; ++it2) + nByteAccum -= it2->second.nByte; + fmap.erase(ret.first, ret.second); + } + } + } + + private: + long long nByteReq; + long long nByteAccum; + }; +} + +void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& purgeState) +{ + char buff[256]; + XrdOucEnv env; + int rdr; + const size_t InfoExtLen = strlen(XrdFileCache::Info::m_infoExtension); // cached var + XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); + + Cache& factory = Cache::GetInstance(); + while ((rdr = iOssDF->Readdir(&buff[0], 256)) >= 0) + { + // printf("readdir [%s]\n", buff); + std::string np = path + "/" + std::string(buff); + size_t fname_len = strlen(&buff[0]); + if (fname_len == 0) + { + // std::cout << "Finish read dir.[" << np <<"] Break loop \n"; + break; + } + + if (strncmp("..", &buff[0], 2) && strncmp(".", &buff[0], 1)) + { + XrdOssDF* dh = factory.GetOss()->newDir(factory.RefConfiguration().m_username.c_str()); + XrdOssDF* fh = factory.GetOss()->newFile(factory.RefConfiguration().m_username.c_str()); + + if (fname_len > InfoExtLen && strncmp(&buff[fname_len - InfoExtLen ], XrdFileCache::Info::m_infoExtension, InfoExtLen) == 0) + { + // XXXX MT - shouldn't we also check if it is currently opened? + + fh->Open(np.c_str(), O_RDONLY, 0600, env); + Info cinfo(factory.RefConfiguration().m_bufferSize); + time_t accessTime; + cinfo.Read(fh); + if (cinfo.GetLatestDetachTime(accessTime, fh)) + { + log->Debug(XrdCl::AppMsg, "FillFileMapRecurse() checking %s accessTime %d ", buff, (int)accessTime); + purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); + } + else + { + // cinfo file does not contain any known accesses, use stat.mtime instead. + + log->Info(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s, trying stat.\n", np.c_str()); + + XrdOss* oss = Cache::GetInstance().GetOss(); + struct stat fstat; + + if (oss->Stat(np.c_str(), &fstat) == XrdOssOK) + { + accessTime = fstat.st_mtime; + log->Info(XrdCl::AppMsg, "FillFileMapRecurse() determined access time for %s via stat: %lld\n", + np.c_str(), accessTime); + + purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); + } + else + { + // This really shouldn't happen ... but if it does remove cinfo and the data file right away. + + log->Warning(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s. Purging directly.\n", + np.c_str()); + + oss->Unlink(np.c_str()); + np = np.substr(0, np.size() - strlen(XrdFileCache::Info::m_infoExtension)); + oss->Unlink(np.c_str()); + } + } + } + else if (dh->Opendir(np.c_str(), env) >= 0) + { + FillFileMapRecurse(dh, np, purgeState); + } + + delete dh; dh = 0; + delete fh; fh = 0; + } + } +} + +void Cache::CacheDirCleanup() +{ + // check state every sleep seconds + const static int sleept = 300; + struct stat fstat; + XrdOucEnv env; + + XrdOss* oss = Cache::GetInstance().GetOss(); + XrdOssVSInfo sP; + + while (1) + { + // get amount of space to erase + long long bytesToRemove = 0; + if (oss->StatVS(&sP, "public", 1) < 0) + { + clLog()->Error(XrdCl::AppMsg, "Cache::CacheDirCleanup() can't get statvs for dir [%s] \n", m_configuration.m_cache_dir.c_str()); + exit(1); + } + else + { + long long ausage = sP.Total - sP.Free; + clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() occupates disk space == %lld", ausage); + if (ausage > m_configuration.m_diskUsageHWM) + { + bytesToRemove = ausage - m_configuration.m_diskUsageLWM; + clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() need space for %lld bytes", bytesToRemove); + } + } + + if (bytesToRemove > 0) + { + // make a sorted map of file patch by access time + XrdOssDF* dh = oss->newDir(m_configuration.m_username.c_str()); + if (dh->Opendir(m_configuration.m_cache_dir.c_str(), env) >= 0) + { + FPurgeState purgeState(bytesToRemove * 5 / 4); // prepare 20% more volume than required + + FillFileMapRecurse(dh, m_configuration.m_cache_dir, purgeState); + + // loop over map and remove files with highest value of access time + for (FPurgeState::map_i it = purgeState.fmap.begin(); it != purgeState.fmap.end(); ++it) + { + // XXXX MT - shouldn't we re-check if the file is currently opened? + + std::string path = it->second.path; + // remove info file + if (oss->Stat(path.c_str(), &fstat) == XrdOssOK) + { + bytesToRemove -= fstat.st_size; + oss->Unlink(path.c_str()); + clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() removed %s size %lld", + path.c_str(), fstat.st_size); + } + + // remove data file + path = path.substr(0, path.size() - strlen(XrdFileCache::Info::m_infoExtension)); + if (oss->Stat(path.c_str(), &fstat) == XrdOssOK) + { + bytesToRemove -= it->second.nByte; + oss->Unlink(path.c_str()); + clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() removed %s bytes %lld, stat_size %lld", + path.c_str(), it->second.nByte, fstat.st_size); + } + + if (bytesToRemove <= 0) + break; + } + } + dh->Close(); + delete dh; dh =0; + } + + sleep(sleept); + } +} From 04c98cdc958982080369ddcd39b47f9d1e49098b Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Sat, 12 Mar 2016 07:25:58 -0800 Subject: [PATCH 072/171] Add changed files missing in the previous commit c00a774. --- src/XrdFileCache.cmake | 3 +- src/XrdFileCache/XrdFileCache.cc | 89 ++++++++++++++-- src/XrdFileCache/XrdFileCache.hh | 105 +++++++++++++++++-- src/XrdFileCache/XrdFileCacheFile.cc | 27 +++-- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 3 +- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 5 +- src/XrdFileCache/XrdFileCacheVRead.cc | 4 +- 7 files changed, 199 insertions(+), 37 deletions(-) diff --git a/src/XrdFileCache.cmake b/src/XrdFileCache.cmake index f241adc7fec..2dc0d84bea9 100644 --- a/src/XrdFileCache.cmake +++ b/src/XrdFileCache.cmake @@ -17,7 +17,8 @@ add_library( ${LIB_XRD_FILECACHE} MODULE XrdFileCache/XrdFileCache.cc XrdFileCache/XrdFileCache.hh - XrdFileCache/XrdFileCacheFactory.cc XrdFileCache/XrdFileCacheFactory.hh + XrdFileCache/XrdFileCacheConfiguration.cc + XrdFileCache/XrdFileCachePurge.cc XrdFileCache/XrdFileCacheFile.cc XrdFileCache/XrdFileCacheFile.hh XrdFileCache/XrdFileCacheVRead.cc XrdFileCache/XrdFileCacheStats.hh diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index a103b032176..7572103dfb3 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -27,15 +27,27 @@ #include "XrdSys/XrdSysTimer.hh" #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucUtils.hh" #include "XrdFileCache.hh" #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheIOFileBlock.hh" -#include "XrdFileCacheFactory.hh" +//#include "XrdFileCacheConfiguration.cc" +//#include "XrdFileCachePurge.cc" using namespace XrdFileCache; + + +Cache * Cache::m_factory = NULL; + +void *CacheDirCleanupThread(void* cache_void) +{ + Cache::GetInstance().CacheDirCleanup(); + return NULL; +} + void *ProcessWriteTaskThread(void* c) { Cache *cache = static_cast(c); @@ -49,12 +61,73 @@ void *PrefetchThread(void* ptr) cache->Prefetch(); return NULL; } + + +extern "C" +{ +XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, + const char *config_filename, + const char *parameters) +{ + XrdSysError err(0, ""); + err.logger(logger); + err.Emsg("Retrieve", "Retrieving a caching proxy factory."); + Cache &factory = Cache::GetInstance(); + if (!factory.Config(logger, config_filename, parameters)) + { + err.Emsg("Retrieve", "Error - unable to create a factory."); + return NULL; + } + err.Emsg("Retrieve", "Success - returning a factory."); + + + pthread_t tid; + XrdSysThread::Run(&tid, CacheDirCleanupThread, NULL, 0, "XrdFileCache CacheDirCleanup"); + return &factory; +} +} + +Cache &Cache::GetInstance() +{ + if (m_factory == NULL) + m_factory = new Cache(); + return *m_factory; +} + +// !AMT will be obsolete in future +XrdOucCache *Cache::Create(Parms & parms, XrdOucCacheIO::aprParms * prParms) +{ + return this; +} + + +//______________________________________________________________________________ + +bool Cache::Decide(XrdOucCacheIO* io) +{ + if (!m_decisionpoints.empty()) + { + std::string filename = io->Path(); + std::vector::const_iterator it; + for (it = m_decisionpoints.begin(); it != m_decisionpoints.end(); ++it) + { + XrdFileCache::Decision *d = *it; + if (!d) continue; + if (!d->Decide(filename, *m_output_fs)) + { + return false; + } + } + } + + return true; +} //______________________________________________________________________________ -Cache::Cache(XrdOucCacheStats & stats) : XrdOucCache(), +Cache::Cache() : XrdOucCache(), + m_log(0, "XrdFileCache_"), m_prefetch_condVar(0), - m_stats(stats), m_RAMblocks_used(0) { pthread_t tid1; @@ -68,11 +141,11 @@ Cache::Cache(XrdOucCacheStats & stats) : XrdOucCache(), XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) { - if (Factory::GetInstance().Decide(io)) + if (Cache::GetInstance().Decide(io)) { clLog()->Info(XrdCl::AppMsg, "Cache::Attach() %s", io->Path()); IO* cio; - if (Factory::GetInstance().RefConfiguration().m_hdfsmode) + if (Cache::GetInstance().RefConfiguration().m_hdfsmode) cio = new IOFileBlock(*io, m_stats, *this); else cio = new IOEntireFile(*io, m_stats, *this); @@ -179,7 +252,7 @@ bool Cache::RequestRAMBlock() { XrdSysMutexHelper lock(&m_RAMblock_mutex); - if ( m_RAMblocks_used < Factory::GetInstance().RefConfiguration().m_NRamBuffers ) + if ( m_RAMblocks_used < Cache::GetInstance().RefConfiguration().m_NRamBuffers ) { m_RAMblocks_used++; return true; @@ -215,7 +288,7 @@ Cache::RegisterPrefetchFile(File* file) { // called from File::Open() - if (Factory::GetInstance().RefConfiguration().m_prefetch) + if (Cache::GetInstance().RefConfiguration().m_prefetch) { XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file BEGIN"); @@ -269,7 +342,7 @@ Cache::GetNextFileToPrefetch() void Cache::Prefetch() { - const static int limitRAM= Factory::GetInstance().RefConfiguration().m_NRamBuffers * 0.7; + const static int limitRAM= Cache::GetInstance().RefConfiguration().m_NRamBuffers * 0.7; XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch thread start"); diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index af616ec0c75..34086a35e1f 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -20,10 +20,15 @@ #include #include +#include "XrdVersion.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOucCache.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdFileCacheFile.hh" +#include "XrdFileCacheDecision.hh" + +class XrdOucStream; +class XrdSysError; namespace XrdCl { class Log; @@ -35,6 +40,38 @@ class IO; namespace XrdFileCache { + //---------------------------------------------------------------------------- + //! Contains parameters configurable from the xrootd config file. + //---------------------------------------------------------------------------- + struct Configuration + { + Configuration() : + m_hdfsmode(false), + m_diskUsageLWM(-1), + m_diskUsageHWM(-1), + m_bufferSize(1024*1024), + m_NRamBuffers(8000), + m_prefetch(false), + m_prefetch_max_blocks(10), + m_hdfsbsize(128*1024*1024) {} + + bool m_hdfsmode; //!< flag for enabling block-level operation + std::string m_cache_dir; //!< path of disk cache + std::string m_username; //!< username passed to oss plugin + + long long m_diskUsageLWM; //!< cache purge low water mark + long long m_diskUsageHWM; //!< cache purge high water mark + + long long m_bufferSize; //!< prefetch buffer size, default 1MB + int m_NRamBuffers; //!< number of total in-memory cache blocks + bool m_prefetch; //!< prefetch enable state + size_t m_prefetch_max_blocks;//!< maximum number of blocks to prefetch per file + + long long m_hdfsbsize; //!< used with m_hdfsmode, default 128MB + }; + + + //---------------------------------------------------------------------------- //! Attaches/creates and detaches/deletes cache-io objects for disk based cache. //---------------------------------------------------------------------------- @@ -44,7 +81,7 @@ namespace XrdFileCache //--------------------------------------------------------------------- //! Constructor //--------------------------------------------------------------------- - Cache(XrdOucCacheStats&); + Cache(); //--------------------------------------------------------------------- //! Obtain a new IO object that fronts existing XrdOucCacheIO. @@ -57,11 +94,49 @@ namespace XrdFileCache virtual int isAttached(); //--------------------------------------------------------------------- - //! \brief Unused abstract method. Plugin instantiation role is given - //! to the Factory class. + // this is an obsolete method + virtual XrdOucCache* Create(XrdOucCache::Parms&, XrdOucCacheIO::aprParms*); + + //-------------------------------------------------------------------- + //! \brief Makes decision if the original XrdOucCacheIO should be cached. + //! + //! @param & URL of file + //! + //! @return decision if IO object will be cached. + //-------------------------------------------------------------------- + bool Decide(XrdOucCacheIO*); + + //------------------------------------------------------------------------ + //! Reference XrdFileCache configuration + //------------------------------------------------------------------------ + const Configuration& RefConfiguration() const { return m_configuration; } + + + //--------------------------------------------------------------------- + //! \brief Parse configuration file + //! + //! @param logger xrootd logger + //! @param config_filename path to configuration file + //! @param parameters optional parameters to be passed + //! + //! @return parse status + //--------------------------------------------------------------------- + bool Config(XrdSysLogger *logger, const char *config_filename, const char *parameters); + + //--------------------------------------------------------------------- + //! Singleton access. + //--------------------------------------------------------------------- + static Cache &GetInstance(); + + //--------------------------------------------------------------------- + //! Version check. + //--------------------------------------------------------------------- + static bool VCheck(XrdVersionInfo &urVersion) { return true; } + + //--------------------------------------------------------------------- + //! Thread function running disk cache purge periodically. //--------------------------------------------------------------------- - virtual XrdOucCache* Create(XrdOucCache::Parms&, XrdOucCacheIO::aprParms*) - { return NULL; } + void CacheDirCleanup(); //--------------------------------------------------------------------- //! Add downloaded block in write queue. @@ -98,13 +173,31 @@ namespace XrdFileCache //! Decrease attached count. Called from IO::Detach(). void Detach(XrdOucCacheIO *); + XrdOss* GetOss() const { return m_output_fs; } + + XrdSysError& GetSysError() { return m_log; } + + private: + bool ConfigParameters(std::string, XrdOucStream&); + bool ConfigXeq(char *, XrdOucStream &); + bool xdlib(XrdOucStream &); + static Cache *m_factory; //!< this object + + XrdSysError m_log; //!< XrdFileCache namespace logger + XrdOucCacheStats m_stats; //!< + XrdOss *m_output_fs; //!< disk cache file system + + std::vector m_decisionpoints; //!< decision plugins + + std::map m_filesInQueue; + + Configuration m_configuration; //!< configurable parameters //! Short log alias. XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } XrdSysCondVar m_prefetch_condVar; //!< central lock for this class - XrdOucCacheStats &m_stats; //!< global cache usage statistics XrdSysMutex m_RAMblock_mutex; //!< central lock for this class int m_RAMblocks_used; diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 41fc3170133..9726888a585 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -35,7 +35,6 @@ #include "XrdSfs/XrdSfsInterface.hh" #include "XrdPosix/XrdPosixFile.hh" #include "XrdPosix/XrdPosix.hh" -#include "XrdFileCacheFactory.hh" #include "XrdFileCache.hh" #include "Xrd/XrdScheduler.hh" @@ -68,14 +67,14 @@ namespace namespace { - Cache* cache() { return Factory::GetInstance().GetCache(); } + Cache* cache() { return &Cache::GetInstance(); } } File::File(XrdOucCacheIO &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : m_input(inputIO), m_output(NULL), m_infoFile(NULL), -m_cfi(Factory::GetInstance().RefConfiguration().m_bufferSize), +m_cfi(Cache::GetInstance().RefConfiguration().m_bufferSize), m_temp_filename(disk_file_path), m_offset(iOffset), m_fileSize(iFileSize), @@ -223,11 +222,11 @@ bool File::Open() { clLog()->Dump(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_input.Path()); - XrdOss &m_output_fs = *Factory::GetInstance().GetOss(); + XrdOss &m_output_fs = *Cache::GetInstance().GetOss(); // Create the data file itself. XrdOucEnv myEnv; - m_output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), m_temp_filename.c_str(), 0600, myEnv, XRDOSS_mkpath); - m_output = m_output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); + m_output_fs.Create(Cache::GetInstance().RefConfiguration().m_username.c_str(), m_temp_filename.c_str(), 0600, myEnv, XRDOSS_mkpath); + m_output = m_output_fs.newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); if (m_output) { int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0600, myEnv); @@ -248,8 +247,8 @@ bool File::Open() // Create the info file std::string ifn = m_temp_filename + Info::m_infoExtension; - m_output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0600, myEnv, XRDOSS_mkpath); - m_infoFile = m_output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); + m_output_fs.Create(Cache::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0600, myEnv, XRDOSS_mkpath); + m_infoFile = m_output_fs.newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); if (m_infoFile) { int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv); @@ -266,11 +265,11 @@ bool File::Open() return false; } - if (m_cfi.Read(m_infoFile, Factory::GetInstance().RefConfiguration().m_prefetch) <= 0) + if (m_cfi.Read(m_infoFile, Cache::GetInstance().RefConfiguration().m_prefetch) <= 0) { int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input.Path()); - m_cfi.ResizeBits(ss, Factory::GetInstance().RefConfiguration().m_prefetch); + m_cfi.ResizeBits(ss, Cache::GetInstance().RefConfiguration().m_prefetch); m_cfi.WriteHeader(m_infoFile); } else @@ -350,7 +349,7 @@ Block* File::RequestBlock(int i, bool prefetch) clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); m_block_map[i] = b; - if (m_prefetchState == kOn && m_block_map.size() > Factory::GetInstance().RefConfiguration().m_prefetch_max_blocks) + if (m_prefetchState == kOn && m_block_map.size() > Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) { m_prefetchState = kHold; cache()->DeRegisterPrefetchFile(this); @@ -829,7 +828,7 @@ void File::free_block(Block* b) cache()->RAMBlockReleased(); } - if (m_prefetchState == kHold && m_block_map.size() < Factory::GetInstance().RefConfiguration().m_prefetch_max_blocks) + if (m_prefetchState == kHold && m_block_map.size() < Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) { m_prefetchState = kOn; cache()->RegisterPrefetchFile(this); @@ -956,7 +955,7 @@ void File::Prefetch() //______________________________________________________________________________ void File::CheckPrefetchStatRAM(Block* b) { - if (Factory::GetInstance().RefConfiguration().m_prefetch) { + if (Cache::GetInstance().RefConfiguration().m_prefetch) { if (b->m_prefetch) { m_prefetchHitCnt++; m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; @@ -967,7 +966,7 @@ void File::CheckPrefetchStatRAM(Block* b) //______________________________________________________________________________ void File::CheckPrefetchStatDisk(int idx) { - if (Factory::GetInstance().RefConfiguration().m_prefetch) { + if (Cache::GetInstance().RefConfiguration().m_prefetch) { if (m_cfi.TestPrefetchBit(idx)) m_prefetchHitCnt++; } diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index e7728db67e2..ee620601077 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -25,7 +25,6 @@ #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheStats.hh" -#include "XrdFileCacheFactory.hh" using namespace XrdFileCache; @@ -39,7 +38,7 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache & c clLog()->Info(XrdCl::AppMsg, "IO::IO() [%p] %s", this, m_io.Path()); XrdCl::URL url(io.Path()); - std::string fname = Factory::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); + std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); m_file = new File(io, fname, 0, io.FSize()); } diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 1b63bf17c11..ae9b8afe206 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -25,7 +25,6 @@ #include "XrdFileCacheIOFileBlock.hh" #include "XrdFileCache.hh" #include "XrdFileCacheStats.hh" -#include "XrdFileCacheFactory.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSfs/XrdSfsInterface.hh" @@ -37,7 +36,7 @@ using namespace XrdFileCache; IOFileBlock::IOFileBlock(XrdOucCacheIO &io, XrdOucCacheStats &statsGlobal, Cache & cache) : IO(io, statsGlobal, cache) { - m_blocksize = Factory::GetInstance().RefConfiguration().m_hdfsbsize; + m_blocksize = Cache::GetInstance().RefConfiguration().m_hdfsbsize; GetBlockSizeFromPath(); } @@ -87,7 +86,7 @@ void IOFileBlock::GetBlockSizeFromPath() File* IOFileBlock::newBlockFile(long long off, int blocksize, XrdOucCacheIO* io) { XrdCl::URL url(io->Path()); - std::string fname = Factory::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); + std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); std::stringstream ss; ss << fname; diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index cbfdb3ae471..451502ebc01 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -1,6 +1,4 @@ - #include "XrdFileCacheFile.hh" -#include "XrdFileCacheFactory.hh" #include "XrdFileCache.hh" #include "XrdFileCacheInfo.hh" @@ -189,7 +187,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d , chunk idx = %d on disk", block_idx,iov_idx ); } else { - if ( Factory::GetInstance().GetCache()->HaveFreeWritingSlots() && Factory::GetInstance().GetCache()->RequestRAMBlock()) + if ( Cache::GetInstance().HaveFreeWritingSlots() && Cache::GetInstance().RequestRAMBlock()) { Block *b = RequestBlock(block_idx, false); if (!b) return false; From d89de8d171e86c26425fc6da6c9e95d2d2d0aa93 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Thu, 17 Mar 2016 22:10:13 -0700 Subject: [PATCH 073/171] Implement async Posix I/O and deferred file open. --- src/XrdOuc/XrdOucCache.hh | 27 ++++ src/XrdPlugins.cmake | 1 + src/XrdPosix.cmake | 6 +- src/XrdPosix/XrdPosixCacheBC.hh | 119 +++++++++++++++ src/XrdPosix/XrdPosixCallBack.cc | 53 +++++++ src/XrdPosix/XrdPosixCallBack.hh | 66 ++++++--- src/XrdPosix/XrdPosixFile.cc | 240 +++++++++++++++++++++++++------ src/XrdPosix/XrdPosixFile.hh | 71 ++++++--- src/XrdPosix/XrdPosixFileRH.cc | 163 +++++++++++++++++++++ src/XrdPosix/XrdPosixFileRH.hh | 81 +++++++++++ src/XrdPosix/XrdPosixObjGaurd.hh | 53 +++++++ src/XrdPosix/XrdPosixObject.cc | 2 +- src/XrdPosix/XrdPosixObject.hh | 20 ++- src/XrdPosix/XrdPosixPrepIO.cc | 100 +++++++++++++ src/XrdPosix/XrdPosixPrepIO.hh | 100 +++++++++++++ src/XrdPosix/XrdPosixXrootd.cc | 235 ++++++++++++++++++++++++++---- src/XrdPosix/XrdPosixXrootd.hh | 33 ++++- src/XrdPss/XrdPssAio.cc | 40 ++---- src/XrdPss/XrdPssAioCB.cc | 107 ++++++++++++++ src/XrdPss/XrdPssAioCB.hh | 64 +++++++++ src/XrdPss/XrdPssConfig.cc | 23 ++- src/XrdUtils.cmake | 1 + src/XrdXrootd/XrdXrootdAio.cc | 1 + 23 files changed, 1456 insertions(+), 150 deletions(-) create mode 100644 src/XrdPosix/XrdPosixCacheBC.hh create mode 100644 src/XrdPosix/XrdPosixCallBack.cc create mode 100644 src/XrdPosix/XrdPosixFileRH.cc create mode 100644 src/XrdPosix/XrdPosixFileRH.hh create mode 100644 src/XrdPosix/XrdPosixObjGaurd.hh create mode 100644 src/XrdPosix/XrdPosixPrepIO.cc create mode 100644 src/XrdPosix/XrdPosixPrepIO.hh create mode 100644 src/XrdPss/XrdPssAioCB.cc create mode 100644 src/XrdPss/XrdPssAioCB.hh diff --git a/src/XrdOuc/XrdOucCache.hh b/src/XrdOuc/XrdOucCache.hh index 0fcedc71773..0841d68131e 100644 --- a/src/XrdOuc/XrdOucCache.hh +++ b/src/XrdOuc/XrdOucCache.hh @@ -139,6 +139,33 @@ private: XrdSysMutex sMutex; }; +/******************************************************************************/ +/* X r d O u c C a c h e I O C B */ +/******************************************************************************/ + +//----------------------------------------------------------------------------- +//! The XrdOucCacheIOCB defines a callback object that must be used to handle +//! asynchronous I/O operations. +//----------------------------------------------------------------------------- + +class XrdOucCacheIOCB +{ +public: + +//------------------------------------------------------------------------------ +//! Handle result from a previous async operation. +//! +//! @param result is result from a previous operation. Successful results are +//! always values >= 0 while errors are negative values and are +//! always '-errno' indicate the reason for the error. +//------------------------------------------------------------------------------ +virtual +void Done(int result) = 0; + + XrdOucCacheIOCB() {} +virtual ~XrdOucCacheIOCB() {} +}; + /******************************************************************************/ /* C l a s s X r d O u c C a c h e I O */ /******************************************************************************/ diff --git a/src/XrdPlugins.cmake b/src/XrdPlugins.cmake index b0bba3dc478..c789693c651 100644 --- a/src/XrdPlugins.cmake +++ b/src/XrdPlugins.cmake @@ -21,6 +21,7 @@ add_library( ${LIB_XRD_PSS} MODULE XrdPss/XrdPssAio.cc + XrdPss/XrdPssAioCB.cc XrdPss/XrdPssAioCB.hh XrdPss/XrdPss.cc XrdPss/XrdPss.hh XrdPss/XrdPssCks.cc XrdPss/XrdPssCks.hh XrdPss/XrdPssConfig.cc ) diff --git a/src/XrdPosix.cmake b/src/XrdPosix.cmake index e3d88b16282..7df4e3a2c4c 100644 --- a/src/XrdPosix.cmake +++ b/src/XrdPosix.cmake @@ -16,13 +16,17 @@ add_library( XrdPosix SHARED XrdPosix/XrdPosixAdmin.cc XrdPosix/XrdPosixAdmin.hh + XrdPosix/XrdPosixCacheBC.hh + XrdPosix/XrdPosixCallBack.cc XrdPosix/XrdPosixCallBack.hh XrdPosix/XrdPosixDir.cc XrdPosix/XrdPosixDir.hh XrdPosix/XrdPosixFile.cc XrdPosix/XrdPosixFile.hh + XrdPosix/XrdPosixFileRH.cc XrdPosix/XrdPosixFileRH.hh XrdPosix/XrdPosixMap.cc XrdPosix/XrdPosixMap.hh XrdPosix/XrdPosixObject.cc XrdPosix/XrdPosixObject.hh + XrdPosix/XrdPosixObjGaurd.hh + XrdPosix/XrdPosixPrepIO.cc XrdPosix/XrdPosixPrepIO.hh XrdPosix/XrdPosixXrootd.cc XrdPosix/XrdPosixXrootd.hh XrdPosix/XrdPosixXrootdPath.cc XrdPosix/XrdPosixXrootdPath.hh - XrdPosix/XrdPosixCallBack.hh XrdPosix/XrdPosixOsDep.hh ) target_link_libraries( diff --git a/src/XrdPosix/XrdPosixCacheBC.hh b/src/XrdPosix/XrdPosixCacheBC.hh new file mode 100644 index 00000000000..5694b9ef42d --- /dev/null +++ b/src/XrdPosix/XrdPosixCacheBC.hh @@ -0,0 +1,119 @@ +#ifndef __XRDPOSIXCACHEBC_HH__ +#define __XRDPOSIXCACHEBC_HH__ +/******************************************************************************/ +/* */ +/* X r d P o s i x C a c h e B C . h h */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "XrdOuc/XrdOucCache2.hh" + +/******************************************************************************/ +/* X r d P o s i x C a c h e B C I O */ +/******************************************************************************/ + +class XrdPosixCacheBCIO : public XrdOucCacheIO2 +{ +public: + +virtual +XrdOucCacheIO2 *Base() {return cacheIO2;} + +virtual +XrdOucCacheIO2 *Detach() {XrdOucCacheIO2 *theCIO = cacheIO2; + cacheIO1->Detach(); + delete this; + return theCIO; + } + +virtual +long long FSize() {return cacheIO1->FSize();} + +virtual +const char *Path() {return cacheIO1->Path();} + +virtual int Read (char *Buffer, long long Offset, int Length) + {return cacheIO1->Read(Buffer, Offset, Length);} + +virtual int ReadV(const XrdOucIOVec *readV, int n) + {return cacheIO1->ReadV(readV, n);} + +virtual int Sync() {return cacheIO1->Sync();} + +virtual int Trunc(long long Offset) {return cacheIO1->Trunc(Offset);} + +virtual int Write(char *Buffer, long long Offset, int Length) + {return cacheIO1->Write(Buffer, Offset, Length);} + +virtual bool ioActive() { return cacheIO1->ioActive();} + +virtual void Preread (long long Offset, int Length, int Opts=0) + {return cacheIO1->Preread(Offset, Length, Opts);} + +virtual void Preread(aprParms &Parms) { cacheIO1->Preread(Parms);} + + XrdPosixCacheBCIO(XrdOucCacheIO *urCIO, XrdOucCacheIO2 *myCIO) + : cacheIO1(urCIO), cacheIO2(myCIO) {} +virtual ~XrdPosixCacheBCIO() {} + +private: +XrdOucCacheIO *cacheIO1; +XrdOucCacheIO2 *cacheIO2; +}; + +/******************************************************************************/ +/* X r d P o s i x C a c h e B C */ +/******************************************************************************/ + +class XrdPosixCacheBC : public XrdOucCache2 +{ +public: +virtual +XrdOucCacheIO2 *Attach(XrdOucCacheIO2 *ioP, int opts=0) + {XrdOucCacheIO *newIOP = v1Cache->Attach(ioP, opts); + if (newIOP == (XrdOucCacheIO *)newIOP) return ioP; + return new XrdPosixCacheBCIO(newIOP, ioP); + } + +virtual int isAttached() {return v1Cache->isAttached();} + +virtual int Rmdir(const char* path) {return v1Cache->Rmdir(path);} + +virtual int Rename(const char* pathO, const char* pathN) + {return v1Cache->Rename(pathO, pathN);} + +virtual int Truncate(const char* path, off_t size) + {return v1Cache->Truncate(path, size);} + +virtual int Unlink(const char* path) {return v1Cache->Unlink(path);} + + XrdPosixCacheBC(XrdOucCache *cP) : v1Cache(cP) {} +virtual ~XrdPosixCacheBC() {} +private: +XrdOucCache *v1Cache; +}; +#endif diff --git a/src/XrdPosix/XrdPosixCallBack.cc b/src/XrdPosix/XrdPosixCallBack.cc new file mode 100644 index 00000000000..548ad5b1731 --- /dev/null +++ b/src/XrdPosix/XrdPosixCallBack.cc @@ -0,0 +1,53 @@ +/******************************************************************************/ +/* */ +/* X r d P o s i x C a l l B a c k . c c */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include + +#include "XrdPosix/XrdPosixCallBack.hh" +#include "XrdPosix/XrdPosixFile.hh" + +/******************************************************************************/ +/* D o n e */ +/******************************************************************************/ + +void XrdPosixCallBackIO::Done(int result) +{ +// Unreference the file so that the callback is able to close it. +// + theFile->unRef(); + +// Diagnose the problem here, if there is a problem +// + if (result < 0) {errno = -result; result = -1;} + +// Invoke the callback +// + Complete((ssize_t)result); +} diff --git a/src/XrdPosix/XrdPosixCallBack.hh b/src/XrdPosix/XrdPosixCallBack.hh index 18197bb0ac8..01794387de6 100644 --- a/src/XrdPosix/XrdPosixCallBack.hh +++ b/src/XrdPosix/XrdPosixCallBack.hh @@ -4,7 +4,7 @@ /* */ /* X r d P o s i x C a l l B a c k . h h */ /* */ -/* (c) 2010 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ @@ -30,22 +30,23 @@ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ -// This abstract class defines the callback interface for file open() calls. -// It is passed when using the XrdPosixXrootd::Open() call. When passed, the -// open request will be done in the background. When a callback object is -// supplied, Open() will *always* return -1. However, if started successfully, -// Open() will return -1 with errno set to EINPROGRESS. Otherwise, errno will -// contain the reason the Open() request immediately failed. Upon completion, -// the callback's Compete() method is invoked. The Result parameter will either -// be a non-negative file descriptor or -errno indicating that the Open() -// failed. Note that the caller is responsible for deleting the callback object -// after it has been invoked. Note that callbacks will be executed in a -// separate thread unless open() is called with O_SYNC or maxThreads is zero. -// WARNING: If O_SYNC or maxThreads is zero, then the callback must *not* -// issue any filesystem calls using the supplied file descriptor. -// Ignoring this will produce undetermined results including possible -// deadlock. Synchrnous callbacks are only meant to support private -// thread management. +#include "XrdOuc/XrdOucCache.hh" + +//----------------------------------------------------------------------------- +//! @brief An abstract class to define a callback for Open() call. +//! +//! This abstract class defines the callback interface for Open() calls. +//! When passed, the request is done in the background. When a callback +//! object is supplied, the method *always* return -1. However, if started +//! successfully, the method sets errno to EINPROGRESS. Otherwise, errno +//! contain the reason the request immediately failed. Upon completion, the +//! the callback's Compete() method is invoked. The Result parameter contains +//! what the method would have returned if it were executed synchronously: +//! for succsessful Open() it is a non-negative file descriptor but on failure +//! -1 with errno indicating why Open() failed. The caller is responsible for +//! deleting the callback object after it has been invoked. Callbacks are +//! executed in a separate thread. +//----------------------------------------------------------------------------- class XrdPosixCallBack { @@ -56,4 +57,35 @@ virtual void Complete(int Result) = 0; XrdPosixCallBack() {} virtual ~XrdPosixCallBack() {} }; + +//----------------------------------------------------------------------------- +//! @brief An abstract class to define a callback for open file requests. +//! +//! This abstract class defines the callback interface for Fsync(), Pread(), +//! Pwrite(), and VRead(). Async I/O is not supported for Read(), Readv(), +//! Write(), and Writev() as these update the file offset associated with the +//! file descriptor and cannot be made async safe. All results are return via +//! the callback object. For immediate errors, the callback is invoked on the +//! calling thread. Any locks held by the calling thread that are obtained by +//! the callback must be recursive locks in order to avoid a deadlock. +//----------------------------------------------------------------------------- + +class XrdPosixFile; + +class XrdPosixCallBackIO : public XrdOucCacheIOCB +{ +public: +friend class XrdPosixXrootd; + +virtual void Complete(ssize_t Result) = 0; + + XrdPosixCallBackIO() : theFile(0) {} +virtual ~XrdPosixCallBackIO() {} + +private: + +void Done(int result); + +XrdPosixFile *theFile; +}; #endif diff --git a/src/XrdPosix/XrdPosixFile.cc b/src/XrdPosix/XrdPosixFile.cc index ce3258a2c03..08e52d03e86 100644 --- a/src/XrdPosix/XrdPosixFile.cc +++ b/src/XrdPosix/XrdPosixFile.cc @@ -39,25 +39,45 @@ #include "XrdPosix/XrdPosixCallBack.hh" #include "XrdPosix/XrdPosixFile.hh" +#include "XrdPosix/XrdPosixFileRH.hh" +#include "XrdPosix/XrdPosixPrepIO.hh" /******************************************************************************/ /* S t a t i c M e m b e r s */ /******************************************************************************/ -XrdOucCache *XrdPosixFile::CacheR = 0; -XrdOucCache *XrdPosixFile::CacheW = 0; +namespace XrdPosixGlobals +{ +extern XrdOucCache2 *theCache; +}; + +namespace +{ +XrdPosixFile *InitDDL() +{ +pthread_t tid; +XrdSysThread::Run(&tid, XrdPosixFile::DelayedDestroy, 0, 0, "PosixFileDestroy"); +return (XrdPosixFile *)0; +} +}; + +XrdSysSemaphore XrdPosixFile::ddSem(0); +XrdSysMutex XrdPosixFile::ddMutex; +XrdPosixFile *XrdPosixFile::ddList = InitDDL(); + char *XrdPosixFile::sfSFX = 0; int XrdPosixFile::sfSLN = 0; +bool XrdPosixFile::ddPosted = false; /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdPosixFile::XrdPosixFile(const char *path, XrdPosixCallBack *cbP, int Opts) - : XCio((XrdOucCacheIO *)this), + : XCio((XrdOucCacheIO2 *)this), PrepIO(0), mySize(0), myMtime(0), myInode(0), myMode(0), theCB(cbP), - fPath(0), + fPath(strdup(path)), cOpt(0), isStream(Opts & isStrm ? 1 : 0) { @@ -89,13 +109,17 @@ XrdPosixFile::~XrdPosixFile() // if (clFile.IsOpen()) {XrdCl::XRootDStatus status = clFile.Close();}; +// Get rid of defered open object +// + if (PrepIO) delete PrepIO; + // Free the path // if (fPath) free(fPath); } /******************************************************************************/ -/* D e l a y e d D e s t r o y */ +/* D e l a y e d D e s t r o y */ /******************************************************************************/ void* XrdPosixFile::DelayedDestroy(void* vpf) @@ -103,40 +127,83 @@ void* XrdPosixFile::DelayedDestroy(void* vpf) // Static function. // Called within a dedicated thread if XrdOucCacheIO is io-active or the // file cannot be closed in a clean fashion for some reason. +// + static const int ddInterval = 30; + static const int maxTries = (3*60)/ddInterval; + char eBuff[2048]; XrdCl::XRootDStatus Status; - int wtCnt = 180, wtChk = 3; - XrdPosixFile* fP = (XrdPosixFile*)vpf; - const char *eTxt = "unknown"; + const char *eTxt; + XrdPosixFile *fCurr, *fNext; + time_t tNow, wakeTime = 0; + bool doWait = false; // Wait for active I/O to complete // - while(fP->XCio->ioActive() && wtCnt) {sleep(wtChk); wtCnt -= wtChk;} +do{if (doWait) + {sleep(ddInterval); + doWait = false; + } else { + ddSem.Wait(); + tNow = time(0); + if (tNow < wakeTime) continue; + wakeTime = tNow + ddInterval; + } -// If it didn't complete we can't delete this object. Otherwise, try to close -// it if it is open. This may not be possible if recovery is taking too long. +// Grab the delayed delete list // - if (!wtCnt) eTxt = "active I/O"; - else {while(fP->clFile.IsOpen() && wtCnt) - {if (fP->Close(Status)) break; - sleep(wtChk); wtCnt -= wtChk; - } - if (wtCnt) eTxt = Status.ToString().c_str(); - } + ddMutex.Lock(); fNext=ddList; ddList=0; ddPosted=false; ddMutex.UnLock(); -// Delete the object if it is safe to do so. Otherwise, issue error message. +// Try to delete all the files on the list. If we exceeded the try limit, +// remove the file from the list and let it sit forever. // - if (wtCnt) delete fP; - else {char eBuff[2048]; - snprintf(eBuff, sizeof(eBuff), - "PosixFile: %s timeout closing %s; object lost!\n", - eTxt, fP->Path()); - std::cerr <nextFile; + if (!(fCurr->XCio->ioActive()) && !fCurr->Refs()) + {if (fCurr->Close(Status)) {delete fCurr; continue;} + else eTxt = Status.ToString().c_str(); + } else eTxt = "active I/O"; + + if (fCurr->numTries > maxTries) + {snprintf(eBuff, sizeof(eBuff), + "PosixFile: %s timeout closing %s; object lost!\n", + eTxt, fCurr->Path()); + std::cerr <Close(Status); + } else { + fCurr->numTries++; + doWait = true; + ddMutex.Lock(); + fCurr->nextFile = ddList; ddList = fCurr; ddPosted = true; + ddMutex.UnLock(); + } + }; + } while(true); return 0; } +/******************************************************************************/ + +void XrdPosixFile::DelayedDestroy(XrdPosixFile *fp) +{ + bool doPost; + +// Place this file on the delayed delete list +// + ddMutex.Lock(); + fp->nextFile = ddList; + ddList = fp; + if (ddPosted) doPost = false; + else {doPost = true; + ddPosted = true; + } + ddMutex.UnLock(); + fp->numTries = 0; + + if (doPost) ddSem.Post(); +} + /******************************************************************************/ /* C l o s e */ /******************************************************************************/ @@ -172,10 +239,8 @@ bool XrdPosixFile::Finalize(XrdCl::XRootDStatus &Status) // Setup the cache if it is to be used // - if (cOpt & XrdOucCache::optRW) - { if (CacheW) XCio = CacheW->Attach((XrdOucCacheIO *)this, cOpt);} - else if (CacheR) XCio = CacheR->Attach((XrdOucCacheIO *)this, cOpt); - + if (XrdPosixGlobals::theCache) + XCio = XrdPosixGlobals::theCache->Attach((XrdOucCacheIO2 *)this, cOpt); return true; } @@ -207,17 +272,6 @@ void XrdPosixFile::HandleResponse(XrdCl::XRootDStatus *status, if (rc) delete this; } -/******************************************************************************/ -/* P a t h */ -/******************************************************************************/ - -const char *XrdPosixFile::Path() -{ - std::string fileUrl; clFile.GetProperty( "LastURL", fileUrl ); - if (!fPath) fPath = strdup(fileUrl.c_str()); - return fPath; -} - /******************************************************************************/ /* R e a d */ /******************************************************************************/ @@ -234,6 +288,24 @@ int XrdPosixFile::Read (char *Buff, long long Offs, int Len) return (Status.IsOK() ? (int)bytes : XrdPosixMap::Result(Status)); } +/******************************************************************************/ + +void XrdPosixFile::Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, + int rlen) +{ + XrdCl::XRootDStatus Status; + XrdPosixFileRH *rhp = XrdPosixFileRH::Alloc(&iocb, this, offs, rlen, + XrdPosixFileRH::isRead); + +// Issue read +// + Status = clFile.Read((uint64_t)offs, (uint32_t)rlen, buff, rhp); + +// Check status +// + if (!Status.IsOK()) rhp->Sched(-XrdPosixMap::Result(Status)); +} + /******************************************************************************/ /* R e a d V */ /******************************************************************************/ @@ -243,12 +315,12 @@ int XrdPosixFile::ReadV (const XrdOucIOVec *readV, int n) XrdCl::XRootDStatus Status; XrdCl::ChunkList chunkVec; XrdCl::VectorReadInfo *vrInfo = 0; - int i, nbytes = 0; + int nbytes = 0; // Copy in the vector (would be nice if we didn't need to do this) // chunkVec.reserve(n); - for (i = 0; i < n; i++) + for (int i = 0; i < n; i++) {nbytes += readV[i].size; chunkVec.push_back(XrdCl::ChunkInfo((uint64_t)readV[i].offset, (uint32_t)readV[i].size, @@ -267,6 +339,36 @@ int XrdPosixFile::ReadV (const XrdOucIOVec *readV, int n) return (Status.IsOK() ? nbytes : XrdPosixMap::Result(Status)); } +/******************************************************************************/ + +void XrdPosixFile::ReadV(XrdOucCacheIOCB &iocb, const XrdOucIOVec *readV, int n) +{ + XrdCl::XRootDStatus Status; + XrdCl::ChunkList chunkVec; + int nbytes = 0; + +// Copy in the vector (would be nice if we didn't need to do this) +// + chunkVec.reserve(n); + for (int i = 0; i < n; i++) + {nbytes += readV[i].size; + chunkVec.push_back(XrdCl::ChunkInfo((uint64_t)readV[i].offset, + (uint32_t)readV[i].size, + (void *)readV[i].data + )); + } + +// Issue the readv. +// + XrdPosixFileRH *rhp = XrdPosixFileRH::Alloc(&iocb, this, 0, nbytes, + XrdPosixFileRH::isReadV); + Status = clFile.VectorRead(chunkVec, (void *)0, rhp); + +// Return appropriate result +// + if (!Status.IsOK()) rhp->Sched(-XrdPosixMap::Result(Status)); +} + /******************************************************************************/ /* S t a t */ /******************************************************************************/ @@ -296,6 +398,58 @@ bool XrdPosixFile::Stat(XrdCl::XRootDStatus &Status, bool force) return true; } +/******************************************************************************/ +/* S y n c */ +/******************************************************************************/ + +void XrdPosixFile::Sync(XrdOucCacheIOCB &iocb) +{ + XrdCl::XRootDStatus Status; + XrdPosixFileRH *rhp = XrdPosixFileRH::Alloc(&iocb, this, 0, 0, + XrdPosixFileRH::nonIO); + +// Issue read +// + Status = clFile.Sync(rhp); + +// Check status +// + if (!Status.IsOK()) rhp->Sched(-XrdPosixMap::Result(Status)); +} + +/******************************************************************************/ +/* W r i t e */ +/******************************************************************************/ + +int XrdPosixFile::Write(char *Buff, long long Offs, int Len) +{ + XrdCl::XRootDStatus Status; + +// Issue read and return appropriately +// + Status = clFile.Write((uint64_t)Offs, (uint32_t)Len, Buff); + + return (Status.IsOK() ? Len : XrdPosixMap::Result(Status)); +} + +/******************************************************************************/ + +void XrdPosixFile::Write(XrdOucCacheIOCB &iocb, char *buff, long long offs, + int wlen) +{ + XrdCl::XRootDStatus Status; + XrdPosixFileRH *rhp = XrdPosixFileRH::Alloc(&iocb, this, offs, wlen, + XrdPosixFileRH::isWrite); + +// Issue read +// + Status = clFile.Write((uint64_t)offs, (uint32_t)wlen, buff, rhp); + +// Check status +// + if (!Status.IsOK()) rhp->Sched(-XrdPosixMap::Result(Status)); +} + /******************************************************************************/ /* D o I t */ /******************************************************************************/ diff --git a/src/XrdPosix/XrdPosixFile.hh b/src/XrdPosix/XrdPosixFile.hh index 10ff3c310f7..d30cf68e8a5 100644 --- a/src/XrdPosix/XrdPosixFile.hh +++ b/src/XrdPosix/XrdPosixFile.hh @@ -42,7 +42,7 @@ #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClXRootDResponses.hh" -#include "XrdOuc/XrdOucCache.hh" +#include "XrdOuc/XrdOucCache2.hh" #include "XrdPosix/XrdPosixMap.hh" #include "XrdPosix/XrdPosixObject.hh" @@ -53,68 +53,95 @@ /******************************************************************************/ class XrdPosixCallBack; +class XrdPosixPrepIO; class XrdPosixFile : public XrdPosixObject, - public XrdOucCacheIO, + public XrdOucCacheIO2, public XrdCl::ResponseHandler, public XrdJob { public: -XrdOucCacheIO *XCio; -XrdCl::File clFile; +XrdOucCacheIO2 *XCio; +XrdPosixPrepIO *PrepIO; +XrdCl::File clFile; long long addOffset(long long offs, int updtSz=0) - {currOffset += offs; + {updMutex.Lock(); + currOffset += offs; if (updtSz && currOffset > (long long)mySize) mySize = currOffset; - return currOffset; + long long retOffset = currOffset; + updMutex.UnLock(); + return retOffset; } static XrdPosixFile *Alloc(const char *path, XrdPosixCallBack *cbP, int Opts); static void* DelayedDestroy(void*); +static void DelayedDestroy(XrdPosixFile *fp); + bool Close(XrdCl::XRootDStatus &Status); bool Finalize(XrdCl::XRootDStatus &Status); - long long FSize() {return static_cast(mySize);} + long long FSize() {AtomicBeg(updMutex); + long long retSize = AtomicGet(mySize); + AtomicEnd(updMutex); + return retSize; + } void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response); void isOpen(); - long long Offset() {return currOffset;} + long long Offset() {AtomicRet(updMutex, currOffset);} - const char *Path(); + const char *Path() {return fPath;} int Read (char *Buff, long long Offs, int Len); + void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, + int rlen); + int ReadV (const XrdOucIOVec *readV, int n); + void ReadV (XrdOucCacheIOCB &iocb, const XrdOucIOVec *readV, + int n); + long long setOffset(long long offs) - {currOffset = offs; - return currOffset; + {updMutex.Lock(); + currOffset = offs; + updMutex.UnLock(); + return offs; } bool Stat(XrdCl::XRootDStatus &Status, bool force=false); int Sync() {return XrdPosixMap::Result(clFile.Sync());} + void Sync(XrdOucCacheIOCB &iocb); + int Trunc(long long Offset) {return XrdPosixMap::Result(clFile.Truncate((uint64_t)Offset));} + void UpdtSize(size_t newsz) + {updMutex.Lock(); + if (newsz > mySize) mySize = newsz; + updMutex.UnLock(); + } + using XrdPosixObject::Who; bool Who(XrdPosixFile **fileP) {*fileP = this; return true;} - int Write(char *Buff, long long Offs, int Len) - {return XrdPosixMap::Result(clFile.Write((uint64_t)Offs, - (uint32_t)Len, Buff)); - } + int Write(char *Buff, long long Offs, int Len); + + void Write(XrdOucCacheIOCB &iocb, char *buff, long long offs, + int wlen); void DoIt(); @@ -124,10 +151,13 @@ static void* DelayedDestroy(void*); ino_t myInode; mode_t myMode; -static XrdOucCache *CacheR; -static XrdOucCache *CacheW; -static char *sfSFX; -static int sfSLN; +static +XrdSysSemaphore ddSem; +static XrdSysMutex ddMutex; +static XrdPosixFile *ddList; +static char *sfSFX; +static int sfSLN; +static bool ddPosted; static const int realFD = 1; static const int isStrm = 2; @@ -140,10 +170,11 @@ private: union {long long currOffset; XrdPosixCallBack *theCB; + XrdPosixFile *nextFile; }; char *fPath; -int cOpt; +union {int cOpt; int numTries;}; char isStream; }; #endif diff --git a/src/XrdPosix/XrdPosixFileRH.cc b/src/XrdPosix/XrdPosixFileRH.cc new file mode 100644 index 00000000000..0d05154a634 --- /dev/null +++ b/src/XrdPosix/XrdPosixFileRH.cc @@ -0,0 +1,163 @@ +/******************************************************************************/ +/* */ +/* X r d P o x F i l e R H . c c */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "Xrd/XrdScheduler.hh" + +#include "XrdOuc/XrdOucCache.hh" + +#include "XrdPosix/XrdPosixFileRH.hh" +#include "XrdPosix/XrdPosixFile.hh" +#include "XrdPosix/XrdPosixMap.hh" + +/******************************************************************************/ +/* S t a t i c M e m b e r s */ +/******************************************************************************/ + +namespace XrdPosixGlobals +{ +extern XrdScheduler *schedP; +}; + +XrdSysMutex XrdPosixFileRH::myMutex; +XrdPosixFileRH *XrdPosixFileRH::freeRH = 0; +int XrdPosixFileRH::numFree = 0; +int XrdPosixFileRH::maxFree = 100; + +/******************************************************************************/ +/* E x t e r n a l L i n k a g e s */ +/******************************************************************************/ + +namespace +{ +void *callDoIt(void *pp) +{ + XrdPosixFileRH *rhp = (XrdPosixFileRH *)pp; + rhp->DoIt(); + return (void *)0; +} +}; + +/******************************************************************************/ +/* A l l o c */ +/******************************************************************************/ + +XrdPosixFileRH *XrdPosixFileRH::Alloc(XrdOucCacheIOCB *cbp, + XrdPosixFile *fp, + long long offs, + int xResult, + ioType typeIO) +{ + XrdPosixFileRH *newCB; + +// Try to allocate an prexisting object otherwise get a new one +// + myMutex.Lock(); + if ((newCB = freeRH)) {freeRH = newCB->next; numFree--;} + else newCB = new XrdPosixFileRH; + myMutex.UnLock(); + +// Initialize the callback and return it +// + newCB->theCB = cbp; + newCB->theFile = fp; + newCB->offset = offs; + newCB->result = xResult; + newCB->typeIO = typeIO; + return newCB; +} + +/******************************************************************************/ +/* H a n d l e R e s p o n s e */ +/******************************************************************************/ + +void XrdPosixFileRH::HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response) +{ + +// Determine ending status. +// + if (!(status->IsOK())) result = -XrdPosixMap::Result(*status); + else if (typeIO == nonIO) result = 0; + else if (typeIO == isRead) + {XrdCl::ChunkInfo *cInfo = 0; + union {uint32_t ubRead; int ibRead;}; + response->Get(cInfo); + ubRead = (cInfo ? cInfo->length : 0); + result = ibRead; + } + else if (typeIO == isWrite) theFile->UpdtSize(offset+result); + +// Get rid of things we don't need +// + delete status; + delete response; + +// Now schedule this callback +// + if (XrdPosixGlobals::schedP) XrdPosixGlobals::schedP->Schedule(this); + else {pthread_t tid; + XrdSysThread::Run(&tid, callDoIt, this, 0, "PosixFileRH"); + } +} + +/******************************************************************************/ +/* R e c y c l e */ +/******************************************************************************/ + +void XrdPosixFileRH::Recycle() +{ +// Perform recycling +// + myMutex.Lock(); + if (numFree >= maxFree) delete this; + else {next = freeRH; + freeRH = this; + numFree++; + } + myMutex.UnLock(); +} + +/******************************************************************************/ +/* S c h e d */ +/******************************************************************************/ + +void XrdPosixFileRH::Sched(int rval) +{ +// Set result +// + result = rval; + +// Now schedule this callback +// + if (XrdPosixGlobals::schedP) XrdPosixGlobals::schedP->Schedule(this); + else {pthread_t tid; + XrdSysThread::Run(&tid, callDoIt, this, 0, "PosixFileRH"); + } +} diff --git a/src/XrdPosix/XrdPosixFileRH.hh b/src/XrdPosix/XrdPosixFileRH.hh new file mode 100644 index 00000000000..d56bfd4d5b1 --- /dev/null +++ b/src/XrdPosix/XrdPosixFileRH.hh @@ -0,0 +1,81 @@ +#ifndef __POSIX_FILERH_HH__ +#define __POSIX_FILERH_HH__ +/******************************************************************************/ +/* */ +/* X r d P o s i x F l e R H . h h */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "XrdCl/XrdClFile.hh" +#include "XrdSys/XrdSysPthread.hh" + +class XrdOucCacheIOCB; +class XrdPosixFile; + +/******************************************************************************/ +/* X r d P o s i x F i l e R H */ +/******************************************************************************/ + +class XrdPosixFileRH : public XrdJob, + public XrdCl::ResponseHandler +{ +public: + +enum ioType {nonIO = 0, isRead = 1, isReadV = 2, isWrite = 3}; + +static XrdPosixFileRH *Alloc(XrdOucCacheIOCB *cbp, XrdPosixFile *fp, + long long offs, int xResult, ioType typeIO); + + void DoIt() {theCB->Done(result); Recycle();} + + void HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response); + + void Recycle(); + +static void SetMax(int mval) {maxFree = mval;} + + void Sched(int result); + +private: + XrdPosixFileRH() : theCB(0),theFile(0),result(0),typeIO(nonIO) {} +virtual ~XrdPosixFileRH() {} + +static XrdSysMutex myMutex; +static XrdPosixFileRH *freeRH; +static int numFree; +static int maxFree; + +union {XrdOucCacheIOCB *theCB; + XrdPosixFileRH *next; + }; +XrdPosixFile *theFile; +long long offset; +int result; +ioType typeIO; +}; +#endif diff --git a/src/XrdPosix/XrdPosixObjGaurd.hh b/src/XrdPosix/XrdPosixObjGaurd.hh new file mode 100644 index 00000000000..b9b9b1d3012 --- /dev/null +++ b/src/XrdPosix/XrdPosixObjGaurd.hh @@ -0,0 +1,53 @@ +#ifndef __XRDPOSIXOBJGAURD_HH__ +#define __XRDPOSIXOBJGAURD_HH__ +/******************************************************************************/ +/* */ +/* X r d P o s i x O b j G a u r d . h h */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "XrdPosix/XrdPosixObject.hh" + +class XrdPosixObjGaurd +{ +public: + +void Init(XrdPosixObject *oP, bool rw=true) + {if (objP) Release(); + objP = oP; + objP->Lock(rw); + } + +void Release() {if (objP) {objP->UnLock(); objP = 0;}} + + XrdPosixObjGaurd(XrdPosixObject *oP, bool rw=true) {Init(oP, rw);} + ~XrdPosixObjGaurd() {Release();} + +private: +XrdPosixObject *objP; +}; +#endif diff --git a/src/XrdPosix/XrdPosixObject.cc b/src/XrdPosix/XrdPosixObject.cc index 0a5d4e511bd..91d1d905ac6 100644 --- a/src/XrdPosix/XrdPosixObject.cc +++ b/src/XrdPosix/XrdPosixObject.cc @@ -257,7 +257,7 @@ void XrdPosixObject::Release(XrdPosixObject *oP, bool needlk) close(oP->fdNum); } -// Zorch the object fd and relese the global lock (object lock still held) +// Zorch the object fd and release the global lock // oP->fdNum = -1; fdMutex.UnLock(); diff --git a/src/XrdPosix/XrdPosixObject.hh b/src/XrdPosix/XrdPosixObject.hh index 81dc76d58bd..0b7b5ebc7b1 100644 --- a/src/XrdPosix/XrdPosixObject.hh +++ b/src/XrdPosix/XrdPosixObject.hh @@ -32,6 +32,7 @@ #include +#include "XrdSys/XrdSysAtomics.hh" #include "XrdSys/XrdSysPthread.hh" class XrdPosixDir; @@ -53,6 +54,21 @@ static XrdPosixFile *File(int fildes, bool glk=false); static int Init(int numfd); + void Lock(bool wr=true) + {if (wr) objMutex.WriteLock(); + else objMutex.ReadLock(); + } + + void Ref() {AtomicBeg(updMutex); + AtomicInc(refCnt); + AtomicEnd(updMutex); + } + int Refs() {AtomicRet(updMutex, refCnt);} + void unRef() {AtomicBeg(updMutex); + AtomicDec(refCnt); + AtomicEnd(updMutex); + } + static void Release(XrdPosixObject *oP, bool needlk=true); static XrdPosixDir *ReleaseDir( int fildes); @@ -71,12 +87,14 @@ virtual bool Who(XrdPosixDir **dirP) {return false;} virtual bool Who(XrdPosixFile **fileP) {return false;} - XrdPosixObject() : fdNum(-1) {} + XrdPosixObject() : fdNum(-1), refCnt(0) {} virtual ~XrdPosixObject() {if (fdNum >= 0) Release(this);} protected: + XrdSysRecMutex updMutex; XrdSysRWLock objMutex; int fdNum; + int refCnt; private: diff --git a/src/XrdPosix/XrdPosixPrepIO.cc b/src/XrdPosix/XrdPosixPrepIO.cc new file mode 100644 index 00000000000..754d35b57c5 --- /dev/null +++ b/src/XrdPosix/XrdPosixPrepIO.cc @@ -0,0 +1,100 @@ +/******************************************************************************/ +/* */ +/* X r d P o s i x P r e p I O . h h */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include +#include + +#include "XrdPosix/XrdPosixObjGaurd.hh" +#include "XrdPosix/XrdPosixPrepIO.hh" + +/******************************************************************************/ +/* G l o b a l s */ +/******************************************************************************/ + +namespace XrdPosixGlobals +{ +extern bool psxDBG; +}; + +/******************************************************************************/ +/* I n i t */ +/******************************************************************************/ + +bool XrdPosixPrepIO::Init(XrdOucCacheIOCB *iocbP) +{ + XrdPosixObjGaurd objGaurd((XrdPosixObject *)fileP); + XrdCl::XRootDStatus Status; + static int maxCalls = 64; + +// Count the number of entries here. We want to catch someone ignoring the +// Update() call and using this object as it is very ineffecient. +// + if (iCalls++ >= maxCalls) + {maxCalls = maxCalls*2; + std::cerr <<"XrdPosix: Unexpected PrepIO calls (" <clFile.IsOpen()) return true; + +// Do not try to open the file if there was previous error +// + if (openRC) return false; + +// Open the file. It is too difficult to do an async open here as there is a +// possible pending async request and doing both is not easy at all. +// + Status = fileP->clFile.Open((std::string)fileP->Path(), clFlags, clMode); + +// If all went well, then we need to do a Stat() call on the underlying file +// + if (Status.IsOK()) fileP->Stat(Status); + +// Make sure all went well +// + if (!Status.IsOK()) + {XrdPosixMap::Result(Status); + openRC = -errno; + if (XrdPosixGlobals::psxDBG && errno != ENOENT && errno != ELOOP) + {char eBuff[2048]; + snprintf(eBuff, sizeof(eBuff), "%s deferred open %s\n", + Status.ToString().c_str(), fileP->Path()); + std::cerr <XCio->Update((XrdOucCacheIO2 &)*fileP); + return true; +} diff --git a/src/XrdPosix/XrdPosixPrepIO.hh b/src/XrdPosix/XrdPosixPrepIO.hh new file mode 100644 index 00000000000..18c51019e67 --- /dev/null +++ b/src/XrdPosix/XrdPosixPrepIO.hh @@ -0,0 +1,100 @@ +#ifndef __XRDPOSIXPREPIO_HH__ +#define __XRDPOSIXPREPIO_HH__ +/******************************************************************************/ +/* */ +/* X r d P o s i x P r e p I O . h h */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "Xrd/XrdJob.hh" +#include "XrdPosix/XrdPosixFile.hh" + +class XrdPosixPrepIO : public XrdOucCacheIO2 +{ +public: + +XrdOucCacheIO *Base() {return this;} // Already defined + +XrdOucCacheIO *Detach() {return this;} // Already defined + +long long FSize() {return (Init() ? FSize() : openRC);} + +bool ioActive() { return false; } // Already defined + +const char *Path() {return fileP->Path();} + +int Read (char *Buffer, long long Offset, int Length) + {return (Init() ? Read(Buffer, Offset, Length) : openRC);} + +void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen) + {if (Init(&iocb)) Read(iocb, buff, offs, rlen); + else iocb.Done(openRC); + } + +int ReadV(const XrdOucIOVec *readV, int n) + {return (Init() ? ReadV(readV, n) : openRC);} + +void ReadV(XrdOucCacheIOCB &iocb, const XrdOucIOVec *readV, int rnum) + {if (Init(&iocb)) ReadV(iocb, readV, rnum); + else iocb.Done(openRC); + } + +int Sync() {return (Init() ? Sync() : openRC);} + +void Sync(XrdOucCacheIOCB &iocb) + {if (Init(&iocb)) Sync(iocb); + else iocb.Done(openRC); + } + +int Trunc(long long Offset) + {return (Init() ? Trunc(Offset) : openRC);} + +int Write(char *Buffer, long long Offset, int Length) + {return (Init() ? Write(Buffer, Offset, Length) : openRC);} + +void Write(XrdOucCacheIOCB &iocb, char *buff, long long offs, int wlen) + {if (Init(&iocb)) Write(iocb, buff, offs, wlen); + else iocb.Done(openRC); + } + + XrdPosixPrepIO(XrdPosixFile *fP, XrdCl::OpenFlags::Flags clflags, + XrdCl::Access::Mode clmode) + : fileP(fP), openRC(0), iCalls(0), + clFlags(clflags), clMode(clmode) {} +virtual ~XrdPosixPrepIO() {} + +private: +bool Init(XrdOucCacheIOCB *iocbP=0); + +XrdPosixFile *fileP; +int openRC; +int iCalls; + +XrdCl::OpenFlags::Flags clFlags; +XrdCl::Access::Mode clMode; +}; +#endif diff --git a/src/XrdPosix/XrdPosixXrootd.cc b/src/XrdPosix/XrdPosixXrootd.cc index cf38f3288eb..13abadf2f36 100644 --- a/src/XrdPosix/XrdPosixXrootd.cc +++ b/src/XrdPosix/XrdPosixXrootd.cc @@ -47,14 +47,18 @@ #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysPlatform.hh" +#include "XrdOuc/XrdOucCache2.hh" #include "XrdOuc/XrdOucCacheDram.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdPosix/XrdPosixAdmin.hh" +#include "XrdPosix/XrdPosixCacheBC.hh" #include "XrdPosix/XrdPosixCallBack.hh" #include "XrdPosix/XrdPosixDir.hh" #include "XrdPosix/XrdPosixFile.hh" +#include "XrdPosix/XrdPosixFileRH.hh" #include "XrdPosix/XrdPosixMap.hh" +#include "XrdPosix/XrdPosixPrepIO.hh" #include "XrdPosix/XrdPosixXrootd.hh" /******************************************************************************/ @@ -63,15 +67,59 @@ namespace XrdPosixGlobals { -XrdScheduler *schedP = 0; +XrdScheduler *schedP = 0; +XrdOucCache2 *theCache = 0; XrdCl::DirListFlags::Flags dlFlag = XrdCl::DirListFlags::None; bool psxDBG = (getenv("XRDPOSIX_DEBUG") != 0); }; XrdOucCache *XrdPosixXrootd::myCache = 0; +XrdOucCache2 *XrdPosixXrootd::myCache2 = 0; int XrdPosixXrootd::baseFD = 0; int XrdPosixXrootd::initDone = 0; +/******************************************************************************/ +/* L o c a l F u n c t i o n s */ +/******************************************************************************/ + +namespace +{ + +/******************************************************************************/ +/* O p e n D e f e r */ +/******************************************************************************/ + +int OpenDefer(XrdPosixFile *fp, + XrdPosixCallBack *cbP, + XrdCl::OpenFlags::Flags XOflags, + XrdCl::Access::Mode XOmode, + bool isStream) +{ + +// Assign a file descriptor to this file +// + if (!(fp->AssignFD(isStream))) + {delete fp; + errno = EMFILE; + return -1; + } + +// Allocate a prepare I/O object to defer this open +// + fp->XCio = fp->PrepIO = new XrdPosixPrepIO(fp, XOflags, XOmode); + +// For sync opens we just need to return the file descriptor +// + if (!cbP) return fp->FDNum(); + +// For async opens do the callback here and return an inprogress +// + cbP->Complete(fp->FDNum()); + errno = EINPROGRESS; + return -1; +} +}; + /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ @@ -148,13 +196,16 @@ int XrdPosixXrootd::Close(int fildes) XrdPosixFile *fP; bool ret; +// Map the file number to the file object. In the prcess we relese the file +// number so no one can reference this file again. +// if (!(fP = XrdPosixObject::ReleaseFile(fildes))) {errno = EBADF; return -1;} // Close the file if there is no active I/O (possible caching). Delete the // object if the close was successful (it might not be). // - if (!(fP->XCio->ioActive())) + if (!fP->Refs() && !(fP->XCio->ioActive())) {if ((ret = fP->Close(Status))) {delete fP; fP = 0;} else if (XrdPosixGlobals::psxDBG) {char eBuff[2048]; @@ -167,13 +218,7 @@ int XrdPosixXrootd::Close(int fildes) // If we still have a handle then we need to do a delayed delete on this // object because either the close failed or there is still active I/O // - if (fP) - {if (XrdPosixGlobals::schedP) XrdPosixGlobals::schedP->Schedule(fP); - else {pthread_t tid; - XrdSysThread::Run(&tid, XrdPosixFile::DelayedDestroy, fP, 0, - "PosixFileDestroy"); - } - } + if (fP) XrdPosixFile::DelayedDestroy(fP); // Return final result // @@ -253,7 +298,7 @@ int XrdPosixXrootd::Fstat(int fildes, struct stat *buf) // Return what little we can // initStat(buf); - buf->st_size = fp->mySize; + buf->st_size = fp->FSize(); buf->st_atime = buf->st_mtime = buf->st_ctime = fp->myMtime; buf->st_blocks = buf->st_size/512+1; buf->st_ino = fp->myInode; @@ -285,6 +330,21 @@ int XrdPosixXrootd::Fsync(int fildes) return 0; } +/******************************************************************************/ + +void XrdPosixXrootd::Fsync(int fildes, XrdPosixCallBackIO *cbp) +{ + XrdPosixFile *fp; + +// Find the file object and do the sync +// + if ((fp = XrdPosixObject::File(fildes))) + {cbp->theFile = fp; + fp->Ref(); fp->UnLock(); + fp->XCio->Sync(*cbp); + } else cbp->Complete(-1); +} + /******************************************************************************/ /* F t r u n c a t e */ /******************************************************************************/ @@ -359,7 +419,7 @@ off_t XrdPosixXrootd::Lseek(int fildes, off_t offset, int whence) if (whence == SEEK_SET) curroffset = fp->setOffset(offset); else if (whence == SEEK_CUR) curroffset = fp->addOffset(offset); else if (whence == SEEK_END) - curroffset = fp->setOffset(fp->mySize+offset); + curroffset = fp->setOffset(fp->FSize()+offset); else return Fault(fp, EINVAL); // All done @@ -442,6 +502,15 @@ int XrdPosixXrootd::Open(const char *path, int oflags, mode_t mode, return -1; } +// If we have a cache, then issue a prepare as the cache may want to defer the +// open request ans we have a lot more work to do. +// + if (myCache2) + {int rc = myCache2->Prepare(path, oflags, mode); + if (rc > 0) return OpenDefer(fp, cbP, XOflags, XOmode, oflags&isStream); + if (rc < 0) {errno = -rc; return -1;} + } + // Open the file (sync or async) // if (!cbP) Status = fp->clFile.Open((std::string)path, XOflags, XOmode); @@ -479,7 +548,7 @@ int XrdPosixXrootd::Open(const char *path, int oflags, mode_t mode, if (fp->Finalize(Status)) return fp->FDNum(); return XrdPosixMap::Result(Status); } - + /******************************************************************************/ /* O p e n d i r */ /******************************************************************************/ @@ -546,6 +615,40 @@ ssize_t XrdPosixXrootd::Pread(int fildes, void *buf, size_t nbyte, off_t offset) return (ssize_t)bytes; } +/******************************************************************************/ + +void XrdPosixXrootd::Pread(int fildes, void *buf, size_t nbyte, off_t offset, + XrdPosixCallBackIO *cbp) +{ + XrdPosixFile *fp; + long long offs; + int iosz; + +// Find the file object +// + if (!(fp = XrdPosixObject::File(fildes))) {cbp->Complete(-1); return;} + +// Make sure the size is not too large +// + if (nbyte > (size_t)0x7fffffff) + {fp->UnLock(); + errno = EOVERFLOW; + cbp->Complete(-1); + return; + } + +// Prepare for the read +// + cbp->theFile = fp; + fp->Ref(); fp->UnLock(); + iosz = static_cast(nbyte); + offs = static_cast(offset); + +// Issue the read +// + fp->XCio->Read(*cbp, (char *)buf, offs, (int)iosz); +} + /******************************************************************************/ /* P w r i t e */ /******************************************************************************/ @@ -573,11 +676,45 @@ ssize_t XrdPosixXrootd::Pwrite(int fildes, const void *buf, size_t nbyte, off_t // All went well // - if (offs+iosz > (long long)fp->mySize) fp->mySize = offs + iosz; + fp->UpdtSize(offs + iosz); fp->UnLock(); return (ssize_t)iosz; } +/******************************************************************************/ + +void XrdPosixXrootd::Pwrite(int fildes, const void *buf, size_t nbyte, + off_t offset, XrdPosixCallBackIO *cbp) +{ + XrdPosixFile *fp; + long long offs; + int iosz; + +// Find the file object +// + if (!(fp = XrdPosixObject::File(fildes))) {cbp->Complete(-1); return;} + +// Make sure the size is not too large +// + if (nbyte > (size_t)0x7fffffff) + {fp->UnLock(); + errno = EOVERFLOW; + cbp->Complete(-1); + return; + } + +// Prepare for the writing +// + cbp->theFile = fp; + fp->Ref(); fp->UnLock(); + iosz = static_cast(nbyte); + offs = static_cast(offset); + +// Issue the read +// + fp->XCio->Write(*cbp, (char *)buf, offs, (int)iosz); +} + /******************************************************************************/ /* R e a d */ /******************************************************************************/ @@ -655,6 +792,22 @@ ssize_t XrdPosixXrootd::VRead(int fildes, const XrdOucIOVec *readV, int n) return bytes; } +/******************************************************************************/ + +void XrdPosixXrootd::VRead(int fildes, const XrdOucIOVec *readV, int n, + XrdPosixCallBackIO *cbp) +{ + XrdPosixFile *fp; + +// Find the file object and issue read +// + if ((fp = XrdPosixObject::File(fildes))) + {cbp->theFile = fp; + fp->Ref(); fp->UnLock(); + fp->XCio->ReadV(*cbp, readV, n); + } else cbp->Complete(-1); +} + /******************************************************************************/ /* R e a d d i r */ /******************************************************************************/ @@ -769,8 +922,8 @@ int XrdPosixXrootd::Rename(const char *oldpath, const char *newpath) std::string urlp = admin.Url.GetPathWithParams(); std::string nurlp = newUrl.GetPathWithParams(); int res = XrdPosixMap::Result(admin.Xrd.Mv(urlp, nurlp)); - if (!res && myCache) - myCache->Rename(urlp.c_str(), nurlp.c_str()); + if (!res && XrdPosixGlobals::theCache) + XrdPosixGlobals::theCache->Rename(urlp.c_str(), nurlp.c_str()); return res; } @@ -808,8 +961,8 @@ int XrdPosixXrootd::Rmdir(const char *path) // std::string urlp = admin.Url.GetPathWithParams(); int res = XrdPosixMap::Result(admin.Xrd.RmDir(urlp)); - if (!res && myCache) - myCache->Rmdir(urlp.c_str()); + if (!res && XrdPosixGlobals::theCache) + XrdPosixGlobals::theCache->Rmdir(urlp.c_str()); return res; } @@ -850,6 +1003,19 @@ int XrdPosixXrootd::Stat(const char *path, struct stat *buf) time_t stMtime; mode_t stFlags; +// Make sure the admin is OK +// + if (!admin.isOK()) return -1; + +// Check if we can get the stat informat directly +// + if (myCache2) + {std::string urlp = admin.Url.GetPathWithParams(); + int rc = myCache2->Stat(urlp.c_str(), *buf); + if (!rc) return 0; + if (rc < 0) {errno = -rc; return -1;} + } + // Issue the stat and verify that all went well // if (!admin.Stat(&stFlags, &stMtime, &stSize, &stId, &stRdev)) return -1; @@ -1008,8 +1174,8 @@ int XrdPosixXrootd::Truncate(const char *path, off_t Size) std::string urlp = admin.Url.GetPathWithParams(); int res = XrdPosixMap::Result(admin.Xrd.Truncate(urlp,tSize)); - if (!res && myCache) { - myCache->Truncate(urlp.c_str(), tSize); + if (!res && XrdPosixGlobals::theCache) { + XrdPosixGlobals::theCache->Truncate(urlp.c_str(), tSize); } return res; @@ -1031,8 +1197,8 @@ int XrdPosixXrootd::Unlink(const char *path) // std::string urlp = admin.Url.GetPathWithParams(); int res = XrdPosixMap::Result(admin.Xrd.Rm(urlp)); - if (!res && myCache) - myCache->Unlink(urlp.c_str()); + if (!res && XrdPosixGlobals::theCache) + XrdPosixGlobals::theCache->Unlink(urlp.c_str()); return res; } @@ -1104,8 +1270,10 @@ void XrdPosixXrootd::initEnv() // Now we must check if we have a new cache over-ride // - if ((evar = getenv("XRDPOSIX_CACHE")) && *evar) initEnv(evar); - else if (myCache) {char ebuf[] = {0}; initEnv(ebuf);} + if (!myCache2) + {if ((evar = getenv("XRDPOSIX_CACHE")) && *evar) initEnv(evar); + else if (myCache) {char ebuf[] = {0}; initEnv(ebuf);} + } else XrdPosixGlobals::theCache = myCache2; } /******************************************************************************/ @@ -1134,6 +1302,7 @@ void XrdPosixXrootd::initEnv(char *eData) XrdOucEnv theEnv(eData); XrdOucCache::Parms myParms; XrdOucCacheIO::aprParms apParms; + XrdOucCache *v1Cache; long long Val; int isRW = 0; char * tP; @@ -1191,9 +1360,9 @@ void XrdPosixXrootd::initEnv(char *eData) // additional but unnecessary locking. // myParms.Options |= XrdOucCache::Serialized; - if (!(XrdPosixFile::CacheR = myCache->Create(myParms, &apParms))) + if (!(v1Cache = myCache->Create(myParms, &apParms))) cerr <<"XrdPosix: " <PutString((std::string)"NetworkStack", (const std::string)ipmode); } +/******************************************************************************/ +/* s e t N u m C B */ +/******************************************************************************/ + +void XrdPosixXrootd::setNumCB(int numcb) +{ + if (numcb >= 0) XrdPosixFileRH::SetMax(numcb); +} + /******************************************************************************/ /* s e t S c h e d */ /******************************************************************************/ diff --git a/src/XrdPosix/XrdPosixXrootd.hh b/src/XrdPosix/XrdPosixXrootd.hh index 3ce4bb0e056..92dce68f2e4 100644 --- a/src/XrdPosix/XrdPosixXrootd.hh +++ b/src/XrdPosix/XrdPosixXrootd.hh @@ -51,8 +51,10 @@ struct XrdOucIOVec; class XrdScheduler; class XrdOucCache; +class XrdOucCache2; class XrdOucEnv; class XrdPosixCallBack; +class XrdPosixCallBackIO; class XrdPosixFile; //----------------------------------------------------------------------------- @@ -106,6 +108,8 @@ static int Fstat(int fildes, struct stat *buf); static int Fsync(int fildes); +static void Fsync(int fildes, XrdPosixCallBackIO *cbp); // Async extension! + //----------------------------------------------------------------------------- //! Ftruncate() conforms to POSIX.1-2001 ftruncate() //----------------------------------------------------------------------------- @@ -175,12 +179,18 @@ static DIR* Opendir(const char *path); static ssize_t Pread(int fildes, void *buf, size_t nbyte, off_t offset); +static void Pread(int fildes, void *buf, size_t nbyte, off_t offset, + XrdPosixCallBackIO *cbp); // Async extension! + //----------------------------------------------------------------------------- //! Pwrite() conforms to POSIX.1-2001 pwrite() //----------------------------------------------------------------------------- static ssize_t Pwrite(int fildes, const void *buf, size_t nbyte, off_t offset); +static void Pwrite(int fildes, const void *buf, size_t nbyte, off_t offset, + XrdPosixCallBackIO *cbp); // Async extension! + //----------------------------------------------------------------------------- //! QueryChksum() is a POSIX extension and returns a file's modification time //! and its associated checksum value. @@ -311,12 +321,17 @@ static int Unlink(const char *path); //! each offset of the specifiued length is placed in buffer. //! @param n the number of elements in the readV vector. //! +//! @param cbp pointer to the callback object for async execution. +//! //! @return Upon success returns the total number of bytes read. Otherwise, -1 //! is returned and errno is appropriately set. //----------------------------------------------------------------------------- static ssize_t VRead(int fildes, const XrdOucIOVec *readV, int n); +static void VRead(int fildes, const XrdOucIOVec *readV, int n, + XrdPosixCallBackIO *cbp); // Async extension! + //----------------------------------------------------------------------------- //! Write() conforms to POSIX.1-2001 write() //----------------------------------------------------------------------------- @@ -342,12 +357,16 @@ static bool myFD(int fd); static void setCache(XrdOucCache *cP); +static void setCache(XrdOucCache2 *cP); + static void setDebug(int val, bool doDebug=false); static void setEnv(const char *kword, int kval); static void setIPV4(bool userv4); +static void setNumCB(int numcb); + static void setSched(XrdScheduler *sP); /* There must be one instance of this object per executable image. Typically, @@ -366,14 +385,14 @@ static void setSched(XrdScheduler *sP); private: -static void initEnv(); -static void initEnv(char *eData); -static void initEnv(XrdOucEnv &, const char *, long long &); -static int Fault(XrdPosixFile *fp, int ecode); -static void initStat(struct stat *buf); -static void initXdev(dev_t &st_dev, dev_t &st_rdev); - +static void initEnv(); +static void initEnv(char *eData); +static void initEnv(XrdOucEnv &, const char *, long long &); +static int Fault(XrdPosixFile *fp, int ecode); +static void initStat(struct stat *buf); +static void initXdev(dev_t &st_dev, dev_t &st_rdev); static XrdOucCache *myCache; +static XrdOucCache2 *myCache2; static int baseFD; static int initDone; }; diff --git a/src/XrdPss/XrdPssAio.cc b/src/XrdPss/XrdPssAio.cc index c33ff7ed0f9..f747bce7667 100644 --- a/src/XrdPss/XrdPssAio.cc +++ b/src/XrdPss/XrdPssAio.cc @@ -31,15 +31,13 @@ #include #include +#include "XrdPosix/XrdPosixXrootd.hh" #include "XrdPss/XrdPss.hh" +#include "XrdPss/XrdPssAioCB.hh" #include "XrdSfs/XrdSfsAio.hh" // All AIO interfaces are defined here. - -// Currently we disable aio support for proxies because the client does not -// support async reads and writes. It should to make proxy I/O more scalable. - /******************************************************************************/ /* F s y n c */ /******************************************************************************/ @@ -53,13 +51,9 @@ int XrdPssFile::Fsync(XrdSfsAio *aiop) { -// Execute this request in a synchronous fashion +// Execute this request in an asynchronous fashion // - if ((aiop->Result = Fsync())) aiop->Result = -errno; - -// Simply call the write completion routine and return as if all went well -// - aiop->doneWrite(); + XrdPosixXrootd::Fsync(fd, XrdPssAioCB::Alloc(aiop, true)); return 0; } @@ -81,15 +75,12 @@ int XrdPssFile::Fsync(XrdSfsAio *aiop) int XrdPssFile::Read(XrdSfsAio *aiop) { -// Execute this request in a synchronous fashion -// - aiop->Result = this->Read((void *)aiop->sfsAio.aio_buf, - (off_t)aiop->sfsAio.aio_offset, - (size_t)aiop->sfsAio.aio_nbytes); - -// Simple call the read completion routine and return as if all went well +// Execute this request in an asynchronous fashion // - aiop->doneRead(); + XrdPosixXrootd::Pread(fd, (void *)aiop->sfsAio.aio_buf, + (size_t)aiop->sfsAio.aio_nbytes, + (off_t)aiop->sfsAio.aio_offset, + XrdPssAioCB::Alloc(aiop, false)); return 0; } @@ -111,14 +102,11 @@ int XrdPssFile::Read(XrdSfsAio *aiop) int XrdPssFile::Write(XrdSfsAio *aiop) { -// Execute this request in a synchronous fashion +// Execute this request in an asynchronous fashion // - aiop->Result = this->Write((const void *)aiop->sfsAio.aio_buf, - (off_t)aiop->sfsAio.aio_offset, - (size_t)aiop->sfsAio.aio_nbytes); - -// Simply call the write completion routine and return as if all went well -// - aiop->doneWrite(); + XrdPosixXrootd::Pwrite(fd, (const void *)aiop->sfsAio.aio_buf, + (size_t)aiop->sfsAio.aio_nbytes, + (off_t)aiop->sfsAio.aio_offset, + XrdPssAioCB::Alloc(aiop, true)); return 0; } diff --git a/src/XrdPss/XrdPssAioCB.cc b/src/XrdPss/XrdPssAioCB.cc new file mode 100644 index 00000000000..d10c33fb90a --- /dev/null +++ b/src/XrdPss/XrdPssAioCB.cc @@ -0,0 +1,107 @@ +/******************************************************************************/ +/* */ +/* X r d P s s A i o C B . c c */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include + +#include "XrdPss/XrdPssAioCB.hh" +#include "XrdSfs/XrdSfsAio.hh" + +/******************************************************************************/ +/* S t a t i c M e m b e r s */ +/******************************************************************************/ + +XrdSysMutex XrdPssAioCB::myMutex; +XrdPssAioCB *XrdPssAioCB::freeCB = 0; +int XrdPssAioCB::numFree = 0; +int XrdPssAioCB::maxFree = 100; + +/******************************************************************************/ +/* A l l o c */ +/******************************************************************************/ + +XrdPssAioCB *XrdPssAioCB::Alloc(XrdSfsAio *aiop, bool isWr) +{ + XrdPssAioCB *newCB; + +// Try to allocate an prexisting object otherwise get a new one +// + myMutex.Lock(); + if ((newCB = freeCB)) {freeCB = newCB->next; numFree--;} + else newCB = new XrdPssAioCB; + myMutex.UnLock(); + +// Initialize the callback and return it +// + newCB->theAIOP = aiop; + newCB->isWrite = isWr; + return newCB; +} + +/******************************************************************************/ +/* C o m p l e t e */ +/******************************************************************************/ + +#include +void XrdPssAioCB::Complete(ssize_t result) +{ + +// Set correct result +// +// std::cerr <<"AIO fin " <<(isWrite ? " write ":" read ") +// <sfsAio.aio_nbytes <<'@' <sfsAio.aio_offset +// <<" result " <Result = (result < 0 ? -errno : result); + +// Invoke the callback +// + if (isWrite) theAIOP->doneWrite(); + else theAIOP->doneRead(); + +// Now recycle ourselves +// + Recycle(); +} + +/******************************************************************************/ +/* R e c y c l e */ +/******************************************************************************/ + +void XrdPssAioCB::Recycle() +{ +// Perform recycling +// + myMutex.Lock(); + if (numFree >= maxFree) delete this; + else {next = freeCB; + freeCB = this; + numFree++; + } + myMutex.UnLock(); +} diff --git a/src/XrdPss/XrdPssAioCB.hh b/src/XrdPss/XrdPssAioCB.hh new file mode 100644 index 00000000000..3a46124867c --- /dev/null +++ b/src/XrdPss/XrdPssAioCB.hh @@ -0,0 +1,64 @@ +#ifndef __PSS_AIOCB_HH__ +#define __PSS_AIOCB_HH__ +/******************************************************************************/ +/* */ +/* X r d P s s A i o C B . h h */ +/* */ +/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "XrdPosix/XrdPosixCallBack.hh" +#include "XrdSys/XrdSysPthread.hh" + +class XrdSfsAio; + +class XrdPssAioCB : public XrdPosixCallBackIO +{ +public: + +static XrdPssAioCB *Alloc(XrdSfsAio *aiop, bool isWr); + +virtual void Complete(ssize_t Result); + + void Recycle(); + +static void SetMax(int mval) {maxFree = mval;} + +private: + XrdPssAioCB() : theAIOP(0), isWrite(false) {} +virtual ~XrdPssAioCB() {} + +static XrdSysMutex myMutex; +static XrdPssAioCB *freeCB; +static int numFree; +static int maxFree; + +union {XrdSfsAio *theAIOP; + XrdPssAioCB *next; + }; +bool isWrite; +}; +#endif diff --git a/src/XrdPss/XrdPssConfig.cc b/src/XrdPss/XrdPssConfig.cc index 93a8b49d807..6722981a70b 100644 --- a/src/XrdPss/XrdPssConfig.cc +++ b/src/XrdPss/XrdPssConfig.cc @@ -56,7 +56,7 @@ #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOuca2x.hh" -#include "XrdOuc/XrdOucCache.hh" +#include "XrdOuc/XrdOucCache2.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucExport.hh" #include "XrdOuc/XrdOucN2NLoader.hh" @@ -209,7 +209,7 @@ int XrdPssSys::Configure(const char *cfn) // Tell xrootd to disable async I/O as it just will slow everything down. // - XrdOucEnv::Export("XRDXROOTD_NOAIO", "1"); +// XrdOucEnv::Export("XRDXROOTD_NOAIO", "1"); // Thell xrootd to disable POSC mode as this is meaningless here // @@ -462,18 +462,31 @@ int XrdPssSys::getCache() { XrdOucPinLoader myLib(&eDest,myVersion,"cachelib",cPath); XrdOucCache *(*ep)(XrdSysLogger *, const char *, const char *); - XrdOucCache *theCache; + union {XrdOucCache2 *theCache2; + XrdOucCache *theCache; + }; + const char *cName; + bool isCache2; + +// First find out if this is a cache2 or old cache library +// + if (myLib.Resolve("?XrdOucGetCache2") == 0) + {cName = "XrdOucGetCache"; isCache2 = false; + } else { + cName = "XrdOucGetCache2"; isCache2 = true; + } // Now get the entry point of the object creator // ep = (XrdOucCache *(*)(XrdSysLogger *, const char *, const char *)) - (myLib.Resolve("XrdOucGetCache")); + (myLib.Resolve(cName)); if (!ep) return 0; // Get the Object now // theCache = ep(eDest.logger(), ConfigFN, cParm); - if (theCache) {XrdPosixXrootd::setCache(theCache);} + if (theCache) + {XrdPosixXrootd::setCache(isCache2 ? theCache2 : theCache);} else eDest.Emsg("Config", "Unable to get cache object from", cPath); return theCache != 0; } diff --git a/src/XrdUtils.cmake b/src/XrdUtils.cmake index 6f4678a309e..f1cab7564d6 100644 --- a/src/XrdUtils.cmake +++ b/src/XrdUtils.cmake @@ -55,6 +55,7 @@ add_library( XrdOuc/XrdOucBackTrace.cc XrdOuc/XrdOucBackTrace.hh XrdOuc/XrdOucBuffer.cc XrdOuc/XrdOucBuffer.hh XrdOuc/XrdOucCache.hh + XrdOuc/XrdOucCache2.hh XrdOuc/XrdOucCacheData.cc XrdOuc/XrdOucCacheData.hh XrdOuc/XrdOucCacheDram.cc XrdOuc/XrdOucCacheDram.hh XrdOuc/XrdOucCacheReal.cc XrdOuc/XrdOucCacheReal.hh diff --git a/src/XrdXrootd/XrdXrootdAio.cc b/src/XrdXrootd/XrdXrootdAio.cc index 4bdc6e590ad..73a4c942845 100644 --- a/src/XrdXrootd/XrdXrootdAio.cc +++ b/src/XrdXrootd/XrdXrootdAio.cc @@ -168,6 +168,7 @@ void XrdXrootdAio::doneWrite() // requests are outstanding. It is impossible to have a zero length with more // requests outstanding. // +//cerr <<"doneWrite left " <numActive <<' ' <myIOLen <myIOLen > 0) {Next = aioReq->aioFree; aioReq->aioFree = this;} else {if (!(aioReq->numActive)) Sched->Schedule((XrdJob *)aioReq); From d02b22c35a92750b4249d088591afe77fa8062dc Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 18 Mar 2016 15:14:06 -0700 Subject: [PATCH 074/171] Moving to XrdOucCache2. --- src/XrdFileCache/XrdFileCache.cc | 4 +- src/XrdFileCache/XrdFileCache.hh | 8 ++- src/XrdFileCache/XrdFileCacheFile.cc | 69 +++++++------------- src/XrdFileCache/XrdFileCacheFile.hh | 21 +++--- src/XrdFileCache/XrdFileCacheIO.hh | 8 +-- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 2 +- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 2 +- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 4 +- src/XrdFileCache/XrdFileCacheIOFileBlock.hh | 6 +- src/XrdFileCache/XrdFileCacheVRead.cc | 17 ++--- 10 files changed, 60 insertions(+), 81 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 7572103dfb3..0d63aff1926 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -65,7 +65,7 @@ void *PrefetchThread(void* ptr) extern "C" { -XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, +XrdOucCache2 *XrdOucGetCache2(XrdSysLogger *logger, const char *config_filename, const char *parameters) { @@ -139,7 +139,7 @@ Cache::Cache() : XrdOucCache(), //______________________________________________________________________________ -XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) +XrdOucCacheIO2 *Cache::Attach(XrdOucCacheIO2 *io, int Options) { if (Cache::GetInstance().Decide(io)) { diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 34086a35e1f..97477a1c5f0 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -22,7 +22,8 @@ #include "XrdVersion.hh" #include "XrdSys/XrdSysPthread.hh" -#include "XrdOuc/XrdOucCache.hh" +#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucCallBack.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdFileCacheFile.hh" #include "XrdFileCacheDecision.hh" @@ -38,6 +39,7 @@ class File; class IO; } + namespace XrdFileCache { //---------------------------------------------------------------------------- @@ -75,7 +77,7 @@ namespace XrdFileCache //---------------------------------------------------------------------------- //! Attaches/creates and detaches/deletes cache-io objects for disk based cache. //---------------------------------------------------------------------------- - class Cache : public XrdOucCache + class Cache : public XrdOucCache2 { public: //--------------------------------------------------------------------- @@ -86,7 +88,7 @@ namespace XrdFileCache //--------------------------------------------------------------------- //! Obtain a new IO object that fronts existing XrdOucCacheIO. //--------------------------------------------------------------------- - virtual XrdOucCacheIO *Attach(XrdOucCacheIO *, int Options=0); + virtual XrdOucCacheIO2 *Attach(XrdOucCacheIO2 *, int Options=0); //--------------------------------------------------------------------- //! Number of cache-io objects atteched through this cache. diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 9726888a585..a670874846a 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -70,7 +70,7 @@ namespace Cache* cache() { return &Cache::GetInstance(); } } -File::File(XrdOucCacheIO &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : +File::File(XrdOucCacheIO2 &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : m_input(inputIO), m_output(NULL), m_infoFile(NULL), @@ -334,7 +334,6 @@ Block* File::RequestBlock(int i, bool prefetch) // catch the block while still in memory. clLog()->Debug(XrdCl::AppMsg, "RequestBlock() %d pOn=(%d)", i, prefetch); - XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; const long long BS = m_cfi.GetBufferSize(); const int last_block = m_cfi.GetSizeInBits() - 1; @@ -344,23 +343,18 @@ Block* File::RequestBlock(int i, bool prefetch) Block *b = new Block(this, off, this_bs, prefetch); // should block be reused to avoid recreation - XrdCl::XRootDStatus status = client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); - if (status.IsOK()) { - clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); - m_block_map[i] = b; + BlockResponseHandler* oucCB = new BlockResponseHandler(b); + m_input.Read(*oucCB, (char*)b->get_buff(), off, (int)this_bs); - if (m_prefetchState == kOn && m_block_map.size() > Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) - { - m_prefetchState = kHold; - cache()->DeRegisterPrefetchFile(this); - } - return b; - } - else { - clLog()->Error(XrdCl::AppMsg, "File::RequestBlock() error %d, this = %p, b=%p, this idx=%d pOn=(%d) %s", status.code, (void*)this, (void*)b, i, prefetch, lPath()); - XrdPosixMap::Result(status); - return 0; + clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); + m_block_map[i] = b; + + if (m_prefetchState == kOn && m_block_map.size() > Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) + { + m_prefetchState = kHold; + cache()->DeRegisterPrefetchFile(this); } + return b; } //------------------------------------------------------------------------------ @@ -368,8 +362,6 @@ Block* File::RequestBlock(int i, bool prefetch) int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, char* req_buf, long long req_off, long long req_size) { - XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; - const long long BS = m_cfi.GetBufferSize(); // XXX Use readv to load more at the same time. @@ -385,16 +377,9 @@ int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, overlap(*ii, BS, req_off, req_size, off, blk_off, size); - XrdCl::Status status = client.Read(*ii * BS + blk_off, size, req_buf + off, handler); - if (!status.IsOK()) - { - clLog()->Error(XrdCl::AppMsg, "File::RequestBlocksDirect error %s\n", lPath()); - XrdPosixMap::Result(status); - return -1; // AMT all reads should be canceled in this case - } - else { - clLog()->Dump(XrdCl::AppMsg, "RequestBlockDirect success %d %ld %s", *ii, size, lPath()); - } + m_input.Read( *handler, req_buf + off, *ii * BS + blk_off, size); + clLog()->Dump(XrdCl::AppMsg, "RequestBlockDirect success %d %ld %s", *ii, size, lPath()); + total += size; } @@ -837,13 +822,13 @@ void File::free_block(Block* b) //------------------------------------------------------------------------------ -void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) +void File::ProcessBlockResponse(Block* b, int res) { m_downloadCond.Lock(); clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %p, %d %s",(void*)b,(int)(b->m_offset/BufferSize()), lPath()); - if (status->IsOK()) + if (res >= 0) { b->m_downloaded = true; clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d finished %d %s",(int)(b->m_offset/BufferSize()), b->is_finished(), lPath()); @@ -862,8 +847,8 @@ void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) { // AMT how long to keep? // when to retry? - clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %p %d error=%d, [%s] %s",(void*)b,(int)(b->m_offset/BufferSize()), status->code, status->GetErrorMessage().c_str(), lPath()); - XrdPosixMap::Result(*status); + clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %p %d error=%d, %s",(void*)b,(int)(b->m_offset/BufferSize()), res, lPath()); + // XrdPosixMap::Result(*status); // AMT could notfiy global cache we dont need RAM for that block b->set_error_and_free(errno); errno = 0; @@ -999,32 +984,26 @@ void File::UnMarkPrefetch() //================== RESPONSE HANDLER ================================== //============================================================================== -void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, - XrdCl::AnyObject *response) +void BlockResponseHandler::Done(int res) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"BlockResponseHandler::HandleResponse()"); - - m_block->m_file->ProcessBlockResponse(m_block, status); + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"BlockResponseHandler::Done()"); - delete status; - delete response; + m_block->m_file->ProcessBlockResponse(m_block, res); delete this; } //------------------------------------------------------------------------------ -void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, - XrdCl::AnyObject *response) +void DirectResponseHandler::Done(int res) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"DirectResponseHandler::HandleRespons()"); + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"DirectResponseHandler::Done()"); XrdSysCondVarHelper _lck(m_cond); --m_to_wait; - if ( ! status->IsOK()) + if (res < 0) { - XrdPosixMap::Result(*status); m_errno = errno; } diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 1a98651cc45..45af499445c 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -21,6 +21,9 @@ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClDefaultEnv.hh" +#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucIOVec.hh" + #include "XrdFileCacheInfo.hh" #include "XrdFileCacheStats.hh" @@ -94,7 +97,7 @@ namespace XrdFileCache private: enum PrefetchState_e { kOn, kHold, kCanceled }; - XrdOucCacheIO &m_input; //!< original data source + XrdOucCacheIO2 &m_input; //!< original data source XrdOssDF *m_output; //!< file handle for data file on disk XrdOssDF *m_infoFile; //!< file handle for data-info file on disk Info m_cfi; //!< download status of file blocks and access statistics @@ -141,7 +144,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - File(XrdOucCacheIO &io, std::string &path, + File(XrdOucCacheIO2 &io, std::string &path, long long offset, long long fileSize); //------------------------------------------------------------------------ @@ -174,7 +177,7 @@ namespace XrdFileCache //---------------------------------------------------------------------- Stats& GetStats() { return m_stats; } - void ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status); + void ProcessBlockResponse(Block* b, int res); void WriteBlockToDisk(Block* b); void Prefetch(); @@ -205,7 +208,7 @@ namespace XrdFileCache char* req_buf, long long req_off, long long req_size); // VRead - bool VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& blks_to_process, ReadVBlockListDisk& blks_on_disk, XrdCl::ChunkList& chunkVec); + bool VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& blks_to_process, ReadVBlockListDisk& blks_on_disk, std::vector& chunkVec); int VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blks_on_disk); int VReadProcessBlocks(const XrdOucIOVec *readV, int n, std::vector& blks_to_process, std::vector& blks_rocessed); @@ -230,18 +233,17 @@ namespace XrdFileCache // ================================================================ - class BlockResponseHandler : public XrdCl::ResponseHandler + class BlockResponseHandler : public XrdOucCacheIOCB { public: Block *m_block; BlockResponseHandler(Block *b) : m_block(b) {} - void HandleResponse(XrdCl::XRootDStatus *status, - XrdCl::AnyObject *response); + virtual void Done(int result); }; - class DirectResponseHandler : public XrdCl::ResponseHandler + class DirectResponseHandler : public XrdOucCacheIOCB { public: XrdSysCondVar m_cond; @@ -254,8 +256,7 @@ namespace XrdFileCache bool is_ok() { XrdSysCondVarHelper _lck(m_cond); return m_to_wait == 0 && m_errno == 0; } bool is_failed() { XrdSysCondVarHelper _lck(m_cond); return m_errno != 0; } - void HandleResponse(XrdCl::XRootDStatus *status, - XrdCl::AnyObject *response); + virtual void Done(int result); }; } diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index df2b0fd18c8..815cf196ffe 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -2,7 +2,7 @@ #define __XRDFILECACHE_CACHE_IO_HH__ #include "XrdFileCache.hh" -#include "XrdOuc/XrdOucCache.hh" +#include "XrdOuc/XrdOucCache2.hh" #include "XrdCl/XrdClDefaultEnv.hh" namespace XrdFileCache @@ -10,10 +10,10 @@ namespace XrdFileCache //---------------------------------------------------------------------------- //! Base cache-io class that implements XrdOucCacheIO abstract methods. //---------------------------------------------------------------------------- - class IO : public XrdOucCacheIO + class IO : public XrdOucCacheIO2 { public: - IO (XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache &cache) : + IO (XrdOucCacheIO2 &io, XrdOucCacheStats &stats, Cache &cache) : m_io(io), m_statsGlobal(stats), m_cache(cache) {} //! Original data source. @@ -36,7 +36,7 @@ namespace XrdFileCache protected: XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - XrdOucCacheIO &m_io; //!< original data source + XrdOucCacheIO2 &m_io; //!< original data source XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics Cache &m_cache; //!< reference to Cache needed in detach }; diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index ee620601077..f20dbf9a7cd 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -31,7 +31,7 @@ using namespace XrdFileCache; //______________________________________________________________________________ -IOEntireFile::IOEntireFile(XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache & cache) +IOEntireFile::IOEntireFile(XrdOucCacheIO2 &io, XrdOucCacheStats &stats, Cache & cache) : IO(io, stats, cache), m_file(0) { diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index c9019bd4db2..59f8704359e 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -43,7 +43,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ - IOEntireFile(XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache &cache); + IOEntireFile(XrdOucCacheIO2 &io, XrdOucCacheStats &stats, Cache &cache); //------------------------------------------------------------------------ //! Destructor diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index ae9b8afe206..921e0a28db1 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -33,7 +33,7 @@ using namespace XrdFileCache; //______________________________________________________________________________ -IOFileBlock::IOFileBlock(XrdOucCacheIO &io, XrdOucCacheStats &statsGlobal, Cache & cache) +IOFileBlock::IOFileBlock(XrdOucCacheIO2 &io, XrdOucCacheStats &statsGlobal, Cache & cache) : IO(io, statsGlobal, cache) { m_blocksize = Cache::GetInstance().RefConfiguration().m_hdfsbsize; @@ -83,7 +83,7 @@ void IOFileBlock::GetBlockSizeFromPath() } //______________________________________________________________________________ -File* IOFileBlock::newBlockFile(long long off, int blocksize, XrdOucCacheIO* io) +File* IOFileBlock::newBlockFile(long long off, int blocksize, XrdOucCacheIO2* io) { XrdCl::URL url(io->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index b11cf139ab6..8fb43736bbe 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -20,7 +20,7 @@ #include #include -#include "XrdOuc/XrdOucCache.hh" +#include "XrdOuc/XrdOucCache2.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdFileCacheIO.hh" @@ -41,7 +41,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - IOFileBlock(XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache &cache); + IOFileBlock(XrdOucCacheIO2 &io, XrdOucCacheStats &stats, Cache &cache); //------------------------------------------------------------------------ //! Destructor. @@ -70,7 +70,7 @@ namespace XrdFileCache XrdSysMutex m_mutex; //!< map mutex void GetBlockSizeFromPath(); - File* newBlockFile(long long off, int blocksize, XrdOucCacheIO* io); + File* newBlockFile(long long off, int blocksize, XrdOucCacheIO2* io); }; } diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index 451502ebc01..5e8103cc561 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -75,7 +75,7 @@ int File::ReadV (const XrdOucIOVec *readV, int n) ReadVBlockListRAM blocks_to_process; std::vector blks_processed; ReadVBlockListDisk blocks_on_disk; - XrdCl::ChunkList chunkVec; + std::vector chunkVec; DirectResponseHandler* direct_handler = 0; if (!VReadPreProcess(readV, n, blocks_to_process, blocks_on_disk, chunkVec)) bytesRead = -1; @@ -86,12 +86,9 @@ int File::ReadV (const XrdOucIOVec *readV, int n) if (bytesRead >= 0) { if (!chunkVec.empty()) { direct_handler = new DirectResponseHandler(1); - XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; // TODO check interface in the client file - XrdCl::XRootDStatus vrStatus = client.VectorRead(chunkVec, (void*) 0, direct_handler); - if (!vrStatus.IsOK()) { - bytesRead = -1; - } + // m_input.VectorRead(chunkVec, (void*) 0, direct_handler); + m_input.ReadV(*direct_handler, &chunkVec[0], chunkVec.size()); } } @@ -125,8 +122,8 @@ int File::ReadV (const XrdOucIOVec *readV, int n) if (direct_handler->m_errno == 0) { - for (XrdCl::ChunkList::iterator i = chunkVec.begin(); i != chunkVec.end(); ++i) - bytesRead += i->length; + for (std::vector::iterator i = chunkVec.begin(); i != chunkVec.end(); ++i) + bytesRead += i->size; } else { @@ -163,7 +160,7 @@ int File::ReadV (const XrdOucIOVec *readV, int n) //______________________________________________________________________________ -bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& blocks_to_process, ReadVBlockListDisk& blocks_on_disk, XrdCl::ChunkList& chunkVec) +bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& blocks_to_process, ReadVBlockListDisk& blocks_on_disk, std::vector& chunkVec) { XrdSysCondVarHelper _lck(m_downloadCond); for (int iov_idx=0; iov_idxDebug(XrdCl::AppMsg, "VReadPreProcess direct read %d", block_idx); } } From 0a7b83e4e309f549d015ec8105e10df3633e9555 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 21 Mar 2016 14:58:07 -0700 Subject: [PATCH 075/171] Fix problem with XrdOucCache2 instantiation in XrdPssSys::getCache(). --- src/XrdPss/XrdPssConfig.cc | 41 +++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/src/XrdPss/XrdPssConfig.cc b/src/XrdPss/XrdPssConfig.cc index 6722981a70b..2b6360195d6 100644 --- a/src/XrdPss/XrdPssConfig.cc +++ b/src/XrdPss/XrdPssConfig.cc @@ -64,6 +64,8 @@ #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucTList.hh" #include "XrdOuc/XrdOucUtils.hh" +#include "XrdOuc/XrdOucCache2.hh" + #include "XrdPosix/XrdPosixXrootd.hh" @@ -461,10 +463,6 @@ int XrdPssSys::ConfigXeq(char *var, XrdOucStream &Config) int XrdPssSys::getCache() { XrdOucPinLoader myLib(&eDest,myVersion,"cachelib",cPath); - XrdOucCache *(*ep)(XrdSysLogger *, const char *, const char *); - union {XrdOucCache2 *theCache2; - XrdOucCache *theCache; - }; const char *cName; bool isCache2; @@ -474,21 +472,32 @@ int XrdPssSys::getCache() {cName = "XrdOucGetCache"; isCache2 = false; } else { cName = "XrdOucGetCache2"; isCache2 = true; - } - -// Now get the entry point of the object creator -// - ep = (XrdOucCache *(*)(XrdSysLogger *, const char *, const char *)) - (myLib.Resolve(cName)); - if (!ep) return 0; + } // Get the Object now // - theCache = ep(eDest.logger(), ConfigFN, cParm); - if (theCache) - {XrdPosixXrootd::setCache(isCache2 ? theCache2 : theCache);} - else eDest.Emsg("Config", "Unable to get cache object from", cPath); - return theCache != 0; + if (isCache2) { + XrdOucCache2 *(*ep)(XrdSysLogger *, const char *, const char *); + ep = (XrdOucCache2 *(*)(XrdSysLogger *, const char *, const char *)) + (myLib.Resolve(cName)); + + if (!ep) return 0; + + XrdOucCache2* theCache2 = ep(eDest.logger(), ConfigFN, cParm); + if (theCache2) XrdPosixXrootd::setCache(theCache2); + return theCache2 != 0; + } + else { + XrdOucCache *(*ep)(XrdSysLogger *, const char *, const char *); + ep = (XrdOucCache *(*)(XrdSysLogger *, const char *, const char *)) + (myLib.Resolve(cName)); + + if (!ep) return 0; + + XrdOucCache* theCache = ep(eDest.logger(), ConfigFN, cParm); + if (theCache) XrdPosixXrootd::setCache(theCache); + return theCache != 0; + } } /******************************************************************************/ From 8f681aae349f0656ee0ce4843a05f4fd45a1b698 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 21 Mar 2016 15:00:40 -0700 Subject: [PATCH 076/171] Set version info relative to XrdOucGetCache2. --- src/XrdFileCache/XrdFileCacheConfiguration.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 7f45d454a83..e3bf3ccb049 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -15,7 +15,7 @@ #include using namespace XrdFileCache; -XrdVERSIONINFO(XrdOucGetCache, XrdFileCache); +XrdVERSIONINFO(XrdOucGetCache2, XrdFileCache); /* Function: xdlib Purpose: To parse the directive: decisionlib [] @@ -95,7 +95,7 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char // Obtain plugin configurator XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(config_filename,&Config,&m_log, - &XrdVERSIONINFOVAR(XrdOucGetCache)); + &XrdVERSIONINFOVAR(XrdOucGetCache2)); if (!ofsCfg) return false; From e6d970fbf758482e126c713b24c3d947ac2924a0 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 21 Mar 2016 15:47:11 -0700 Subject: [PATCH 077/171] Add asynchronous read. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 5 +++++ src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index f20dbf9a7cd..25fb8cdbff3 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -65,6 +65,11 @@ XrdOucCacheIO *IOEntireFile::Detach() return io; } +void IOEntireFile::Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen) +{ + iocb.Done(IOEntireFile::Read(buff, offs, rlen)); +} + int IOEntireFile::Read (char *buff, long long off, int size) { clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() [%p] %lld@%d %s", this, off, size, m_io.Path()); diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index 59f8704359e..1c2db014e9e 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -61,6 +61,19 @@ namespace XrdFileCache //--------------------------------------------------------------------- virtual int Read(char *Buffer, long long Offset, int Length); + //--------------------------------------------------------------------- + //! Asynchronous read. + //! + //! @param callback + //! @param Buffer + //! @param Offset + //! @param Length + //! + //! @return number of bytes read + //--------------------------------------------------------------------- + virtual void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen); + + //--------------------------------------------------------------------- //! Pass ReadV request to the corresponding File object. //! From 703b1bffd11a97ed005b2e4040bf051cad722248 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 21 Mar 2016 15:52:25 -0700 Subject: [PATCH 078/171] Fix white space created in the previous commit. --- src/XrdPss/XrdPssConfig.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/XrdPss/XrdPssConfig.cc b/src/XrdPss/XrdPssConfig.cc index 2b6360195d6..c0293cd7ac1 100644 --- a/src/XrdPss/XrdPssConfig.cc +++ b/src/XrdPss/XrdPssConfig.cc @@ -66,7 +66,6 @@ #include "XrdOuc/XrdOucUtils.hh" #include "XrdOuc/XrdOucCache2.hh" - #include "XrdPosix/XrdPosixXrootd.hh" /******************************************************************************/ From dc7c4970fc35518de8c269c00c5610e1322effb9 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Mon, 21 Mar 2016 18:22:42 -0700 Subject: [PATCH 079/171] Remove unused variable. --- src/XrdPosix/XrdPosixXrootd.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/XrdPosix/XrdPosixXrootd.cc b/src/XrdPosix/XrdPosixXrootd.cc index 13abadf2f36..7576bb7f045 100644 --- a/src/XrdPosix/XrdPosixXrootd.cc +++ b/src/XrdPosix/XrdPosixXrootd.cc @@ -1304,7 +1304,6 @@ void XrdPosixXrootd::initEnv(char *eData) XrdOucCacheIO::aprParms apParms; XrdOucCache *v1Cache; long long Val; - int isRW = 0; char * tP; // Get numeric type variable (errors force a default) @@ -1350,7 +1349,7 @@ void XrdPosixXrootd::initEnv(char *eData) myParms.Options |= XrdOucCache::logStats; if ((tP = theEnv.Get("optpr")) && *tP && *tP != '0') myParms.Options |= XrdOucCache::canPreRead; - if ((tP = theEnv.Get("optwr")) && *tP && *tP != '0') isRW = 1; +// if ((tP = theEnv.Get("optwr")) && *tP && *tP != '0') isRW = 1; // Use the default cache if one was not provided // From 090db989e2a43c78991d5a73789798713d5ff420 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Tue, 22 Mar 2016 18:11:05 -0700 Subject: [PATCH 080/171] Corrections for partially implemented deferred open. --- src/XrdPosix/XrdPosixFile.cc | 17 ++++++++++------- src/XrdPosix/XrdPosixFile.hh | 2 +- src/XrdPosix/XrdPosixXrootd.cc | 8 ++++++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/XrdPosix/XrdPosixFile.cc b/src/XrdPosix/XrdPosixFile.cc index 08e52d03e86..92ebe77722d 100644 --- a/src/XrdPosix/XrdPosixFile.cc +++ b/src/XrdPosix/XrdPosixFile.cc @@ -225,22 +225,25 @@ bool XrdPosixFile::Close(XrdCl::XRootDStatus &Status) /* F i n a l i z e */ /******************************************************************************/ -bool XrdPosixFile::Finalize(XrdCl::XRootDStatus &Status) +bool XrdPosixFile::Finalize(XrdCl::XRootDStatus *Status) { + XrdOucCacheIO2 *ioP; + // Indicate that we are at the start of the file // currOffset = 0; // Complete initialization. If the stat() fails, the caller will unwind the -// whole open process (ick). +// whole open process (ick). In the process get correct I/O vector. - if (!Stat(Status)) - return false; + if (!Status) ioP = (XrdOucCacheIO2 *)PrepIO; + else if (Stat(*Status)) ioP = (XrdOucCacheIO2 *)this; + else return false; // Setup the cache if it is to be used // if (XrdPosixGlobals::theCache) - XCio = XrdPosixGlobals::theCache->Attach((XrdOucCacheIO2 *)this, cOpt); + XCio = XrdPosixGlobals::theCache->Attach(ioP, cOpt); return true; } @@ -258,8 +261,8 @@ void XrdPosixFile::HandleResponse(XrdCl::XRootDStatus *status, // If no errors occured, complete the open // - if (!(status->IsOK())) rc = XrdPosixMap::Result(*status); - else if (!Finalize(Status)) rc = XrdPosixMap::Result(Status); + if (!(status->IsOK())) rc = XrdPosixMap::Result(*status); + else if (!Finalize(&Status)) rc = XrdPosixMap::Result(Status); // Issue callback with the correct result // diff --git a/src/XrdPosix/XrdPosixFile.hh b/src/XrdPosix/XrdPosixFile.hh index d30cf68e8a5..d9e2adf052c 100644 --- a/src/XrdPosix/XrdPosixFile.hh +++ b/src/XrdPosix/XrdPosixFile.hh @@ -84,7 +84,7 @@ static void DelayedDestroy(XrdPosixFile *fp); bool Close(XrdCl::XRootDStatus &Status); - bool Finalize(XrdCl::XRootDStatus &Status); + bool Finalize(XrdCl::XRootDStatus *Status); long long FSize() {AtomicBeg(updMutex); long long retSize = AtomicGet(mySize); diff --git a/src/XrdPosix/XrdPosixXrootd.cc b/src/XrdPosix/XrdPosixXrootd.cc index 7576bb7f045..84b94dbecfc 100644 --- a/src/XrdPosix/XrdPosixXrootd.cc +++ b/src/XrdPosix/XrdPosixXrootd.cc @@ -106,7 +106,11 @@ int OpenDefer(XrdPosixFile *fp, // Allocate a prepare I/O object to defer this open // - fp->XCio = fp->PrepIO = new XrdPosixPrepIO(fp, XOflags, XOmode); + fp->PrepIO = new XrdPosixPrepIO(fp, XOflags, XOmode); + +// Finalize this file object. A null argument indicates it is defered. +// + fp->Finalize(0); // For sync opens we just need to return the file descriptor // @@ -545,7 +549,7 @@ int XrdPosixXrootd::Open(const char *path, int oflags, mode_t mode, // finalization is defered until the callback happens. // if (cbP) {errno = EINPROGRESS; return -1;} - if (fp->Finalize(Status)) return fp->FDNum(); + if (fp->Finalize(&Status)) return fp->FDNum(); return XrdPosixMap::Result(Status); } From d6b41680477b7fda31cf817c0ffa93b9f4155d98 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Thu, 24 Mar 2016 00:51:01 -0700 Subject: [PATCH 081/171] More fixes to defered open. --- src/XrdPosix/XrdPosixFile.hh | 4 ++++ src/XrdPosix/XrdPosixObjGaurd.hh | 16 ++++++------- src/XrdPosix/XrdPosixPrepIO.cc | 2 +- src/XrdPosix/XrdPosixPrepIO.hh | 20 ++++++++-------- src/XrdPosix/XrdPosixXrootd.cc | 40 ++++++++++++++++++++++++++++---- 5 files changed, 58 insertions(+), 24 deletions(-) diff --git a/src/XrdPosix/XrdPosixFile.hh b/src/XrdPosix/XrdPosixFile.hh index d9e2adf052c..116c52ac536 100644 --- a/src/XrdPosix/XrdPosixFile.hh +++ b/src/XrdPosix/XrdPosixFile.hh @@ -97,6 +97,10 @@ static void DelayedDestroy(XrdPosixFile *fp); void isOpen(); + void updLock() {updMutex.Lock();} + + void updUnLock() {updMutex.UnLock();} + long long Offset() {AtomicRet(updMutex, currOffset);} const char *Path() {return fPath;} diff --git a/src/XrdPosix/XrdPosixObjGaurd.hh b/src/XrdPosix/XrdPosixObjGaurd.hh index b9b9b1d3012..75a293b81d7 100644 --- a/src/XrdPosix/XrdPosixObjGaurd.hh +++ b/src/XrdPosix/XrdPosixObjGaurd.hh @@ -30,24 +30,24 @@ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ -#include "XrdPosix/XrdPosixObject.hh" +#include "XrdPosix/XrdPosixFile.hh" class XrdPosixObjGaurd { public: -void Init(XrdPosixObject *oP, bool rw=true) - {if (objP) Release(); - objP = oP; - objP->Lock(rw); +void Init(XrdPosixFile *fP) + {if (gaurdP) gaurdP->updUnLock(); + gaurdP = fP; + gaurdP->updLock(); } -void Release() {if (objP) {objP->UnLock(); objP = 0;}} +void Release() {if (gaurdP) {gaurdP->updUnLock(); gaurdP = 0;}} - XrdPosixObjGaurd(XrdPosixObject *oP, bool rw=true) {Init(oP, rw);} + XrdPosixObjGaurd(XrdPosixFile *fP) {Init(fP);} ~XrdPosixObjGaurd() {Release();} private: -XrdPosixObject *objP; +XrdPosixFile *gaurdP; }; #endif diff --git a/src/XrdPosix/XrdPosixPrepIO.cc b/src/XrdPosix/XrdPosixPrepIO.cc index 754d35b57c5..ab10e4ad280 100644 --- a/src/XrdPosix/XrdPosixPrepIO.cc +++ b/src/XrdPosix/XrdPosixPrepIO.cc @@ -49,7 +49,7 @@ extern bool psxDBG; bool XrdPosixPrepIO::Init(XrdOucCacheIOCB *iocbP) { - XrdPosixObjGaurd objGaurd((XrdPosixObject *)fileP); + XrdPosixObjGaurd objGaurd(fileP); XrdCl::XRootDStatus Status; static int maxCalls = 64; diff --git a/src/XrdPosix/XrdPosixPrepIO.hh b/src/XrdPosix/XrdPosixPrepIO.hh index 18c51019e67..ff3f7402113 100644 --- a/src/XrdPosix/XrdPosixPrepIO.hh +++ b/src/XrdPosix/XrdPosixPrepIO.hh @@ -41,43 +41,43 @@ XrdOucCacheIO *Base() {return this;} // Already defined XrdOucCacheIO *Detach() {return this;} // Already defined -long long FSize() {return (Init() ? FSize() : openRC);} +long long FSize() {return (Init() ? fileP->FSize() : openRC);} bool ioActive() { return false; } // Already defined const char *Path() {return fileP->Path();} int Read (char *Buffer, long long Offset, int Length) - {return (Init() ? Read(Buffer, Offset, Length) : openRC);} + {return (Init() ? fileP->Read(Buffer, Offset, Length) : openRC);} void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen) - {if (Init(&iocb)) Read(iocb, buff, offs, rlen); + {if (Init(&iocb)) fileP->Read(iocb, buff, offs, rlen); else iocb.Done(openRC); } int ReadV(const XrdOucIOVec *readV, int n) - {return (Init() ? ReadV(readV, n) : openRC);} + {return (Init() ? fileP->ReadV(readV, n) : openRC);} void ReadV(XrdOucCacheIOCB &iocb, const XrdOucIOVec *readV, int rnum) - {if (Init(&iocb)) ReadV(iocb, readV, rnum); + {if (Init(&iocb)) fileP->ReadV(iocb, readV, rnum); else iocb.Done(openRC); } -int Sync() {return (Init() ? Sync() : openRC);} +int Sync() {return (Init() ? fileP->Sync() : openRC);} void Sync(XrdOucCacheIOCB &iocb) - {if (Init(&iocb)) Sync(iocb); + {if (Init(&iocb)) fileP->Sync(iocb); else iocb.Done(openRC); } int Trunc(long long Offset) - {return (Init() ? Trunc(Offset) : openRC);} + {return (Init() ? fileP->Trunc(Offset) : openRC);} int Write(char *Buffer, long long Offset, int Length) - {return (Init() ? Write(Buffer, Offset, Length) : openRC);} + {return (Init() ? fileP->Write(Buffer,Offset,Length) : openRC);} void Write(XrdOucCacheIOCB &iocb, char *buff, long long offs, int wlen) - {if (Init(&iocb)) Write(iocb, buff, offs, wlen); + {if (Init(&iocb)) fileP->Write(iocb, buff, offs, wlen); else iocb.Done(openRC); } diff --git a/src/XrdPosix/XrdPosixXrootd.cc b/src/XrdPosix/XrdPosixXrootd.cc index 84b94dbecfc..77f83299a4d 100644 --- a/src/XrdPosix/XrdPosixXrootd.cc +++ b/src/XrdPosix/XrdPosixXrootd.cc @@ -294,15 +294,42 @@ int XrdPosixXrootd::endPoint(int FD, char *Buff, int Blen) int XrdPosixXrootd::Fstat(int fildes, struct stat *buf) { XrdPosixFile *fp; + long long theSize; // Find the file object // if (!(fp = XrdPosixObject::File(fildes))) return -1; -// Return what little we can +// First initialize the stat buffer // initStat(buf); - buf->st_size = fp->FSize(); + +// Check if we can get the stat information from the cache. +// + if (myCache2) + {int rc = myCache2->Stat(fp->Path(), *buf); + if (rc <= 0) + {fp->UnLock(); + if (!rc) return 0; + errno = -rc; + return -1; + } + } + +// We need to trat getting the size separately as this file may still not be +// open. This gets resolved via the cache layer. +// + theSize = fp->XCio->FSize(); + if (theSize < 0) + {fp->UnLock(); + errno = static_cast(-theSize); + return -1; + } + + +// Return what little we can +// + buf->st_size = theSize; buf->st_atime = buf->st_mtime = buf->st_ctime = fp->myMtime; buf->st_blocks = buf->st_size/512+1; buf->st_ino = fp->myInode; @@ -1009,9 +1036,13 @@ int XrdPosixXrootd::Stat(const char *path, struct stat *buf) // Make sure the admin is OK // - if (!admin.isOK()) return -1; + if (!admin.isOK()) return -1; -// Check if we can get the stat informat directly +// Initialize the stat buffer +// + initStat(buf); + +// Check if we can get the stat informatation from the cache // if (myCache2) {std::string urlp = admin.Url.GetPathWithParams(); @@ -1026,7 +1057,6 @@ int XrdPosixXrootd::Stat(const char *path, struct stat *buf) // Return what little we can // - initStat(buf); buf->st_size = stSize; buf->st_blocks = stSize/512+1; buf->st_atime = buf->st_mtime = buf->st_ctime = stMtime; From 40cbdd97f13e32eea8422b3235f8e658f509d473 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Thu, 24 Mar 2016 17:16:45 -0700 Subject: [PATCH 082/171] Implement Fstat() at the cache level. --- src/XrdOuc/XrdOucCache2.hh | 19 +++++++++-- src/XrdPosix/XrdPosixCacheBC.hh | 2 ++ src/XrdPosix/XrdPosixFile.cc | 25 +++++++++++++++ src/XrdPosix/XrdPosixFile.hh | 2 ++ src/XrdPosix/XrdPosixPrepIO.hh | 5 +++ src/XrdPosix/XrdPosixXrootd.cc | 57 ++++++++++++--------------------- 6 files changed, 72 insertions(+), 38 deletions(-) diff --git a/src/XrdOuc/XrdOucCache2.hh b/src/XrdOuc/XrdOucCache2.hh index 4439e641c0d..a27396f5e28 100644 --- a/src/XrdOuc/XrdOucCache2.hh +++ b/src/XrdOuc/XrdOucCache2.hh @@ -61,6 +61,21 @@ class XrdOucCacheIO2 : public virtual XrdOucCacheIO { public: +//------------------------------------------------------------------------------ +//! Perform an fstat() operation (defaults to passthrough). +//! +//! @param sbuff reference to the stat buffer to be filled in. Only fields +//! st_size, st_blocks, st_mtime (st_atime and st_ctime may be +//! set to st_mtime), st_ino, and st_mode need to be set. All +//! other fields are preset and should not be changed. +//! +//! @return <0 - fstat failed, value is -errno. +//! =0 - fstat succeeded, sbuff holds stat information. +//! >0 - fstat could not be done, forward operation to next level. +//------------------------------------------------------------------------------ + +virtual int Fstat(struct stat &sbuff) {(void)sbuff; return 1;} + //------------------------------------------------------------------------------ //! Perform an asynchronous read (defaults to synchrnous). //! @@ -227,8 +242,8 @@ int Prepare(const char *url, int oflags, mode_t mode) //! @param url pointer to the url whose stat information is wanted. //! @param sbuff reference to the stat buffer to be filled in. Only fields //! st_size, st_blocks, st_mtime (st_atime and st_ctime may be -//! set to st_mtime), st_ino, st_rdev, and st_mode need to be -//! set. All other fields should be set to zero. +//! set to st_mtime), st_ino, and st_mode need to be set. All +//! other fields are preset and should not be changed. //! //! @return <0 - Stat failed, value is -errno. //! =0 - Stat succeeded, sbuff holds stat information. diff --git a/src/XrdPosix/XrdPosixCacheBC.hh b/src/XrdPosix/XrdPosixCacheBC.hh index 5694b9ef42d..54d05125876 100644 --- a/src/XrdPosix/XrdPosixCacheBC.hh +++ b/src/XrdPosix/XrdPosixCacheBC.hh @@ -53,6 +53,8 @@ XrdOucCacheIO2 *Detach() {XrdOucCacheIO2 *theCIO = cacheIO2; virtual long long FSize() {return cacheIO1->FSize();} +virtual int Fstat(struct stat &buf) {return cacheIO2->Fstat(buf);} + virtual const char *Path() {return cacheIO1->Path();} diff --git a/src/XrdPosix/XrdPosixFile.cc b/src/XrdPosix/XrdPosixFile.cc index 92ebe77722d..988af11d8e3 100644 --- a/src/XrdPosix/XrdPosixFile.cc +++ b/src/XrdPosix/XrdPosixFile.cc @@ -248,6 +248,31 @@ bool XrdPosixFile::Finalize(XrdCl::XRootDStatus *Status) return true; } +/******************************************************************************/ +/* F s t a t */ +/******************************************************************************/ + +int XrdPosixFile::Fstat(struct stat &buf) +{ + long long theSize; + +// The size is treated differently here as it may come from a cache and may +// actually trigger a file open if the open was deferred. +// + theSize = XCio->FSize(); + if (theSize < 0) return static_cast(theSize); + +// Return what little we can +// + buf.st_size = theSize; + buf.st_atime = buf.st_mtime = buf.st_ctime = myMtime; + buf.st_blocks = buf.st_size/512+1; + buf.st_ino = myInode; + buf.st_rdev = myRdev; + buf.st_mode = myMode; + return 0; +} + /******************************************************************************/ /* H a n d l e R e s p o n s e */ /******************************************************************************/ diff --git a/src/XrdPosix/XrdPosixFile.hh b/src/XrdPosix/XrdPosixFile.hh index 116c52ac536..1f12025494b 100644 --- a/src/XrdPosix/XrdPosixFile.hh +++ b/src/XrdPosix/XrdPosixFile.hh @@ -92,6 +92,8 @@ static void DelayedDestroy(XrdPosixFile *fp); return retSize; } + int Fstat(struct stat &buf); + void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response); diff --git a/src/XrdPosix/XrdPosixPrepIO.hh b/src/XrdPosix/XrdPosixPrepIO.hh index ff3f7402113..42d601b91ea 100644 --- a/src/XrdPosix/XrdPosixPrepIO.hh +++ b/src/XrdPosix/XrdPosixPrepIO.hh @@ -43,8 +43,13 @@ XrdOucCacheIO *Detach() {return this;} // Already defined long long FSize() {return (Init() ? fileP->FSize() : openRC);} +int Fstat(struct stat &buf) + {return (Init() ? fileP->Fstat(buf) : openRC);} + bool ioActive() { return false; } // Already defined +int Open() {Init(); return openRC;} + const char *Path() {return fileP->Path();} int Read (char *Buffer, long long Offset, int Length) diff --git a/src/XrdPosix/XrdPosixXrootd.cc b/src/XrdPosix/XrdPosixXrootd.cc index 77f83299a4d..1658419ff49 100644 --- a/src/XrdPosix/XrdPosixXrootd.cc +++ b/src/XrdPosix/XrdPosixXrootd.cc @@ -294,7 +294,7 @@ int XrdPosixXrootd::endPoint(int FD, char *Buff, int Blen) int XrdPosixXrootd::Fstat(int fildes, struct stat *buf) { XrdPosixFile *fp; - long long theSize; + int rc; // Find the file object // @@ -306,40 +306,21 @@ int XrdPosixXrootd::Fstat(int fildes, struct stat *buf) // Check if we can get the stat information from the cache. // - if (myCache2) - {int rc = myCache2->Stat(fp->Path(), *buf); - if (rc <= 0) - {fp->UnLock(); - if (!rc) return 0; - errno = -rc; - return -1; - } - } - -// We need to trat getting the size separately as this file may still not be -// open. This gets resolved via the cache layer. -// - theSize = fp->XCio->FSize(); - if (theSize < 0) + rc = fp->XCio->Fstat(*buf); + if (rc <= 0) {fp->UnLock(); - errno = static_cast(-theSize); + if (!rc) return 0; + errno = -rc; return -1; } - -// Return what little we can -// - buf->st_size = theSize; - buf->st_atime = buf->st_mtime = buf->st_ctime = fp->myMtime; - buf->st_blocks = buf->st_size/512+1; - buf->st_ino = fp->myInode; - buf->st_rdev = fp->myRdev; - buf->st_mode = fp->myMode; - -// All done +// At this point we can call the file's Fstat() and if the file is not open +// it will be opened. // + rc = fp->Fstat(*buf); fp->UnLock(); - return 0; + if (rc < 0) {errno = -rc; rc = -1;} + return rc; } /******************************************************************************/ @@ -445,13 +426,17 @@ off_t XrdPosixXrootd::Lseek(int fildes, off_t offset, int whence) // if (!(fp = XrdPosixObject::File(fildes))) return -1; -// Set the new offset +// Set the new offset. Note that SEEK_END requires that the file be opened. +// An open may occur by calling the FSize() method via the cache pointer. // - if (whence == SEEK_SET) curroffset = fp->setOffset(offset); - else if (whence == SEEK_CUR) curroffset = fp->addOffset(offset); - else if (whence == SEEK_END) - curroffset = fp->setOffset(fp->FSize()+offset); - else return Fault(fp, EINVAL); + if (whence == SEEK_SET) curroffset = fp->setOffset(offset); + else if (whence == SEEK_CUR) curroffset = fp->addOffset(offset); + else if (whence == SEEK_END) + {curroffset = fp->XCio->FSize(); + if (curroffset < 0) return Fault(fp,static_cast(-curroffset)); + curroffset = fp->setOffset(curroffset+offset); + } + else return Fault(fp, EINVAL); // All done // @@ -1395,7 +1380,7 @@ void XrdPosixXrootd::initEnv(char *eData) myParms.Options |= XrdOucCache::Serialized; if (!(v1Cache = myCache->Create(myParms, &apParms))) cerr <<"XrdPosix: " < Date: Mon, 28 Mar 2016 15:00:03 -0700 Subject: [PATCH 083/171] Pass input source as a pointer instead of reference. --- src/XrdFileCache/XrdFileCache.cc | 42 +++++++++++++++--- src/XrdFileCache/XrdFileCache.hh | 7 +++ src/XrdFileCache/XrdFileCacheFile.cc | 27 +++++++----- src/XrdFileCache/XrdFileCacheFile.hh | 4 +- src/XrdFileCache/XrdFileCacheIO.hh | 11 ++--- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 32 ++++++++------ src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 2 +- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 46 ++++++++++---------- src/XrdFileCache/XrdFileCacheIOFileBlock.hh | 6 +-- src/XrdFileCache/XrdFileCacheVRead.cc | 2 +- 10 files changed, 113 insertions(+), 66 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 0d63aff1926..383579cb59b 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -30,11 +30,9 @@ #include "XrdOuc/XrdOucUtils.hh" #include "XrdFileCache.hh" +#include "XrdFileCacheInfo.hh" #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheIOFileBlock.hh" -//#include "XrdFileCacheConfiguration.cc" -//#include "XrdFileCachePurge.cc" - using namespace XrdFileCache; @@ -146,9 +144,9 @@ XrdOucCacheIO2 *Cache::Attach(XrdOucCacheIO2 *io, int Options) clLog()->Info(XrdCl::AppMsg, "Cache::Attach() %s", io->Path()); IO* cio; if (Cache::GetInstance().RefConfiguration().m_hdfsmode) - cio = new IOFileBlock(*io, m_stats, *this); + cio = new IOFileBlock(io, m_stats, *this); else - cio = new IOEntireFile(*io, m_stats, *this); + cio = new IOEntireFile(io, m_stats, *this); return cio; } @@ -316,6 +314,7 @@ Cache::DeRegisterPrefetchFile(File* file) } m_prefetch_condVar.UnLock(); } + //______________________________________________________________________________ File* @@ -336,6 +335,39 @@ Cache::GetNextFileToPrefetch() return f; } +//______________________________________________________________________________ + +int +Cache::Prepare(const char *url, int oflags, mode_t mode) +{ + return 1; +} + +//______________________________________________________________________________ + +int +Cache::Stat(const char *curl, struct stat &sbuff) +{ + XrdCl::URL url(curl); + std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); + fname += ".cinfo"; + + XrdOucEnv myEnv; + XrdOssDF* df = m_output_fs->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); + int res = df->Open(fname.c_str(), O_RDONLY, 0600, myEnv); + if (res != 0) + return 1; + + df->Fstat(&sbuff); + + Info cinfo(0); + cinfo.Read(df); + sbuff.st_size = cinfo.GetSizeInBits(); + delete df; + return 0; +} + + //______________________________________________________________________________ diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 97477a1c5f0..e23aeefadf0 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -99,6 +99,13 @@ namespace XrdFileCache // this is an obsolete method virtual XrdOucCache* Create(XrdOucCache::Parms&, XrdOucCacheIO::aprParms*); + // Virtual function of XrdOucCache2. Used for deferred open. + virtual int Prepare(const char *url, int oflags, mode_t mode); + + + // virtual function of XrdOucCache2::Stat() + virtual int Stat(const char *url, struct stat &sbuff); + //-------------------------------------------------------------------- //! \brief Makes decision if the original XrdOucCacheIO should be cached. //! diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index a670874846a..5bd2704c3aa 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -70,7 +70,7 @@ namespace Cache* cache() { return &Cache::GetInstance(); } } -File::File(XrdOucCacheIO2 &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : +File::File(XrdOucCacheIO2 *inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : m_input(inputIO), m_output(NULL), m_infoFile(NULL), @@ -91,9 +91,9 @@ m_prefetchHitCnt(0), m_prefetchScore(1), m_prefetchCurrentCnt(0) { - clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); + clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input->Path()); if (!Open()) { - clLog()->Error(XrdCl::AppMsg, "File::File() Open failed %s !!!", m_input.Path()); + clLog()->Error(XrdCl::AppMsg, "File::File() Open failed %s !!!", m_input->Path()); } } @@ -220,7 +220,7 @@ bool File::InitiateClose() bool File::Open() { - clLog()->Dump(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_input.Path()); + clLog()->Dump(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_input->Path()); XrdOss &m_output_fs = *Cache::GetInstance().GetOss(); // Create the data file itself. @@ -232,7 +232,7 @@ bool File::Open() int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0600, myEnv); if (res < 0) { - clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data-FD for %s %s", m_temp_filename.c_str(), m_input.Path()); + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data-FD for %s %s", m_temp_filename.c_str(), m_input->Path()); delete m_output; m_output = 0; @@ -254,7 +254,7 @@ bool File::Open() int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv); if (res < 0) { - clLog()->Error(XrdCl::AppMsg, "File::Open() can't get info-FD %s %s", ifn.c_str(), m_input.Path()); + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get info-FD %s %s", ifn.c_str(), m_input->Path()); delete m_infoFile; m_infoFile = 0; return false; @@ -267,14 +267,17 @@ bool File::Open() if (m_cfi.Read(m_infoFile, Cache::GetInstance().RefConfiguration().m_prefetch) <= 0) { + m_fileSize = m_input->FSize(); int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; - clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input.Path()); + clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input->Path()); m_cfi.ResizeBits(ss, Cache::GetInstance().RefConfiguration().m_prefetch); m_cfi.WriteHeader(m_infoFile); } else { - clLog()->Debug(XrdCl::AppMsg, "Info file read from disk: %s", m_input.Path()); + m_fileSize = m_cfi.GetSizeInBits(); + printf("%lld file size \n", m_fileSize); + clLog()->Debug(XrdCl::AppMsg, "Info file read from disk: %s", m_input->Path()); } @@ -339,12 +342,12 @@ Block* File::RequestBlock(int i, bool prefetch) const int last_block = m_cfi.GetSizeInBits() - 1; long long off = i * BS; - long long this_bs = (i == last_block) ? m_input.FSize() - off : BS; + long long this_bs = (i == last_block) ? m_input->FSize() - off : BS; Block *b = new Block(this, off, this_bs, prefetch); // should block be reused to avoid recreation BlockResponseHandler* oucCB = new BlockResponseHandler(b); - m_input.Read(*oucCB, (char*)b->get_buff(), off, (int)this_bs); + m_input->Read(*oucCB, (char*)b->get_buff(), off, (int)this_bs); clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); m_block_map[i] = b; @@ -377,7 +380,7 @@ int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, overlap(*ii, BS, req_off, req_size, off, blk_off, size); - m_input.Read( *handler, req_buf + off, *ii * BS + blk_off, size); + m_input->Read( *handler, req_buf + off, *ii * BS + blk_off, size); clLog()->Dump(XrdCl::AppMsg, "RequestBlockDirect success %d %ld %s", *ii, size, lPath()); total += size; @@ -684,7 +687,7 @@ void File::WriteBlockToDisk(Block* b) int retval = 0; // write block buffer into disk file long long offset = b->m_offset - m_offset; - long long size = (b->m_offset + m_cfi.GetBufferSize()) > m_input.FSize() ? (m_input.FSize() - b->m_offset) : m_cfi.GetBufferSize(); + long long size = (b->m_offset + m_cfi.GetBufferSize()) > m_input->FSize() ? (m_input->FSize() - b->m_offset) : m_cfi.GetBufferSize(); int buffer_remaining = size; int buffer_offset = 0; int cnt = 0; diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 45af499445c..6857edf9957 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -97,7 +97,7 @@ namespace XrdFileCache private: enum PrefetchState_e { kOn, kHold, kCanceled }; - XrdOucCacheIO2 &m_input; //!< original data source + XrdOucCacheIO2 *m_input; //!< original data source XrdOssDF *m_output; //!< file handle for data file on disk XrdOssDF *m_infoFile; //!< file handle for data-info file on disk Info m_cfi; //!< download status of file blocks and access statistics @@ -144,7 +144,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - File(XrdOucCacheIO2 &io, std::string &path, + File(XrdOucCacheIO2 *io, std::string &path, long long offset, long long fileSize); //------------------------------------------------------------------------ diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index 815cf196ffe..a91d90d32f8 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -13,17 +13,17 @@ namespace XrdFileCache class IO : public XrdOucCacheIO2 { public: - IO (XrdOucCacheIO2 &io, XrdOucCacheStats &stats, Cache &cache) : + IO (XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache) : m_io(io), m_statsGlobal(stats), m_cache(cache) {} //! Original data source. - virtual XrdOucCacheIO *Base() { return &m_io; } + virtual XrdOucCacheIO *Base() { return m_io; } //! Original data source URL. - virtual long long FSize() { return m_io.FSize(); } + virtual long long FSize() { return m_io->FSize(); } //! Original data source URL. - virtual const char *Path() { return m_io.Path(); } + virtual const char *Path() { return m_io->Path(); } virtual int Sync() { return 0; } @@ -32,11 +32,12 @@ namespace XrdFileCache virtual int Write(char *Buffer, long long Offset, int Length) { errno = ENOTSUP; return -1; } + virtual void Update(XrdOucCacheIO2 &iocp) { m_io = &iocp; } protected: XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - XrdOucCacheIO2 &m_io; //!< original data source + XrdOucCacheIO2 *m_io; //!< original data source XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics Cache &m_cache; //!< reference to Cache needed in detach }; diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 25fb8cdbff3..6da14758973 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -31,16 +31,16 @@ using namespace XrdFileCache; //______________________________________________________________________________ -IOEntireFile::IOEntireFile(XrdOucCacheIO2 &io, XrdOucCacheStats &stats, Cache & cache) +IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & cache) : IO(io, stats, cache), m_file(0) { - clLog()->Info(XrdCl::AppMsg, "IO::IO() [%p] %s", this, m_io.Path()); + clLog()->Info(XrdCl::AppMsg, "IO::IO() [%p] %s", this, m_io->Path()); - XrdCl::URL url(io.Path()); + XrdCl::URL url(m_io->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); - m_file = new File(io, fname, 0, io.FSize()); + m_file = new File(io, fname, 0, 0); } IOEntireFile::~IOEntireFile() @@ -55,7 +55,7 @@ XrdOucCacheIO *IOEntireFile::Detach() { m_statsGlobal.Add(m_file->GetStats()); - XrdOucCacheIO * io = &m_io; + XrdOucCacheIO * io = m_io; delete m_file; m_file = 0; @@ -72,24 +72,28 @@ void IOEntireFile::Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int int IOEntireFile::Read (char *buff, long long off, int size) { - clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() [%p] %lld@%d %s", this, off, size, m_io.Path()); + + + clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() [%p] %lld@%d %s", this, off, size, m_io->Path()); + + return m_io->Read(buff, off, size); // protect from reads over the file size - if (off >= m_io.FSize()) - return 0; + // if (off >= m_io->FSize()) + // return 0; if (off < 0) { errno = EINVAL; return -1; } - if (off + size > m_io.FSize()) - size = m_io.FSize() - off; + //if (off + size > m_io->FSize()) + // size = m_io->FSize() - off; ssize_t bytes_read = 0; ssize_t retval = 0; retval = m_file->Read(buff, off, size); - clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() read from File retval = %d %s", retval, m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() read from File retval = %d %s", retval, m_io->Path()); if (retval >= 0) { bytes_read += retval; @@ -97,11 +101,11 @@ int IOEntireFile::Read (char *buff, long long off, int size) size -= retval; if (size > 0) - clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::Read() missed %d bytes %s", size, m_io.Path()); + clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::Read() missed %d bytes %s", size, m_io->Path()); } else { - clLog()->Error(XrdCl::AppMsg, "IOEntireFile::Read(), origin bytes read %d %s", retval, m_io.Path()); + clLog()->Error(XrdCl::AppMsg, "IOEntireFile::Read(), origin bytes read %d %s", retval, m_io->Path()); } return (retval < 0) ? retval : bytes_read; @@ -113,7 +117,7 @@ int IOEntireFile::Read (char *buff, long long off, int size) */ int IOEntireFile::ReadV (const XrdOucIOVec *readV, int n) { - clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::ReadV(), get %d requests %s", n, m_io.Path()); + clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::ReadV(), get %d requests %s", n, m_io->Path()); return m_file->ReadV(readV, n); diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index 1c2db014e9e..f0d112249d5 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -43,7 +43,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ - IOEntireFile(XrdOucCacheIO2 &io, XrdOucCacheStats &stats, Cache &cache); + IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache); //------------------------------------------------------------------------ //! Destructor diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 921e0a28db1..9151cbdccdf 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -33,7 +33,7 @@ using namespace XrdFileCache; //______________________________________________________________________________ -IOFileBlock::IOFileBlock(XrdOucCacheIO2 &io, XrdOucCacheStats &statsGlobal, Cache & cache) +IOFileBlock::IOFileBlock(XrdOucCacheIO2 *io, XrdOucCacheStats &statsGlobal, Cache & cache) : IO(io, statsGlobal, cache) { m_blocksize = Cache::GetInstance().RefConfiguration().m_hdfsbsize; @@ -43,8 +43,8 @@ IOFileBlock::IOFileBlock(XrdOucCacheIO2 &io, XrdOucCacheStats &statsGlobal, Cach //______________________________________________________________________________ XrdOucCacheIO* IOFileBlock::Detach() { - clLog()->Info(XrdCl::AppMsg, "IOFileBlock::Detach() %s", m_io.Path()); - XrdOucCacheIO * io = &m_io; + clLog()->Info(XrdCl::AppMsg, "IOFileBlock::Detach() %s", m_io->Path()); + XrdOucCacheIO * io = m_io; for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) @@ -62,7 +62,7 @@ XrdOucCacheIO* IOFileBlock::Detach() void IOFileBlock::GetBlockSizeFromPath() { const static std::string tag = "hdfsbsize="; - std::string path= m_io.Path(); + std::string path= m_io->Path(); size_t pos1 = path.find(tag); size_t t = tag.length(); if ( pos1 != path.npos) @@ -78,14 +78,14 @@ void IOFileBlock::GetBlockSizeFromPath() m_blocksize = atoi(path.substr(pos1).c_str()); } - clLog()->Debug(XrdCl::AppMsg, "FileBlock::GetBlockSizeFromPath(), blocksize = %lld. %s", m_blocksize, m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "FileBlock::GetBlockSizeFromPath(), blocksize = %lld. %s", m_blocksize, m_io->Path()); } } //______________________________________________________________________________ -File* IOFileBlock::newBlockFile(long long off, int blocksize, XrdOucCacheIO2* io) +File* IOFileBlock::newBlockFile(long long off, int blocksize) { - XrdCl::URL url(io->Path()); + XrdCl::URL url(m_io->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); std::stringstream ss; @@ -96,8 +96,8 @@ File* IOFileBlock::newBlockFile(long long off, int blocksize, XrdOucCacheIO2* i ss << &offExt[0]; fname = ss.str(); - clLog()->Debug(XrdCl::AppMsg, "FileBlock::FileBlock(), create XrdFileCacheFile. %s", m_io.Path()); - File* prefetch = new File(*io, fname, off, blocksize); + clLog()->Debug(XrdCl::AppMsg, "FileBlock::FileBlock(), create XrdFileCacheFile. %s", m_io->Path()); + File* prefetch = new File(m_io, fname, off, blocksize); return prefetch; } @@ -118,21 +118,21 @@ bool IOFileBlock::ioActive() int IOFileBlock::Read (char *buff, long long off, int size) { // protect from reads over the file size - if (off >= m_io.FSize()) + if (off >= m_io->FSize()) return 0; if (off < 0) { errno = EINVAL; return -1; } - if (off + size > m_io.FSize()) - size = m_io.FSize() - off; + if (off + size > m_io->FSize()) + size = m_io->FSize() - off; long long off0 = off; int idx_first = off0/m_blocksize; int idx_last = (off0+size-1)/m_blocksize; int bytes_read = 0; - clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() %lld@%d block range [%d-%d] \n %s", off, size, idx_first, idx_last, m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() %lld@%d block range [%d-%d] \n %s", off, size, idx_first, idx_last, m_io->Path()); for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx ) { @@ -148,14 +148,14 @@ int IOFileBlock::Read (char *buff, long long off, int size) { size_t pbs = m_blocksize; // check if this is last block - int lastIOFileBlock = (m_io.FSize()-1)/m_blocksize; + int lastIOFileBlock = (m_io->FSize()-1)/m_blocksize; if (blockIdx == lastIOFileBlock ) { - pbs = m_io.FSize() - blockIdx*m_blocksize; - clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() last block, change output file size to %lld \n %s", pbs, m_io.Path()); + pbs = m_io->FSize() - blockIdx*m_blocksize; + clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() last block, change output file size to %lld \n %s", pbs, m_io->Path()); } - fb = newBlockFile(blockIdx*m_blocksize, pbs, &m_io); + fb = newBlockFile(blockIdx*m_blocksize, pbs); m_blocks.insert(std::pair(blockIdx, (File*) fb)); } m_mutex.UnLock(); @@ -167,12 +167,12 @@ int IOFileBlock::Read (char *buff, long long off, int size) if (blockIdx == idx_first) { readBlockSize = (blockIdx + 1) *m_blocksize - off0; - clLog()->Debug(XrdCl::AppMsg, "Read partially till the end of the block %s", m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "Read partially till the end of the block %s", m_io->Path()); } else if (blockIdx == idx_last) { readBlockSize = (off0+size) - blockIdx*m_blocksize; - clLog()->Debug(XrdCl::AppMsg, "Read partially from beginning of block %s", m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "Read partially from beginning of block %s", m_io->Path()); } else { @@ -181,14 +181,14 @@ int IOFileBlock::Read (char *buff, long long off, int size) } assert(readBlockSize); - clLog()->Info(XrdCl::AppMsg, "IOFileBlock::Read() block[%d] read-block-size[%d], offset[%lld] %s", blockIdx, readBlockSize, off, m_io.Path()); + clLog()->Info(XrdCl::AppMsg, "IOFileBlock::Read() block[%d] read-block-size[%d], offset[%lld] %s", blockIdx, readBlockSize, off, m_io->Path()); long long min = blockIdx*m_blocksize; if ( off < min) { assert(0); } assert(off+readBlockSize <= (min + m_blocksize)); int retvalBlock = fb->Read(buff, off, readBlockSize); - clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() Block read returned %d %s", retvalBlock , m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() Block read returned %d %s", retvalBlock , m_io->Path()); if (retvalBlock == readBlockSize ) { bytes_read += retvalBlock; @@ -196,12 +196,12 @@ int IOFileBlock::Read (char *buff, long long off, int size) off += retvalBlock; } else if (retvalBlock > 0) { - clLog()->Warning(XrdCl::AppMsg, "IOFileBlock::Read() incomplete read, missing bytes %d %s", readBlockSize-retvalBlock, m_io.Path()); + clLog()->Warning(XrdCl::AppMsg, "IOFileBlock::Read() incomplete read, missing bytes %d %s", readBlockSize-retvalBlock, m_io->Path()); return bytes_read + retvalBlock; } else { - clLog()->Error(XrdCl::AppMsg, "IOFileBlock::Read() read error, retval %d %s", retvalBlock, m_io.Path()); + clLog()->Error(XrdCl::AppMsg, "IOFileBlock::Read() read error, retval %d %s", retvalBlock, m_io->Path()); return retvalBlock; } } diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index 8fb43736bbe..b6820ebff49 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -41,7 +41,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - IOFileBlock(XrdOucCacheIO2 &io, XrdOucCacheStats &stats, Cache &cache); + IOFileBlock(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache); //------------------------------------------------------------------------ //! Destructor. @@ -66,11 +66,11 @@ namespace XrdFileCache private: long long m_blocksize; //!< size of file-block - std::map m_blocks; //!< map of created blocks + std::map m_blocks; //!< map of created blocks XrdSysMutex m_mutex; //!< map mutex void GetBlockSizeFromPath(); - File* newBlockFile(long long off, int blocksize, XrdOucCacheIO2* io); + File* newBlockFile(long long off, int blocksize); }; } diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index 5e8103cc561..0c3e5f289dd 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -88,7 +88,7 @@ int File::ReadV (const XrdOucIOVec *readV, int n) direct_handler = new DirectResponseHandler(1); // TODO check interface in the client file // m_input.VectorRead(chunkVec, (void*) 0, direct_handler); - m_input.ReadV(*direct_handler, &chunkVec[0], chunkVec.size()); + m_input->ReadV(*direct_handler, &chunkVec[0], chunkVec.size()); } } From af5b29e0c2643bbb410041fd053d66a1bb503fb2 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 30 Mar 2016 14:37:29 -0700 Subject: [PATCH 084/171] Put file size in the info file. --- src/XrdFileCache/XrdFileCacheInfo.cc | 25 ++++++++++++++++--------- src/XrdFileCache/XrdFileCacheInfo.hh | 12 ++++++++++++ src/XrdFileCache/XrdFileCachePrint.cc | 4 ++-- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index b08d8cbc51b..40d7e270582 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -37,7 +37,7 @@ using namespace XrdFileCache; Info::Info(long long iBufferSize) : - m_version(0), + m_version(1), m_bufferSize(iBufferSize), m_sizeInBits(0), m_buff_fetched(0), m_buff_write_called(0), m_buff_prefetch(0), @@ -53,6 +53,13 @@ Info::~Info() if (m_buff_prefetch) free(m_buff_prefetch); } +//______________________________________________________________________________ +void Info::SetFileSize(long long fs) +{ + m_fileSize = fs; + ResizeBits(m_fileSize/m_bufferSize); +} + //______________________________________________________________________________ @@ -82,15 +89,15 @@ int Info::Read(XrdOssDF* fp, bool init_prefetch_buff ) off += fp->Read(&m_bufferSize, off, sizeof(long long)); if (off <= 0) return off; - int sb; - off += fp->Read(&sb, off, sizeof(int)); - ResizeBits(sb); + + off += fp->Read(&m_fileSize, off, sizeof(long long)); + ResizeBits(m_fileSize/m_bufferSize); off += fp->Read(m_buff_fetched, off, GetSizeInBytes()); assert (off == GetHeaderSize()); memcpy(m_buff_write_called, m_buff_fetched, GetSizeInBytes()); - m_complete = IsAnythingEmptyInRng(0, sb-1) ? false : true; + m_complete = IsAnythingEmptyInRng(0, m_sizeInBits - 1) ? false : true; off += fp->Read(&m_accessCnt, off, sizeof(int)); @@ -110,8 +117,8 @@ int Info::Read(XrdOssDF* fp, bool init_prefetch_buff ) int Info::GetHeaderSize() const { - // version + buffersize + download-status-array-size + download-status-array - return sizeof(int) + sizeof(long long) + sizeof(int) + GetSizeInBytes(); + // version + buffersize + file-size + download-status-array + return sizeof(int) + sizeof(long long) + sizeof(long long) + GetSizeInBytes(); } //______________________________________________________________________________ @@ -124,8 +131,8 @@ void Info::WriteHeader(XrdOssDF* fp) off += fp->Write(&m_version, off, sizeof(int)); off += fp->Write(&m_bufferSize, off, sizeof(long long)); - int nb = GetSizeInBits(); - off += fp->Write(&nb, off, sizeof(int)); + + off += fp->Write(&m_fileSize, off, sizeof(long long)); off += fp->Write(m_buff_write_called, off, GetSizeInBytes()); flr = XrdOucSxeq::Release(fp->getFD()); diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index e4378710d94..adeca61e311 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -85,6 +85,7 @@ namespace XrdFileCache //--------------------------------------------------------------------- void SetBitPrefetch(int i); + void SetFileSize(long long); //--------------------------------------------------------------------- //! \brief Reserve buffer for fileSize/bufferSize bytes @@ -127,6 +128,11 @@ namespace XrdFileCache //--------------------------------------------------------------------- int GetSizeInBits() const; + //--------------------------------------------------------------------- + //! Get file size + //--------------------------------------------------------------------- + long long GetFileSize() const; + //---------------------------------------------------------------------- //! Get header size. //---------------------------------------------------------------------- @@ -195,6 +201,7 @@ namespace XrdFileCache int m_version; //!< info version long long m_bufferSize; //!< prefetch buffer size + long long m_fileSize; //!< number of file blocks int m_sizeInBits; //!< number of file blocks unsigned char *m_buff_fetched; //!< download state vector unsigned char *m_buff_write_called; //!< disk written state vector @@ -246,6 +253,11 @@ namespace XrdFileCache return m_sizeInBits; } + inline long long Info::GetFileSize() const + { + return m_fileSize; + } + inline bool Info::IsComplete() const { return m_complete; diff --git a/src/XrdFileCache/XrdFileCachePrint.cc b/src/XrdFileCache/XrdFileCachePrint.cc index a87fb5956d5..b86bbce8e5d 100644 --- a/src/XrdFileCache/XrdFileCachePrint.cc +++ b/src/XrdFileCache/XrdFileCachePrint.cc @@ -71,8 +71,8 @@ void Print::printFile(const std::string& path) int cntd = 0; for (int i = 0; i < cfi.GetSizeInBits(); ++i) if (cfi.TestBit(i)) cntd++; - printf("version == %d, bufferSize %lld nBlocks %d nDownloaded %d %s\n", - cfi.GetVersion(), cfi.GetBufferSize(), cfi.GetSizeInBits(), cntd, + printf("version == %d, fileSize %lld, bufferSize %lld nBlocks %d nDownloaded %d %s\n", + cfi.GetVersion(), cfi.GetFileSize(),cfi.GetBufferSize(), cfi.GetSizeInBits(), cntd, (cfi.GetSizeInBits() == cntd) ? "complete" : ""); if (m_verbose) { From 946cc296448c3c7f48117ee601981e40ace32080 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 30 Mar 2016 14:40:06 -0700 Subject: [PATCH 085/171] Implement Fstat. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 32 +++++++++++++++++--- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 3 ++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 6da14758973..c1fd211c9d1 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -17,6 +17,7 @@ //---------------------------------------------------------------------------------- #include +#include //#include "XrdClient/XrdClientConst.hh" #include "XrdSys/XrdSysError.hh" @@ -26,6 +27,8 @@ #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheStats.hh" +#include "XrdOuc/XrdOucEnv.hh" + using namespace XrdFileCache; //______________________________________________________________________________ @@ -46,6 +49,31 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & IOEntireFile::~IOEntireFile() {} +int IOEntireFile::Fstat(struct stat &sbuff) +{ + XrdCl::URL url(m_io->Path()); + std::string name = url.GetPath(); + name += ".cinfo"; + printf("AMT IOEntireFile::Fstat get stat for path%s \n", name.c_str()); + if (m_cache.GetOss()->Stat(name.c_str(), &sbuff) == XrdOssOK) { + XrdOssDF* infoFile = m_cache.GetOss()->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); + XrdOucEnv myEnv; + int res = infoFile->Open(name.c_str(), O_RDONLY, 0600, myEnv); + if (res < 0) return res; + Info info(0); + if (info.Read(infoFile) < 0) return -1; + + sbuff.st_size = info.GetFileSize(); + printf("AMT IONETIREFILE::Stat %ld\n", sbuff.st_size); + infoFile->Close(); + delete infoFile; + return 0; + } + else { + return m_io->Fstat(sbuff); + } +} + bool IOEntireFile::ioActive() { return m_file->InitiateClose(); @@ -72,12 +100,8 @@ void IOEntireFile::Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int int IOEntireFile::Read (char *buff, long long off, int size) { - - clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() [%p] %lld@%d %s", this, off, size, m_io->Path()); - return m_io->Read(buff, off, size); - // protect from reads over the file size // if (off >= m_io->FSize()) // return 0; diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index f0d112249d5..e486a0a0c6c 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -95,6 +95,9 @@ namespace XrdFileCache //! Called to check if destruction needs to be done in a separate task. virtual bool ioActive(); + virtual int Fstat(struct stat &sbuff); + + private: File* m_file; }; From e6b3b820a66e9ffade513e1dd267b3ffbf82d208 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 30 Mar 2016 14:41:17 -0700 Subject: [PATCH 086/171] Implement virtual ouc-cache Prepare(). --- src/XrdFileCache/XrdFileCache.cc | 39 ++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 383579cb59b..b460ebc3cd1 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -336,11 +336,31 @@ Cache::GetNextFileToPrefetch() } //______________________________________________________________________________ +//! Preapare the cache for a file open request. This method is called prior to +//! actually opening a file. This method is meant to allow defering an open +//! request or implementing the full I/O stack in the cache layer. +//! @return <0 Error has occurred, return value is -errno; fail open request. +//! =0 Continue with open() request. +//! >0 Defer open but treat the file as actually being open. Use the +//! XrdOucCacheIO2::Open() method to open the file at a later time. +//------------------------------------------------------------------------------ int Cache::Prepare(const char *url, int oflags, mode_t mode) { - return 1; + std::string curl(url); + XrdCl::URL xx(curl); + const std::string& spath = xx.GetPath(); + + struct stat buf; + int res = m_output_fs->Stat(spath.c_str(), &buf); + if (res == 0) { + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch defer open %s", spath.c_str()); + return 1; + } + else { + return 0; + } } //______________________________________________________________________________ @@ -348,22 +368,7 @@ Cache::Prepare(const char *url, int oflags, mode_t mode) int Cache::Stat(const char *curl, struct stat &sbuff) { - XrdCl::URL url(curl); - std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); - fname += ".cinfo"; - - XrdOucEnv myEnv; - XrdOssDF* df = m_output_fs->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); - int res = df->Open(fname.c_str(), O_RDONLY, 0600, myEnv); - if (res != 0) - return 1; - - df->Fstat(&sbuff); - - Info cinfo(0); - cinfo.Read(df); - sbuff.st_size = cinfo.GetSizeInBits(); - delete df; + assert(0 && "Cache::Stat() should not be called."); return 0; } From d2563c986dac82f38c0b312a4b2bc19920636eca Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 30 Mar 2016 14:42:35 -0700 Subject: [PATCH 087/171] Use file size in info file. --- src/XrdFileCache/XrdFileCacheFile.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 5bd2704c3aa..4b5a5bd2c92 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -207,9 +207,10 @@ bool File::InitiateClose() m_stateCond.Lock(); m_stopping = true; m_stateCond.UnLock(); - m_prefetchState = kCanceled; - if (m_cfi.IsComplete()) return false; // AMT maybe map size is here more meaningfull, but might hold block state lock - return true; + m_prefetchState = kCanceled; + m_cfi.CheckComplete(); + bool complete = m_cfi.IsComplete(); + return !complete; } //______________________________________________________________________________ @@ -270,8 +271,9 @@ bool File::Open() m_fileSize = m_input->FSize(); int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input->Path()); - m_cfi.ResizeBits(ss, Cache::GetInstance().RefConfiguration().m_prefetch); + m_cfi.SetFileSize(m_fileSize); m_cfi.WriteHeader(m_infoFile); + m_infoFile->Fsync(); } else { From d5e176481641b7d2313ff95dac9d1f71ea0740c2 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 1 Apr 2016 16:00:42 -0700 Subject: [PATCH 088/171] Clear block map repetitively in InitiateClose(). --- src/XrdFileCache/XrdFileCacheFile.cc | 92 +++++++++++----------------- 1 file changed, 37 insertions(+), 55 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 4b5a5bd2c92..f4dad66d455 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -109,55 +109,6 @@ File::~File() { clLog()->Debug(XrdCl::AppMsg, "File::~File() enter %p %s", (void*)this, lPath()); - // assert if in debug mode - if (XrdCl::DefaultEnv::GetLog()->GetLevel() >= XrdCl::Log::DebugMsg ) { - m_stateCond.Lock(); - assert (m_stopping == true); - m_stateCond.UnLock(); - } - - cache()->RemoveWriteQEntriesFor(this); - - clLog()->Info(XrdCl::AppMsg, "File::~File() check write queues ...%s", lPath()); - - while (true) - { - m_stateCond.Lock(); - bool isPrefetching = (m_prefetchCurrentCnt > 0); - m_stateCond.UnLock(); - - if (isPrefetching == false) - { - m_downloadCond.Lock(); - // remove failed blocks - BlockMap_i itr = m_block_map.begin(); - while (itr != m_block_map.end()) { - if (itr->second->is_failed() && itr->second->m_refcnt == 1) { - BlockMap_i toErase = itr; - ++itr; - free_block(toErase->second); - } - else { - ++itr; - } - } - - bool blockMapEmpty = m_block_map.empty(); - int blocksize = (int)m_block_map.size(); - m_downloadCond.UnLock(); - if ( blockMapEmpty) - break; - clLog()->Info(XrdCl::AppMsg, "File::~File() mapsize %d %s", blocksize,lPath()); - for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it) { - Block* b = it->second; - clLog()->Debug(XrdCl::AppMsg, "File::~File() block idx=%d p=%d rcnt=%d dwnd=%d %s", - b->m_offset/m_cfi.GetBufferSize(), b->m_prefetch, b->m_refcnt, b->m_downloaded, lPath()); - } - } - XrdSysTimer::Wait(10); - } - clLog()->Debug(XrdCl::AppMsg, "File::~File finished with writing %s",lPath() ); - // Wait disk sync bool do_sync = false; @@ -202,15 +153,46 @@ bool File::InitiateClose() // Retruns true if delay is needed clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start %s", lPath()); - cache()->DeRegisterPrefetchFile(this); m_stateCond.Lock(); - m_stopping = true; + bool firsttime = false; + if (!m_stopping) { + firsttime = true; + m_prefetchState = kCanceled; + cache()->DeRegisterPrefetchFile(this); + m_stopping = true; + } + m_stateCond.UnLock(); + + if (firsttime) cache()->RemoveWriteQEntriesFor(this); + + m_stateCond.Lock(); + bool isPrefetching = (m_prefetchCurrentCnt > 0); m_stateCond.UnLock(); - m_prefetchState = kCanceled; - m_cfi.CheckComplete(); - bool complete = m_cfi.IsComplete(); - return !complete; + + if (isPrefetching == false) + { + m_downloadCond.Lock(); + // remove failed blocks + BlockMap_i itr = m_block_map.begin(); + while (itr != m_block_map.end()) { + if (itr->second->is_failed() && itr->second->m_refcnt == 1) { + BlockMap_i toErase = itr; + ++itr; + free_block(toErase->second); + } + else { + ++itr; + } + } + bool blockMapEmpty = m_block_map.empty(); + m_downloadCond.UnLock(); + + if ( blockMapEmpty) + return false; + } + + return true; } //______________________________________________________________________________ From 5610690894532c33aa00e9f7c5bd486062c6ef56 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 6 Apr 2016 14:33:56 -0700 Subject: [PATCH 089/171] Fix restoration of file size (bug from the previous commit). --- src/XrdFileCache/XrdFileCacheInfo.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index 40d7e270582..68c4fdad8f0 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -57,7 +57,7 @@ Info::~Info() void Info::SetFileSize(long long fs) { m_fileSize = fs; - ResizeBits(m_fileSize/m_bufferSize); + ResizeBits((m_fileSize-1)/m_bufferSize + 1) ; } //______________________________________________________________________________ @@ -89,9 +89,9 @@ int Info::Read(XrdOssDF* fp, bool init_prefetch_buff ) off += fp->Read(&m_bufferSize, off, sizeof(long long)); if (off <= 0) return off; - - off += fp->Read(&m_fileSize, off, sizeof(long long)); - ResizeBits(m_fileSize/m_bufferSize); + long long fs; + off += fp->Read(&fs, off, sizeof(long long)); + SetFileSize(fs); off += fp->Read(m_buff_fetched, off, GetSizeInBytes()); assert (off == GetHeaderSize()); @@ -131,7 +131,6 @@ void Info::WriteHeader(XrdOssDF* fp) off += fp->Write(&m_version, off, sizeof(int)); off += fp->Write(&m_bufferSize, off, sizeof(long long)); - off += fp->Write(&m_fileSize, off, sizeof(long long)); off += fp->Write(m_buff_write_called, off, GetSizeInBytes()); From a9c3e987c247cdb391cccc601de98361d3bfb221 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 6 Apr 2016 14:39:10 -0700 Subject: [PATCH 090/171] Cache local stat. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 58 ++++++++++++++------ src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 2 + 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index c1fd211c9d1..ff2e6705322 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -36,44 +36,68 @@ using namespace XrdFileCache; IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & cache) : IO(io, stats, cache), - m_file(0) + m_file(0), + m_localStat(0) { clLog()->Info(XrdCl::AppMsg, "IO::IO() [%p] %s", this, m_io->Path()); XrdCl::URL url(m_io->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); - m_file = new File(io, fname, 0, 0); + struct stat st; + Fstat(st); + m_file = new File(io, fname, 0, st.st_size); } IOEntireFile::~IOEntireFile() -{} +{ + + delete m_localStat; +} int IOEntireFile::Fstat(struct stat &sbuff) { XrdCl::URL url(m_io->Path()); std::string name = url.GetPath(); name += ".cinfo"; - printf("AMT IOEntireFile::Fstat get stat for path%s \n", name.c_str()); - if (m_cache.GetOss()->Stat(name.c_str(), &sbuff) == XrdOssOK) { - XrdOssDF* infoFile = m_cache.GetOss()->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); - XrdOucEnv myEnv; - int res = infoFile->Open(name.c_str(), O_RDONLY, 0600, myEnv); - if (res < 0) return res; - Info info(0); - if (info.Read(infoFile) < 0) return -1; - - sbuff.st_size = info.GetFileSize(); - printf("AMT IONETIREFILE::Stat %ld\n", sbuff.st_size); - infoFile->Close(); - delete infoFile; - return 0; + + struct stat* ls = getValidLocalStat(name.c_str()); + if (ls) { + memcpy(&sbuff, ls, sizeof(struct stat)); + return 0; } else { return m_io->Fstat(sbuff); } } + +struct stat* IOEntireFile::getValidLocalStat(const char* path) +{ + if (!m_localStat) { + m_localStat = new struct stat; + memset(m_localStat, 0, sizeof(struct stat)); + if (m_cache.GetOss()->Stat(path, m_localStat) == XrdOssOK) { + m_localStat->st_size = 0; + XrdOssDF* infoFile = m_cache.GetOss()->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); + XrdOucEnv myEnv; + int res = infoFile->Open(path, O_RDONLY, 0600, myEnv); + if (res >= 0) { + Info info(0); + if (info.Read(infoFile) > 0) { + m_localStat->st_size = info.GetFileSize(); + } + } + infoFile->Close(); + delete infoFile; + } + } + if (m_localStat->st_size) + return m_localStat; + else return 0; + +} + bool IOEntireFile::ioActive() { return m_file->InitiateClose(); diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index e486a0a0c6c..cbd7de232a1 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -100,6 +100,8 @@ namespace XrdFileCache private: File* m_file; + struct stat *m_localStat; + struct stat* getValidLocalStat(const char* path); }; } From 3b0881293a684d22b7e971884c7deb7422ea3fb5 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 6 Apr 2016 14:47:41 -0700 Subject: [PATCH 091/171] InitateClose(): Don't remove this blocks from global write queue. --- src/XrdFileCache/XrdFileCacheFile.cc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index f4dad66d455..1dea4964408 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -95,6 +95,7 @@ m_prefetchCurrentCnt(0) if (!Open()) { clLog()->Error(XrdCl::AppMsg, "File::File() Open failed %s !!!", m_input->Path()); } + printf("AMT open ended in File\n"); } void File::BlockRemovedFromWriteQ(Block* b) @@ -153,7 +154,6 @@ bool File::InitiateClose() // Retruns true if delay is needed clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start %s", lPath()); - m_stateCond.Lock(); bool firsttime = false; if (!m_stopping) { @@ -164,8 +164,6 @@ bool File::InitiateClose() } m_stateCond.UnLock(); - if (firsttime) cache()->RemoveWriteQEntriesFor(this); - m_stateCond.Lock(); bool isPrefetching = (m_prefetchCurrentCnt > 0); m_stateCond.UnLock(); @@ -173,6 +171,13 @@ bool File::InitiateClose() if (isPrefetching == false) { m_downloadCond.Lock(); + /* + for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it) { + Block* b = it->second; + clLog()->Debug(XrdCl::AppMsg, "File::InitiateClose() block idx=%d p=%d rcnt=%d dwnd=%d %s", + b->m_offset/m_cfi.GetBufferSize(), b->m_prefetch, b->m_refcnt, b->m_downloaded, lPath()); + } + */ // remove failed blocks BlockMap_i itr = m_block_map.begin(); while (itr != m_block_map.end()) { @@ -185,6 +190,7 @@ bool File::InitiateClose() ++itr; } } + bool blockMapEmpty = m_block_map.empty(); m_downloadCond.UnLock(); @@ -259,8 +265,6 @@ bool File::Open() } else { - m_fileSize = m_cfi.GetSizeInBits(); - printf("%lld file size \n", m_fileSize); clLog()->Debug(XrdCl::AppMsg, "Info file read from disk: %s", m_input->Path()); } @@ -641,7 +645,6 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) delete direct_handler; } - clLog()->Debug(XrdCl::AppMsg, "File::Read() before assert %s.", lPath()); assert(iUserSize >= bytes_read); // Last, stamp and release blocks, release file. From 824a88f69524d4c724fdef98ea5b894b2d53f6bd Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 7 Apr 2016 12:23:45 -0700 Subject: [PATCH 092/171] Add missing initialization. --- src/XrdPosix/XrdPosixObjGaurd.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdPosix/XrdPosixObjGaurd.hh b/src/XrdPosix/XrdPosixObjGaurd.hh index 75a293b81d7..1c12365b866 100644 --- a/src/XrdPosix/XrdPosixObjGaurd.hh +++ b/src/XrdPosix/XrdPosixObjGaurd.hh @@ -44,7 +44,7 @@ void Init(XrdPosixFile *fP) void Release() {if (gaurdP) {gaurdP->updUnLock(); gaurdP = 0;}} - XrdPosixObjGaurd(XrdPosixFile *fP) {Init(fP);} + XrdPosixObjGaurd(XrdPosixFile *fP):fP(0) {Init(fP);} ~XrdPosixObjGaurd() {Release();} private: From 0c53ef557c846d08be368350607e763466425a5e Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 7 Apr 2016 12:44:29 -0700 Subject: [PATCH 093/171] Use cached values of file path and file size instead of access them through XrdPosixFile. --- src/XrdFileCache/XrdFileCacheFile.cc | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 1dea4964408..4291240b053 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -91,11 +91,10 @@ m_prefetchHitCnt(0), m_prefetchScore(1), m_prefetchCurrentCnt(0) { - clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input->Path()); + clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_temp_filename.c_str()); if (!Open()) { - clLog()->Error(XrdCl::AppMsg, "File::File() Open failed %s !!!", m_input->Path()); + clLog()->Error(XrdCl::AppMsg, "File::File() Open failed %s !!!", m_temp_filename.c_str()); } - printf("AMT open ended in File\n"); } void File::BlockRemovedFromWriteQ(Block* b) @@ -155,9 +154,7 @@ bool File::InitiateClose() clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start %s", lPath()); m_stateCond.Lock(); - bool firsttime = false; if (!m_stopping) { - firsttime = true; m_prefetchState = kCanceled; cache()->DeRegisterPrefetchFile(this); m_stopping = true; @@ -209,7 +206,7 @@ bool File::InitiateClose() bool File::Open() { - clLog()->Dump(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_input->Path()); + clLog()->Dump(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_temp_filename.c_str()); XrdOss &m_output_fs = *Cache::GetInstance().GetOss(); // Create the data file itself. @@ -221,7 +218,7 @@ bool File::Open() int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0600, myEnv); if (res < 0) { - clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data-FD for %s %s", m_temp_filename.c_str(), m_input->Path()); + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data-FD for %s %s", m_temp_filename.c_str(), m_temp_filename.c_str()); delete m_output; m_output = 0; @@ -243,7 +240,7 @@ bool File::Open() int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv); if (res < 0) { - clLog()->Error(XrdCl::AppMsg, "File::Open() can't get info-FD %s %s", ifn.c_str(), m_input->Path()); + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get info-FD %s %s", ifn.c_str(), m_temp_filename.c_str()); delete m_infoFile; m_infoFile = 0; return false; @@ -256,16 +253,16 @@ bool File::Open() if (m_cfi.Read(m_infoFile, Cache::GetInstance().RefConfiguration().m_prefetch) <= 0) { - m_fileSize = m_input->FSize(); + m_fileSize = m_fileSize; int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; - clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input->Path()); + clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_temp_filename.c_str()); m_cfi.SetFileSize(m_fileSize); m_cfi.WriteHeader(m_infoFile); m_infoFile->Fsync(); } else { - clLog()->Debug(XrdCl::AppMsg, "Info file read from disk: %s", m_input->Path()); + clLog()->Debug(XrdCl::AppMsg, "Info file read from disk: %s", m_temp_filename.c_str()); } @@ -330,7 +327,7 @@ Block* File::RequestBlock(int i, bool prefetch) const int last_block = m_cfi.GetSizeInBits() - 1; long long off = i * BS; - long long this_bs = (i == last_block) ? m_input->FSize() - off : BS; + long long this_bs = (i == last_block) ? m_fileSize - off : BS; Block *b = new Block(this, off, this_bs, prefetch); // should block be reused to avoid recreation @@ -674,7 +671,7 @@ void File::WriteBlockToDisk(Block* b) int retval = 0; // write block buffer into disk file long long offset = b->m_offset - m_offset; - long long size = (b->m_offset + m_cfi.GetBufferSize()) > m_input->FSize() ? (m_input->FSize() - b->m_offset) : m_cfi.GetBufferSize(); + long long size = (b->m_offset + m_cfi.GetBufferSize()) > m_fileSize ? (m_fileSize - b->m_offset) : m_cfi.GetBufferSize(); int buffer_remaining = size; int buffer_offset = 0; int cnt = 0; From 7ccb47b15be7f395dd5fbb0b4c3637a4c55306ba Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 7 Apr 2016 13:35:47 -0700 Subject: [PATCH 094/171] Add missing initialization. --- src/XrdPosix/XrdPosixObjGaurd.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdPosix/XrdPosixObjGaurd.hh b/src/XrdPosix/XrdPosixObjGaurd.hh index 1c12365b866..2a0c711096d 100644 --- a/src/XrdPosix/XrdPosixObjGaurd.hh +++ b/src/XrdPosix/XrdPosixObjGaurd.hh @@ -44,7 +44,7 @@ void Init(XrdPosixFile *fP) void Release() {if (gaurdP) {gaurdP->updUnLock(); gaurdP = 0;}} - XrdPosixObjGaurd(XrdPosixFile *fP):fP(0) {Init(fP);} + XrdPosixObjGaurd(XrdPosixFile *fP): gaurdP(0) {Init(fP);} ~XrdPosixObjGaurd() {Release();} private: From 5f2f083a7c64abf0b6966cc723c24e93f9e4aeb0 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 8 Apr 2016 15:19:41 -0700 Subject: [PATCH 095/171] Initiate buffer for prefetch score statistics. --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- src/XrdFileCache/XrdFileCacheInfo.cc | 7 ++++--- src/XrdFileCache/XrdFileCacheInfo.hh | 5 +++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 4291240b053..f80b2a1fa35 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -74,7 +74,7 @@ File::File(XrdOucCacheIO2 *inputIO, std::string& disk_file_path, long long iOffs m_input(inputIO), m_output(NULL), m_infoFile(NULL), -m_cfi(Cache::GetInstance().RefConfiguration().m_bufferSize), +m_cfi(Cache::GetInstance().RefConfiguration().m_bufferSize, Cache::GetInstance().RefConfiguration().m_prefetch), m_temp_filename(disk_file_path), m_offset(iOffset), m_fileSize(iFileSize), diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index 68c4fdad8f0..c141aab6989 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -36,9 +36,10 @@ const char* XrdFileCache::Info::m_infoExtension = ".cinfo"; using namespace XrdFileCache; -Info::Info(long long iBufferSize) : +Info::Info(long long iBufferSize, bool prefetchBuffer) : m_version(1), m_bufferSize(iBufferSize), + m_hasPrefetchBuffer(prefetchBuffer), m_sizeInBits(0), m_buff_fetched(0), m_buff_write_called(0), m_buff_prefetch(0), m_accessCnt(0), @@ -63,14 +64,14 @@ void Info::SetFileSize(long long fs) //______________________________________________________________________________ -void Info::ResizeBits(int s, bool init_prefetch_buff) +void Info::ResizeBits(int s) { m_sizeInBits = s; m_buff_fetched = (unsigned char*)malloc(GetSizeInBytes()); m_buff_write_called = (unsigned char*)malloc(GetSizeInBytes()); memset(m_buff_fetched, 0, GetSizeInBytes()); memset(m_buff_write_called, 0, GetSizeInBytes()); - if (init_prefetch_buff) { + if (m_hasPrefetchBuffer) { m_buff_prefetch = (unsigned char*)malloc(GetSizeInBytes()); memset(m_buff_prefetch, 0, GetSizeInBytes()); } diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index adeca61e311..8eee78d4deb 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -59,7 +59,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - Info(long long bufferSize); + Info(long long bufferSize, bool prefetchBuffer = false); //------------------------------------------------------------------------ //! Destructor. @@ -92,7 +92,7 @@ namespace XrdFileCache //! //! @param n number of file blocks //--------------------------------------------------------------------- - void ResizeBits(int n, bool prefetch_stat = false); + void ResizeBits(int n); //--------------------------------------------------------------------- //! \brief Rea load content from cinfo file into this object @@ -201,6 +201,7 @@ namespace XrdFileCache int m_version; //!< info version long long m_bufferSize; //!< prefetch buffer size + bool m_hasPrefetchBuffer; //!< constains current prefetch score long long m_fileSize; //!< number of file blocks int m_sizeInBits; //!< number of file blocks unsigned char *m_buff_fetched; //!< download state vector From c5ffa70d2aa52a7f5ffe38779410a013fd94f4c5 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Fri, 8 Apr 2016 16:56:21 -0700 Subject: [PATCH 096/171] Corectly spell guard! --- src/XrdPosix.cmake | 2 +- ...rdPosixObjGaurd.hh => XrdPosixObjGuard.hh} | 22 +++++++++---------- src/XrdPosix/XrdPosixPrepIO.cc | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) rename src/XrdPosix/{XrdPosixObjGaurd.hh => XrdPosixObjGuard.hh} (84%) diff --git a/src/XrdPosix.cmake b/src/XrdPosix.cmake index 7df4e3a2c4c..f47b5cce1d0 100644 --- a/src/XrdPosix.cmake +++ b/src/XrdPosix.cmake @@ -23,7 +23,7 @@ add_library( XrdPosix/XrdPosixFileRH.cc XrdPosix/XrdPosixFileRH.hh XrdPosix/XrdPosixMap.cc XrdPosix/XrdPosixMap.hh XrdPosix/XrdPosixObject.cc XrdPosix/XrdPosixObject.hh - XrdPosix/XrdPosixObjGaurd.hh + XrdPosix/XrdPosixObjGuard.hh XrdPosix/XrdPosixPrepIO.cc XrdPosix/XrdPosixPrepIO.hh XrdPosix/XrdPosixXrootd.cc XrdPosix/XrdPosixXrootd.hh XrdPosix/XrdPosixXrootdPath.cc XrdPosix/XrdPosixXrootdPath.hh diff --git a/src/XrdPosix/XrdPosixObjGaurd.hh b/src/XrdPosix/XrdPosixObjGuard.hh similarity index 84% rename from src/XrdPosix/XrdPosixObjGaurd.hh rename to src/XrdPosix/XrdPosixObjGuard.hh index 2a0c711096d..ed5a9d36f58 100644 --- a/src/XrdPosix/XrdPosixObjGaurd.hh +++ b/src/XrdPosix/XrdPosixObjGuard.hh @@ -1,8 +1,8 @@ -#ifndef __XRDPOSIXOBJGAURD_HH__ -#define __XRDPOSIXOBJGAURD_HH__ +#ifndef __XRDPOSIXOBJGUARD_HH__ +#define __XRDPOSIXOBJGUARD_HH__ /******************************************************************************/ /* */ -/* X r d P o s i x O b j G a u r d . h h */ +/* X r d P o s i x O b j G u a r d . h h */ /* */ /* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ @@ -32,22 +32,22 @@ #include "XrdPosix/XrdPosixFile.hh" -class XrdPosixObjGaurd +class XrdPosixObjGuard { public: void Init(XrdPosixFile *fP) - {if (gaurdP) gaurdP->updUnLock(); - gaurdP = fP; - gaurdP->updLock(); + {if (guardP) guardP->updUnLock(); + guardP = fP; + guardP->updLock(); } -void Release() {if (gaurdP) {gaurdP->updUnLock(); gaurdP = 0;}} +void Release() {if (guardP) {guardP->updUnLock(); guardP = 0;}} - XrdPosixObjGaurd(XrdPosixFile *fP): gaurdP(0) {Init(fP);} - ~XrdPosixObjGaurd() {Release();} + XrdPosixObjGuard(XrdPosixFile *fP): guardP(0) {Init(fP);} + ~XrdPosixObjGuard() {Release();} private: -XrdPosixFile *gaurdP; +XrdPosixFile *guardP; }; #endif diff --git a/src/XrdPosix/XrdPosixPrepIO.cc b/src/XrdPosix/XrdPosixPrepIO.cc index ab10e4ad280..e1f26c99cfc 100644 --- a/src/XrdPosix/XrdPosixPrepIO.cc +++ b/src/XrdPosix/XrdPosixPrepIO.cc @@ -31,7 +31,7 @@ #include #include -#include "XrdPosix/XrdPosixObjGaurd.hh" +#include "XrdPosix/XrdPosixObjGuard.hh" #include "XrdPosix/XrdPosixPrepIO.hh" /******************************************************************************/ @@ -49,7 +49,7 @@ extern bool psxDBG; bool XrdPosixPrepIO::Init(XrdOucCacheIOCB *iocbP) { - XrdPosixObjGaurd objGaurd(fileP); + XrdPosixObjGuard objGuard(fileP); XrdCl::XRootDStatus Status; static int maxCalls = 64; From cea8ba5b90392c809fc155f2abd5f911bab9b007 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 11 Apr 2016 14:05:22 -0700 Subject: [PATCH 097/171] Sync disk every 100 new blocks. --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index f80b2a1fa35..95a9798e8ac 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -726,7 +726,7 @@ void File::WriteBlockToDisk(Block* b) ++m_non_flushed_cnt; } - if (m_non_flushed_cnt >= 100 && (m_cfi.IsComplete() && m_non_flushed_cnt > 0)) + if (m_non_flushed_cnt >= 100 ) { schedule_sync = true; m_in_sync = true; From 35c5fb907858be7fcb15a5faf6232899ba5e4148 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 12 Apr 2016 12:30:26 -0700 Subject: [PATCH 098/171] Remove obosolete parameter cachedir. --- src/XrdFileCache/XrdFileCacheConfiguration.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index e3bf3ccb049..4d3b86f2338 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -156,11 +156,9 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char int loff = 0; char buff[2048]; loff = snprintf(buff, sizeof(buff), "result\n" - "\tpfc.cachedir %s\n" "\tpfc.blocksize %lld\n" "\tpfc.prefetch %d\n" "\tpfc.nram %d\n\n", - m_configuration.m_cache_dir.c_str() , m_configuration.m_bufferSize, m_configuration.m_prefetch, // AMT not sure what parsing should be m_configuration.m_NRamBuffers ); From 214ccbcd190195054f26ebd0aaaf68bd3fce8b90 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 12 Apr 2016 12:48:28 -0700 Subject: [PATCH 099/171] Reject incompatible versions. --- src/XrdFileCache/XrdFileCacheInfo.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index c141aab6989..944f18c1583 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -86,7 +86,13 @@ int Info::Read(XrdOssDF* fp, bool init_prefetch_buff ) // before File::Run() starts int off = 0; - off += fp->Read(&m_version, off, sizeof(int)); + int version; + off += fp->Read(&version, off, sizeof(int)); + if (version != m_version) { + clLog()->Dump(XrdCl::AppMsg, "Info:::Read(), incomatible file version"); + return 0; + } + off += fp->Read(&m_bufferSize, off, sizeof(long long)); if (off <= 0) return off; From fe2f44cdae608e96462b5b377290fad951ab2e65 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 12 Apr 2016 12:52:46 -0700 Subject: [PATCH 100/171] Remove obosolete function argument. --- src/XrdFileCache/XrdFileCacheInfo.cc | 4 ++-- src/XrdFileCache/XrdFileCacheInfo.hh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index 944f18c1583..d4f343724b8 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -80,7 +80,7 @@ void Info::ResizeBits(int s) //______________________________________________________________________________ -int Info::Read(XrdOssDF* fp, bool init_prefetch_buff ) +int Info::Read(XrdOssDF* fp) { // does not need lock, called only in File::Open // before File::Run() starts @@ -111,7 +111,7 @@ int Info::Read(XrdOssDF* fp, bool init_prefetch_buff ) clLog()->Dump(XrdCl::AppMsg, "Info:::Read() complete %d access_cnt %d", m_complete, m_accessCnt); - if (init_prefetch_buff) { + if (m_hasPrefetchBuffer) { m_buff_prefetch = (unsigned char*)malloc(GetSizeInBytes()); memset(m_buff_prefetch, 0, GetSizeInBytes()); } diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index 8eee78d4deb..9345e75cef7 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -101,7 +101,7 @@ namespace XrdFileCache //! //! @return number of bytes read //--------------------------------------------------------------------- - int Read(XrdOssDF* fp, bool init_prefetch = false); + int Read(XrdOssDF* fp); //--------------------------------------------------------------------- //! Write number of blocks and read buffer size From 5434c47668b6f9703e32fc1ee31bd3a7e323ed26 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 12 Apr 2016 12:53:02 -0700 Subject: [PATCH 101/171] Remove obosolete function argument. --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 95a9798e8ac..502bd1c1e19 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -251,7 +251,7 @@ bool File::Open() return false; } - if (m_cfi.Read(m_infoFile, Cache::GetInstance().RefConfiguration().m_prefetch) <= 0) + if (m_cfi.Read(m_infoFile) { m_fileSize = m_fileSize; int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; From e98386aa13295011f2019d7cc60e7afdb5f56bc4 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 12 Apr 2016 13:46:50 -0700 Subject: [PATCH 102/171] Aligh code. --- src/XrdFileCache/XrdFileCacheInfo.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index 9345e75cef7..ea7b6fc1c78 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -59,7 +59,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - Info(long long bufferSize, bool prefetchBuffer = false); + Info(long long bufferSize, bool prefetchBuffer = false); //------------------------------------------------------------------------ //! Destructor. From fdb98ab4e6f1b654928eb37a37514a1e06074fb1 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 12 Apr 2016 13:48:27 -0700 Subject: [PATCH 103/171] Merge configuration parameters prefetch and prefetch_max_blocks. --- src/XrdFileCache/XrdFileCache.hh | 3 +-- src/XrdFileCache/XrdFileCacheConfiguration.cc | 21 ++++++++++++------- src/XrdFileCache/XrdFileCacheFile.cc | 6 +++--- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index e23aeefadf0..0341a6f0397 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -52,8 +52,7 @@ namespace XrdFileCache m_diskUsageLWM(-1), m_diskUsageHWM(-1), m_bufferSize(1024*1024), - m_NRamBuffers(8000), - m_prefetch(false), + m_NRamBuffers(8000), m_prefetch_max_blocks(10), m_hdfsbsize(128*1024*1024) {} diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 4d3b86f2338..61a634a670d 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -251,13 +251,20 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) } else if (part == "prefetch" ) { - int p = ::atoi(config.GetWord()); - if (p > 0) { - printf("prefetch enabled, max blocks per file=%d\n", p); - m_configuration.m_prefetch = true; - m_configuration.m_prefetch_max_blocks = p; - } else { - m_configuration.m_prefetch = false; + const char* params = config.GetWord(); + if (params) { + int p = ::atoi(config.GetWord()); + if (p > 0) { + printf("prefetch enabled, max blocks per file=%d\n", p); + m_configuration.m_prefetch_max_blocks = p; + } else { + m_log.Emsg("Config", "Prefetch is disabled"); + m_configuration.m_prefetch_max_blocks = 0; + } + } + { + m_log.Emsg("Config", "Error setting prefetch level."); + return false; } } else if (part == "nram" ) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 502bd1c1e19..6c8a09ccbbc 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -74,7 +74,7 @@ File::File(XrdOucCacheIO2 *inputIO, std::string& disk_file_path, long long iOffs m_input(inputIO), m_output(NULL), m_infoFile(NULL), -m_cfi(Cache::GetInstance().RefConfiguration().m_bufferSize, Cache::GetInstance().RefConfiguration().m_prefetch), +m_cfi(Cache::GetInstance().RefConfiguration().m_bufferSize, Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks > 0), m_temp_filename(disk_file_path), m_offset(iOffset), m_fileSize(iFileSize), @@ -927,7 +927,7 @@ void File::Prefetch() //______________________________________________________________________________ void File::CheckPrefetchStatRAM(Block* b) { - if (Cache::GetInstance().RefConfiguration().m_prefetch) { + if (Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) { if (b->m_prefetch) { m_prefetchHitCnt++; m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; @@ -938,7 +938,7 @@ void File::CheckPrefetchStatRAM(Block* b) //______________________________________________________________________________ void File::CheckPrefetchStatDisk(int idx) { - if (Cache::GetInstance().RefConfiguration().m_prefetch) { + if (Cache::GetInstance().RefConfiguration().m_prefetch_max_block) { if (m_cfi.TestPrefetchBit(idx)) m_prefetchHitCnt++; } From 071bafa6509f46fd4324f37c41572f9bc25fcb71 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 12 Apr 2016 14:51:15 -0700 Subject: [PATCH 104/171] Define diskusage as an absolute value (GB or TB). This is merge from master. --- src/XrdFileCache/XrdFileCacheConfiguration.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 61a634a670d..dcfe44f6662 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -225,12 +225,12 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) errno = 0; float lwmf = strtod(minV.c_str(), &eP); if (errno || eP == minV.c_str()) { - m_log.Emsg("Cache::ConfigParameters() error parsing diskusage parameter ", minV.c_str()); + m_log.Emsg("Factory::ConfigParameters() error parsing diskusage parameter ", minV.c_str()); return false; } float hwmf = strtod(maxV.c_str(), &eP); if (errno || eP == maxV.c_str()) { - m_log.Emsg("Cache::ConfigParameters() error parsing diskusage parameter ", maxV.c_str()); + m_log.Emsg("Factory::ConfigParameters() error parsing diskusage parameter ", maxV.c_str()); return false; } From 23dbb6dd0be81104012acb3d1f6c7b40f92997ba Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 12 Apr 2016 15:22:18 -0700 Subject: [PATCH 105/171] Set max RAM usage in absolute units. --- src/XrdFileCache/XrdFileCache.hh | 6 ++++-- src/XrdFileCache/XrdFileCacheConfiguration.cc | 11 +++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 0341a6f0397..e4daec0756a 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -52,7 +52,8 @@ namespace XrdFileCache m_diskUsageLWM(-1), m_diskUsageHWM(-1), m_bufferSize(1024*1024), - m_NRamBuffers(8000), + m_RamAbsAvailable(8*1024*1024), + m_NRamBuffers(-1), m_prefetch_max_blocks(10), m_hdfsbsize(128*1024*1024) {} @@ -64,7 +65,8 @@ namespace XrdFileCache long long m_diskUsageHWM; //!< cache purge high water mark long long m_bufferSize; //!< prefetch buffer size, default 1MB - int m_NRamBuffers; //!< number of total in-memory cache blocks + int m_RamAbsAvailable; //!< available from configuration + int m_NRamBuffers; //!< number of total in-memory cache blocks, cached bool m_prefetch; //!< prefetch enable state size_t m_prefetch_max_blocks;//!< maximum number of blocks to prefetch per file diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index dcfe44f6662..f09ae3017d2 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -151,6 +151,8 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char } } + // get number of available RAM blocks after process configuration + m_configuration.m_NRamBuffers = static_cast(m_configuration.m_RamAbsAvailable/ m_configuration.m_bufferSize); if (retval) { int loff = 0; @@ -158,7 +160,7 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char loff = snprintf(buff, sizeof(buff), "result\n" "\tpfc.blocksize %lld\n" "\tpfc.prefetch %d\n" - "\tpfc.nram %d\n\n", + "\tpfc.nramblocks %d\n\n", m_configuration.m_bufferSize, m_configuration.m_prefetch, // AMT not sure what parsing should be m_configuration.m_NRamBuffers ); @@ -269,7 +271,12 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) } else if (part == "nram" ) { - m_configuration.m_NRamBuffers = ::atoi(config.GetWord()); + long long minRAM = 1024* 1024 * 1024;; + long long maxRAM = 100 * minRAM; + if ( XrdOuca2x::a2sz(m_log, "get RAM available", config.GetWord(), &m_RamAbsAvailable, minRAM, maxRAM)) + { + return false; + } } else if ( part == "hdfsmode" ) { From 9da1021570a148b6657f515d075f8832e9845458 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 13 Apr 2016 13:12:07 -0700 Subject: [PATCH 106/171] Fix compilation error. --- src/XrdFileCache/XrdFileCache.hh | 2 +- src/XrdFileCache/XrdFileCacheConfiguration.cc | 2 +- src/XrdFileCache/XrdFileCacheFile.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index e4daec0756a..0a386a4a7ef 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -65,7 +65,7 @@ namespace XrdFileCache long long m_diskUsageHWM; //!< cache purge high water mark long long m_bufferSize; //!< prefetch buffer size, default 1MB - int m_RamAbsAvailable; //!< available from configuration + long long m_RamAbsAvailable; //!< available from configuration int m_NRamBuffers; //!< number of total in-memory cache blocks, cached bool m_prefetch; //!< prefetch enable state size_t m_prefetch_max_blocks;//!< maximum number of blocks to prefetch per file diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index f09ae3017d2..21dea396f46 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -273,7 +273,7 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) { long long minRAM = 1024* 1024 * 1024;; long long maxRAM = 100 * minRAM; - if ( XrdOuca2x::a2sz(m_log, "get RAM available", config.GetWord(), &m_RamAbsAvailable, minRAM, maxRAM)) + if ( XrdOuca2x::a2sz(m_log, "get RAM available", config.GetWord(), &m_configuration.m_RamAbsAvailable, minRAM, maxRAM)) { return false; } diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 6c8a09ccbbc..9a920ba8a5e 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -251,7 +251,7 @@ bool File::Open() return false; } - if (m_cfi.Read(m_infoFile) + if (m_cfi.Read(m_infoFile)) { m_fileSize = m_fileSize; int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; @@ -938,7 +938,7 @@ void File::CheckPrefetchStatRAM(Block* b) //______________________________________________________________________________ void File::CheckPrefetchStatDisk(int idx) { - if (Cache::GetInstance().RefConfiguration().m_prefetch_max_block) { + if (Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) { if (m_cfi.TestPrefetchBit(idx)) m_prefetchHitCnt++; } From cacf8e798b9ae5838a31704f2d5a19a674570bc0 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 13 Apr 2016 15:18:24 -0700 Subject: [PATCH 107/171] Rename member for prefetch list. --- src/XrdFileCache/XrdFileCache.cc | 14 +++++++------- src/XrdFileCache/XrdFileCache.hh | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index b460ebc3cd1..7d20865af71 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -291,7 +291,7 @@ Cache::RegisterPrefetchFile(File* file) XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file BEGIN"); m_prefetch_condVar.Lock(); - m_files.push_back(file); + m_prefetchList.push_back(file); m_prefetch_condVar.Signal(); m_prefetch_condVar.UnLock(); XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file End"); @@ -306,9 +306,9 @@ Cache::DeRegisterPrefetchFile(File* file) // called from last line File::InitiateClose() m_prefetch_condVar.Lock(); - for (FileList::iterator it = m_files.begin(); it != m_files.end(); ++it) { + for (FileList::iterator it = m_prefetchList.begin(); it != m_prefetchList.end(); ++it) { if (*it == file) { - m_files.erase(it); + m_prefetchList.erase(it); break; } } @@ -321,15 +321,15 @@ File* Cache::GetNextFileToPrefetch() { m_prefetch_condVar.Lock(); - if (m_files.empty()) { + if (m_prefetchList.empty()) { m_prefetch_condVar.Wait(); } - // std::sort(m_files.begin(), m_files.end(), myobject); + // std::sort(m_prefetchList.begin(), m_prefetchList.end(), myobject); - size_t l = m_files.size(); + size_t l = m_prefetchList.size(); int idx = rand() % l; - File* f = m_files[idx]; + File* f = m_prefetchList[idx]; f->MarkPrefetch(); m_prefetch_condVar.UnLock(); return f; diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 0a386a4a7ef..a2b650d4f81 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -223,8 +223,8 @@ namespace XrdFileCache WriteQ s_writeQ; // prefetching - typedef std::vector FileList; - FileList m_files; + typedef std::vector PrefetchList; + PrefetchList m_prefetchList; }; } From ef713b59193b327be726663ea479e0ba34eb8ecc Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 15 Apr 2016 13:02:58 -0700 Subject: [PATCH 108/171] Rename write queue member. --- src/XrdFileCache/XrdFileCache.cc | 42 ++++++++++++++++---------------- src/XrdFileCache/XrdFileCache.hh | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 7d20865af71..a7b3ca57ea7 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -176,11 +176,11 @@ bool Cache::HaveFreeWritingSlots() { const static size_t maxWriteWaits=100000; - if ( s_writeQ.size < maxWriteWaits) { + if ( m_writeQ.size < maxWriteWaits) { return true; } else { - XrdCl::DefaultEnv::GetLog()->Info(XrdCl::AppMsg, "Cache::HaveFreeWritingSlots() negative", s_writeQ.size); + XrdCl::DefaultEnv::GetLog()->Info(XrdCl::AppMsg, "Cache::HaveFreeWritingSlots() negative", m_writeQ.size); return false; } } @@ -189,22 +189,22 @@ void Cache::AddWriteTask(Block* b, bool fromRead) { XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::AddWriteTask() bOff=%ld", b->m_offset); - s_writeQ.condVar.Lock(); + m_writeQ.condVar.Lock(); if (fromRead) - s_writeQ.queue.push_back(b); + m_writeQ.queue.push_back(b); else - s_writeQ.queue.push_front(b); // AMT should this not be the opposite - s_writeQ.size++; - s_writeQ.condVar.Signal(); - s_writeQ.condVar.UnLock(); + m_writeQ.queue.push_front(b); // AMT should this not be the opposite + m_writeQ.size++; + m_writeQ.condVar.Signal(); + m_writeQ.condVar.UnLock(); } //______________________________________________________________________________ void Cache::RemoveWriteQEntriesFor(File *iFile) { - s_writeQ.condVar.Lock(); - std::list::iterator i = s_writeQ.queue.begin(); - while (i != s_writeQ.queue.end()) + m_writeQ.condVar.Lock(); + std::list::iterator i = m_writeQ.queue.begin(); + while (i != m_writeQ.queue.end()) { if ((*i)->m_file == iFile) { @@ -212,15 +212,15 @@ void Cache::RemoveWriteQEntriesFor(File *iFile) XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Remove entries for %p path %s", (void*)(*i), iFile->lPath()); std::list::iterator j = i++; iFile->BlockRemovedFromWriteQ(*j); - s_writeQ.queue.erase(j); - --s_writeQ.size; + m_writeQ.queue.erase(j); + --m_writeQ.size; } else { ++i; } } - s_writeQ.condVar.UnLock(); + m_writeQ.condVar.UnLock(); } //______________________________________________________________________________ @@ -229,16 +229,16 @@ Cache::ProcessWriteTasks() { while (true) { - s_writeQ.condVar.Lock(); - while (s_writeQ.queue.empty()) + m_writeQ.condVar.Lock(); + while (m_writeQ.queue.empty()) { - s_writeQ.condVar.Wait(); + m_writeQ.condVar.Wait(); } - Block* block = s_writeQ.queue.front(); // AMT should not be back ??? - s_writeQ.queue.pop_front(); - s_writeQ.size--; + Block* block = m_writeQ.queue.front(); // AMT should not be back ??? + m_writeQ.queue.pop_front(); + m_writeQ.size--; XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::ProcessWriteTasks for %p path %s", (void*)(block), block->m_file->lPath()); - s_writeQ.condVar.UnLock(); + m_writeQ.condVar.UnLock(); block->m_file->WriteBlockToDisk(block); } diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index a2b650d4f81..1e19c1566d4 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -220,7 +220,7 @@ namespace XrdFileCache std::list queue; //!< container }; - WriteQ s_writeQ; + WriteQ m_writeQ; // prefetching typedef std::vector PrefetchList; From 5b3ddc66cc4cfa51a796b79fdfaae8aa9ec847a0 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 15 Apr 2016 14:23:53 -0700 Subject: [PATCH 109/171] Rename parameters for prefetch and available ram. --- src/XrdFileCache/XrdFileCacheConfiguration.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 21dea396f46..3017425c515 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -251,11 +251,11 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) return false; } } - else if (part == "prefetch" ) + else if (part == "prefetch_max_blocks" ) { const char* params = config.GetWord(); if (params) { - int p = ::atoi(config.GetWord()); + int p = ::atoi(params); if (p > 0) { printf("prefetch enabled, max blocks per file=%d\n", p); m_configuration.m_prefetch_max_blocks = p; @@ -264,12 +264,13 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) m_configuration.m_prefetch_max_blocks = 0; } } + else { m_log.Emsg("Config", "Error setting prefetch level."); return false; } } - else if (part == "nram" ) + else if (part == "ram" ) { long long minRAM = 1024* 1024 * 1024;; long long maxRAM = 100 * minRAM; From 71ab90a243f5ac3f2b961b1364fa2679c9d80b83 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 15 Apr 2016 15:09:17 -0700 Subject: [PATCH 110/171] Handle case where new file is attached while a CacheIO object with the same path is still ioActive --- src/XrdFileCache/XrdFileCache.cc | 31 +++++++++++++++++++- src/XrdFileCache/XrdFileCache.hh | 13 ++++++-- src/XrdFileCache/XrdFileCacheIO.hh | 4 ++- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 31 +++++++++++++++----- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 1 + 5 files changed, 68 insertions(+), 12 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index a7b3ca57ea7..09b1c9cedac 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -168,6 +168,16 @@ int Cache::isAttached() void Cache::Detach(XrdOucCacheIO* io) { clLog()->Info(XrdCl::AppMsg, "Cache::Detach() %s", io->Path()); + std::map::iterator it = m_active.begin(); + while (it != m_active.end() ) + { + if (it->second.io == io) { + m_active.erase(it++); + } + else { + ++it; + } + } delete io; } @@ -265,6 +275,25 @@ Cache::RAMBlockReleased() m_RAMblocks_used--; } +//============================================================================== +//======================= File relinquish at process of dying =================== +//====================================================================== +File* Cache::GetFileForLocalPath(std::string path, IO* io) +{ + typedef std::map ActiveMap_t; + ActiveMap_t::iterator it = m_active.find(path); + if (it == m_active.end()) + { + return 0; + } + else { + File* file = it->second.file; + it->second.io->RelinquishFile(file); + return file; + } +} + + //============================================================================== //======================= PREFETCH =================================== @@ -306,7 +335,7 @@ Cache::DeRegisterPrefetchFile(File* file) // called from last line File::InitiateClose() m_prefetch_condVar.Lock(); - for (FileList::iterator it = m_prefetchList.begin(); it != m_prefetchList.end(); ++it) { + for (PrefetchList::iterator it = m_prefetchList.begin(); it != m_prefetchList.end(); ++it) { if (*it == file) { m_prefetchList.erase(it); break; diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 1e19c1566d4..84489a7b889 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -186,13 +186,14 @@ namespace XrdFileCache XrdOss* GetOss() const { return m_output_fs; } XrdSysError& GetSysError() { return m_log; } - + + File* GetFileForLocalPath(std::string, IO*); private: bool ConfigParameters(std::string, XrdOucStream&); bool ConfigXeq(char *, XrdOucStream &); bool xdlib(XrdOucStream &); - static Cache *m_factory; //!< this object + static Cache *m_factory; //!< this object XrdSysError m_log; //!< XrdFileCache namespace logger XrdOucCacheStats m_stats; //!< @@ -222,6 +223,14 @@ namespace XrdFileCache WriteQ m_writeQ; + struct DiskNetIO + { + IO* io; + File* file; + }; + + std::map m_active; + // prefetching typedef std::vector PrefetchList; PrefetchList m_prefetchList; diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index a91d90d32f8..ff2fa866848 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -32,7 +32,9 @@ namespace XrdFileCache virtual int Write(char *Buffer, long long Offset, int Length) { errno = ENOTSUP; return -1; } - virtual void Update(XrdOucCacheIO2 &iocp) { m_io = &iocp; } + virtual void Update(XrdOucCacheIO2 &iocp) { m_io = &iocp; } + + virtual void RelinquishFile(File*) {} protected: XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index ff2e6705322..bcd1fa053c4 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -44,11 +44,16 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & XrdCl::URL url(m_io->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); - struct stat st; - Fstat(st); - m_file = new File(io, fname, 0, st.st_size); + if (!Cache::GetInstance().GetFileForLocalPath(fname, this)) + { + struct stat st; + Fstat(st); + m_file = new File(io, fname, 0, st.st_size); + } } + + IOEntireFile::~IOEntireFile() { @@ -71,6 +76,12 @@ int IOEntireFile::Fstat(struct stat &sbuff) } } +void IOEntireFile::RelinquishFile(File* f) +{ + assert(m_file == f); + m_file = 0; +} + struct stat* IOEntireFile::getValidLocalStat(const char* path) { @@ -100,17 +111,21 @@ struct stat* IOEntireFile::getValidLocalStat(const char* path) bool IOEntireFile::ioActive() { - return m_file->InitiateClose(); + if (!m_file) + return false; + else + return m_file->InitiateClose(); } XrdOucCacheIO *IOEntireFile::Detach() { - m_statsGlobal.Add(m_file->GetStats()); - XrdOucCacheIO * io = m_io; - delete m_file; - m_file = 0; + if (m_file) { + m_statsGlobal.Add(m_file->GetStats()); + delete m_file; + m_file = 0; + } // This will delete us! m_cache.Detach(this); diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index cbd7de232a1..e3e8fe9f1a8 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -97,6 +97,7 @@ namespace XrdFileCache virtual int Fstat(struct stat &sbuff); + virtual void RelinquishFile(File*); private: File* m_file; From 62b6585a95b68cabee594fa886d28a9c09aea0e9 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 15 Apr 2016 15:11:20 -0700 Subject: [PATCH 111/171] Fix error in Info initialization. --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 9a920ba8a5e..e341e700efd 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -251,7 +251,7 @@ bool File::Open() return false; } - if (m_cfi.Read(m_infoFile)) + if (m_cfi.Read(m_infoFile) <= 0) { m_fileSize = m_fileSize; int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; From 3246f3fe30b5cdabcb26e999f856bf259dd0f613 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 18 Apr 2016 10:06:33 -0700 Subject: [PATCH 112/171] Update. --- src/XrdFileCache/README | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/XrdFileCache/README b/src/XrdFileCache/README index f76251a22da..44244e0db95 100644 --- a/src/XrdFileCache/README +++ b/src/XrdFileCache/README @@ -114,12 +114,11 @@ CONFIGURATION pfc.blocksize: prefetch buffer size, default 1M -pfc.nread: number of in memory cached blocks reserved for read tasks +pfc.ram [bytes[g]]: maximum allowed RAM usage for caching proxy -pfc.nprefetch: number of in memory cached blocks reserved for prefetch tasks +pfc.prefetch : prefetch level, default is 10. Value zero disables prefetching. -pfc.diskusage : high / low watermarks for disk cache -purge operation (default 0.9 and 0.95) +pfc.diskusage diskusage boundaries, can be specified relative in percantage or in g or T bytes pfc.user : username used by XrdOss plugin @@ -135,7 +134,7 @@ Examples a) Enable proxy file prefetching: pps.cachelib libXrdFileCache.so pfc.localroot /data/xrd-file-cache -pfc.nprefetch 1 +pfc.ram 5g b) enable file block mode, with block size 64 MB: From b2dd2b2b5dba6a91e4a5d70f9e270c5bf6163d01 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 18 Apr 2016 10:09:24 -0700 Subject: [PATCH 113/171] Use 'prefetch' string to name a parameter for prefetch level. --- src/XrdFileCache/XrdFileCacheConfiguration.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 3017425c515..5d21489932e 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -196,7 +196,6 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) { - printf("part %s \n", part.c_str()); XrdSysError err(0, ""); if ( part == "user" ) { @@ -251,7 +250,7 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) return false; } } - else if (part == "prefetch_max_blocks" ) + else if (part == "prefetch" ) { const char* params = config.GetWord(); if (params) { From 4c1b188fadd92a3d01bdf6aa784bbab0dbdda4fa Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 18 Apr 2016 11:30:13 -0700 Subject: [PATCH 114/171] Remove obsolete paramter. --- src/XrdFileCache/XrdFileCache.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 84489a7b889..ce09eca6ab7 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -67,7 +67,6 @@ namespace XrdFileCache long long m_bufferSize; //!< prefetch buffer size, default 1MB long long m_RamAbsAvailable; //!< available from configuration int m_NRamBuffers; //!< number of total in-memory cache blocks, cached - bool m_prefetch; //!< prefetch enable state size_t m_prefetch_max_blocks;//!< maximum number of blocks to prefetch per file long long m_hdfsbsize; //!< used with m_hdfsmode, default 128MB From aa5a918af119b39f4b998e2d387ee4e38f36c9ea Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 18 Apr 2016 11:31:03 -0700 Subject: [PATCH 115/171] Run prefetch thread after parsing configuration. --- src/XrdFileCache/XrdFileCache.cc | 15 ++++++++------- src/XrdFileCache/XrdFileCacheConfiguration.cc | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 09b1c9cedac..f3ec32cabf4 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -78,6 +78,12 @@ XrdOucCache2 *XrdOucGetCache2(XrdSysLogger *logger, } err.Emsg("Retrieve", "Success - returning a factory."); + pthread_t tid1; + XrdSysThread::Run(&tid1, ProcessWriteTaskThread, (void*)(&factory), 0, "XrdFileCache WriteTasks "); + + pthread_t tid2; + XrdSysThread::Run(&tid2, PrefetchThread, (void*)(&factory), 0, "XrdFileCache Prefetch "); + pthread_t tid; XrdSysThread::Run(&tid, CacheDirCleanupThread, NULL, 0, "XrdFileCache CacheDirCleanup"); @@ -128,11 +134,6 @@ Cache::Cache() : XrdOucCache(), m_prefetch_condVar(0), m_RAMblocks_used(0) { - pthread_t tid1; - XrdSysThread::Run(&tid1, ProcessWriteTaskThread, (void*)this, 0, "XrdFileCache WriteTasks "); - - pthread_t tid2; - XrdSysThread::Run(&tid2, PrefetchThread, (void*)this, 0, "XrdFileCache Prefetch "); } //______________________________________________________________________________ @@ -315,7 +316,7 @@ Cache::RegisterPrefetchFile(File* file) { // called from File::Open() - if (Cache::GetInstance().RefConfiguration().m_prefetch) + if (Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) { XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file BEGIN"); @@ -408,7 +409,7 @@ Cache::Stat(const char *curl, struct stat &sbuff) void Cache::Prefetch() { - const static int limitRAM= Cache::GetInstance().RefConfiguration().m_NRamBuffers * 0.7; + int limitRAM= Cache::GetInstance().RefConfiguration().m_NRamBuffers * 0.7; XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch thread start"); diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 5d21489932e..fd9ca0e672b 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -159,10 +159,10 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char char buff[2048]; loff = snprintf(buff, sizeof(buff), "result\n" "\tpfc.blocksize %lld\n" - "\tpfc.prefetch %d\n" + "\tpfc.prefetch %ld\n" "\tpfc.nramblocks %d\n\n", m_configuration.m_bufferSize, - m_configuration.m_prefetch, // AMT not sure what parsing should be + m_configuration.m_prefetch_max_blocks, // AMT not sure what parsing should be m_configuration.m_NRamBuffers ); if (m_configuration.m_hdfsmode) From 17bdd96076480ea59e22f3e29bde05a153d8829c Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 18 Apr 2016 12:22:33 -0700 Subject: [PATCH 116/171] Make RAM usage a required configuration parameter. --- src/XrdFileCache/XrdFileCache.hh | 2 +- src/XrdFileCache/XrdFileCacheConfiguration.cc | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index ce09eca6ab7..847fece3788 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -52,7 +52,7 @@ namespace XrdFileCache m_diskUsageLWM(-1), m_diskUsageHWM(-1), m_bufferSize(1024*1024), - m_RamAbsAvailable(8*1024*1024), + m_RamAbsAvailable(0), m_NRamBuffers(-1), m_prefetch_max_blocks(10), m_hdfsbsize(128*1024*1024) {} diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index fd9ca0e672b..513e71bd2fb 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -152,7 +152,13 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char } // get number of available RAM blocks after process configuration + if (m_configuration.m_RamAbsAvailable == 0 ) + { + m_log.Emsg("Error", "RAM usage not specified. Please set pfc.ram value in configuration file."); + return false; + } m_configuration.m_NRamBuffers = static_cast(m_configuration.m_RamAbsAvailable/ m_configuration.m_bufferSize); + if (retval) { int loff = 0; From 14d5713034eb095b87482206d583e37eabeea77b Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 19 Apr 2016 14:53:36 -0700 Subject: [PATCH 117/171] Fix possible crash when scheduler calls disk sync at the time of destruction. --- src/XrdFileCache/XrdFileCacheFile.cc | 43 +++++++++++++--------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index e341e700efd..d9c01a2d3b4 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -110,21 +110,6 @@ File::~File() clLog()->Debug(XrdCl::AppMsg, "File::~File() enter %p %s", (void*)this, lPath()); - // Wait disk sync - bool do_sync = false; - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - if (m_non_flushed_cnt > 0 || !m_writes_during_sync.empty()) - { - do_sync = true; - m_in_sync = true; - clLog()->Info(XrdCl::AppMsg, "File::~File sync unflushed %d\n", m_non_flushed_cnt); - } - } - if (do_sync) - { - Sync(); - } // write statistics in *cinfo file AppendIOStatToFileInfo(); @@ -142,7 +127,11 @@ File::~File() delete m_infoFile; m_infoFile = NULL; } - delete m_syncer; + + m_syncStatusMutex.Lock(); + bool syncEmpty = m_writes_during_sync.empty(); + m_syncStatusMutex.UnLock(); + if (!syncEmpty) Sync(); // print just for curiosity clLog()->Debug(XrdCl::AppMsg, "File::~File() ended, prefetch score ...%d/%d=%.2f", m_prefetchHitCnt, m_prefetchReadCnt, m_prefetchScore); @@ -192,7 +181,15 @@ bool File::InitiateClose() m_downloadCond.UnLock(); if ( blockMapEmpty) - return false; + { + // file is not active when block map is empty and sync is done + XrdSysMutexHelper _lck(&m_syncStatusMutex); + if (m_in_sync) { + delete m_syncer; + m_syncer = NULL; + return false; + } + } } return true; @@ -724,13 +721,13 @@ void File::WriteBlockToDisk(Block* b) { m_cfi.SetBitWriteCalled(pfIdx); ++m_non_flushed_cnt; - } + if (m_non_flushed_cnt >= 100 ) + { + schedule_sync = true; + m_in_sync = true; + m_non_flushed_cnt = 0; + } - if (m_non_flushed_cnt >= 100 ) - { - schedule_sync = true; - m_in_sync = true; - m_non_flushed_cnt = 0; } } From cc97df629de0d21ff9345d22fa598191a8fdc292 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 19 Apr 2016 15:00:43 -0700 Subject: [PATCH 118/171] Fix logical error. --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index d9c01a2d3b4..1dd89c223e2 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -184,7 +184,7 @@ bool File::InitiateClose() { // file is not active when block map is empty and sync is done XrdSysMutexHelper _lck(&m_syncStatusMutex); - if (m_in_sync) { + if (m_in_sync == false) { delete m_syncer; m_syncer = NULL; return false; From e2bdc0cdd645f2572fc15c1175219f978b9af64d Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Wed, 20 Apr 2016 14:47:59 -0500 Subject: [PATCH 119/171] Make sure info about blocks since the last sync is also written out. --- src/XrdFileCache/XrdFileCacheFile.cc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 1dd89c223e2..259c7744cdb 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -109,9 +109,17 @@ File::~File() { clLog()->Debug(XrdCl::AppMsg, "File::~File() enter %p %s", (void*)this, lPath()); - + m_syncStatusMutex.Lock(); + bool needs_sync = ! m_writes_during_sync.empty(); + m_syncStatusMutex.UnLock(); + if (needs_sync || m_non_flushed_cnt > 0) + { + Sync(); + m_cfi.WriteHeader(m_infoFile); + } // write statistics in *cinfo file AppendIOStatToFileInfo(); + m_infoFile->Fsync(); clLog()->Info(XrdCl::AppMsg, "File::~File close data file %p",(void*)this , lPath()); if (m_output) @@ -128,11 +136,6 @@ File::~File() m_infoFile = NULL; } - m_syncStatusMutex.Lock(); - bool syncEmpty = m_writes_during_sync.empty(); - m_syncStatusMutex.UnLock(); - if (!syncEmpty) Sync(); - // print just for curiosity clLog()->Debug(XrdCl::AppMsg, "File::~File() ended, prefetch score ...%d/%d=%.2f", m_prefetchHitCnt, m_prefetchReadCnt, m_prefetchScore); } From 3270162346aee225288958325e0896f486cd6e71 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 20 Apr 2016 15:37:52 -0700 Subject: [PATCH 120/171] Improve printout of configuration status. --- src/XrdFileCache/XrdFileCacheConfiguration.cc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 513e71bd2fb..0449724db9a 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -163,13 +163,16 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char { int loff = 0; char buff[2048]; - loff = snprintf(buff, sizeof(buff), "result\n" - "\tpfc.blocksize %lld\n" - "\tpfc.prefetch %ld\n" - "\tpfc.nramblocks %d\n\n", + float rg = (m_configuration.m_RamAbsAvailable)/float(1024*1024*1024); + loff = snprintf(buff, sizeof(buff), "Config effective %s pfc configuration:\n" + " pfc.blocksize %lld\n" + " pfc.prefetch %ld\n" + " pfc.ram %.fg", + config_filename, m_configuration.m_bufferSize, m_configuration.m_prefetch_max_blocks, // AMT not sure what parsing should be - m_configuration.m_NRamBuffers ); + rg); + if (m_configuration.m_hdfsmode) { @@ -188,10 +191,10 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char loff += snprintf(&buff[loff], strlen(unameBuff), "%s", unameBuff); } - m_log.Emsg("Config", buff); + m_log.Say( buff); } - m_log.Emsg("Config", "Configuration = ", retval ? "Success" : "Fail"); + m_log.Say("------ File Caching Proxy interface initialization ", retval ? "completed" : "failed"); if (ofsCfg) delete ofsCfg; return retval; From 43c0be0ee05ac134cd6be56aad1b6ae64bb16b7b Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 27 Apr 2016 07:24:45 -0700 Subject: [PATCH 121/171] Fix bug in selecting file to prefetch. --- src/XrdFileCache/XrdFileCache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index f3ec32cabf4..0e1906a4f6f 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -351,7 +351,7 @@ File* Cache::GetNextFileToPrefetch() { m_prefetch_condVar.Lock(); - if (m_prefetchList.empty()) { + while (m_prefetchList.empty()) { m_prefetch_condVar.Wait(); } From 543836940a2b74b58812390df3e0a700045a9dd6 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 9 May 2016 13:15:09 -0700 Subject: [PATCH 122/171] Add check if reading local stat from cinfo failed. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 22 +++++++++----------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index bcd1fa053c4..68f84ce5c96 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -86,27 +86,25 @@ void IOEntireFile::RelinquishFile(File* f) struct stat* IOEntireFile::getValidLocalStat(const char* path) { if (!m_localStat) { - m_localStat = new struct stat; - memset(m_localStat, 0, sizeof(struct stat)); - if (m_cache.GetOss()->Stat(path, m_localStat) == XrdOssOK) { - m_localStat->st_size = 0; + struct stat tmpStat; + if (m_cache.GetOss()->Stat(path, &tmpStat) == XrdOssOK) { XrdOssDF* infoFile = m_cache.GetOss()->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); XrdOucEnv myEnv; int res = infoFile->Open(path, O_RDONLY, 0600, myEnv); if (res >= 0) { - Info info(0); - if (info.Read(infoFile) > 0) { - m_localStat->st_size = info.GetFileSize(); - } + Info info(0); + if (info.Read(infoFile) > 0) { + tmpStat.st_size = info.GetFileSize(); + m_localStat = new struct stat; + memcpy(m_localStat, &tmpStat, sizeof(struct stat)); + } } infoFile->Close(); delete infoFile; } } - if (m_localStat->st_size) - return m_localStat; - else return 0; - + + return m_localStat; } bool IOEntireFile::ioActive() From 5d8d8fef19377005a26fd4f8fa4727782ee98858 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 12 May 2016 15:56:03 -0700 Subject: [PATCH 123/171] Implement virtual Cache::Stat(). --- src/XrdFileCache/XrdFileCache.cc | 39 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 0e1906a4f6f..4246f26660f 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -394,14 +394,43 @@ Cache::Prepare(const char *url, int oflags, mode_t mode) } //______________________________________________________________________________ +// virtual method of XrdOucCache2::Stat() +//! +//! @return <0 - Stat failed, value is -errno. +//! =0 - Stat succeeded, sbuff holds stat information. +//! >0 - Stat could not be done, forward operation to next level. +//------------------------------------------------------------------------------ -int -Cache::Stat(const char *curl, struct stat &sbuff) +int Cache::Stat(const char *curl, struct stat &sbuff) { - assert(0 && "Cache::Stat() should not be called."); - return 0; -} + XrdCl::URL url(curl); + std::string name = url.GetPath(); + + if (m_output_fs->Stat(name.c_str(), &sbuff) == XrdOssOK) { + if ( S_ISDIR(sbuff.st_mode)) { + return 0; + } + else { + bool success = false; + XrdOssDF* infoFile = m_output_fs->newFile(m_configuration.m_username.c_str()); + XrdOucEnv myEnv; + name += ".cinfo"; + int res = infoFile->Open(name.c_str(), O_RDONLY, 0600, myEnv); + if (res >= 0) { + Info info(0); + if (info.Read(infoFile) > 0) { + sbuff.st_size = info.GetFileSize(); + success = true; + } + } + infoFile->Close(); + delete infoFile; + return success ? 0 : 1; + } + } + return 1; +} //______________________________________________________________________________ From d67631b16736c24fa56d8c8989df4196e939d870 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 20 May 2016 16:22:14 -0700 Subject: [PATCH 124/171] Properly set disk read and write offsets in hdfs healing mode (files with nonzero offsets). --- src/XrdFileCache/XrdFileCacheFile.cc | 18 ++++++++++++------ src/XrdFileCache/XrdFileCacheFile.hh | 3 ++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 259c7744cdb..3c6103e61dd 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -396,9 +396,8 @@ int File::ReadBlocksFromDisk(std::list& blocks, overlap(*ii, BS, req_off, req_size, off, blk_off, size); - long long rs = m_output->Read(req_buf + off, *ii * BS + blk_off, size); + long long rs = m_output->Read(req_buf + off, *ii * BS + blk_off -m_offset, size); clLog()->Dump(XrdCl::AppMsg, "File::ReadBlocksFromDisk block %d size %d %s", *ii, size, lPath()); - if (rs < 0) { clLog()->Error(XrdCl::AppMsg, "File::ReadBlocksFromDisk neg retval %ld (%ld@%d) %s", rs, *ii * BS + blk_off, lPath()); @@ -464,7 +463,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) m_stats.m_BytesRam++; // AMT what if block fails } // On disk? - else if (m_cfi.TestBit(block_idx)) + else if (m_cfi.TestBit(offsetIdx(block_idx))) { clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p read from disk %d %s", (void*)iUserBuff, block_idx, lPath()); blks_on_disk.push_back(block_idx); @@ -671,7 +670,7 @@ void File::WriteBlockToDisk(Block* b) int retval = 0; // write block buffer into disk file long long offset = b->m_offset - m_offset; - long long size = (b->m_offset + m_cfi.GetBufferSize()) > m_fileSize ? (m_fileSize - b->m_offset) : m_cfi.GetBufferSize(); + long long size = (offset + m_cfi.GetBufferSize()) > m_fileSize ? (m_fileSize - offset) : m_cfi.GetBufferSize(); int buffer_remaining = size; int buffer_offset = 0; int cnt = 0; @@ -752,7 +751,7 @@ void File::Sync() XrdSysMutexHelper _lck(&m_syncStatusMutex); for (std::vector::iterator i = m_writes_during_sync.begin(); i != m_writes_during_sync.end(); ++i) { - m_cfi.SetBitWriteCalled(*i); + m_cfi.SetBitWriteCalled(offsetIdx(*i)); } written_while_in_sync = m_non_flushed_cnt = (int) m_writes_during_sync.size(); m_writes_during_sync.clear(); @@ -861,6 +860,12 @@ const char* File::lPath() const return m_temp_filename.c_str(); } +//______________________________________________________________________________ +int File::offsetIdx(int iIdx) +{ + return iIdx - m_offset/m_cfi.GetBufferSize(); +} + //______________________________________________________________________________ void File::AppendIOStatToFileInfo() { @@ -896,6 +901,7 @@ void File::Prefetch() // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch test bit %d", f); if (!m_cfi.TestBit(f)) { + f += m_offset/m_cfi.GetBufferSize(); BlockMap_i bi = m_block_map.find(f); if (bi == m_block_map.end()) { clLog()->Dump(XrdCl::AppMsg, "File::Prefetch take block %d %s", f, lPath()); @@ -939,7 +945,7 @@ void File::CheckPrefetchStatRAM(Block* b) void File::CheckPrefetchStatDisk(int idx) { if (Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) { - if (m_cfi.TestPrefetchBit(idx)) + if (m_cfi.TestPrefetchBit(offsetIdx(idx))) m_prefetchHitCnt++; } } diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 6857edf9957..3f000a1913e 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -227,7 +227,8 @@ namespace XrdFileCache void inc_ref_count(Block*); void dec_ref_count(Block*); void free_block(Block*); - + + int offsetIdx(int idx); }; From de4dacbf957a5e7172482727f17bd99fcf180ca8 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 23 May 2016 10:22:02 -0700 Subject: [PATCH 125/171] In hdfs healing mode forward stat query to origin. --- src/XrdFileCache/XrdFileCache.cc | 44 +++++++++++++++++--------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 4246f26660f..8f80c92c207 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -403,32 +403,34 @@ Cache::Prepare(const char *url, int oflags, mode_t mode) int Cache::Stat(const char *curl, struct stat &sbuff) { - XrdCl::URL url(curl); - std::string name = url.GetPath(); + if (m_configuration.m_hdfsmode == false) + { + XrdCl::URL url(curl); + std::string name = url.GetPath(); - if (m_output_fs->Stat(name.c_str(), &sbuff) == XrdOssOK) { - if ( S_ISDIR(sbuff.st_mode)) { - return 0; - } - else { - bool success = false; - XrdOssDF* infoFile = m_output_fs->newFile(m_configuration.m_username.c_str()); - XrdOucEnv myEnv; - name += ".cinfo"; - int res = infoFile->Open(name.c_str(), O_RDONLY, 0600, myEnv); - if (res >= 0) { - Info info(0); - if (info.Read(infoFile) > 0) { - sbuff.st_size = info.GetFileSize(); - success = true; + if (m_output_fs->Stat(name.c_str(), &sbuff) == XrdOssOK) { + if ( S_ISDIR(sbuff.st_mode)) { + return 0; + } + else { + bool success = false; + XrdOssDF* infoFile = m_output_fs->newFile(m_configuration.m_username.c_str()); + XrdOucEnv myEnv; + name += ".cinfo"; + int res = infoFile->Open(name.c_str(), O_RDONLY, 0600, myEnv); + if (res >= 0) { + Info info(0); + if (info.Read(infoFile) > 0) { + sbuff.st_size = info.GetFileSize(); + success = true; + } } + infoFile->Close(); + delete infoFile; + return success ? 0 : 1; } - infoFile->Close(); - delete infoFile; - return success ? 0 : 1; } } - return 1; } From c7af3d04ce302b65d3d936c869b524a4cb0d2170 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 23 May 2016 10:22:54 -0700 Subject: [PATCH 126/171] In hdfs healing mode forward stat query to origin. --- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 4 ++++ src/XrdFileCache/XrdFileCacheIOFileBlock.hh | 1 + 2 files changed, 5 insertions(+) diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 9151cbdccdf..30cdade0825 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -58,6 +58,10 @@ XrdOucCacheIO* IOFileBlock::Detach() return io; } +int IOFileBlock::Fstat(struct stat &sbuff) +{ + return m_io->Fstat(sbuff); +} //______________________________________________________________________________ void IOFileBlock::GetBlockSizeFromPath() { diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index b6820ebff49..c5f05a5fb29 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -64,6 +64,7 @@ namespace XrdFileCache //! Called to check if destruction needs to be done in a separate task. virtual bool ioActive(); + virtual int Fstat(struct stat &sbuff); private: long long m_blocksize; //!< size of file-block std::map m_blocks; //!< map of created blocks From 864c95119f3fce6c51d272ea53bb006c47852a97 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 23 May 2016 10:32:49 -0700 Subject: [PATCH 127/171] Include file offset (hdfs healing mode). --- src/XrdFileCache/XrdFileCacheVRead.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index 0c3e5f289dd..a7ab30a58f7 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -178,7 +178,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b inc_ref_count(bi->second); clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d in map", block_idx); } - else if (m_cfi.TestBit(block_idx)) + else if (m_cfi.TestBit(offsetIdx(block_idx))) { blocks_on_disk.AddEntry(block_idx, iov_idx); clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d , chunk idx = %d on disk", block_idx,iov_idx ); @@ -229,7 +229,7 @@ int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blo clLog()->Debug(XrdCl::AppMsg, "VReadFromDisk block=%d chunk=%d", blockIdx, chunkIdx); overlap(blockIdx, m_cfi.GetBufferSize(), readV[chunkIdx].offset, readV[chunkIdx].size, off, blk_off, size); - int rs = m_output->Read(readV[chunkIdx].data + off, blockIdx*m_cfi.GetBufferSize() + blk_off, size); + int rs = m_output->Read(readV[chunkIdx].data + off, blockIdx*m_cfi.GetBufferSize() + blk_off - m_offset, size); if (rs >=0 ) { bytes_read += rs; } From 03d1dd652a5c3c38747e898a7115e132bd52535b Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 24 May 2016 11:26:33 -0700 Subject: [PATCH 128/171] FileBlockMode: recycle File object during delayed destruction. --- src/XrdFileCache/XrdFileCache.cc | 46 ++++++++++++-------- src/XrdFileCache/XrdFileCache.hh | 8 +++- src/XrdFileCache/XrdFileCacheFile.hh | 2 + src/XrdFileCache/XrdFileCacheIO.hh | 2 +- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 10 +---- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 43 +++++++++++++----- src/XrdFileCache/XrdFileCacheIOFileBlock.hh | 3 +- 7 files changed, 74 insertions(+), 40 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 8f80c92c207..4fbaf823563 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -169,19 +169,24 @@ int Cache::isAttached() void Cache::Detach(XrdOucCacheIO* io) { clLog()->Info(XrdCl::AppMsg, "Cache::Detach() %s", io->Path()); - std::map::iterator it = m_active.begin(); - while (it != m_active.end() ) + + // Cache owns File objects + XrdSysMutexHelper lock(&m_active_mutex); + std::vector::iterator it = m_active.begin(); + while ( it != m_active.end() ) { - if (it->second.io == io) { - m_active.erase(it++); + if (it->io == io) { + it->io->RelinquishFile(it->file); + delete it->file; + m_active.erase(it); } - else { + else ++it; - } } delete io; } + //______________________________________________________________________________ bool Cache::HaveFreeWritingSlots() @@ -276,26 +281,31 @@ Cache::RAMBlockReleased() m_RAMblocks_used--; } +void +Cache::AddActive(IO* io, File* file) +{ + XrdSysMutexHelper lock(&m_active_mutex); + m_active.push_back(DiskNetIO(io, file)); +} + //============================================================================== //======================= File relinquish at process of dying =================== //====================================================================== -File* Cache::GetFileForLocalPath(std::string path, IO* io) +File* Cache::GetFileWithLocalPath(std::string path, IO* iIo) { - typedef std::map ActiveMap_t; - ActiveMap_t::iterator it = m_active.find(path); - if (it == m_active.end()) + XrdSysMutexHelper lock(&m_active_mutex); + for ( std::vector::iterator it = m_active.begin(); it != m_active.end(); ++it) { - return 0; - } - else { - File* file = it->second.file; - it->second.io->RelinquishFile(file); - return file; + if (!strcmp(path.c_str(), it->file->lPath())) + { + it->io->RelinquishFile(it->file); + it->io = iIo; + return it->file; + } } + return 0; } - - //============================================================================== //======================= PREFETCH =================================== //============================================================================== diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 847fece3788..6cd289018ef 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -186,7 +186,9 @@ namespace XrdFileCache XrdSysError& GetSysError() { return m_log; } - File* GetFileForLocalPath(std::string, IO*); + File* GetFileWithLocalPath(std::string, IO* io); + + void AddActive(IO*, File*); private: bool ConfigParameters(std::string, XrdOucStream&); @@ -224,11 +226,13 @@ namespace XrdFileCache struct DiskNetIO { + DiskNetIO(IO* iIO, File* iFile): io(iIO), file(iFile){} IO* io; File* file; }; - std::map m_active; + std::vector m_active; + XrdSysMutex m_active_mutex; // prefetching typedef std::vector PrefetchList; diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 3f000a1913e..79d74f02555 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -140,6 +140,7 @@ namespace XrdFileCache float m_prefetchScore; //cached int m_prefetchCurrentCnt; + public: //------------------------------------------------------------------------ //! Constructor. @@ -189,6 +190,7 @@ namespace XrdFileCache //! Log path const char* lPath() const; + std::string GetLocalPath(); private: bool overlap(int blk, // block to query long long blk_size, // diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index ff2fa866848..088e2858cd1 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -34,7 +34,7 @@ namespace XrdFileCache virtual void Update(XrdOucCacheIO2 &iocp) { m_io = &iocp; } - virtual void RelinquishFile(File*) {} + virtual void RelinquishFile(File*) = 0; protected: XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 68f84ce5c96..c2cf4c6d68f 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -44,11 +44,12 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & XrdCl::URL url(m_io->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); - if (!Cache::GetInstance().GetFileForLocalPath(fname, this)) + if (!(m_file = Cache::GetInstance().GetFileWithLocalPath(fname, this))) { struct stat st; Fstat(st); m_file = new File(io, fname, 0, st.st_size); + Cache::GetInstance().AddActive(this, m_file); } } @@ -56,7 +57,6 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & IOEntireFile::~IOEntireFile() { - delete m_localStat; } @@ -119,12 +119,6 @@ XrdOucCacheIO *IOEntireFile::Detach() { XrdOucCacheIO * io = m_io; - if (m_file) { - m_statsGlobal.Add(m_file->GetStats()); - delete m_file; - m_file = 0; - } - // This will delete us! m_cache.Detach(this); return io; diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 30cdade0825..cdcc139e654 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -50,7 +50,6 @@ XrdOucCacheIO* IOFileBlock::Detach() for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { m_statsGlobal.Add(it->second->GetStats()); - delete it->second; } m_cache.Detach(this); // This will delete us! @@ -58,10 +57,6 @@ XrdOucCacheIO* IOFileBlock::Detach() return io; } -int IOFileBlock::Fstat(struct stat &sbuff) -{ - return m_io->Fstat(sbuff); -} //______________________________________________________________________________ void IOFileBlock::GetBlockSizeFromPath() { @@ -101,21 +96,49 @@ File* IOFileBlock::newBlockFile(long long off, int blocksize) fname = ss.str(); clLog()->Debug(XrdCl::AppMsg, "FileBlock::FileBlock(), create XrdFileCacheFile. %s", m_io->Path()); - File* prefetch = new File(m_io, fname, off, blocksize); + + File* file; + if (!(file = Cache::GetInstance().GetFileWithLocalPath(fname, this))) + { + file = new File(m_io, fname, off, m_io->FSize()); + Cache::GetInstance().AddActive(this, file); + } + + return file; +} - return prefetch; +//______________________________________________________________________________ +void IOFileBlock::RelinquishFile(File* f) +{ + // called from Cache::Detach() or Cache::GetFileWithLocalPath() + // the object is in process of dying + + XrdSysMutexHelper lock(&m_mutex); + for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) + { + if (it->second == f) + { + m_blocks.erase(it++); + break; + } + else + { + ++it; + } + } } //______________________________________________________________________________ bool IOFileBlock::ioActive() { - bool res = false; + XrdSysMutexHelper lock(&m_mutex); + for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { if (it->second->InitiateClose()) - res = true; + return true; } - return res; + return false; } //______________________________________________________________________________ diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index c5f05a5fb29..dda7ff88f4d 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -64,7 +64,8 @@ namespace XrdFileCache //! Called to check if destruction needs to be done in a separate task. virtual bool ioActive(); - virtual int Fstat(struct stat &sbuff); + virtual void RelinquishFile(File*); + private: long long m_blocksize; //!< size of file-block std::map m_blocks; //!< map of created blocks From 660d82d4bc57e423b23723ab489ebf9d9a4016a2 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 27 May 2016 13:26:54 -0700 Subject: [PATCH 129/171] Introduce trace macros. --- src/XrdFileCache/XrdFileCache.cc | 8 ++++++-- src/XrdFileCache/XrdFileCache.hh | 6 ++++++ src/XrdFileCache/XrdFileCacheFile.cc | 7 ++++--- src/XrdFileCache/XrdFileCacheFile.hh | 1 + src/XrdFileCache/XrdFileCacheIO.hh | 4 +++- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 4fbaf823563..674b164b3a2 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -28,18 +28,19 @@ #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucUtils.hh" +#include "XrdOuc/XrdOucTrace.hh" #include "XrdFileCache.hh" +#include "XrdFileCacheTrace.hh" #include "XrdFileCacheInfo.hh" #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheIOFileBlock.hh" using namespace XrdFileCache; - - Cache * Cache::m_factory = NULL; + void *CacheDirCleanupThread(void* cache_void) { Cache::GetInstance().CacheDirCleanup(); @@ -131,9 +132,12 @@ bool Cache::Decide(XrdOucCacheIO* io) Cache::Cache() : XrdOucCache(), m_log(0, "XrdFileCache_"), + m_trace(0), + m_traceID("Manager"), m_prefetch_condVar(0), m_RAMblocks_used(0) { + m_trace = new XrdOucTrace(&m_log); } //______________________________________________________________________________ diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 6cd289018ef..56c82844c66 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -30,6 +30,7 @@ class XrdOucStream; class XrdSysError; +class XrdOucTrace; namespace XrdCl { class Log; @@ -190,6 +191,8 @@ namespace XrdFileCache void AddActive(IO*, File*); + + XrdOucTrace* GetTrace() { return m_trace; } private: bool ConfigParameters(std::string, XrdOucStream&); bool ConfigXeq(char *, XrdOucStream &); @@ -197,6 +200,9 @@ namespace XrdFileCache static Cache *m_factory; //!< this object XrdSysError m_log; //!< XrdFileCache namespace logger + XrdOucTrace* m_trace; + const char* m_traceID; + XrdOucCacheStats m_stats; //!< XrdOss *m_output_fs; //!< disk cache file system diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 3c6103e61dd..885ccd6f925 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -18,7 +18,7 @@ #include "XrdFileCacheFile.hh" - +#include "XrdFileCacheTrace.hh" #include #include @@ -40,6 +40,7 @@ using namespace XrdFileCache; + namespace XrdPosixGlobals { extern XrdScheduler *schedP; @@ -80,7 +81,6 @@ m_offset(iOffset), m_fileSize(iFileSize), m_stopping(false), m_stateCond(0), // We will explicitly lock the condition before use. - m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), m_non_flushed_cnt(0), m_in_sync(false), @@ -89,7 +89,8 @@ m_prefetchState(kOn), m_prefetchReadCnt(0), m_prefetchHitCnt(0), m_prefetchScore(1), -m_prefetchCurrentCnt(0) +m_prefetchCurrentCnt(0), +m_traceID("File") { clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_temp_filename.c_str()); if (!Open()) { diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 79d74f02555..9185ae642fa 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -140,6 +140,7 @@ namespace XrdFileCache float m_prefetchScore; //cached int m_prefetchCurrentCnt; + const char* m_traceID; public: //------------------------------------------------------------------------ diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index 088e2858cd1..36007a154ee 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -14,7 +14,7 @@ namespace XrdFileCache { public: IO (XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache) : - m_io(io), m_statsGlobal(stats), m_cache(cache) {} + m_io(io), m_statsGlobal(stats), m_cache(cache), m_traceID("IO"){} //! Original data source. virtual XrdOucCacheIO *Base() { return m_io; } @@ -42,6 +42,8 @@ namespace XrdFileCache XrdOucCacheIO2 *m_io; //!< original data source XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics Cache &m_cache; //!< reference to Cache needed in detach + + const char* m_traceID; }; } From 8969ab318af7daa5e4d3068ff25aa3e2f652e313 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 27 May 2016 15:17:03 -0700 Subject: [PATCH 130/171] Parse trace level. --- src/XrdFileCache/XrdFileCache.hh | 1 + src/XrdFileCache/XrdFileCacheConfiguration.cc | 42 ++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 56c82844c66..5f337a7b7f2 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -197,6 +197,7 @@ namespace XrdFileCache bool ConfigParameters(std::string, XrdOucStream&); bool ConfigXeq(char *, XrdOucStream &); bool xdlib(XrdOucStream &); + bool xtrace(XrdOucStream &); static Cache *m_factory; //!< this object XrdSysError m_log; //!< XrdFileCache namespace logger diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 0449724db9a..69c9665be7b 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -1,4 +1,5 @@ #include "XrdFileCache.hh" +#include "XrdFileCacheTrace.hh" #include "XrdOss/XrdOss.hh" #include "XrdOss/XrdOssCache.hh" @@ -65,6 +66,37 @@ bool Cache::xdlib(XrdOucStream &Config) return true; } +/* Function: xtrace + + Purpose: To parse the directive: trace + Output: true upon success or false upon failure. + */ +bool Cache::xtrace(XrdOucStream &Config) +{ + char *val; + static struct traceopts {const char *opname; int opval;} tropts[] = + { + {"none", 0}, + {"error", 1}, + {"warning", 2}, + {"info", 3}, + {"debug", 4}, + {"dump", 5} + }; + int numopts = sizeof(tropts)/sizeof(struct traceopts); + + if (!(val = Config.GetWord())) + {m_log.Emsg("Config", "trace option not specified"); return 1;} + + for (int i = 0; i < numopts; i++) + { + if (!strcmp(val, tropts[i].opname)) + m_trace->What = tropts[i].opval; + return true; + } + return 0; +} + //______________________________________________________________________________ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char *parameters) @@ -123,7 +155,11 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char } else if (!strcmp(var,"pfc.decisionlib")) { - xdlib(Config); + retval = xdlib(Config); + } + else if (!strcmp(var,"pfc.trace")) + { + retval = xtrace(Config); } else if (!strncmp(var,"pfc.", 4)) { @@ -159,6 +195,10 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char } m_configuration.m_NRamBuffers = static_cast(m_configuration.m_RamAbsAvailable/ m_configuration.m_bufferSize); + // Set tracing to debug if this is set in environment + char* cenv = getenv("XRDDEBUG"); + if (cenv && !strcmp(cenv,"1")) m_trace->What = 4; + if (retval) { int loff = 0; From dcb407d70522aa8758cbac3a22ab20ba2ec998ed Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 3 Jun 2016 11:30:38 -0700 Subject: [PATCH 131/171] Replace XrdCl::Log with XrdOucTrace --- src/XrdFileCache/XrdFileCache.cc | 28 ++-- src/XrdFileCache/XrdFileCache.hh | 5 +- src/XrdFileCache/XrdFileCacheConfiguration.cc | 32 ++-- src/XrdFileCache/XrdFileCacheFile.cc | 138 ++++++++---------- src/XrdFileCache/XrdFileCacheFile.hh | 8 +- src/XrdFileCache/XrdFileCacheIO.hh | 7 +- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 24 +-- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 23 +-- src/XrdFileCache/XrdFileCacheInfo.cc | 37 +++-- src/XrdFileCache/XrdFileCacheInfo.hh | 15 +- src/XrdFileCache/XrdFileCachePrint.cc | 7 +- src/XrdFileCache/XrdFileCachePurge.cc | 37 ++--- src/XrdFileCache/XrdFileCacheTrace.hh | 42 ++++++ src/XrdFileCache/XrdFileCacheVRead.cc | 25 ++-- 14 files changed, 230 insertions(+), 198 deletions(-) create mode 100644 src/XrdFileCache/XrdFileCacheTrace.hh diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 674b164b3a2..accaaf8445b 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -146,18 +146,19 @@ XrdOucCacheIO2 *Cache::Attach(XrdOucCacheIO2 *io, int Options) { if (Cache::GetInstance().Decide(io)) { - clLog()->Info(XrdCl::AppMsg, "Cache::Attach() %s", io->Path()); + TRACE(Debug, "Cache::Attach() " << io->Path()); IO* cio; if (Cache::GetInstance().RefConfiguration().m_hdfsmode) cio = new IOFileBlock(io, m_stats, *this); else cio = new IOEntireFile(io, m_stats, *this); + TRACE(Info, "Cache::Attach() " << io->Path()); return cio; } else { - clLog()->Info(XrdCl::AppMsg, "Cache::Attach() reject %s", io->Path()); + TRACE(Info, "Cache::Attach() decision decline " << io->Path()); } return io; } @@ -172,7 +173,7 @@ int Cache::isAttached() void Cache::Detach(XrdOucCacheIO* io) { - clLog()->Info(XrdCl::AppMsg, "Cache::Detach() %s", io->Path()); + TRACE(Info, "Cache::Detach() " << io->Path()); // Cache owns File objects XrdSysMutexHelper lock(&m_active_mutex); @@ -200,7 +201,7 @@ Cache::HaveFreeWritingSlots() return true; } else { - XrdCl::DefaultEnv::GetLog()->Info(XrdCl::AppMsg, "Cache::HaveFreeWritingSlots() negative", m_writeQ.size); + TRACE(Dump, "Cache::HaveFreeWritingSlots() negative " << m_writeQ.size); return false; } } @@ -208,7 +209,7 @@ Cache::HaveFreeWritingSlots() void Cache::AddWriteTask(Block* b, bool fromRead) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::AddWriteTask() bOff=%ld", b->m_offset); + TRACE(Dump, "Cache::AddWriteTask() bOff=%ld " << b->m_offset); m_writeQ.condVar.Lock(); if (fromRead) m_writeQ.queue.push_back(b); @@ -228,10 +229,9 @@ void Cache::RemoveWriteQEntriesFor(File *iFile) { if ((*i)->m_file == iFile) { - - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Remove entries for %p path %s", (void*)(*i), iFile->lPath()); + TRACE(Dump, "Cache::Remove entries for " << (void*)(*i) << " path " << iFile->lPath()); std::list::iterator j = i++; - iFile->BlockRemovedFromWriteQ(*j); + iFile->BlockRemovedFromWriteQ(*j); m_writeQ.queue.erase(j); --m_writeQ.size; } @@ -257,7 +257,7 @@ Cache::ProcessWriteTasks() Block* block = m_writeQ.queue.front(); // AMT should not be back ??? m_writeQ.queue.pop_front(); m_writeQ.size--; - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::ProcessWriteTasks for %p path %s", (void*)(block), block->m_file->lPath()); + TRACE(Dump, "Cache::ProcessWriteTasks for %p " << (void*)(block) << " path " << block->m_file->lPath()); m_writeQ.condVar.UnLock(); block->m_file->WriteBlockToDisk(block); @@ -332,13 +332,10 @@ Cache::RegisterPrefetchFile(File* file) if (Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) { - - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file BEGIN"); m_prefetch_condVar.Lock(); m_prefetchList.push_back(file); m_prefetch_condVar.Signal(); m_prefetch_condVar.UnLock(); - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file End"); } } @@ -399,7 +396,7 @@ Cache::Prepare(const char *url, int oflags, mode_t mode) struct stat buf; int res = m_output_fs->Stat(spath.c_str(), &buf); if (res == 0) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch defer open %s", spath.c_str()); + TRACE( Dump, "Cache::Prefetch defer open " << spath); return 1; } else { @@ -433,7 +430,7 @@ int Cache::Stat(const char *curl, struct stat &sbuff) name += ".cinfo"; int res = infoFile->Open(name.c_str(), O_RDONLY, 0600, myEnv); if (res >= 0) { - Info info(0); + Info info(m_trace, 0); if (info.Read(infoFile) > 0) { sbuff.st_size = info.GetFileSize(); success = true; @@ -455,9 +452,6 @@ void Cache::Prefetch() { int limitRAM= Cache::GetInstance().RefConfiguration().m_NRamBuffers * 0.7; - - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch thread start"); - while (true) { m_RAMblock_mutex.Lock(); bool doPrefetch = (m_RAMblocks_used < limitRAM && HaveFreeWritingSlots()); diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 5f337a7b7f2..89249d83873 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -192,7 +192,7 @@ namespace XrdFileCache void AddActive(IO*, File*); - XrdOucTrace* GetTrace() { return m_trace; } + XrdOucTrace* GetTrace() { return m_trace; } private: bool ConfigParameters(std::string, XrdOucStream&); bool ConfigXeq(char *, XrdOucStream &); @@ -213,9 +213,6 @@ namespace XrdFileCache Configuration m_configuration; //!< configurable parameters - //! Short log alias. - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - XrdSysCondVar m_prefetch_condVar; //!< central lock for this class XrdSysMutex m_RAMblock_mutex; //!< central lock for this class diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 69c9665be7b..06e3bc9b1f0 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -34,7 +34,7 @@ bool Cache::xdlib(XrdOucStream &Config) std::string libp; if (!(val = Config.GetWord()) || !val[0]) { - clLog()->Info(XrdCl::AppMsg, " Cache::Config() decisionlib not specified; always caching files"); + TRACE(Info," Cache::Config() decisionlib not specified; always caching files"); return true; } else @@ -55,14 +55,13 @@ bool Cache::xdlib(XrdOucStream &Config) Decision * d = ep(m_log); if (!d) { - clLog()->Error(XrdCl::AppMsg, "Cache::Config() decisionlib was not able to create a decision object"); + TRACE(Error, "Cache::Config() decisionlib was not able to create a decision object"); return false; } if (params) d->ConfigDecision(params); m_decisionpoints.push_back(d); - clLog()->Info(XrdCl::AppMsg, "Cache::Config() successfully created decision lib from %s", libp.c_str()); return true; } @@ -90,11 +89,12 @@ bool Cache::xtrace(XrdOucStream &Config) for (int i = 0; i < numopts; i++) { - if (!strcmp(val, tropts[i].opname)) + if (!strcmp(val, tropts[i].opname)) { m_trace->What = tropts[i].opval; - return true; + return true; + } } - return 0; + return false; } //______________________________________________________________________________ @@ -112,14 +112,14 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char if (!config_filename || !*config_filename) { - clLog()->Warning(XrdCl::AppMsg, "Cache::Config() configuration file not specified."); + TRACE(Error, "Cache::Config() configuration file not specified."); return false; } int fd; if ( (fd = open(config_filename, O_RDONLY, 0)) < 0) { - clLog()->Error(XrdCl::AppMsg, "Cache::Config() can't open configuration file %s", config_filename); + TRACE( Error, "Cache::Config() can't open configuration file " << config_filename); return false; } @@ -138,7 +138,7 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char } else { - clLog()->Error(XrdCl::AppMsg, "Cache::Config() Unable to create an OSS object"); + TRACE(Error, "Cache::Config() Unable to create an OSS object"); m_output_fs = 0; return false; } @@ -169,7 +169,7 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char if (!retval) { retval = false; - clLog()->Error(XrdCl::AppMsg, "Cache::Config() error in parsing"); + TRACE(Error, "Cache::Config() error in parsing"); break; } @@ -183,15 +183,14 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char if (m_output_fs->StatVS(&sP, "public", 1) >= 0) { m_configuration.m_diskUsageLWM = static_cast(0.90 * sP.Total + 0.5); m_configuration.m_diskUsageHWM = static_cast(0.95 * sP.Total + 0.5); - clLog()->Debug(XrdCl::AppMsg, "Default disk usage [%lld, %lld]", m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM); } } // get number of available RAM blocks after process configuration if (m_configuration.m_RamAbsAvailable == 0 ) { - m_log.Emsg("Error", "RAM usage not specified. Please set pfc.ram value in configuration file."); - return false; + TRACE(Error, "RAM usage not specified. Please set pfc.ram value in configuration file."); + return false; } m_configuration.m_NRamBuffers = static_cast(m_configuration.m_RamAbsAvailable/ m_configuration.m_bufferSize); @@ -207,12 +206,13 @@ bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char loff = snprintf(buff, sizeof(buff), "Config effective %s pfc configuration:\n" " pfc.blocksize %lld\n" " pfc.prefetch %ld\n" - " pfc.ram %.fg", + " pfc.ram %.fg" + " pfc.trace %d", config_filename, m_configuration.m_bufferSize, m_configuration.m_prefetch_max_blocks, // AMT not sure what parsing should be - rg); - + rg, + m_trace->What); if (m_configuration.m_hdfsmode) { diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 885ccd6f925..273896615aa 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -75,7 +75,7 @@ File::File(XrdOucCacheIO2 *inputIO, std::string& disk_file_path, long long iOffs m_input(inputIO), m_output(NULL), m_infoFile(NULL), -m_cfi(Cache::GetInstance().RefConfiguration().m_bufferSize, Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks > 0), +m_cfi(Cache::GetInstance().GetTrace(), Cache::GetInstance().RefConfiguration().m_bufferSize, Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks > 0), m_temp_filename(disk_file_path), m_offset(iOffset), m_fileSize(iFileSize), @@ -92,24 +92,19 @@ m_prefetchScore(1), m_prefetchCurrentCnt(0), m_traceID("File") { - clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_temp_filename.c_str()); - if (!Open()) { - clLog()->Error(XrdCl::AppMsg, "File::File() Open failed %s !!!", m_temp_filename.c_str()); - } + Open(); } void File::BlockRemovedFromWriteQ(Block* b) { m_downloadCond.Lock(); dec_ref_count(b); - clLog()->Dump(XrdCl::AppMsg, "File::BlockRemovedFromWriteQ() check write queues %p %d...%s", (void*)b, b->m_offset/m_cfi.GetBufferSize(), lPath()); + TRACEF(Dump, "File::BlockRemovedFromWriteQ() check write queues block = " << (void*)b << " idx= " << b->m_offset/m_cfi.GetBufferSize()); m_downloadCond.UnLock(); } File::~File() { - clLog()->Debug(XrdCl::AppMsg, "File::~File() enter %p %s", (void*)this, lPath()); - m_syncStatusMutex.Lock(); bool needs_sync = ! m_writes_during_sync.empty(); m_syncStatusMutex.UnLock(); @@ -122,7 +117,6 @@ File::~File() AppendIOStatToFileInfo(); m_infoFile->Fsync(); - clLog()->Info(XrdCl::AppMsg, "File::~File close data file %p",(void*)this , lPath()); if (m_output) { m_output->Close(); @@ -131,20 +125,20 @@ File::~File() } if (m_infoFile) { - clLog()->Info(XrdCl::AppMsg, "File::~File close info file"); m_infoFile->Close(); delete m_infoFile; m_infoFile = NULL; } // print just for curiosity - clLog()->Debug(XrdCl::AppMsg, "File::~File() ended, prefetch score ...%d/%d=%.2f", m_prefetchHitCnt, m_prefetchReadCnt, m_prefetchScore); + TRACEF(Debug, "File::~File() ended, prefetch score = " << m_prefetchScore); } bool File::InitiateClose() { // Retruns true if delay is needed - clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start %s", lPath()); + + TRACEF(Debug, "File::Initiate close start"); m_stateCond.Lock(); if (!m_stopping) { @@ -161,19 +155,23 @@ bool File::InitiateClose() if (isPrefetching == false) { m_downloadCond.Lock(); - /* + + /* + // dump print for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it) { Block* b = it->second; - clLog()->Debug(XrdCl::AppMsg, "File::InitiateClose() block idx=%d p=%d rcnt=%d dwnd=%d %s", - b->m_offset/m_cfi.GetBufferSize(), b->m_prefetch, b->m_refcnt, b->m_downloaded, lPath()); + TRACEF(Dump, "File::InitiateClose() block idx = " << b->m_offset/m_cfi.GetBufferSize() << " prefetch = " << b->preferch << " refcnt " << b->refcnt); + } */ + // remove failed blocks BlockMap_i itr = m_block_map.begin(); while (itr != m_block_map.end()) { if (itr->second->is_failed() && itr->second->m_refcnt == 1) { BlockMap_i toErase = itr; ++itr; + TRACEF(Debug, "Remove failed block " << itr->second->m_offset/m_cfi.GetBufferSize()); free_block(toErase->second); } else { @@ -207,7 +205,7 @@ bool File::InitiateClose() bool File::Open() { - clLog()->Dump(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_temp_filename.c_str()); + TRACEF(Dump, "File::Open() open file for disk cache "); XrdOss &m_output_fs = *Cache::GetInstance().GetOss(); // Create the data file itself. @@ -219,16 +217,15 @@ bool File::Open() int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0600, myEnv); if (res < 0) { - clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data-FD for %s %s", m_temp_filename.c_str(), m_temp_filename.c_str()); + TRACEF(Error, "File::Open() can't open data file"); delete m_output; m_output = 0; - return false; } } else { - clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data holder "); + TRACEF(Error, "File::Open() can't get file handle"); return false; } @@ -241,7 +238,7 @@ bool File::Open() int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv); if (res < 0) { - clLog()->Error(XrdCl::AppMsg, "File::Open() can't get info-FD %s %s", ifn.c_str(), m_temp_filename.c_str()); + TRACEF(Error, "File::Open() can't open info file"); delete m_infoFile; m_infoFile = 0; return false; @@ -256,17 +253,16 @@ bool File::Open() { m_fileSize = m_fileSize; int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; - clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_temp_filename.c_str()); + TRACEF(Debug, "Creating new file info, data size = " << m_fileSize << " num blocks = " << ss); m_cfi.SetFileSize(m_fileSize); m_cfi.WriteHeader(m_infoFile); m_infoFile->Fsync(); } else { - clLog()->Debug(XrdCl::AppMsg, "Info file read from disk: %s", m_temp_filename.c_str()); + TRACEF(Debug, "Successfully opened existing info file"); } - cache()->RegisterPrefetchFile(this); return true; } @@ -321,7 +317,6 @@ Block* File::RequestBlock(int i, bool prefetch) // // Reference count is 0 so increase it in calling function if you want to // catch the block while still in memory. - clLog()->Debug(XrdCl::AppMsg, "RequestBlock() %d pOn=(%d)", i, prefetch); const long long BS = m_cfi.GetBufferSize(); @@ -332,10 +327,10 @@ Block* File::RequestBlock(int i, bool prefetch) Block *b = new Block(this, off, this_bs, prefetch); // should block be reused to avoid recreation + TRACEF(Dump, "File::RequestBlock() " << i << "prefetch" << prefetch << "address " << (void*)b); BlockResponseHandler* oucCB = new BlockResponseHandler(b); m_input->Read(*oucCB, (char*)b->get_buff(), off, (int)this_bs); - clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); m_block_map[i] = b; if (m_prefetchState == kOn && m_block_map.size() > Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) @@ -367,7 +362,7 @@ int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, overlap(*ii, BS, req_off, req_size, off, blk_off, size); m_input->Read( *handler, req_buf + off, *ii * BS + blk_off, size); - clLog()->Dump(XrdCl::AppMsg, "RequestBlockDirect success %d %ld %s", *ii, size, lPath()); + TRACEF(Dump, "RequestBlockDirect success, idx = " << *ii << " size = " << size); total += size; } @@ -380,8 +375,7 @@ int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, int File::ReadBlocksFromDisk(std::list& blocks, char* req_buf, long long req_off, long long req_size) { - - clLog()->Dump(XrdCl::AppMsg, "File::ReadBlocksFromDisk %ld %s", blocks.size(), lPath()); + TRACEF(Dump, "File::ReadBlocksFromDisk " << blocks.size()); const long long BS = m_cfi.GetBufferSize(); long long total = 0; @@ -398,17 +392,17 @@ int File::ReadBlocksFromDisk(std::list& blocks, overlap(*ii, BS, req_off, req_size, off, blk_off, size); long long rs = m_output->Read(req_buf + off, *ii * BS + blk_off -m_offset, size); - clLog()->Dump(XrdCl::AppMsg, "File::ReadBlocksFromDisk block %d size %d %s", *ii, size, lPath()); + TRACEF(Dump, "File::ReadBlocksFromDisk block idx = " << *ii << " size= " << size); if (rs < 0) { - clLog()->Error(XrdCl::AppMsg, "File::ReadBlocksFromDisk neg retval %ld (%ld@%d) %s", rs, *ii * BS + blk_off, lPath()); + TRACEF(Error, "File::ReadBlocksFromDisk neg retval = " << rs << " idx = " << *ii ); return rs; } // AMT I think we should exit in this case too if (rs !=size) { - clLog()->Error(XrdCl::AppMsg, "File::ReadBlocksFromDisk incomplete %ld (%ld@%d) %s", rs, *ii * BS + blk_off, lPath()); + TRACEF(Error, "File::ReadBlocksFromDisk incomplete size = " << rs << " idx = " << *ii); return -1; } @@ -449,7 +443,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) for (int block_idx = idx_first; block_idx <= idx_last; ++block_idx) { - clLog()->Dump(XrdCl::AppMsg, "--- File::Read() idx %d %s \n", block_idx, lPath()); + TRACEF(Dump, "File::Read() idx " << block_idx); BlockMap_i bi = m_block_map.find(block_idx); // In RAM or incoming? @@ -459,14 +453,14 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // XXXX Or just push it and handle errors in one place later? inc_ref_count(bi->second); - clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p inc_ref_count for existing block %p %d %s", (void*)iUserBuff, (void*)bi->second, block_idx, lPath()); + TRACEF(Dump, "File::Read() " << iUserBuff << "inc_ref_count for existing block << " << bi->second << " idx = " << block_idx); blks_to_process.push_front(bi->second); m_stats.m_BytesRam++; // AMT what if block fails } // On disk? else if (m_cfi.TestBit(offsetIdx(block_idx))) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p read from disk %d %s", (void*)iUserBuff, block_idx, lPath()); + TRACEF(Dump, "File::Read() read from disk " << (void*)iUserBuff << " idx = " << block_idx); blks_on_disk.push_back(block_idx); m_stats.m_BytesDisk++; } @@ -476,7 +470,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Is there room for one more RAM Block? if ( cache()->HaveFreeWritingSlots() && cache()->RequestRAMBlock()) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p inc_ref_count new %d %s", (void*)iUserBuff, block_idx, lPath()); + TRACEF(Dump, "File::Read() inc_ref_count new " << (void*)iUserBuff << " idx = " << block_idx); Block *b = RequestBlock(block_idx, false); // assert(b); if (!b) { @@ -490,7 +484,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // Nope ... read this directly without caching. else { - clLog()->Debug(XrdCl::AppMsg, "File::Read() direct block %d %s", block_idx, lPath()); + TRACEF(Dump, "File::Read() direct block " << block_idx); blks_direct.push_back(block_idx); m_stats.m_BytesMissed++; } @@ -525,13 +519,13 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) delete direct_handler; return -1; // AMT ??? } - clLog()->Dump(XrdCl::AppMsg, "File::Read() direct read %d. %s", direct_size, lPath()); + TRACEF(Dump, "File::Read() direct read finished, size = " << direct_size); } // Second, read blocks from disk. if ((!blks_on_disk.empty()) && (bytes_read >= 0)) { int rc = ReadBlocksFromDisk(blks_on_disk, iUserBuff, iUserOff, iUserSize); - clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from disk %d. %s", (void*)iUserBuff, rc, lPath()); + TRACEF(Dump, "File::Read() " << (void*)iUserBuff <<" from disk finished size = " << rc); if (rc >= 0) { bytes_read += rc; @@ -539,7 +533,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) else { bytes_read = rc; - clLog()->Error(XrdCl::AppMsg, "File::Read() failed to read from disk. %s", lPath()); + TRACEF(Error, "File::Read() failed read from disk"); // AMT commented line below should not be an immediate return, can have block refcount increased and map increased // return rc; } @@ -556,10 +550,9 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) BlockList_i bi = blks_to_process.begin(); while (bi != blks_to_process.end()) { - // clLog()->Dump(XrdCl::AppMsg, "File::Read() searcing for block %p finished", (void*)(*bi)); if ((*bi)->is_finished()) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() found finished block %p %s", (void*)(*bi), lPath()); + TRACEF(Dump, "File::Read() requested block downloaded " << (void*)(*bi)); finished.push_back(*bi); BlockList_i bj = bi++; blks_to_process.erase(bj); @@ -572,18 +565,16 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) if (finished.empty()) { - - clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block begin %s", lPath()); + TRACEF(Dump, "File::Read() wait block begin" ); m_downloadCond.Wait(); - clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block end %s", lPath()); + TRACEF(Dump, "File::Read() wait block end"); continue; } } - clLog()->Dump(XrdCl::AppMsg, "File::Read() bytes read before processing blocks %d %s\n", bytes_read, lPath()); BlockList_i bi = finished.begin(); while (bi != finished.end()) @@ -597,7 +588,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) // clLog()->Dump(XrdCl::AppMsg, "File::Read() Block finished ok."); overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); - clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from finished block %d , size %d end %s", (void*)iUserBuff, (*bi)->m_offset/BS, size_to_copy, lPath()); + TRACEF(Dump, "File::Read() ub=" << (void*)iUserBuff << " from finished block " << (*bi)->m_offset/BS << " size " << size_to_copy); memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); bytes_read += size_to_copy; @@ -605,7 +596,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } else // it has failed ... krap up. { - clLog()->Error(XrdCl::AppMsg, "File::Read() Block finished with error %s.", lPath()); + TRACEF(Error, "File::Read(), block "<< (*bi)->m_offset/BS << "finished with error "); bytes_read = -1; errno = (*bi)->m_errno; break; @@ -617,12 +608,10 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) finished.clear(); } - clLog()->Dump(XrdCl::AppMsg, "File::Read() bytes read after processing blocks %d %s\n", bytes_read, lPath()); - // Fourth, make sure all direct requests have arrived if ((direct_handler != 0) && (bytes_read >= 0 )) { - clLog()->Debug(XrdCl::AppMsg, "File::Read() waiting for direct requests %s.", lPath()); + TRACEF(Dump, "File::Read() waiting for direct requests "); XrdSysCondVarHelper _lck(direct_handler->m_cond); if (direct_handler->m_to_wait > 0) @@ -655,7 +644,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) for (BlockList_i bi = blks_processed.begin(); bi != blks_processed.end(); ++bi) { - clLog()->Dump(XrdCl::AppMsg, "File::Read() dec_ref_count b=%p, %d %s", (void*)(*bi), ((*bi)->m_offset/BufferSize()), lPath()); + TRACEF(Dump, "File::Read() dec_ref_count " << (void*)(*bi) << " idx = " << (int)((*bi)->m_offset/BufferSize())); dec_ref_count(*bi); // XXXX stamp block } @@ -686,18 +675,17 @@ void File::WriteBlockToDisk(Block* b) if (buffer_remaining) { - clLog()->Warning(XrdCl::AppMsg, "File::WriteToDisk() reattempt[%d] writing missing %ld for block %d %s", - cnt, buffer_remaining, b->m_offset, lPath()); + TRACEF(Warning, "File::WriteToDisk() reattempt " << cnt << " writing missing " << buffer_remaining << " for block offset " << b->m_offset); } if (cnt > PREFETCH_MAX_ATTEMPTS) { - clLog()->Error(XrdCl::AppMsg, "File::WriteToDisk() write failed too manny attempts %s", lPath()); + TRACEF(Error, "File::WriteToDisk() write block with off = " << b->m_offset <<" failed too manny attempts "); return; } } // set bit fetched - clLog()->Dump(XrdCl::AppMsg, "File::WriteToDisk() success set bit for block [%ld] size [%d] %s", b->m_offset, size, lPath()); + TRACEF(Dump, "File::WriteToDisk() success set bit for block " << b->m_offset << " size " << size); int pfIdx = (b->m_offset - m_offset)/m_cfi.GetBufferSize(); m_downloadCond.Lock(); @@ -744,7 +732,7 @@ void File::WriteBlockToDisk(Block* b) void File::Sync() { - clLog()->Dump(XrdCl::AppMsg, "File::Sync %s", lPath()); + TRACEF( Dump, "File::Sync()"); m_output->Fsync(); m_cfi.WriteHeader(m_infoFile); int written_while_in_sync; @@ -758,7 +746,7 @@ void File::Sync() m_writes_during_sync.clear(); m_in_sync = false; } - clLog()->Dump(XrdCl::AppMsg, "File::Sync %d blocks written during sync.", written_while_in_sync); + TRACEF(Dump, "File::Sync() "<< written_while_in_sync << " blocks written during sync."); m_infoFile->Fsync(); } @@ -768,7 +756,7 @@ void File::inc_ref_count(Block* b) { // Method always called under lock b->m_refcnt++; - clLog()->Dump(XrdCl::AppMsg, "File::inc_ref_count b=%p, %d %s ",(void*)b, b->m_refcnt, lPath()); + TRACEF(Dump, "File::inc_ref_count " << b << " refcnt " << b->m_refcnt); } //______________________________________________________________________________ @@ -788,12 +776,13 @@ void File::dec_ref_count(Block* b) void File::free_block(Block* b) { int i = b->m_offset/BufferSize(); - clLog()->Dump(XrdCl::AppMsg, "File::free_block block (%p) %d %s ", (void*)b, i, lPath()); + TRACEF(Dump, "File::free_block block " << b << " idx = " << i); delete m_block_map[i]; size_t ret = m_block_map.erase(i); if (ret != 1) { - clLog()->Error(XrdCl::AppMsg, "File::OnBlockZeroRefCount did not erase %d from map.", i); + // assert might be a better option than a warning + TRACEF(Warning, "File::OnBlockZeroRefCount did not erase " << i << " from map"); } else { @@ -814,13 +803,13 @@ void File::ProcessBlockResponse(Block* b, int res) m_downloadCond.Lock(); - clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %p, %d %s",(void*)b,(int)(b->m_offset/BufferSize()), lPath()); + TRACEF(Dump, "File::ProcessBlockResponse " << (void*)b << " " << b->m_offset/BufferSize()); if (res >= 0) { b->m_downloaded = true; - clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d finished %d %s",(int)(b->m_offset/BufferSize()), b->is_finished(), lPath()); + TRACEF(Dump, "File::ProcessBlockResponse " << (int)(b->m_offset/BufferSize()) << " finished " << b->is_finished()); if (!m_stopping) { // AMT theoretically this should be under state lock, but then are double locks - clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse inc_ref_count %d %s\n", (int)(b->m_offset/BufferSize()), lPath()); + TRACEF(Dump, "File::ProcessBlockResponse inc_ref_count " << (int)(b->m_offset/BufferSize())); inc_ref_count(b); cache()->AddWriteTask(b, true); } @@ -834,7 +823,7 @@ void File::ProcessBlockResponse(Block* b, int res) { // AMT how long to keep? // when to retry? - clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %p %d error=%d, %s",(void*)b,(int)(b->m_offset/BufferSize()), res, lPath()); + TRACEF(Error, "File::ProcessBlockResponse block " << b << " " << (int)(b->m_offset/BufferSize()) << " error=" << res); // XrdPosixMap::Result(*status); // AMT could notfiy global cache we dont need RAM for that block b->set_error_and_free(errno); @@ -882,7 +871,7 @@ void File::AppendIOStatToFileInfo() } else { - clLog()->Warning(XrdCl::AppMsg, "File::AppendIOStatToFileInfo() info file not opened %s", lPath()); + TRACEF(Warning, "File::AppendIOStatToFileInfo() info file not opened"); } } @@ -891,7 +880,7 @@ void File::Prefetch() { if (m_prefetchState == kOn) { - // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status \n"); + TRACEF(Dump, "File::Prefetch enter to check download status"); XrdSysCondVarHelper _lck(m_downloadCond); // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN %s \n", lPath()); @@ -905,7 +894,7 @@ void File::Prefetch() f += m_offset/m_cfi.GetBufferSize(); BlockMap_i bi = m_block_map.find(f); if (bi == m_block_map.end()) { - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch take block %d %s", f, lPath()); + TRACEF(Dump, "File::Prefetch take block " << f); cache()->RequestRAMBlock(); RequestBlock(f, true); m_prefetchReadCnt++; @@ -916,15 +905,13 @@ void File::Prefetch() } } if (!found) { - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch no free blcok found "); + TRACEF(Dump, "File::Prefetch no free block found "); m_cfi.CheckComplete(); - // assert (m_cfi.IsComplete()); // it is possible all missing blocks are in map but downlaoded status is still not complete - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch -- unlikely to happen ... file seem to be complete %s", lPath()); + // assert (m_cfi.IsComplete()); // remove block from map cache()->DeRegisterPrefetchFile(this); } - clLog()->Dump(XrdCl::AppMsg, "File::Prefetch end"); } UnMarkPrefetch(); @@ -974,14 +961,18 @@ void File::UnMarkPrefetch() m_stateCond.UnLock(); } + +XrdOucTrace* File::GetTrace() +{ + return Cache::GetInstance().GetTrace(); +} + //============================================================================== //================== RESPONSE HANDLER ================================== //============================================================================== void BlockResponseHandler::Done(int res) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"BlockResponseHandler::Done()"); - m_block->m_file->ProcessBlockResponse(m_block, res); delete this; @@ -991,7 +982,6 @@ void BlockResponseHandler::Done(int res) void DirectResponseHandler::Done(int res) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"DirectResponseHandler::Done()"); XrdSysCondVarHelper _lck(m_cond); --m_to_wait; diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 9185ae642fa..116cf737c9f 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -191,7 +191,9 @@ namespace XrdFileCache //! Log path const char* lPath() const; - std::string GetLocalPath(); + std::string GetLocalPath() { return m_temp_filename; } + + XrdOucTrace* GetTrace(); private: bool overlap(int blk, // block to query long long blk_size, // @@ -215,7 +217,6 @@ namespace XrdFileCache int VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blks_on_disk); int VReadProcessBlocks(const XrdOucIOVec *readV, int n, std::vector& blks_to_process, std::vector& blks_rocessed); - long long BufferSize(); void AppendIOStatToFileInfo(); @@ -224,9 +225,6 @@ namespace XrdFileCache void UnMarkPrefetch(); - //! Short log alias. - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - void inc_ref_count(Block*); void dec_ref_count(Block*); void free_block(Block*); diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index 36007a154ee..3b272788ddc 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -1,10 +1,13 @@ #ifndef __XRDFILECACHE_CACHE_IO_HH__ #define __XRDFILECACHE_CACHE_IO_HH__ +class XrdOucTrace; + #include "XrdFileCache.hh" #include "XrdOuc/XrdOucCache2.hh" #include "XrdCl/XrdClDefaultEnv.hh" + namespace XrdFileCache { //---------------------------------------------------------------------------- @@ -36,9 +39,9 @@ namespace XrdFileCache virtual void RelinquishFile(File*) = 0; - protected: - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } + XrdOucTrace* GetTrace() {return m_cache.GetTrace();} + protected: XrdOucCacheIO2 *m_io; //!< original data source XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics Cache &m_cache; //!< reference to Cache needed in detach diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index c2cf4c6d68f..b2bbea978e8 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -19,13 +19,13 @@ #include #include -//#include "XrdClient/XrdClientConst.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSfs/XrdSfsInterface.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheStats.hh" +#include "XrdFileCacheTrace.hh" #include "XrdOuc/XrdOucEnv.hh" @@ -39,8 +39,6 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & m_file(0), m_localStat(0) { - clLog()->Info(XrdCl::AppMsg, "IO::IO() [%p] %s", this, m_io->Path()); - XrdCl::URL url(m_io->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); @@ -92,7 +90,7 @@ struct stat* IOEntireFile::getValidLocalStat(const char* path) XrdOucEnv myEnv; int res = infoFile->Open(path, O_RDONLY, 0600, myEnv); if (res >= 0) { - Info info(0); + Info info(m_cache.GetTrace(), 0); if (info.Read(infoFile) > 0) { tmpStat.st_size = info.GetFileSize(); m_localStat = new struct stat; @@ -131,24 +129,16 @@ void IOEntireFile::Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int int IOEntireFile::Read (char *buff, long long off, int size) { - clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() [%p] %lld@%d %s", this, off, size, m_io->Path()); - - // protect from reads over the file size - // if (off >= m_io->FSize()) - // return 0; + TRACEIO(Dump, "IOEntireFile::Read() "<< this << " off: " << off << " size: " << size ); if (off < 0) { errno = EINVAL; return -1; } - //if (off + size > m_io->FSize()) - // size = m_io->FSize() - off; - ssize_t bytes_read = 0; ssize_t retval = 0; retval = m_file->Read(buff, off, size); - clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() read from File retval = %d %s", retval, m_io->Path()); if (retval >= 0) { bytes_read += retval; @@ -156,11 +146,11 @@ int IOEntireFile::Read (char *buff, long long off, int size) size -= retval; if (size > 0) - clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::Read() missed %d bytes %s", size, m_io->Path()); + TRACEIO(Warning, "IOEntireFile::Read() bytes missed " << size ); } else { - clLog()->Error(XrdCl::AppMsg, "IOEntireFile::Read(), origin bytes read %d %s", retval, m_io->Path()); + TRACEIO(Warning, "IOEntireFile::Read() pass to origin bytes ret " << retval ); } return (retval < 0) ? retval : bytes_read; @@ -172,8 +162,6 @@ int IOEntireFile::Read (char *buff, long long off, int size) */ int IOEntireFile::ReadV (const XrdOucIOVec *readV, int n) { - clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::ReadV(), get %d requests %s", n, m_io->Path()); - - + TRACE(Dump, "IO::ReadV(), get " << n << " requests, " << m_io->Path()); return m_file->ReadV(readV, n); } diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index cdcc139e654..82639a1abea 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -25,6 +25,7 @@ #include "XrdFileCacheIOFileBlock.hh" #include "XrdFileCache.hh" #include "XrdFileCacheStats.hh" +#include "XrdFileCacheTrace.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSfs/XrdSfsInterface.hh" @@ -43,7 +44,7 @@ IOFileBlock::IOFileBlock(XrdOucCacheIO2 *io, XrdOucCacheStats &statsGlobal, Cach //______________________________________________________________________________ XrdOucCacheIO* IOFileBlock::Detach() { - clLog()->Info(XrdCl::AppMsg, "IOFileBlock::Detach() %s", m_io->Path()); + TRACEIO(Info, "IOFileBlock::Detach() " ); XrdOucCacheIO * io = m_io; @@ -77,7 +78,7 @@ void IOFileBlock::GetBlockSizeFromPath() m_blocksize = atoi(path.substr(pos1).c_str()); } - clLog()->Debug(XrdCl::AppMsg, "FileBlock::GetBlockSizeFromPath(), blocksize = %lld. %s", m_blocksize, m_io->Path()); + TRACEIO(Debug, "FileBlock::GetBlockSizeFromPath(), blocksize = " << m_blocksize ); } } @@ -95,7 +96,7 @@ File* IOFileBlock::newBlockFile(long long off, int blocksize) ss << &offExt[0]; fname = ss.str(); - clLog()->Debug(XrdCl::AppMsg, "FileBlock::FileBlock(), create XrdFileCacheFile. %s", m_io->Path()); + TRACEIO(Debug, "FileBlock::FileBlock(), create XrdFileCacheFile "); File* file; if (!(file = Cache::GetInstance().GetFileWithLocalPath(fname, this))) @@ -159,7 +160,7 @@ int IOFileBlock::Read (char *buff, long long off, int size) int idx_first = off0/m_blocksize; int idx_last = (off0+size-1)/m_blocksize; int bytes_read = 0; - clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() %lld@%d block range [%d-%d] \n %s", off, size, idx_first, idx_last, m_io->Path()); + TRACEIO(Debug, "IOFileBlock::Read() "<< off << "@" << size << " block range ["<< idx_first << ", " << idx_last << "]"); for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx ) { @@ -179,7 +180,7 @@ int IOFileBlock::Read (char *buff, long long off, int size) if (blockIdx == lastIOFileBlock ) { pbs = m_io->FSize() - blockIdx*m_blocksize; - clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() last block, change output file size to %lld \n %s", pbs, m_io->Path()); + // TRACEIO(Dump, "IOFileBlock::Read() last block, change output file size to " << pbs); } fb = newBlockFile(blockIdx*m_blocksize, pbs); @@ -194,12 +195,12 @@ int IOFileBlock::Read (char *buff, long long off, int size) if (blockIdx == idx_first) { readBlockSize = (blockIdx + 1) *m_blocksize - off0; - clLog()->Debug(XrdCl::AppMsg, "Read partially till the end of the block %s", m_io->Path()); + TRACEIO(Dump, "Read partially till the end of the block"); } else if (blockIdx == idx_last) { readBlockSize = (off0+size) - blockIdx*m_blocksize; - clLog()->Debug(XrdCl::AppMsg, "Read partially from beginning of block %s", m_io->Path()); + TRACEIO(Dump, "Read partially till the end of the block %s"); } else { @@ -208,14 +209,14 @@ int IOFileBlock::Read (char *buff, long long off, int size) } assert(readBlockSize); - clLog()->Info(XrdCl::AppMsg, "IOFileBlock::Read() block[%d] read-block-size[%d], offset[%lld] %s", blockIdx, readBlockSize, off, m_io->Path()); + TRACEIO(Debug, "IOFileBlock::Read() block[ " << blockIdx << "] read-block-size[" << readBlockSize << "], offset[" << readBlockSize << "] off = " << off ); long long min = blockIdx*m_blocksize; if ( off < min) { assert(0); } assert(off+readBlockSize <= (min + m_blocksize)); int retvalBlock = fb->Read(buff, off, readBlockSize); - clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() Block read returned %d %s", retvalBlock , m_io->Path()); + TRACEIO(Dump, "IOFileBlock::Read() Block read returned " << retvalBlock); if (retvalBlock == readBlockSize ) { bytes_read += retvalBlock; @@ -223,12 +224,12 @@ int IOFileBlock::Read (char *buff, long long off, int size) off += retvalBlock; } else if (retvalBlock > 0) { - clLog()->Warning(XrdCl::AppMsg, "IOFileBlock::Read() incomplete read, missing bytes %d %s", readBlockSize-retvalBlock, m_io->Path()); + TRACEIO(Warning, "IOFileBlock::Read() incomplete read, missing bytes " << readBlockSize-retvalBlock); return bytes_read + retvalBlock; } else { - clLog()->Error(XrdCl::AppMsg, "IOFileBlock::Read() read error, retval %d %s", retvalBlock, m_io->Path()); + TRACEIO(Error, "IOFileBlock::Read() read error, retval" << retvalBlock); return retvalBlock; } } diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index d4f343724b8..40c0b1f7461 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -25,18 +25,23 @@ #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucSxeq.hh" +#include "XrdOuc/XrdOucTrace.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdFileCacheInfo.hh" #include "XrdFileCache.hh" +#include "XrdFileCacheStats.hh" +#include "XrdFileCacheTrace.hh" const char* XrdFileCache::Info::m_infoExtension = ".cinfo"; +const char* XrdFileCache::Info::m_traceID = "cinfo"; #define BIT(n) (1ULL << (n)) using namespace XrdFileCache; -Info::Info(long long iBufferSize, bool prefetchBuffer) : +Info::Info(XrdOucTrace* trace, long long iBufferSize, bool prefetchBuffer) : + m_trace(trace), m_version(1), m_bufferSize(iBufferSize), m_hasPrefetchBuffer(prefetchBuffer), @@ -89,7 +94,7 @@ int Info::Read(XrdOssDF* fp) int version; off += fp->Read(&version, off, sizeof(int)); if (version != m_version) { - clLog()->Dump(XrdCl::AppMsg, "Info:::Read(), incomatible file version"); + TRACE(Error, "Info:::Read(), incomatible file version"); return 0; } @@ -108,7 +113,7 @@ int Info::Read(XrdOssDF* fp) off += fp->Read(&m_accessCnt, off, sizeof(int)); - clLog()->Dump(XrdCl::AppMsg, "Info:::Read() complete %d access_cnt %d", m_complete, m_accessCnt); + TRACE(Dump, "Info:::Read() complete "<< m_complete << " access_cnt " << m_accessCnt); if (m_hasPrefetchBuffer) { @@ -132,7 +137,7 @@ int Info::GetHeaderSize() const void Info::WriteHeader(XrdOssDF* fp) { int flr = XrdOucSxeq::Serialize(fp->getFD(), XrdOucSxeq::noWait); - if (flr) clLog()->Error(XrdCl::AppMsg, "WriteHeader() lock failed %s \n", strerror(errno)); + if (flr) TRACE(Error, "Info::WriteHeader() lock failed " << strerror(errno)); long long off = 0; off += fp->Write(&m_version, off, sizeof(int)); @@ -142,7 +147,7 @@ void Info::WriteHeader(XrdOssDF* fp) off += fp->Write(m_buff_write_called, off, GetSizeInBytes()); flr = XrdOucSxeq::Release(fp->getFD()); - if (flr) clLog()->Error(XrdCl::AppMsg, "WriteHeader() un-lock failed \n"); + if (flr) TRACE(Error, "Info::WriteHeader() un-lock failed"); assert (off == GetHeaderSize()); } @@ -150,10 +155,13 @@ void Info::WriteHeader(XrdOssDF* fp) //______________________________________________________________________________ void Info::AppendIOStat(AStat& as, XrdOssDF* fp) { - clLog()->Info(XrdCl::AppMsg, "Info:::AppendIOStat()"); + TRACE(Dump, "Info:::AppendIOStat()"); int flr = XrdOucSxeq::Serialize(fp->getFD(), 0); - if (flr) clLog()->Error(XrdCl::AppMsg, "AppendIOStat() lock failed \n"); + if (flr) { + TRACE(Error, "Info::AppendIOStat() lock failed"); + return; + } m_accessCnt++; long long off = GetHeaderSize(); @@ -162,7 +170,10 @@ void Info::AppendIOStat(AStat& as, XrdOssDF* fp) long long ws = fp->Write(&as, off, sizeof(AStat)); flr = XrdOucSxeq::Release(fp->getFD()); - if (flr) clLog()->Error(XrdCl::AppMsg, "AppenIOStat() un-lock failed \n"); + if (flr) { + TRACE(Error, "Info::AppenIOStat() un-lock failed"); + return; + } if ( ws != sizeof(AStat)) { assert(0); } } @@ -173,7 +184,10 @@ bool Info::GetLatestDetachTime(time_t& t, XrdOssDF* fp) const bool res = false; int flr = XrdOucSxeq::Serialize(fp->getFD(), XrdOucSxeq::Share); - if (flr) clLog()->Error(XrdCl::AppMsg, "Info::GetLatestAttachTime() lock failed \n"); + if (flr) { + TRACE(Error, "Info::GetLatestAttachTime() lock failed"); + return false; + } if (m_accessCnt) { AStat stat; @@ -186,13 +200,14 @@ bool Info::GetLatestDetachTime(time_t& t, XrdOssDF* fp) const } else { - clLog()->Error(XrdCl::AppMsg, " Info::GetLatestAttachTime() can't get latest access stat. read bytes = %d", res); + TRACE(Error, " Info::GetLatestAttachTime() can't get latest access stat "); + return false; } } flr = XrdOucSxeq::Release(fp->getFD()); - if (flr) clLog()->Error(XrdCl::AppMsg, "Info::GetLatestAttachTime() lock failed \n"); + if (flr) TRACE(Error, "Info::GetLatestAttachTime() lock failed"); return res; } diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index ea7b6fc1c78..ce114205885 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -22,8 +22,9 @@ #include #include +class XrdOucTrace; + #include "XrdSys/XrdSysPthread.hh" -#include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClDefaultEnv.hh" @@ -59,7 +60,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - Info(long long bufferSize, bool prefetchBuffer = false); + Info(XrdOucTrace* trace, long long bufferSize, bool prefetchBuffer = false); //------------------------------------------------------------------------ //! Destructor. @@ -190,14 +191,12 @@ namespace XrdFileCache const static char* m_infoExtension; + const static char* m_traceID; + XrdOucTrace* GetTrace() const {return m_trace;} protected: - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - - //--------------------------------------------------------------------- - //! Cache statistics and time of access. - //--------------------------------------------------------------------- + XrdOucTrace* m_trace; int m_version; //!< info version long long m_bufferSize; //!< prefetch buffer size @@ -208,7 +207,7 @@ namespace XrdFileCache unsigned char *m_buff_write_called; //!< disk written state vector unsigned char *m_buff_prefetch; //!< prefetch state vector int m_accessCnt; //!< number of written AStat structs - bool m_complete; //!< cached + bool m_complete; //!< cached }; inline bool Info::TestBit(int i) const diff --git a/src/XrdFileCache/XrdFileCachePrint.cc b/src/XrdFileCache/XrdFileCachePrint.cc index b86bbce8e5d..8e6d922590a 100644 --- a/src/XrdFileCache/XrdFileCachePrint.cc +++ b/src/XrdFileCache/XrdFileCachePrint.cc @@ -23,6 +23,7 @@ #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucArgs.hh" +#include "XrdOuc/XrdOucTrace.hh" #include "XrdOfs/XrdOfsConfigPI.hh" #include "XrdSys/XrdSysLogger.hh" #include "XrdFileCacheInfo.hh" @@ -57,7 +58,11 @@ void Print::printFile(const std::string& path) printf("printing %s ...\n", path.c_str()); XrdOssDF* fh = m_oss->newFile(m_ossUser); fh->Open((path).c_str(),O_RDONLY, 0600, m_env); - Info cfi(0); + + XrdSysLogger log; + XrdSysError err(&log); + XrdOucTrace tr(&err); tr.What = 1; + Info cfi(&tr, 0); long long off = cfi.Read(fh); std::vector statv; diff --git a/src/XrdFileCache/XrdFileCachePurge.cc b/src/XrdFileCache/XrdFileCachePurge.cc index 19ed5a3dd3d..f6ed4d19fdc 100644 --- a/src/XrdFileCache/XrdFileCachePurge.cc +++ b/src/XrdFileCache/XrdFileCachePurge.cc @@ -1,9 +1,11 @@ #include "XrdFileCache.hh" +#include "XrdFileCacheTrace.hh" using namespace XrdFileCache; #include #include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucTrace.hh" namespace { @@ -48,6 +50,11 @@ namespace long long nByteReq; long long nByteAccum; }; + +XrdOucTrace* GetTrace() +{ + // needed for logging macros + return Cache::GetInstance().GetTrace(); } void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& purgeState) @@ -56,8 +63,8 @@ void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& XrdOucEnv env; int rdr; const size_t InfoExtLen = strlen(XrdFileCache::Info::m_infoExtension); // cached var - XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); + static const char* m_traceID = "Purge"; Cache& factory = Cache::GetInstance(); while ((rdr = iOssDF->Readdir(&buff[0], 256)) >= 0) { @@ -80,19 +87,19 @@ void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& // XXXX MT - shouldn't we also check if it is currently opened? fh->Open(np.c_str(), O_RDONLY, 0600, env); - Info cinfo(factory.RefConfiguration().m_bufferSize); + Info cinfo(Cache::GetInstance().GetTrace(), factory.RefConfiguration().m_bufferSize); time_t accessTime; cinfo.Read(fh); if (cinfo.GetLatestDetachTime(accessTime, fh)) { - log->Debug(XrdCl::AppMsg, "FillFileMapRecurse() checking %s accessTime %d ", buff, (int)accessTime); + TRACE(Dump, "FillFileMapRecurse() checking " << buff << " accessTime " << accessTime); purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); } else { // cinfo file does not contain any known accesses, use stat.mtime instead. - log->Info(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s, trying stat.\n", np.c_str()); + TRACE(Warning, "FillFileMapRecurse() could not get access time for " << np << ", trying stat"); XrdOss* oss = Cache::GetInstance().GetOss(); struct stat fstat; @@ -100,18 +107,14 @@ void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& if (oss->Stat(np.c_str(), &fstat) == XrdOssOK) { accessTime = fstat.st_mtime; - log->Info(XrdCl::AppMsg, "FillFileMapRecurse() determined access time for %s via stat: %lld\n", - np.c_str(), accessTime); - + TRACE(Dump, "FillFileMapRecurse() have access time for " << np << " via stat: " << accessTime); purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); } else { // This really shouldn't happen ... but if it does remove cinfo and the data file right away. - log->Warning(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s. Purging directly.\n", - np.c_str()); - + TRACE(Warning, "FillFileMapRecurse() could not get access time for " << np); oss->Unlink(np.c_str()); np = np.substr(0, np.size() - strlen(XrdFileCache::Info::m_infoExtension)); oss->Unlink(np.c_str()); @@ -128,7 +131,7 @@ void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& } } } - +} void Cache::CacheDirCleanup() { // check state every sleep seconds @@ -145,17 +148,17 @@ void Cache::CacheDirCleanup() long long bytesToRemove = 0; if (oss->StatVS(&sP, "public", 1) < 0) { - clLog()->Error(XrdCl::AppMsg, "Cache::CacheDirCleanup() can't get statvs for dir [%s] \n", m_configuration.m_cache_dir.c_str()); + TRACE(Error, "Cache::CacheDirCleanup() can't get statvs for dir " << m_configuration.m_cache_dir.c_str()); exit(1); } else { long long ausage = sP.Total - sP.Free; - clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() occupates disk space == %lld", ausage); + TRACE(Debug, "Cache::CacheDirCleanup() occupates disk space == " << ausage); if (ausage > m_configuration.m_diskUsageHWM) { bytesToRemove = ausage - m_configuration.m_diskUsageLWM; - clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() need space for %lld bytes", bytesToRemove); + TRACE(Debug, "Cache::CacheDirCleanup() need space for " << bytesToRemove << " bytes"); } } @@ -180,8 +183,7 @@ void Cache::CacheDirCleanup() { bytesToRemove -= fstat.st_size; oss->Unlink(path.c_str()); - clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() removed %s size %lld", - path.c_str(), fstat.st_size); + TRACE(Info, "Cache::CacheDirCleanup() removed file:" << path << " size: " << fstat.st_size); } // remove data file @@ -190,8 +192,7 @@ void Cache::CacheDirCleanup() { bytesToRemove -= it->second.nByte; oss->Unlink(path.c_str()); - clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() removed %s bytes %lld, stat_size %lld", - path.c_str(), it->second.nByte, fstat.st_size); + TRACE(Info, "Cache::CacheDirCleanup() removed file: %s " << path << " size " << it->second.nByte); } if (bytesToRemove <= 0) diff --git a/src/XrdFileCache/XrdFileCacheTrace.hh b/src/XrdFileCache/XrdFileCacheTrace.hh new file mode 100644 index 00000000000..77e75353f69 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheTrace.hh @@ -0,0 +1,42 @@ +#ifndef _XRDFILECACHE_TRACE_H +#define _XRDFILECACHE_TRACE_H + +// Trace flags +// +#define TRACE_None 0 +#define TRACE_Error 1 +#define TRACE_Warning 2 +#define TRACE_Info 3 +#define TRACE_Debug 4 +#define TRACE_Dump 5 + +#ifndef NODEBUG + +#include "XrdSys/XrdSysHeaders.hh" +#include "XrdOuc/XrdOucTrace.hh" + +#ifndef XRD_TRACE +#define XRD_TRACE GetTrace()-> +#endif + +#define TRACE(act, x) \ + if (XRD_TRACE What >= TRACE_ ## act) \ + {XRD_TRACE Beg(m_traceID); cerr <= TRACE_ ## act) \ + {XRD_TRACE Beg(m_traceID); cerr <Path(); XRD_TRACE End();} + +#define TRACEF(act, x) \ + if (XRD_TRACE What >= TRACE_ ## act) \ + {XRD_TRACE Beg(m_traceID); cerr <Debug(XrdCl::AppMsg, "ReadV for %d chunks", n); + TRACEF(Dump, "ReadV for %d chunks " << n); int bytesRead = 0; @@ -153,7 +154,7 @@ int File::ReadV (const XrdOucIOVec *readV, int n) delete i->arr; - clLog()->Debug(XrdCl::AppMsg, "VRead exit, total = %d", bytesRead); + TRACEF(Dump, "VRead exit, total = " << bytesRead); return bytesRead; } @@ -170,18 +171,18 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b for (int block_idx = blck_idx_first; block_idx <= blck_idx_last; ++block_idx) { - clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess chunk %lld@%lld", readV[iov_idx].size, readV[iov_idx].offset); + TRACEF(Dump, "VReadPreProcess chunk "<< readV[iov_idx].size << "@"<< readV[iov_idx].offset); BlockMap_i bi = m_block_map.find(block_idx); if (bi != m_block_map.end()) { if ( blocks_to_process.AddEntry(bi->second, iov_idx)) inc_ref_count(bi->second); - clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d in map", block_idx); + TRACEF(Dump, "VReadPreProcess block "<< block_idx <<" in map"); } else if (m_cfi.TestBit(offsetIdx(block_idx))) { blocks_on_disk.AddEntry(block_idx, iov_idx); - clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d , chunk idx = %d on disk", block_idx,iov_idx ); + TRACEF(Dump, "VReadPreProcess block "<< block_idx <<" , chunk idx = " << iov_idx << " on disk"); } else { if ( Cache::GetInstance().HaveFreeWritingSlots() && Cache::GetInstance().RequestRAMBlock()) @@ -189,7 +190,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b Block *b = RequestBlock(block_idx, false); if (!b) return false; blocks_to_process.AddEntry(b, iov_idx); - clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess request block %d", block_idx); + TRACEF(Dump, "VReadPreProcess request block " << block_idx); inc_ref_count(b); } else { @@ -199,7 +200,7 @@ bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& b const long long BS = m_cfi.GetBufferSize(); overlap(block_idx, BS, readV[iov_idx].offset, readV[iov_idx].size, off, blk_off, size); chunkVec.push_back(XrdOucIOVec2(readV[iov_idx].data+off, BS*block_idx + blk_off,size)); - clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess direct read %d", block_idx); + TRACEF(Dump, "VReadPreProcess direct read " << block_idx); } } } @@ -226,7 +227,7 @@ int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blo long long size; // size to copy - clLog()->Debug(XrdCl::AppMsg, "VReadFromDisk block=%d chunk=%d", blockIdx, chunkIdx); + TRACEF(Dump, "VReadFromDisk block= " << blockIdx <<" chunk=" << chunkIdx); overlap(blockIdx, m_cfi.GetBufferSize(), readV[chunkIdx].offset, readV[chunkIdx].size, off, blk_off, size); int rs = m_output->Read(readV[chunkIdx].data + off, blockIdx*m_cfi.GetBufferSize() + blk_off - m_offset, size); @@ -234,10 +235,8 @@ int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blo bytes_read += rs; } else { - // ofs read shoul set the errno - - clLog()->Error(XrdCl::AppMsg, "VReadFromDisk FAILED block=%d chunk=%d off=%lld, blk_off=%lld, size=%lld, chunfoff =%lld", blockIdx, chunkIdx, off, blk_off, size,readV[chunkIdx].offset ); - + // ofs read should set the errno + TRACEF(Error, "VReadFromDisk FAILED block=" << blockIdx << " chunk=" << chunkIdx << " off= " << off << " blk_off=" << blk_off << " size = " << size << "chunOff " << readV[chunkIdx].offset); return -1; } } @@ -311,6 +310,6 @@ int File::VReadProcessBlocks(const XrdOucIOVec *readV, int n, finished.clear(); } - clLog()->Info(XrdCl::AppMsg, "VReadProcessBlocks total read == %d", bytes_read); + TRACEF(Dump, "VReadProcessBlocks total read " << bytes_read); return bytes_read; } From 6d1b14415d3265a39334ea55fddf5f8784e56ea2 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 3 Jun 2016 12:08:15 -0700 Subject: [PATCH 132/171] Remove abosolete argument in Info constructor --- src/XrdFileCache/XrdFileCacheFile.cc | 3 ++- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 2 +- src/XrdFileCache/XrdFileCacheInfo.cc | 13 ++++++++++--- src/XrdFileCache/XrdFileCacheInfo.hh | 3 ++- src/XrdFileCache/XrdFileCachePrint.cc | 2 +- src/XrdFileCache/XrdFileCachePurge.cc | 2 +- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 273896615aa..6edd96789da 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -75,7 +75,7 @@ File::File(XrdOucCacheIO2 *inputIO, std::string& disk_file_path, long long iOffs m_input(inputIO), m_output(NULL), m_infoFile(NULL), -m_cfi(Cache::GetInstance().GetTrace(), Cache::GetInstance().RefConfiguration().m_bufferSize, Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks > 0), +m_cfi(Cache::GetInstance().GetTrace(), Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks > 0), m_temp_filename(disk_file_path), m_offset(iOffset), m_fileSize(iFileSize), @@ -254,6 +254,7 @@ bool File::Open() m_fileSize = m_fileSize; int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; TRACEF(Debug, "Creating new file info, data size = " << m_fileSize << " num blocks = " << ss); + m_cfi.SetBufferSize(Cache::GetInstance().RefConfiguration().m_bufferSize); m_cfi.SetFileSize(m_fileSize); m_cfi.WriteHeader(m_infoFile); m_infoFile->Fsync(); diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index b2bbea978e8..9429b987910 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -90,7 +90,7 @@ struct stat* IOEntireFile::getValidLocalStat(const char* path) XrdOucEnv myEnv; int res = infoFile->Open(path, O_RDONLY, 0600, myEnv); if (res >= 0) { - Info info(m_cache.GetTrace(), 0); + Info info(m_cache.GetTrace()); if (info.Read(infoFile) > 0) { tmpStat.st_size = info.GetFileSize(); m_localStat = new struct stat; diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index 40c0b1f7461..cda22389e63 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -34,16 +34,16 @@ #include "XrdFileCacheTrace.hh" const char* XrdFileCache::Info::m_infoExtension = ".cinfo"; -const char* XrdFileCache::Info::m_traceID = "cinfo"; +const char* XrdFileCache::Info::m_traceID = "Cinfo"; #define BIT(n) (1ULL << (n)) using namespace XrdFileCache; -Info::Info(XrdOucTrace* trace, long long iBufferSize, bool prefetchBuffer) : +Info::Info(XrdOucTrace* trace, bool prefetchBuffer) : m_trace(trace), m_version(1), - m_bufferSize(iBufferSize), + m_bufferSize(-1), m_hasPrefetchBuffer(prefetchBuffer), m_sizeInBits(0), m_buff_fetched(0), m_buff_write_called(0), m_buff_prefetch(0), @@ -59,6 +59,13 @@ Info::~Info() if (m_buff_prefetch) free(m_buff_prefetch); } +//______________________________________________________________________________ +void Info::SetBufferSize(long long bs) +{ + // Needed only info is created first time in File::Open() + m_bufferSize = bs; +} + //______________________________________________________________________________ void Info::SetFileSize(long long fs) { diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index ce114205885..0652e83bdb8 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -60,7 +60,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - Info(XrdOucTrace* trace, long long bufferSize, bool prefetchBuffer = false); + Info(XrdOucTrace* trace, bool prefetchBuffer = false); //------------------------------------------------------------------------ //! Destructor. @@ -86,6 +86,7 @@ namespace XrdFileCache //--------------------------------------------------------------------- void SetBitPrefetch(int i); + void SetBufferSize(long long); void SetFileSize(long long); //--------------------------------------------------------------------- diff --git a/src/XrdFileCache/XrdFileCachePrint.cc b/src/XrdFileCache/XrdFileCachePrint.cc index 8e6d922590a..d2952118cd4 100644 --- a/src/XrdFileCache/XrdFileCachePrint.cc +++ b/src/XrdFileCache/XrdFileCachePrint.cc @@ -62,7 +62,7 @@ void Print::printFile(const std::string& path) XrdSysLogger log; XrdSysError err(&log); XrdOucTrace tr(&err); tr.What = 1; - Info cfi(&tr, 0); + Info cfi(&tr); long long off = cfi.Read(fh); std::vector statv; diff --git a/src/XrdFileCache/XrdFileCachePurge.cc b/src/XrdFileCache/XrdFileCachePurge.cc index f6ed4d19fdc..d880a003fb2 100644 --- a/src/XrdFileCache/XrdFileCachePurge.cc +++ b/src/XrdFileCache/XrdFileCachePurge.cc @@ -87,7 +87,7 @@ void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& // XXXX MT - shouldn't we also check if it is currently opened? fh->Open(np.c_str(), O_RDONLY, 0600, env); - Info cinfo(Cache::GetInstance().GetTrace(), factory.RefConfiguration().m_bufferSize); + Info cinfo(Cache::GetInstance().GetTrace()); time_t accessTime; cinfo.Read(fh); if (cinfo.GetLatestDetachTime(accessTime, fh)) From d73754e448fd1975392b480bb987f4e8142906b5 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 3 Jun 2016 14:07:49 -0700 Subject: [PATCH 133/171] Pass blocksize to File in newBlockFile(). --- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 82639a1abea..cd30cbaca62 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -101,7 +101,7 @@ File* IOFileBlock::newBlockFile(long long off, int blocksize) File* file; if (!(file = Cache::GetInstance().GetFileWithLocalPath(fname, this))) { - file = new File(m_io, fname, off, m_io->FSize()); + file = new File(m_io, fname, off, blocksize); Cache::GetInstance().AddActive(this, file); } From 7b8fb092ee71d9923f55d1d12282f6aae002733e Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 8 Jun 2016 11:09:20 -0700 Subject: [PATCH 134/171] Document trace option. --- src/XrdFileCache/README | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/XrdFileCache/README b/src/XrdFileCache/README index 44244e0db95..b0f00ca723c 100644 --- a/src/XrdFileCache/README +++ b/src/XrdFileCache/README @@ -129,6 +129,8 @@ pfc.osslib [] path to alternative plign for output file system pfc.decisionlib [] path to decision library and plugin parameters +pfc.trace default level is none, xrootd option -d sets debug level + Examples a) Enable proxy file prefetching: From 51a95d9f5e99c7f6e5aab7ea582d1427344f0524 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 8 Jun 2016 14:25:03 -0700 Subject: [PATCH 135/171] Protect from invalid reads. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 76 +++++++++++++------- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 5 +- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 9429b987910..8eaac8c7c71 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -45,7 +45,12 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & if (!(m_file = Cache::GetInstance().GetFileWithLocalPath(fname, this))) { struct stat st; - Fstat(st); + int res = Fstat(st); + + // this should not happen, but make a printout to see it + if (!res) + TRACEIO(Error, "IOEntireFile const, could not get valid stat"); + m_file = new File(io, fname, 0, st.st_size); Cache::GetInstance().AddActive(this, m_file); } @@ -64,16 +69,22 @@ int IOEntireFile::Fstat(struct stat &sbuff) std::string name = url.GetPath(); name += ".cinfo"; - struct stat* ls = getValidLocalStat(name.c_str()); - if (ls) { - memcpy(&sbuff, ls, sizeof(struct stat)); - return 0; + if(!m_localStat) { + int ret = initCachedStat(name.c_str()); + return ret; } else { - return m_io->Fstat(sbuff); + memcpy(&sbuff, m_localStat, sizeof(struct stat)); + return 0; } } + +long long IOEntireFile::FSize() +{ + return m_localStat->st_size; +} + void IOEntireFile::RelinquishFile(File* f) { assert(m_file == f); @@ -81,28 +92,37 @@ void IOEntireFile::RelinquishFile(File* f) } -struct stat* IOEntireFile::getValidLocalStat(const char* path) +int IOEntireFile::initCachedStat(const char* path) { - if (!m_localStat) { - struct stat tmpStat; - if (m_cache.GetOss()->Stat(path, &tmpStat) == XrdOssOK) { - XrdOssDF* infoFile = m_cache.GetOss()->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); - XrdOucEnv myEnv; - int res = infoFile->Open(path, O_RDONLY, 0600, myEnv); - if (res >= 0) { - Info info(m_cache.GetTrace()); - if (info.Read(infoFile) > 0) { - tmpStat.st_size = info.GetFileSize(); - m_localStat = new struct stat; - memcpy(m_localStat, &tmpStat, sizeof(struct stat)); - } + // called indirectly from this constructor first time + + m_localStat = new struct stat; + struct stat tmpStat; + if (m_cache.GetOss()->Stat(path, &tmpStat) == XrdOssOK) { + XrdOssDF* infoFile = m_cache.GetOss()->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); + XrdOucEnv myEnv; + int res = infoFile->Open(path, O_RDONLY, 0600, myEnv); + if (res >= 0) { + Info info(m_cache.GetTrace()); + if (info.Read(infoFile) > 0) { + tmpStat.st_size = info.GetFileSize(); + memcpy(m_localStat, &tmpStat, sizeof(struct stat)); + } + else { + TRACEIO(Error, "IOEntireFile::initCachedStat failed to read file size from info file"); + res = -1; } - infoFile->Close(); - delete infoFile; } + infoFile->Close(); + delete infoFile; + TRACEIO(Debug, "IOEntireFile::initCachedStat successfuly read size from info file = " << m_localStat->st_size); + return res; } - - return m_localStat; + else { + int res = m_io->Fstat(*m_localStat); + TRACEIO(Debug, "IOEntireFile::initCachedStat get stat from client res= " << res << "size = " << m_localStat->st_size); + return res; + } } bool IOEntireFile::ioActive() @@ -130,11 +150,19 @@ void IOEntireFile::Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int int IOEntireFile::Read (char *buff, long long off, int size) { TRACEIO(Dump, "IOEntireFile::Read() "<< this << " off: " << off << " size: " << size ); + + // protect from reads over the file size + if (off >= FSize()) + return 0; if (off < 0) { errno = EINVAL; return -1; } + if (off + size > FSize()) + size = FSize() - off; + + ssize_t bytes_read = 0; ssize_t retval = 0; diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index e3e8fe9f1a8..927be01f281 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -97,12 +97,15 @@ namespace XrdFileCache virtual int Fstat(struct stat &sbuff); + virtual long long FSize(); + virtual void RelinquishFile(File*); + private: File* m_file; struct stat *m_localStat; - struct stat* getValidLocalStat(const char* path); + int initCachedStat(const char* path); }; } From c022e29bd8f72545a2a1775c1729ee9f7e9edb27 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 8 Jun 2016 14:31:52 -0700 Subject: [PATCH 136/171] Default debug level is warning. --- src/XrdFileCache/README | 2 +- src/XrdFileCache/XrdFileCache.cc | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/XrdFileCache/README b/src/XrdFileCache/README index b0f00ca723c..e4e360a5fb5 100644 --- a/src/XrdFileCache/README +++ b/src/XrdFileCache/README @@ -129,7 +129,7 @@ pfc.osslib [] path to alternative plign for output file system pfc.decisionlib [] path to decision library and plugin parameters -pfc.trace default level is none, xrootd option -d sets debug level +pfc.trace default level is warning, xrootd option -d sets debug level Examples diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index accaaf8445b..bfe21453dc7 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -138,6 +138,8 @@ Cache::Cache() : XrdOucCache(), m_RAMblocks_used(0) { m_trace = new XrdOucTrace(&m_log); + // default log level is Warning + m_trace->What = 2; } //______________________________________________________________________________ From 2c5e9d1e216bdaaf85053e1aebee35d2044b1743 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 10:31:47 -0700 Subject: [PATCH 137/171] Print severity with message. --- src/XrdFileCache/XrdFileCacheTrace.hh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheTrace.hh b/src/XrdFileCache/XrdFileCacheTrace.hh index 77e75353f69..979c87e3275 100644 --- a/src/XrdFileCache/XrdFileCacheTrace.hh +++ b/src/XrdFileCache/XrdFileCacheTrace.hh @@ -10,6 +10,14 @@ #define TRACE_Debug 4 #define TRACE_Dump 5 + +#define TRACE_STR_None "" +#define TRACE_STR_Error "error " +#define TRACE_STR_Warning "warning " +#define TRACE_STR_Info "info " +#define TRACE_STR_Debug "debug " +#define TRACE_STR_Dump "dump " + #ifndef NODEBUG #include "XrdSys/XrdSysHeaders.hh" @@ -21,15 +29,15 @@ #define TRACE(act, x) \ if (XRD_TRACE What >= TRACE_ ## act) \ - {XRD_TRACE Beg(m_traceID); cerr <= TRACE_ ## act) \ - {XRD_TRACE Beg(m_traceID); cerr <Path(); XRD_TRACE End();} + {XRD_TRACE Beg(m_traceID); cerr << TRACE_STR_##act <Path(); XRD_TRACE End();} #define TRACEF(act, x) \ if (XRD_TRACE What >= TRACE_ ## act) \ - {XRD_TRACE Beg(m_traceID); cerr < Date: Thu, 9 Jun 2016 10:32:35 -0700 Subject: [PATCH 138/171] Add log message in Read() when 0 bytes read. --- src/XrdFileCache/XrdFileCacheInfo.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index cda22389e63..b9c2951d8ac 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -99,9 +99,13 @@ int Info::Read(XrdOssDF* fp) int off = 0; int version; - off += fp->Read(&version, off, sizeof(int)); + off = fp->Read(&version, off, sizeof(int)); + if (off <= 0) { + TRACE(Warning, "Info:::Read() failed"); + return 0; + } if (version != m_version) { - TRACE(Error, "Info:::Read(), incomatible file version"); + TRACE(Error, "Info:::Read(), incompatible file version"); return 0; } From 0bc5f09417c99a006fd87e5ebad8b323be5523f1 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 10:32:57 -0700 Subject: [PATCH 139/171] Add getter for file size. --- src/XrdFileCache/XrdFileCacheFile.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 116cf737c9f..cbc75cb2404 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -194,6 +194,8 @@ namespace XrdFileCache std::string GetLocalPath() { return m_temp_filename; } XrdOucTrace* GetTrace(); + + long long GetFileSize() { return m_fileSize;} private: bool overlap(int blk, // block to query long long blk_size, // From 2dc921b87f9172b58f6449240c33ebdd15125f03 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 10:34:05 -0700 Subject: [PATCH 140/171] In Open() assert info file exists and its size is nonzero. --- src/XrdFileCache/XrdFileCacheFile.cc | 38 ++++++++++++++++------------ 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 6edd96789da..2d02a6ae156 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -231,10 +231,15 @@ bool File::Open() // Create the info file std::string ifn = m_temp_filename + Info::m_infoExtension; + + struct stat infoStat; + bool fileExisted = (Cache::GetInstance().GetOss()->Stat(ifn.c_str(), &infoStat) == XrdOssOK); + m_output_fs.Create(Cache::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0600, myEnv, XRDOSS_mkpath); m_infoFile = m_output_fs.newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); if (m_infoFile) { + if (fileExisted) assert(infoStat.st_size > 0); int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv); if (res < 0) { @@ -243,27 +248,28 @@ bool File::Open() m_infoFile = 0; return false; } + else { + if (fileExisted) + { + int res = m_cfi.Read(m_infoFile); + TRACEF(Debug, "Reading existing info file bytes = " << res); + } + else { + m_fileSize = m_fileSize; + int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; + TRACEF(Debug, "Creating new file info, data size = " << m_fileSize << " num blocks = " << ss); + m_cfi.SetBufferSize(Cache::GetInstance().RefConfiguration().m_bufferSize); + m_cfi.SetFileSize(m_fileSize); + m_cfi.WriteHeader(m_infoFile); + m_infoFile->Fsync(); + } + } } else { + // this should be a rare case wher FD can't be created return false; } - - if (m_cfi.Read(m_infoFile) <= 0) - { - m_fileSize = m_fileSize; - int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; - TRACEF(Debug, "Creating new file info, data size = " << m_fileSize << " num blocks = " << ss); - m_cfi.SetBufferSize(Cache::GetInstance().RefConfiguration().m_bufferSize); - m_cfi.SetFileSize(m_fileSize); - m_cfi.WriteHeader(m_infoFile); - m_infoFile->Fsync(); - } - else - { - TRACEF(Debug, "Successfully opened existing info file"); - } - cache()->RegisterPrefetchFile(this); return true; } From 035ce727e1f6ed5b817f57e8e08c7715b64365af Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 10:35:12 -0700 Subject: [PATCH 141/171] Init stat when File is recycled. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 51 ++++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 8eaac8c7c71..aad3288f9a8 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -48,12 +48,15 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & int res = Fstat(st); // this should not happen, but make a printout to see it - if (!res) - TRACEIO(Error, "IOEntireFile const, could not get valid stat"); + if (res) + TRACEIO(Error, "IOEntireFile::IOEntireFile, could not get valid stat"); m_file = new File(io, fname, 0, st.st_size); Cache::GetInstance().AddActive(this, m_file); } + else { + // use system stat to init m_localStat if m_file is still in the map + } } @@ -69,24 +72,25 @@ int IOEntireFile::Fstat(struct stat &sbuff) std::string name = url.GetPath(); name += ".cinfo"; + int res = 0; if(!m_localStat) { - int ret = initCachedStat(name.c_str()); - return ret; - } - else { - memcpy(&sbuff, m_localStat, sizeof(struct stat)); - return 0; + res = initCachedStat(name.c_str()); + if (res) return res; } + + memcpy(&sbuff, m_localStat, sizeof(struct stat)); + return 0; } long long IOEntireFile::FSize() { - return m_localStat->st_size; + return m_file->GetFileSize(); } void IOEntireFile::RelinquishFile(File* f) { + TRACEIO(Info, "IOEntireFile::RelinquishFile"); assert(m_file == f); m_file = 0; } @@ -96,33 +100,40 @@ int IOEntireFile::initCachedStat(const char* path) { // called indirectly from this constructor first time - m_localStat = new struct stat; + int res = -1; struct stat tmpStat; + if (m_cache.GetOss()->Stat(path, &tmpStat) == XrdOssOK) { XrdOssDF* infoFile = m_cache.GetOss()->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); XrdOucEnv myEnv; - int res = infoFile->Open(path, O_RDONLY, 0600, myEnv); - if (res >= 0) { + if (infoFile->Open(path, O_RDONLY, 0600, myEnv) > 0) { Info info(m_cache.GetTrace()); + printf("reading info file ..\n"); if (info.Read(infoFile) > 0) { tmpStat.st_size = info.GetFileSize(); - memcpy(m_localStat, &tmpStat, sizeof(struct stat)); + TRACEIO(Info, "IOEntireFile::initCachedStat successfuly read size from info file = " << tmpStat.st_size); + res = 0; } else { + // file exist but can't read it TRACEIO(Error, "IOEntireFile::initCachedStat failed to read file size from info file"); - res = -1; } } infoFile->Close(); delete infoFile; - TRACEIO(Debug, "IOEntireFile::initCachedStat successfuly read size from info file = " << m_localStat->st_size); - return res; } - else { - int res = m_io->Fstat(*m_localStat); - TRACEIO(Debug, "IOEntireFile::initCachedStat get stat from client res= " << res << "size = " << m_localStat->st_size); - return res; + + if (res) { + res = m_io->Fstat(tmpStat); + TRACEIO(Debug, "IOEntireFile::initCachedStat get stat from client res= " << res << "size = " << tmpStat.st_size); } + + if (res == 0) + { + m_localStat = new struct stat; + memcpy(m_localStat, &tmpStat, sizeof(struct stat)); + } + return res; } bool IOEntireFile::ioActive() From 0013238b0206d2b7fe16cb3e5b80787de565a39c Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 13:01:04 -0700 Subject: [PATCH 142/171] Improve readability of relinquish file process. --- src/XrdFileCache/XrdFileCache.cc | 14 +++++++++----- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 8 +++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index bfe21453dc7..380c797e317 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -183,9 +183,12 @@ void Cache::Detach(XrdOucCacheIO* io) while ( it != m_active.end() ) { if (it->io == io) { - it->io->RelinquishFile(it->file); - delete it->file; + if (it->file) { + it->io->RelinquishFile(it->file); + delete it->file; + } m_active.erase(it); + break; } else ++it; @@ -304,9 +307,10 @@ File* Cache::GetFileWithLocalPath(std::string path, IO* iIo) { if (!strcmp(path.c_str(), it->file->lPath())) { - it->io->RelinquishFile(it->file); - it->io = iIo; - return it->file; + File *ff = it->file; + it->io->RelinquishFile(ff); + it->file = 0; + return ff; } } return 0; diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index aad3288f9a8..c19a0e24d8a 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -52,11 +52,9 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & TRACEIO(Error, "IOEntireFile::IOEntireFile, could not get valid stat"); m_file = new File(io, fname, 0, st.st_size); - Cache::GetInstance().AddActive(this, m_file); - } - else { - // use system stat to init m_localStat if m_file is still in the map } + + Cache::GetInstance().AddActive(this, m_file); } @@ -138,7 +136,7 @@ int IOEntireFile::initCachedStat(const char* path) bool IOEntireFile::ioActive() { - if (!m_file) + if ( ! m_file) return false; else return m_file->InitiateClose(); From 132f53b7bc9fc2000cbcd60332f523374a7cf2f9 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 13:03:32 -0700 Subject: [PATCH 143/171] Rename File::initateClose() to ioActive() --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- src/XrdFileCache/XrdFileCacheFile.hh | 2 +- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 2 +- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 2d02a6ae156..e9e9a29262a 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -134,7 +134,7 @@ File::~File() TRACEF(Debug, "File::~File() ended, prefetch score = " << m_prefetchScore); } -bool File::InitiateClose() +bool File::ioActive() { // Retruns true if delay is needed diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index cbc75cb2404..d838fbafc88 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -167,7 +167,7 @@ namespace XrdFileCache //! \brief Initiate close. Return true if still IO active. //! Used in XrdPosixXrootd::Close() //---------------------------------------------------------------------- - bool InitiateClose(); + bool ioActive(); //---------------------------------------------------------------------- //! Sync file cache inf o and output data with disk diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index c19a0e24d8a..13acc030aaf 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -139,7 +139,7 @@ bool IOEntireFile::ioActive() if ( ! m_file) return false; else - return m_file->InitiateClose(); + return m_file->ioActive(); } XrdOucCacheIO *IOEntireFile::Detach() diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index cd30cbaca62..7fe0d3df8e1 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -135,7 +135,7 @@ bool IOFileBlock::ioActive() XrdSysMutexHelper lock(&m_mutex); for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { - if (it->second->InitiateClose()) + if (it->second->ioActive()) return true; } From 1fcddb909667515bf95e10b1eda453602d2e6647 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 13:06:45 -0700 Subject: [PATCH 144/171] Rename Info::CheckComplete() to Info::UpdateDownloadCompleteStaus(). --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- src/XrdFileCache/XrdFileCacheInfo.hh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index e9e9a29262a..5159dc398de 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -913,7 +913,7 @@ void File::Prefetch() } if (!found) { TRACEF(Dump, "File::Prefetch no free block found "); - m_cfi.CheckComplete(); + m_cfi.UpdateDownloadCompleteStatus(); // it is possible all missing blocks are in map but downlaoded status is still not complete // assert (m_cfi.IsComplete()); // remove block from map diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index 0652e83bdb8..80406c7f03a 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -178,7 +178,7 @@ namespace XrdFileCache //--------------------------------------------------------------------- //! Update complete status //--------------------------------------------------------------------- - void CheckComplete(); + void UpdateDownloadCompleteStatus(); //--------------------------------------------------------------------- //! Get number of accesses @@ -272,7 +272,7 @@ namespace XrdFileCache return false; } - inline void Info::CheckComplete() + inline void Info::UpdateDownloadCompleteStatus() { m_complete = !IsAnythingEmptyInRng(0, m_sizeInBits-1); } From ebff8599518ed0a1f569ba2c290b5db7e4fe72ef Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 14:21:46 -0700 Subject: [PATCH 145/171] Remove m_stopped varaible. Add stopped prefetch state. --- src/XrdFileCache/XrdFileCacheFile.cc | 94 ++++++++++++++-------------- src/XrdFileCache/XrdFileCacheFile.hh | 4 +- 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 5159dc398de..f2537de9377 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -79,7 +79,6 @@ m_cfi(Cache::GetInstance().GetTrace(), Cache::GetInstance().RefConfiguration().m m_temp_filename(disk_file_path), m_offset(iOffset), m_fileSize(iFileSize), -m_stopping(false), m_stateCond(0), // We will explicitly lock the condition before use. m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), m_non_flushed_cnt(0), @@ -141,10 +140,9 @@ bool File::ioActive() TRACEF(Debug, "File::Initiate close start"); m_stateCond.Lock(); - if (!m_stopping) { - m_prefetchState = kCanceled; - cache()->DeRegisterPrefetchFile(this); - m_stopping = true; + if (m_prefetchState != kStopped) { + m_prefetchState = kStopped; + cache()->DeRegisterPrefetchFile(this); } m_stateCond.UnLock(); @@ -253,6 +251,11 @@ bool File::Open() { int res = m_cfi.Read(m_infoFile); TRACEF(Debug, "Reading existing info file bytes = " << res); + m_downloadCond.Lock(); + // this method is called from constructor, no need to lock downloadStaus + bool complete = m_cfi.IsComplete(); + if (complete) m_prefetchState = kComplete; + } else { m_fileSize = m_fileSize; @@ -270,7 +273,7 @@ bool File::Open() // this should be a rare case wher FD can't be created return false; } - cache()->RegisterPrefetchFile(this); + if (m_prefetchState != kComplete) cache()->RegisterPrefetchFile(this); return true; } @@ -814,17 +817,9 @@ void File::ProcessBlockResponse(Block* b, int res) if (res >= 0) { b->m_downloaded = true; - TRACEF(Dump, "File::ProcessBlockResponse " << (int)(b->m_offset/BufferSize()) << " finished " << b->is_finished()); - if (!m_stopping) { // AMT theoretically this should be under state lock, but then are double locks - TRACEF(Dump, "File::ProcessBlockResponse inc_ref_count " << (int)(b->m_offset/BufferSize())); - inc_ref_count(b); - cache()->AddWriteTask(b, true); - } - else { - // there is no refcount +/- to remove dropped prefetched blocks on destruction - if (b->m_prefetch && (b->m_refcnt == 0)) - free_block(b); - } + TRACEF(Dump, "File::ProcessBlockResponse inc_ref_count " << (int)(b->m_offset/BufferSize())); + inc_ref_count(b); + cache()->AddWriteTask(b, true); } else { @@ -885,42 +880,47 @@ void File::AppendIOStatToFileInfo() //______________________________________________________________________________ void File::Prefetch() { - if (m_prefetchState == kOn) { - TRACEF(Dump, "File::Prefetch enter to check download status"); + XrdSysCondVarHelper _lck(m_stateCond); + if (m_prefetchState == kComplete ) { + cache()->DeRegisterPrefetchFile(this); + return; + } + else if (m_prefetchState == kHold || m_prefetchState == kStopped) + return; + } + + // check index not on disk and not in RAM + TRACEF(Dump, "File::Prefetch enter to check download status"); + bool found = false; + for (int f=0; f < m_cfi.GetSizeInBits(); ++f) + { XrdSysCondVarHelper _lck(m_downloadCond); - // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN %s \n", lPath()); - - // check index not on disk and not in RAM - bool found = false; - for (int f=0; f < m_cfi.GetSizeInBits(); ++f) - { - // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch test bit %d", f); - if (!m_cfi.TestBit(f)) - { - f += m_offset/m_cfi.GetBufferSize(); - BlockMap_i bi = m_block_map.find(f); - if (bi == m_block_map.end()) { - TRACEF(Dump, "File::Prefetch take block " << f); - cache()->RequestRAMBlock(); - RequestBlock(f, true); - m_prefetchReadCnt++; - m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; - found = true; - break; - } + if (!m_cfi.TestBit(f)) + { + f += m_offset/m_cfi.GetBufferSize(); + BlockMap_i bi = m_block_map.find(f); + if (bi == m_block_map.end()) { + TRACEF(Dump, "File::Prefetch take block " << f); + cache()->RequestRAMBlock(); + RequestBlock(f, true); + m_prefetchReadCnt++; + m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; + found = true; + break; } } - if (!found) { - TRACEF(Dump, "File::Prefetch no free block found "); - m_cfi.UpdateDownloadCompleteStatus(); - // it is possible all missing blocks are in map but downlaoded status is still not complete - // assert (m_cfi.IsComplete()); - // remove block from map - cache()->DeRegisterPrefetchFile(this); - } } + + if (!found) { + TRACEF(Dump, "File::Prefetch no free block found "); + m_stateCond.Lock(); + m_prefetchState = kComplete; + m_stateCond.UnLock(); + cache()->DeRegisterPrefetchFile(this); + } + UnMarkPrefetch(); } diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index d838fbafc88..413b82cb415 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -95,7 +95,7 @@ namespace XrdFileCache class File { private: - enum PrefetchState_e { kOn, kHold, kCanceled }; + enum PrefetchState_e { kOn, kHold, kStopped, kComplete }; XrdOucCacheIO2 *m_input; //!< original data source XrdOssDF *m_output; //!< file handle for data file on disk @@ -106,8 +106,6 @@ namespace XrdFileCache long long m_offset; //!< offset of cached file for block-based operation long long m_fileSize; //!< size of cached disk file for block-based operation - bool m_stopping; //!< run thread should be stopped - XrdSysCondVar m_stateCond; //!< state condition variable // fsync From 9f8bd80fc20e2b6e5d65c8f6d9609b0e958bb1f7 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 14:32:42 -0700 Subject: [PATCH 146/171] Remove Mark/UnMark prefech functions. --- src/XrdFileCache/XrdFileCache.cc | 2 +- src/XrdFileCache/XrdFileCacheFile.cc | 26 +------------------------- src/XrdFileCache/XrdFileCacheFile.hh | 4 ---- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 380c797e317..ced8ab84cab 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -377,7 +377,7 @@ Cache::GetNextFileToPrefetch() size_t l = m_prefetchList.size(); int idx = rand() % l; File* f = m_prefetchList[idx]; - f->MarkPrefetch(); + m_prefetch_condVar.UnLock(); return f; } diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index f2537de9377..f7fc20214db 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -882,11 +882,7 @@ void File::Prefetch() { { XrdSysCondVarHelper _lck(m_stateCond); - if (m_prefetchState == kComplete ) { - cache()->DeRegisterPrefetchFile(this); - return; - } - else if (m_prefetchState == kHold || m_prefetchState == kStopped) + if (m_prefetchState != kOn) return; } @@ -920,8 +916,6 @@ void File::Prefetch() m_stateCond.UnLock(); cache()->DeRegisterPrefetchFile(this); } - - UnMarkPrefetch(); } @@ -951,24 +945,6 @@ float File::GetPrefetchScore() const return m_prefetchScore; } -//______________________________________________________________________________ -void File::MarkPrefetch() -{ - m_stateCond.Lock(); - m_prefetchCurrentCnt++; - m_stateCond.UnLock(); - -} - -//______________________________________________________________________________ -void File::UnMarkPrefetch() -{ - m_stateCond.Lock(); - m_prefetchCurrentCnt--; - m_stateCond.UnLock(); -} - - XrdOucTrace* File::GetTrace() { return Cache::GetInstance().GetTrace(); diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 413b82cb415..9c5f5693f71 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -184,8 +184,6 @@ namespace XrdFileCache float GetPrefetchScore() const; - void MarkPrefetch(); - //! Log path const char* lPath() const; @@ -223,8 +221,6 @@ namespace XrdFileCache void CheckPrefetchStatRAM(Block* b); void CheckPrefetchStatDisk(int idx); - void UnMarkPrefetch(); - void inc_ref_count(Block*); void dec_ref_count(Block*); void free_block(Block*); From e422df428c06b8798a1f956fdf27a420fcd71552 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 14:43:07 -0700 Subject: [PATCH 147/171] Delete disk syncer in destructor. --- src/XrdFileCache/XrdFileCacheFile.cc | 68 +++++++++++++--------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index f7fc20214db..262e72f3bdf 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -88,7 +88,6 @@ m_prefetchState(kOn), m_prefetchReadCnt(0), m_prefetchHitCnt(0), m_prefetchScore(1), -m_prefetchCurrentCnt(0), m_traceID("File") { Open(); @@ -116,6 +115,9 @@ File::~File() AppendIOStatToFileInfo(); m_infoFile->Fsync(); + delete m_syncer; + m_syncer = NULL; + if (m_output) { m_output->Close(); @@ -147,48 +149,40 @@ bool File::ioActive() m_stateCond.UnLock(); m_stateCond.Lock(); - bool isPrefetching = (m_prefetchCurrentCnt > 0); m_stateCond.UnLock(); - if (isPrefetching == false) - { - m_downloadCond.Lock(); - - /* - // dump print - for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it) { - Block* b = it->second; - TRACEF(Dump, "File::InitiateClose() block idx = " << b->m_offset/m_cfi.GetBufferSize() << " prefetch = " << b->preferch << " refcnt " << b->refcnt); - + m_downloadCond.Lock(); + /* + // high debug print + for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it) { + Block* b = it->second; + TRACEF(Dump, "File::InitiateClose() block idx = " << b->m_offset/m_cfi.GetBufferSize() << " prefetch = " << b->preferch << " refcnt " << b->refcnt); + + } + */ + // remove failed blocks + BlockMap_i itr = m_block_map.begin(); + while (itr != m_block_map.end()) { + if (itr->second->is_failed() && itr->second->m_refcnt == 1) { + BlockMap_i toErase = itr; + ++itr; + TRACEF(Debug, "Remove failed block " << itr->second->m_offset/m_cfi.GetBufferSize()); + free_block(toErase->second); } - */ - - // remove failed blocks - BlockMap_i itr = m_block_map.begin(); - while (itr != m_block_map.end()) { - if (itr->second->is_failed() && itr->second->m_refcnt == 1) { - BlockMap_i toErase = itr; - ++itr; - TRACEF(Debug, "Remove failed block " << itr->second->m_offset/m_cfi.GetBufferSize()); - free_block(toErase->second); - } - else { - ++itr; - } + else { + ++itr; } + } - bool blockMapEmpty = m_block_map.empty(); - m_downloadCond.UnLock(); + bool blockMapEmpty = m_block_map.empty(); + m_downloadCond.UnLock(); - if ( blockMapEmpty) - { - // file is not active when block map is empty and sync is done - XrdSysMutexHelper _lck(&m_syncStatusMutex); - if (m_in_sync == false) { - delete m_syncer; - m_syncer = NULL; - return false; - } + if ( blockMapEmpty) + { + // file is not active when block map is empty and sync is done + XrdSysMutexHelper _lck(&m_syncStatusMutex); + if (m_in_sync == false) { + return false; } } From b5688d6f3e993ecebee17f8905e867dd4969cd55 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 14:54:17 -0700 Subject: [PATCH 148/171] Set prefetch state when file is recycled by some other IO. --- src/XrdFileCache/XrdFileCacheFile.cc | 13 +++++++++---- src/XrdFileCache/XrdFileCacheFile.hh | 2 ++ src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 6 +++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 262e72f3bdf..bef067d6274 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -146,11 +146,11 @@ bool File::ioActive() m_prefetchState = kStopped; cache()->DeRegisterPrefetchFile(this); } - m_stateCond.UnLock(); - m_stateCond.Lock(); m_stateCond.UnLock(); + + // remove failed blocks and check if map is empty m_downloadCond.Lock(); /* // high debug print @@ -160,7 +160,6 @@ bool File::ioActive() } */ - // remove failed blocks BlockMap_i itr = m_block_map.begin(); while (itr != m_block_map.end()) { if (itr->second->is_failed() && itr->second->m_refcnt == 1) { @@ -191,7 +190,13 @@ bool File::ioActive() //______________________________________________________________________________ - +void File::WakeUp() +{ + // called if this object is recycled by other IO + m_stateCond.Lock(); + if (m_prefetchState != kComplete) m_prefetchState = kOn; + m_stateCond.UnLock(); +} //============================================================================== diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 9c5f5693f71..9f7c7d597bd 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -192,6 +192,8 @@ namespace XrdFileCache XrdOucTrace* GetTrace(); long long GetFileSize() { return m_fileSize;} + + void WakeUp(); private: bool overlap(int blk, // block to query long long blk_size, // diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 13acc030aaf..6734b51539e 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -42,8 +42,12 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & XrdCl::URL url(m_io->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); - if (!(m_file = Cache::GetInstance().GetFileWithLocalPath(fname, this))) + if ((m_file = Cache::GetInstance().GetFileWithLocalPath(fname, this))) { + m_file->WakeUp(); + } + else { + struct stat st; int res = Fstat(st); From 525975795fae168ca940711787fc2520cf1b6b3c Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 9 Jun 2016 15:13:46 -0700 Subject: [PATCH 149/171] Complete lock/unlock in Open (change relate to previous commit) --- src/XrdFileCache/XrdFileCacheFile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index bef067d6274..9c1adb35a19 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -254,7 +254,7 @@ bool File::Open() // this method is called from constructor, no need to lock downloadStaus bool complete = m_cfi.IsComplete(); if (complete) m_prefetchState = kComplete; - + m_downloadCond.UnLock(); } else { m_fileSize = m_fileSize; From 27f4b0b629b56b20b76ca19dfd36dab5925c809a Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 10 Jun 2016 14:30:12 -0700 Subject: [PATCH 150/171] Propagate input change from IO to File. This problem solves warning XrdPosix: Unexpected PrepIO call. --- src/XrdFileCache/XrdFileCacheFile.cc | 9 +++--- src/XrdFileCache/XrdFileCacheFile.hh | 7 +++-- src/XrdFileCache/XrdFileCacheIO.cc | 31 ++++++++++++++++++++ src/XrdFileCache/XrdFileCacheIO.hh | 19 ++++++++---- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 16 +++++----- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 1 - src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 21 +++++++------ src/XrdFileCache/XrdFileCacheTrace.hh | 2 +- src/XrdFileCache/XrdFileCacheVRead.cc | 3 +- 9 files changed, 76 insertions(+), 33 deletions(-) create mode 100644 src/XrdFileCache/XrdFileCacheIO.cc diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 9c1adb35a19..a2665d5ce59 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -18,6 +18,7 @@ #include "XrdFileCacheFile.hh" +#include "XrdFileCacheIO.hh" #include "XrdFileCacheTrace.hh" #include @@ -71,8 +72,8 @@ namespace Cache* cache() { return &Cache::GetInstance(); } } -File::File(XrdOucCacheIO2 *inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : -m_input(inputIO), +File::File(IO *io, std::string& disk_file_path, long long iOffset, long long iFileSize) : +m_io(io), m_output(NULL), m_infoFile(NULL), m_cfi(Cache::GetInstance().GetTrace(), Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks > 0), @@ -338,7 +339,7 @@ Block* File::RequestBlock(int i, bool prefetch) TRACEF(Dump, "File::RequestBlock() " << i << "prefetch" << prefetch << "address " << (void*)b); BlockResponseHandler* oucCB = new BlockResponseHandler(b); - m_input->Read(*oucCB, (char*)b->get_buff(), off, (int)this_bs); + m_io->GetInput()->Read(*oucCB, (char*)b->get_buff(), off, (int)this_bs); m_block_map[i] = b; @@ -370,7 +371,7 @@ int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, overlap(*ii, BS, req_off, req_size, off, blk_off, size); - m_input->Read( *handler, req_buf + off, *ii * BS + blk_off, size); + m_io->GetInput()->Read( *handler, req_buf + off, *ii * BS + blk_off, size); TRACEF(Dump, "RequestBlockDirect success, idx = " << *ii << " size = " << size); total += size; diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 9f7c7d597bd..68aa388b211 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -42,7 +42,8 @@ namespace XrdFileCache { class BlockResponseHandler; class DirectResponseHandler; - + class IO; + struct ReadVBlockListRAM; struct ReadVChunkListRAM; struct ReadVBlockListDisk; @@ -97,7 +98,7 @@ namespace XrdFileCache private: enum PrefetchState_e { kOn, kHold, kStopped, kComplete }; - XrdOucCacheIO2 *m_input; //!< original data source + IO *m_io; //!< original data source XrdOssDF *m_output; //!< file handle for data file on disk XrdOssDF *m_infoFile; //!< file handle for data-info file on disk Info m_cfi; //!< download status of file blocks and access statistics @@ -144,7 +145,7 @@ namespace XrdFileCache //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - File(XrdOucCacheIO2 *io, std::string &path, + File(IO *io, std::string &path, long long offset, long long fileSize); //------------------------------------------------------------------------ diff --git a/src/XrdFileCache/XrdFileCacheIO.cc b/src/XrdFileCache/XrdFileCacheIO.cc new file mode 100644 index 00000000000..3eefceaa685 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheIO.cc @@ -0,0 +1,31 @@ +#include "XrdFileCacheIO.hh" +#include "XrdSys/XrdSysAtomics.hh" +#include "XrdPosix/XrdPosixFile.hh" + +using namespace XrdFileCache; + +IO::IO(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache): +m_statsGlobal(stats), m_cache(cache), m_traceID("IO"), m_io(io) +{ + m_path = m_io->Path(); +} + +void IO::Update(XrdOucCacheIO2 &iocp) +{ + SetInput(&iocp); +} + + +void IO::SetInput(XrdOucCacheIO2* x) +{ + updMutex.Lock(); + m_io = x; + updMutex.UnLock(); +} + +XrdOucCacheIO2* IO::GetInput() +{ + AtomicBeg(updMutex); + return m_io; + AtomicEnd(updMutex); +} diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index 3b272788ddc..73376bc38b8 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -6,7 +6,7 @@ class XrdOucTrace; #include "XrdFileCache.hh" #include "XrdOuc/XrdOucCache2.hh" #include "XrdCl/XrdClDefaultEnv.hh" - +#include "XrdSys/XrdSysPthread.hh" namespace XrdFileCache { @@ -16,8 +16,7 @@ namespace XrdFileCache class IO : public XrdOucCacheIO2 { public: - IO (XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache) : - m_io(io), m_statsGlobal(stats), m_cache(cache), m_traceID("IO"){} + IO (XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache); //! Original data source. virtual XrdOucCacheIO *Base() { return m_io; } @@ -35,18 +34,26 @@ namespace XrdFileCache virtual int Write(char *Buffer, long long Offset, int Length) { errno = ENOTSUP; return -1; } - virtual void Update(XrdOucCacheIO2 &iocp) { m_io = &iocp; } + virtual void Update(XrdOucCacheIO2 &iocp); virtual void RelinquishFile(File*) = 0; XrdOucTrace* GetTrace() {return m_cache.GetTrace();} + + XrdOucCacheIO2* GetInput(); - protected: - XrdOucCacheIO2 *m_io; //!< original data source + protected: XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics Cache &m_cache; //!< reference to Cache needed in detach const char* m_traceID; + std::string m_path; + const char* GetPath() { return m_path.c_str(); } + + private: + XrdOucCacheIO2 *m_io; //!< original data source + XrdSysRecMutex updMutex; + void SetInput(XrdOucCacheIO2*); }; } diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 6734b51539e..9dcbf28ebb9 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -39,7 +39,7 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & m_file(0), m_localStat(0) { - XrdCl::URL url(m_io->Path()); + XrdCl::URL url(GetInput()->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); if ((m_file = Cache::GetInstance().GetFileWithLocalPath(fname, this))) @@ -55,10 +55,11 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & if (res) TRACEIO(Error, "IOEntireFile::IOEntireFile, could not get valid stat"); - m_file = new File(io, fname, 0, st.st_size); + m_file = new File(this, fname, 0, st.st_size); } Cache::GetInstance().AddActive(this, m_file); + std::cout << " IOEntireFile::IOEntireFile " << this << std::endl; } @@ -70,7 +71,7 @@ IOEntireFile::~IOEntireFile() int IOEntireFile::Fstat(struct stat &sbuff) { - XrdCl::URL url(m_io->Path()); + XrdCl::URL url(GetPath()); std::string name = url.GetPath(); name += ".cinfo"; @@ -108,9 +109,8 @@ int IOEntireFile::initCachedStat(const char* path) if (m_cache.GetOss()->Stat(path, &tmpStat) == XrdOssOK) { XrdOssDF* infoFile = m_cache.GetOss()->newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); XrdOucEnv myEnv; - if (infoFile->Open(path, O_RDONLY, 0600, myEnv) > 0) { + if (infoFile->Open(path, O_RDONLY, 0600, myEnv) == XrdOssOK) { Info info(m_cache.GetTrace()); - printf("reading info file ..\n"); if (info.Read(infoFile) > 0) { tmpStat.st_size = info.GetFileSize(); TRACEIO(Info, "IOEntireFile::initCachedStat successfuly read size from info file = " << tmpStat.st_size); @@ -126,7 +126,7 @@ int IOEntireFile::initCachedStat(const char* path) } if (res) { - res = m_io->Fstat(tmpStat); + res = GetInput()->Fstat(tmpStat); TRACEIO(Debug, "IOEntireFile::initCachedStat get stat from client res= " << res << "size = " << tmpStat.st_size); } @@ -148,7 +148,7 @@ bool IOEntireFile::ioActive() XrdOucCacheIO *IOEntireFile::Detach() { - XrdOucCacheIO * io = m_io; + XrdOucCacheIO * io = GetInput(); // This will delete us! m_cache.Detach(this); @@ -203,6 +203,6 @@ int IOEntireFile::Read (char *buff, long long off, int size) */ int IOEntireFile::ReadV (const XrdOucIOVec *readV, int n) { - TRACE(Dump, "IO::ReadV(), get " << n << " requests, " << m_io->Path()); + TRACEIO(Dump, "IO::ReadV(), get " << n << " requests" ); return m_file->ReadV(readV, n); } diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index 927be01f281..46f1048ef39 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -101,7 +101,6 @@ namespace XrdFileCache virtual void RelinquishFile(File*); - private: File* m_file; struct stat *m_localStat; diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 7fe0d3df8e1..cf464f56e3b 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -45,7 +45,7 @@ IOFileBlock::IOFileBlock(XrdOucCacheIO2 *io, XrdOucCacheStats &statsGlobal, Cach XrdOucCacheIO* IOFileBlock::Detach() { TRACEIO(Info, "IOFileBlock::Detach() " ); - XrdOucCacheIO * io = m_io; + XrdOucCacheIO * io = GetInput(); for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) @@ -62,7 +62,7 @@ XrdOucCacheIO* IOFileBlock::Detach() void IOFileBlock::GetBlockSizeFromPath() { const static std::string tag = "hdfsbsize="; - std::string path= m_io->Path(); + std::string path= GetInput()->Path(); size_t pos1 = path.find(tag); size_t t = tag.length(); if ( pos1 != path.npos) @@ -85,7 +85,7 @@ void IOFileBlock::GetBlockSizeFromPath() //______________________________________________________________________________ File* IOFileBlock::newBlockFile(long long off, int blocksize) { - XrdCl::URL url(m_io->Path()); + XrdCl::URL url(GetInput()->Path()); std::string fname = Cache::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); std::stringstream ss; @@ -101,7 +101,7 @@ File* IOFileBlock::newBlockFile(long long off, int blocksize) File* file; if (!(file = Cache::GetInstance().GetFileWithLocalPath(fname, this))) { - file = new File(m_io, fname, off, blocksize); + file = new File(this, fname, off, blocksize); Cache::GetInstance().AddActive(this, file); } @@ -146,15 +146,18 @@ bool IOFileBlock::ioActive() int IOFileBlock::Read (char *buff, long long off, int size) { // protect from reads over the file size - if (off >= m_io->FSize()) + + long long fileSize = GetInput()->FSize(); + + if (off >= fileSize) return 0; if (off < 0) { errno = EINVAL; return -1; } - if (off + size > m_io->FSize()) - size = m_io->FSize() - off; + if (off + size > fileSize) + size = fileSize - off; long long off0 = off; int idx_first = off0/m_blocksize; @@ -176,10 +179,10 @@ int IOFileBlock::Read (char *buff, long long off, int size) { size_t pbs = m_blocksize; // check if this is last block - int lastIOFileBlock = (m_io->FSize()-1)/m_blocksize; + int lastIOFileBlock = (fileSize-1)/m_blocksize; if (blockIdx == lastIOFileBlock ) { - pbs = m_io->FSize() - blockIdx*m_blocksize; + pbs = fileSize - blockIdx*m_blocksize; // TRACEIO(Dump, "IOFileBlock::Read() last block, change output file size to " << pbs); } diff --git a/src/XrdFileCache/XrdFileCacheTrace.hh b/src/XrdFileCache/XrdFileCacheTrace.hh index 979c87e3275..9bf4e44ecd4 100644 --- a/src/XrdFileCache/XrdFileCacheTrace.hh +++ b/src/XrdFileCache/XrdFileCacheTrace.hh @@ -33,7 +33,7 @@ #define TRACEIO(act, x) \ if (XRD_TRACE What >= TRACE_ ## act) \ - {XRD_TRACE Beg(m_traceID); cerr << TRACE_STR_##act <Path(); XRD_TRACE End();} + {XRD_TRACE Beg(m_traceID); cerr << TRACE_STR_##act <= TRACE_ ## act) \ diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index 1a5fbe69067..c903dc3bd0d 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -4,6 +4,7 @@ #include "XrdFileCacheInfo.hh" #include "XrdFileCacheStats.hh" +#include "XrdFileCacheIO.hh" #include "XrdOss/XrdOss.hh" #include "XrdCl/XrdClDefaultEnv.hh" @@ -89,7 +90,7 @@ int File::ReadV (const XrdOucIOVec *readV, int n) direct_handler = new DirectResponseHandler(1); // TODO check interface in the client file // m_input.VectorRead(chunkVec, (void*) 0, direct_handler); - m_input->ReadV(*direct_handler, &chunkVec[0], chunkVec.size()); + m_io->GetInput()->ReadV(*direct_handler, &chunkVec[0], chunkVec.size()); } } From 3f56363ace7aebec14ce9892168ae16c5a9aa942 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 10 Jun 2016 14:34:13 -0700 Subject: [PATCH 151/171] protect against invalid reads: check IO has relinquished File. --- src/XrdFileCache/XrdFileCache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index ced8ab84cab..897a53a952c 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -305,7 +305,7 @@ File* Cache::GetFileWithLocalPath(std::string path, IO* iIo) XrdSysMutexHelper lock(&m_active_mutex); for ( std::vector::iterator it = m_active.begin(); it != m_active.end(); ++it) { - if (!strcmp(path.c_str(), it->file->lPath())) + if (it->file && (!strcmp(path.c_str(), it->file->lPath()))) { File *ff = it->file; it->io->RelinquishFile(ff); From 7bc109d86e5cfef82c2c57f9fdc82c813ce7da91 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 10 Jun 2016 14:36:37 -0700 Subject: [PATCH 152/171] Remove debug print. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 9dcbf28ebb9..95b69c1592e 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -59,7 +59,6 @@ IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & } Cache::GetInstance().AddActive(this, m_file); - std::cout << " IOEntireFile::IOEntireFile " << this << std::endl; } From 63694a5ad21fe290c97505d92836ef9b829c409e Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Mon, 13 Jun 2016 10:40:45 -0700 Subject: [PATCH 153/171] Add XrdFileCacheIO.cc source --- src/XrdFileCache.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache.cmake b/src/XrdFileCache.cmake index 2dc0d84bea9..37b44045748 100644 --- a/src/XrdFileCache.cmake +++ b/src/XrdFileCache.cmake @@ -23,7 +23,7 @@ add_library( XrdFileCache/XrdFileCacheVRead.cc XrdFileCache/XrdFileCacheStats.hh XrdFileCache/XrdFileCacheInfo.cc XrdFileCache/XrdFileCacheInfo.hh - XrdFileCache/XrdFileCacheIO.hh + XrdFileCache/XrdFileCacheIO.cc XrdFileCache/XrdFileCacheIO.hh XrdFileCache/XrdFileCacheIOEntireFile.cc XrdFileCache/XrdFileCacheIOEntireFile.hh XrdFileCache/XrdFileCacheIOFileBlock.cc XrdFileCache/XrdFileCacheIOFileBlock.hh XrdFileCache/XrdFileCacheDecision.hh) From 3c56d49cf636899eedd14f518169a453d0b42082 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 15 Jun 2016 09:58:35 -0700 Subject: [PATCH 154/171] In destructor properly check if Info struct is created. --- src/XrdFileCache/XrdFileCacheFile.cc | 46 ++++++++++++++++------------ 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index a2665d5ce59..7e8ea60bc5d 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -104,20 +104,29 @@ void File::BlockRemovedFromWriteQ(Block* b) File::~File() { - m_syncStatusMutex.Lock(); - bool needs_sync = ! m_writes_during_sync.empty(); - m_syncStatusMutex.UnLock(); - if (needs_sync || m_non_flushed_cnt > 0) + if (m_infoFile) { - Sync(); - m_cfi.WriteHeader(m_infoFile); - } - // write statistics in *cinfo file - AppendIOStatToFileInfo(); - m_infoFile->Fsync(); + m_syncStatusMutex.Lock(); - delete m_syncer; - m_syncer = NULL; + bool needs_sync = ! m_writes_during_sync.empty(); + m_syncStatusMutex.UnLock(); + if (needs_sync || m_non_flushed_cnt > 0) + { + Sync(); + m_cfi.WriteHeader(m_infoFile); + } + + // write statistics in *cinfo file + AppendIOStatToFileInfo(); + m_infoFile->Fsync(); + + m_syncStatusMutex.UnLock(); + + + m_infoFile->Close(); + delete m_infoFile; + m_infoFile = NULL; + } if (m_output) { @@ -125,12 +134,9 @@ File::~File() delete m_output; m_output = NULL; } - if (m_infoFile) - { - m_infoFile->Close(); - delete m_infoFile; - m_infoFile = NULL; - } + + delete m_syncer; + m_syncer = NULL; // print just for curiosity TRACEF(Debug, "File::~File() ended, prefetch score = " << m_prefetchScore); @@ -215,7 +221,7 @@ bool File::Open() int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0600, myEnv); if (res < 0) { - TRACEF(Error, "File::Open() can't open data file"); + TRACEF(Error, "File::Open() can't open data file, " << strerror(errno)); delete m_output; m_output = 0; return false; @@ -241,7 +247,7 @@ bool File::Open() int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv); if (res < 0) { - TRACEF(Error, "File::Open() can't open info file"); + TRACEF(Error, "File::Open() can't open info file, " << strerror(errno)); delete m_infoFile; m_infoFile = 0; return false; From 73933db20529b848b8f62a31d8747605999ec749 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 15 Jun 2016 10:22:31 -0700 Subject: [PATCH 155/171] Check XrdOss::Create() return value in File::Open(). --- src/XrdFileCache/XrdFileCacheFile.cc | 90 +++++++++++++--------------- 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 7e8ea60bc5d..ae257135a5b 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -214,24 +214,21 @@ bool File::Open() XrdOss &m_output_fs = *Cache::GetInstance().GetOss(); // Create the data file itself. XrdOucEnv myEnv; - m_output_fs.Create(Cache::GetInstance().RefConfiguration().m_username.c_str(), m_temp_filename.c_str(), 0600, myEnv, XRDOSS_mkpath); - m_output = m_output_fs.newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); - if (m_output) - { - int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0600, myEnv); - if (res < 0) - { - TRACEF(Error, "File::Open() can't open data file, " << strerror(errno)); - delete m_output; - m_output = 0; - return false; - } + if ( m_output_fs.Create(Cache::GetInstance().RefConfiguration().m_username.c_str(), m_temp_filename.c_str(), 0600, myEnv, XRDOSS_mkpath) !=XrdOssOK) + { + TRACEF(Error, "File::Open() can't create data file, " << strerror(errno)); + return false; } - else + + m_output = m_output_fs.newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); + if (m_output->Open(m_temp_filename.c_str(), O_RDWR, 0600, myEnv) != XrdOssOK) { - TRACEF(Error, "File::Open() can't get file handle"); + TRACEF(Error, "File::Open() can't get FD for data file, " << strerror(errno)); + delete m_output; + m_output = 0; return false; } + // Create the info file std::string ifn = m_temp_filename + Info::m_infoExtension; @@ -239,46 +236,41 @@ bool File::Open() struct stat infoStat; bool fileExisted = (Cache::GetInstance().GetOss()->Stat(ifn.c_str(), &infoStat) == XrdOssOK); - m_output_fs.Create(Cache::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0600, myEnv, XRDOSS_mkpath); - m_infoFile = m_output_fs.newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); - if (m_infoFile) + if (m_output_fs.Create(Cache::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0600, myEnv, XRDOSS_mkpath) != XrdOssOK) { - if (fileExisted) assert(infoStat.st_size > 0); - int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv); - if (res < 0) - { - TRACEF(Error, "File::Open() can't open info file, " << strerror(errno)); - delete m_infoFile; - m_infoFile = 0; - return false; - } - else { - if (fileExisted) - { - int res = m_cfi.Read(m_infoFile); - TRACEF(Debug, "Reading existing info file bytes = " << res); - m_downloadCond.Lock(); - // this method is called from constructor, no need to lock downloadStaus - bool complete = m_cfi.IsComplete(); - if (complete) m_prefetchState = kComplete; - m_downloadCond.UnLock(); - } - else { - m_fileSize = m_fileSize; - int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; - TRACEF(Debug, "Creating new file info, data size = " << m_fileSize << " num blocks = " << ss); - m_cfi.SetBufferSize(Cache::GetInstance().RefConfiguration().m_bufferSize); - m_cfi.SetFileSize(m_fileSize); - m_cfi.WriteHeader(m_infoFile); - m_infoFile->Fsync(); - } - } + TRACEF(Error, "File::Open() can't create info file, " << strerror(errno)); + return false; } - else + m_infoFile = m_output_fs.newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); + if (fileExisted) assert(infoStat.st_size > 0); + if (m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv) != XrdOssOK) { - // this should be a rare case wher FD can't be created + TRACEF(Error, "File::Open() can't get FD info file, " << strerror(errno)); + delete m_infoFile; + m_infoFile = 0; return false; } + + if (fileExisted) + { + int res = m_cfi.Read(m_infoFile); + TRACEF(Debug, "Reading existing info file bytes = " << res); + m_downloadCond.Lock(); + // this method is called from constructor, no need to lock downloadStaus + bool complete = m_cfi.IsComplete(); + if (complete) m_prefetchState = kComplete; + m_downloadCond.UnLock(); + } + else { + m_fileSize = m_fileSize; + int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; + TRACEF(Debug, "Creating new file info, data size = " << m_fileSize << " num blocks = " << ss); + m_cfi.SetBufferSize(Cache::GetInstance().RefConfiguration().m_bufferSize); + m_cfi.SetFileSize(m_fileSize); + m_cfi.WriteHeader(m_infoFile); + m_infoFile->Fsync(); + } + if (m_prefetchState != kComplete) cache()->RegisterPrefetchFile(this); return true; } From 59b130dc5c9988c774183ac022e8e73e93fc8aee Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 15 Jun 2016 10:27:03 -0700 Subject: [PATCH 156/171] Replace assert with an error message. --- src/XrdFileCache/XrdFileCacheFile.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index ae257135a5b..cd287208fd6 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -236,13 +236,18 @@ bool File::Open() struct stat infoStat; bool fileExisted = (Cache::GetInstance().GetOss()->Stat(ifn.c_str(), &infoStat) == XrdOssOK); + // AMT: the folowing below is a sanity check, it is not expected to happen. Could be an assert + if (fileExisted && (infoStat.st_size == 0)) { + TRACEF(Error, "File::Open() info file stored zero data file size"); + return false; + } + if (m_output_fs.Create(Cache::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0600, myEnv, XRDOSS_mkpath) != XrdOssOK) { TRACEF(Error, "File::Open() can't create info file, " << strerror(errno)); return false; } m_infoFile = m_output_fs.newFile(Cache::GetInstance().RefConfiguration().m_username.c_str()); - if (fileExisted) assert(infoStat.st_size > 0); if (m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv) != XrdOssOK) { TRACEF(Error, "File::Open() can't get FD info file, " << strerror(errno)); From 9e1a9e3cc078c296ef702865bc5c00bbca7c0175 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 15 Jun 2016 10:41:57 -0700 Subject: [PATCH 157/171] Add error message in initCachedStat, if open of info file fails --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 95b69c1592e..9f2e734783b 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -120,6 +120,9 @@ int IOEntireFile::initCachedStat(const char* path) TRACEIO(Error, "IOEntireFile::initCachedStat failed to read file size from info file"); } } + else { + TRACEIO(Error, "IOEntireFile::initCachedStat can't open info file " << strerror(errno)); + } infoFile->Close(); delete infoFile; } From f39041e3fe330cdc4e995e0df61fbd29acf27997 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Wed, 15 Jun 2016 10:52:15 -0700 Subject: [PATCH 158/171] Add warning with errno if can't open info file. --- src/XrdFileCache/XrdFileCachePurge.cc | 62 ++++++++++++++------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/src/XrdFileCache/XrdFileCachePurge.cc b/src/XrdFileCache/XrdFileCachePurge.cc index d880a003fb2..b22aaf67d44 100644 --- a/src/XrdFileCache/XrdFileCachePurge.cc +++ b/src/XrdFileCache/XrdFileCachePurge.cc @@ -86,42 +86,46 @@ void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& { // XXXX MT - shouldn't we also check if it is currently opened? - fh->Open(np.c_str(), O_RDONLY, 0600, env); - Info cinfo(Cache::GetInstance().GetTrace()); - time_t accessTime; - cinfo.Read(fh); - if (cinfo.GetLatestDetachTime(accessTime, fh)) - { - TRACE(Dump, "FillFileMapRecurse() checking " << buff << " accessTime " << accessTime); - purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); - } - else - { - // cinfo file does not contain any known accesses, use stat.mtime instead. - - TRACE(Warning, "FillFileMapRecurse() could not get access time for " << np << ", trying stat"); - - XrdOss* oss = Cache::GetInstance().GetOss(); - struct stat fstat; - - if (oss->Stat(np.c_str(), &fstat) == XrdOssOK) + if (fh->Open(np.c_str(), O_RDONLY, 0600, env) == XrdOssOK) { + Info cinfo(Cache::GetInstance().GetTrace()); + time_t accessTime; + cinfo.Read(fh); + if (cinfo.GetLatestDetachTime(accessTime, fh)) { - accessTime = fstat.st_mtime; - TRACE(Dump, "FillFileMapRecurse() have access time for " << np << " via stat: " << accessTime); + TRACE(Dump, "FillFileMapRecurse() checking " << buff << " accessTime " << accessTime); purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); } else { - // This really shouldn't happen ... but if it does remove cinfo and the data file right away. - - TRACE(Warning, "FillFileMapRecurse() could not get access time for " << np); - oss->Unlink(np.c_str()); - np = np.substr(0, np.size() - strlen(XrdFileCache::Info::m_infoExtension)); - oss->Unlink(np.c_str()); + // cinfo file does not contain any known accesses, use stat.mtime instead. + + TRACE(Warning, "FillFileMapRecurse() could not get access time for " << np << ", trying stat"); + + XrdOss* oss = Cache::GetInstance().GetOss(); + struct stat fstat; + + if (oss->Stat(np.c_str(), &fstat) == XrdOssOK) + { + accessTime = fstat.st_mtime; + TRACE(Dump, "FillFileMapRecurse() have access time for " << np << " via stat: " << accessTime); + purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); + } + else + { + // This really shouldn't happen ... but if it does remove cinfo and the data file right away. + + TRACE(Warning, "FillFileMapRecurse() could not get access time for " << np); + oss->Unlink(np.c_str()); + np = np.substr(0, np.size() - strlen(XrdFileCache::Info::m_infoExtension)); + oss->Unlink(np.c_str()); + } } } + else { + TRACE(Warning, "FillFileMapRecurse() cant open " << np << " " << strerror(errno)); + } } - else if (dh->Opendir(np.c_str(), env) >= 0) + else if (dh->Opendir(np.c_str(), env) == XrdOssOK) { FillFileMapRecurse(dh, np, purgeState); } @@ -166,7 +170,7 @@ void Cache::CacheDirCleanup() { // make a sorted map of file patch by access time XrdOssDF* dh = oss->newDir(m_configuration.m_username.c_str()); - if (dh->Opendir(m_configuration.m_cache_dir.c_str(), env) >= 0) + if (dh->Opendir(m_configuration.m_cache_dir.c_str(), env) == XrdOssOK) { FPurgeState purgeState(bytesToRemove * 5 / 4); // prepare 20% more volume than required From 9118903f75457239f23afdf6ed4207fff0a9c559 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Thu, 16 Jun 2016 03:20:10 -0700 Subject: [PATCH 159/171] Add Location() method to CacheIO2 to get file location. --- src/XrdOuc/XrdOucCache2.hh | 9 +++++++++ src/XrdPosix/XrdPosixCacheBC.hh | 3 +++ src/XrdPosix/XrdPosixFile.cc | 31 +++++++++++++++++++++++++++++-- src/XrdPosix/XrdPosixFile.hh | 5 ++++- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/XrdOuc/XrdOucCache2.hh b/src/XrdOuc/XrdOucCache2.hh index a27396f5e28..f7487056df1 100644 --- a/src/XrdOuc/XrdOucCache2.hh +++ b/src/XrdOuc/XrdOucCache2.hh @@ -76,6 +76,15 @@ public: virtual int Fstat(struct stat &sbuff) {(void)sbuff; return 1;} +//----------------------------------------------------------------------------- +//! Get the file's location (i.e. endpoint hostname and port) +//! +//! @return A pointer to the file's location. It remains valid until the file +//! is closed. A null string means the file is not open or is unknown. +//----------------------------------------------------------------------------- +virtual +const char *Location() {return "";} + //------------------------------------------------------------------------------ //! Perform an asynchronous read (defaults to synchrnous). //! diff --git a/src/XrdPosix/XrdPosixCacheBC.hh b/src/XrdPosix/XrdPosixCacheBC.hh index 54d05125876..20b65ff88d6 100644 --- a/src/XrdPosix/XrdPosixCacheBC.hh +++ b/src/XrdPosix/XrdPosixCacheBC.hh @@ -55,6 +55,9 @@ long long FSize() {return cacheIO1->FSize();} virtual int Fstat(struct stat &buf) {return cacheIO2->Fstat(buf);} +virtual +const char *Location() {return cacheIO2->Location();} + virtual const char *Path() {return cacheIO1->Path();} diff --git a/src/XrdPosix/XrdPosixFile.cc b/src/XrdPosix/XrdPosixFile.cc index 988af11d8e3..cf245f3f6a7 100644 --- a/src/XrdPosix/XrdPosixFile.cc +++ b/src/XrdPosix/XrdPosixFile.cc @@ -59,6 +59,8 @@ pthread_t tid; XrdSysThread::Run(&tid, XrdPosixFile::DelayedDestroy, 0, 0, "PosixFileDestroy"); return (XrdPosixFile *)0; } + +std::string dsProperty("DataServer"); }; XrdSysSemaphore XrdPosixFile::ddSem(0); @@ -77,7 +79,7 @@ XrdPosixFile::XrdPosixFile(const char *path, XrdPosixCallBack *cbP, int Opts) : XCio((XrdOucCacheIO2 *)this), PrepIO(0), mySize(0), myMtime(0), myInode(0), myMode(0), theCB(cbP), - fPath(strdup(path)), + fPath(strdup(path)), fLoc(0), cOpt(0), isStream(Opts & isStrm ? 1 : 0) { @@ -113,9 +115,10 @@ XrdPosixFile::~XrdPosixFile() // if (PrepIO) delete PrepIO; -// Free the path +// Free the path and location information // if (fPath) free(fPath); + if (fLoc) free(fLoc); } /******************************************************************************/ @@ -300,6 +303,30 @@ void XrdPosixFile::HandleResponse(XrdCl::XRootDStatus *status, if (rc) delete this; } +/******************************************************************************/ +/* L o c a t i o n */ +/******************************************************************************/ + +const char *XrdPosixFile::Location() +{ + +// If the file is not open, then we have no location +// + if (!clFile.IsOpen()) return 0; + +// If we have no location info, get it +// + if (!fLoc) + {std::string currNode; + if (clFile.GetProperty(dsProperty, currNode)) + fLoc = strdup(currNode.c_str()); + } + +// Return location information +// + return fLoc; +} + /******************************************************************************/ /* R e a d */ /******************************************************************************/ diff --git a/src/XrdPosix/XrdPosixFile.hh b/src/XrdPosix/XrdPosixFile.hh index 1f12025494b..6116b147def 100644 --- a/src/XrdPosix/XrdPosixFile.hh +++ b/src/XrdPosix/XrdPosixFile.hh @@ -76,7 +76,7 @@ XrdCl::File clFile; return retOffset; } -static XrdPosixFile *Alloc(const char *path, XrdPosixCallBack *cbP, int Opts); +//atic XrdPosixFile *Alloc(const char *path, XrdPosixCallBack *cbP, int Opts); static void* DelayedDestroy(void*); @@ -94,6 +94,8 @@ static void DelayedDestroy(XrdPosixFile *fp); int Fstat(struct stat &buf); + const char *Location(); + void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response); @@ -180,6 +182,7 @@ union {long long currOffset; }; char *fPath; +char *fLoc; union {int cOpt; int numTries;}; char isStream; }; From 8eca724b1e1d1bde94e70caf7995c40a56fc34bb Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 16 Jun 2016 09:56:14 -0700 Subject: [PATCH 160/171] Fix in Detach for hdfs mode. --- src/XrdFileCache/XrdFileCache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 897a53a952c..8adbf650c90 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -188,7 +188,7 @@ void Cache::Detach(XrdOucCacheIO* io) delete it->file; } m_active.erase(it); - break; + (m_configuration.m_hdfsmode == false) break; } else ++it; From 9bfd662dfa60c0710a49e60498a9cedf919556cd Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 16 Jun 2016 10:00:32 -0700 Subject: [PATCH 161/171] Fix compilation error from the previous commit --- src/XrdFileCache/XrdFileCache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 8adbf650c90..a748d2a2f74 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -188,7 +188,7 @@ void Cache::Detach(XrdOucCacheIO* io) delete it->file; } m_active.erase(it); - (m_configuration.m_hdfsmode == false) break; + if (m_configuration.m_hdfsmode == false) break; } else ++it; From 56e86a795263a958908a6b66cc946235c3e73d00 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 16 Jun 2016 10:01:43 -0700 Subject: [PATCH 162/171] Fix index offset i Sync(). This applies only to hdfs healing mode. --- src/XrdFileCache/XrdFileCacheFile.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index cd287208fd6..bfc60bbe88a 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -268,12 +268,12 @@ bool File::Open() } else { m_fileSize = m_fileSize; - int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; - TRACEF(Debug, "Creating new file info, data size = " << m_fileSize << " num blocks = " << ss); m_cfi.SetBufferSize(Cache::GetInstance().RefConfiguration().m_bufferSize); m_cfi.SetFileSize(m_fileSize); m_cfi.WriteHeader(m_infoFile); m_infoFile->Fsync(); + int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; + TRACEF(Debug, "Creating new file info, data size = " << m_fileSize << " num blocks = " << ss); } if (m_prefetchState != kComplete) cache()->RegisterPrefetchFile(this); @@ -753,7 +753,7 @@ void File::Sync() XrdSysMutexHelper _lck(&m_syncStatusMutex); for (std::vector::iterator i = m_writes_during_sync.begin(); i != m_writes_during_sync.end(); ++i) { - m_cfi.SetBitWriteCalled(offsetIdx(*i)); + m_cfi.SetBitWriteCalled(*i); } written_while_in_sync = m_non_flushed_cnt = (int) m_writes_during_sync.size(); m_writes_during_sync.clear(); From 39734236539ee57ba182ae37607f7249a1e6c6b3 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Thu, 16 Jun 2016 11:40:21 -0700 Subject: [PATCH 163/171] Change in severity of read messages. --- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index cf464f56e3b..58309a2ef14 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -163,7 +163,7 @@ int IOFileBlock::Read (char *buff, long long off, int size) int idx_first = off0/m_blocksize; int idx_last = (off0+size-1)/m_blocksize; int bytes_read = 0; - TRACEIO(Debug, "IOFileBlock::Read() "<< off << "@" << size << " block range ["<< idx_first << ", " << idx_last << "]"); + TRACEIO(Dump, "IOFileBlock::Read() "<< off << "@" << size << " block range ["<< idx_first << ", " << idx_last << "]"); for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx ) { @@ -212,7 +212,7 @@ int IOFileBlock::Read (char *buff, long long off, int size) } assert(readBlockSize); - TRACEIO(Debug, "IOFileBlock::Read() block[ " << blockIdx << "] read-block-size[" << readBlockSize << "], offset[" << readBlockSize << "] off = " << off ); + TRACEIO(Dump, "IOFileBlock::Read() block[ " << blockIdx << "] read-block-size[" << readBlockSize << "], offset[" << readBlockSize << "] off = " << off ); long long min = blockIdx*m_blocksize; if ( off < min) { assert(0); } From 2b3a370e8b6138a2b733957673cd3a4a5e844567 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Thu, 16 Jun 2016 22:38:07 -0700 Subject: [PATCH 164/171] Add location printout into Attach() and Update(). --- src/XrdFileCache/XrdFileCache.cc | 3 ++- src/XrdFileCache/XrdFileCacheIO.cc | 11 +++++------ src/XrdFileCache/XrdFileCacheIO.hh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index a748d2a2f74..29dae2819da 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -155,7 +155,8 @@ XrdOucCacheIO2 *Cache::Attach(XrdOucCacheIO2 *io, int Options) else cio = new IOEntireFile(io, m_stats, *this); - TRACE(Info, "Cache::Attach() " << io->Path()); + TRACE(Info, "Cache::Attach() " << io->Path() << " location: " << + (io->Location() && io->Location()[0] != 0) ? io->Location() : ""); return cio; } else diff --git a/src/XrdFileCache/XrdFileCacheIO.cc b/src/XrdFileCache/XrdFileCacheIO.cc index 3eefceaa685..142c028a6f6 100644 --- a/src/XrdFileCache/XrdFileCacheIO.cc +++ b/src/XrdFileCache/XrdFileCacheIO.cc @@ -1,5 +1,5 @@ #include "XrdFileCacheIO.hh" -#include "XrdSys/XrdSysAtomics.hh" +#include "XrdFileCacheTrace.hh" #include "XrdPosix/XrdPosixFile.hh" using namespace XrdFileCache; @@ -13,19 +13,18 @@ m_statsGlobal(stats), m_cache(cache), m_traceID("IO"), m_io(io) void IO::Update(XrdOucCacheIO2 &iocp) { SetInput(&iocp); + TRACE(Info, "IO::Update() " << Path() << " location: " << + (Location() && Location()[0] != 0) ? Location() : ""); } - void IO::SetInput(XrdOucCacheIO2* x) { - updMutex.Lock(); + XrdSysMutexHelper lock(&updMutex); m_io = x; - updMutex.UnLock(); } XrdOucCacheIO2* IO::GetInput() { - AtomicBeg(updMutex); + XrdSysMutexHelper lock(&updMutex); return m_io; - AtomicEnd(updMutex); } diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index 73376bc38b8..aadfa824231 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -52,7 +52,7 @@ namespace XrdFileCache private: XrdOucCacheIO2 *m_io; //!< original data source - XrdSysRecMutex updMutex; + XrdSysMutex updMutex; void SetInput(XrdOucCacheIO2*); }; } From e3118d8e37a02bd686f455137417872613974b89 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Fri, 17 Jun 2016 00:44:54 -0700 Subject: [PATCH 165/171] Remove unnecessary cast. --- src/XrdPosix/XrdPosixPrepIO.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdPosix/XrdPosixPrepIO.cc b/src/XrdPosix/XrdPosixPrepIO.cc index e1f26c99cfc..0c4b93b62d1 100644 --- a/src/XrdPosix/XrdPosixPrepIO.cc +++ b/src/XrdPosix/XrdPosixPrepIO.cc @@ -95,6 +95,6 @@ bool XrdPosixPrepIO::Init(XrdOucCacheIOCB *iocbP) // Inform the cache that we have now have a new I/O object // - fileP->XCio->Update((XrdOucCacheIO2 &)*fileP); + fileP->XCio->Update(*fileP); return true; } From 170580600597c7aad92aa20801cc53ecd31b2d26 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Fri, 17 Jun 2016 00:45:17 -0700 Subject: [PATCH 166/171] Improve printing of Location() in Info trace. --- src/XrdFileCache/XrdFileCache.cc | 5 +++-- src/XrdFileCache/XrdFileCacheIO.cc | 5 +++-- src/XrdFileCache/XrdFileCacheTrace.hh | 11 ++++++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 29dae2819da..26fc24c17fa 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -155,8 +155,9 @@ XrdOucCacheIO2 *Cache::Attach(XrdOucCacheIO2 *io, int Options) else cio = new IOEntireFile(io, m_stats, *this); - TRACE(Info, "Cache::Attach() " << io->Path() << " location: " << - (io->Location() && io->Location()[0] != 0) ? io->Location() : ""); + TRACE_PC(Info, const char* loc = io->Location(), + "Cache::Attach() " << io->Path() << " location: " << + ((loc && loc[0] != 0) ? loc : "")); return cio; } else diff --git a/src/XrdFileCache/XrdFileCacheIO.cc b/src/XrdFileCache/XrdFileCacheIO.cc index 142c028a6f6..6e00288544b 100644 --- a/src/XrdFileCache/XrdFileCacheIO.cc +++ b/src/XrdFileCache/XrdFileCacheIO.cc @@ -13,8 +13,9 @@ m_statsGlobal(stats), m_cache(cache), m_traceID("IO"), m_io(io) void IO::Update(XrdOucCacheIO2 &iocp) { SetInput(&iocp); - TRACE(Info, "IO::Update() " << Path() << " location: " << - (Location() && Location()[0] != 0) ? Location() : ""); + TRACE_PC(Info, const char* loc = Location(), + "IO::Update() " << Path() << " location: " << + ((loc && loc[0] != 0) ? loc : "")); } void IO::SetInput(XrdOucCacheIO2* x) diff --git a/src/XrdFileCache/XrdFileCacheTrace.hh b/src/XrdFileCache/XrdFileCacheTrace.hh index 9bf4e44ecd4..55118f5e4b2 100644 --- a/src/XrdFileCache/XrdFileCacheTrace.hh +++ b/src/XrdFileCache/XrdFileCacheTrace.hh @@ -31,6 +31,10 @@ if (XRD_TRACE What >= TRACE_ ## act) \ {XRD_TRACE Beg(m_traceID); cerr << TRACE_STR_##act <= TRACE_ ## act) \ + {pre_code; XRD_TRACE Beg(m_traceID); cerr << TRACE_STR_##act <= TRACE_ ## act) \ {XRD_TRACE Beg(m_traceID); cerr << TRACE_STR_##act <= TRACE_ ## act) \ {XRD_TRACE Beg(m_traceID); cerr << TRACE_STR_##act << x << " " << GetLocalPath(); XRD_TRACE End();} - - - #else #define TRACE(act,x) +#define TRACE_PC(act, pre_code, x) +#define TRACEIO(act, x) +#define TRACEF(act, x) + #endif #endif From 6b2bab1448877aac501f9178d4cb884bedf97e96 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 17 Jun 2016 13:26:21 -0700 Subject: [PATCH 167/171] Remove obsolete method. --- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 5 ----- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 13 ------------- 2 files changed, 18 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 9f2e734783b..d2f8d20500c 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -157,11 +157,6 @@ XrdOucCacheIO *IOEntireFile::Detach() return io; } -void IOEntireFile::Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen) -{ - iocb.Done(IOEntireFile::Read(buff, offs, rlen)); -} - int IOEntireFile::Read (char *buff, long long off, int size) { TRACEIO(Dump, "IOEntireFile::Read() "<< this << " off: " << off << " size: " << size ); diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index 46f1048ef39..132281efcd3 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -61,19 +61,6 @@ namespace XrdFileCache //--------------------------------------------------------------------- virtual int Read(char *Buffer, long long Offset, int Length); - //--------------------------------------------------------------------- - //! Asynchronous read. - //! - //! @param callback - //! @param Buffer - //! @param Offset - //! @param Length - //! - //! @return number of bytes read - //--------------------------------------------------------------------- - virtual void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen); - - //--------------------------------------------------------------------- //! Pass ReadV request to the corresponding File object. //! From 71afe1de18c07e522073837d01592f040c291b43 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 17 Jun 2016 13:31:32 -0700 Subject: [PATCH 168/171] Remove trace mesage in Detach becuse similar message is printed already in manager. --- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 58309a2ef14..02caf71d210 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -44,9 +44,7 @@ IOFileBlock::IOFileBlock(XrdOucCacheIO2 *io, XrdOucCacheStats &statsGlobal, Cach //______________________________________________________________________________ XrdOucCacheIO* IOFileBlock::Detach() { - TRACEIO(Info, "IOFileBlock::Detach() " ); - XrdOucCacheIO * io = GetInput(); - + XrdOucCacheIO * io = GetInput(); for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { From 8dfb6d86c23705bf47a6a11f1b030d044670feb5 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Fri, 17 Jun 2016 14:42:19 -0700 Subject: [PATCH 169/171] Store access statistics in bytes instead of number of blocks --- src/XrdFileCache/XrdFileCacheFile.cc | 9 +++------ src/XrdFileCache/XrdFileCacheVRead.cc | 5 +++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index bfc60bbe88a..f57ecd986a6 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -421,10 +421,10 @@ int File::ReadBlocksFromDisk(std::list& blocks, total += rs; - CheckPrefetchStatDisk(*ii); } + m_stats.m_BytesDisk += total; return total; } @@ -468,14 +468,12 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) inc_ref_count(bi->second); TRACEF(Dump, "File::Read() " << iUserBuff << "inc_ref_count for existing block << " << bi->second << " idx = " << block_idx); blks_to_process.push_front(bi->second); - m_stats.m_BytesRam++; // AMT what if block fails } // On disk? else if (m_cfi.TestBit(offsetIdx(block_idx))) { TRACEF(Dump, "File::Read() read from disk " << (void*)iUserBuff << " idx = " << block_idx); blks_on_disk.push_back(block_idx); - m_stats.m_BytesDisk++; } // Then we have to get it ... else @@ -492,14 +490,12 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) } inc_ref_count(b); blks_to_process.push_back(b); - m_stats.m_BytesRam++; } // Nope ... read this directly without caching. else { TRACEF(Dump, "File::Read() direct block " << block_idx); blks_direct.push_back(block_idx); - m_stats.m_BytesMissed++; } } @@ -604,7 +600,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) TRACEF(Dump, "File::Read() ub=" << (void*)iUserBuff << " from finished block " << (*bi)->m_offset/BS << " size " << size_to_copy); memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); bytes_read += size_to_copy; - + m_stats.m_BytesRam += size_to_copy; CheckPrefetchStatRAM(*bi); } else // it has failed ... krap up. @@ -635,6 +631,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) if (direct_handler->m_errno == 0) { bytes_read += direct_size; + m_stats.m_BytesMissed += direct_size; } else { diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index c903dc3bd0d..f0ac1d1936a 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -125,7 +125,10 @@ int File::ReadV (const XrdOucIOVec *readV, int n) if (direct_handler->m_errno == 0) { for (std::vector::iterator i = chunkVec.begin(); i != chunkVec.end(); ++i) + { bytesRead += i->size; + m_stats.m_BytesMissed += i->size; + } } else { @@ -234,6 +237,7 @@ int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blo int rs = m_output->Read(readV[chunkIdx].data + off, blockIdx*m_cfi.GetBufferSize() + blk_off - m_offset, size); if (rs >=0 ) { bytes_read += rs; + m_stats.m_BytesDisk += rs; } else { // ofs read should set the errno @@ -295,6 +299,7 @@ int File::VReadProcessBlocks(const XrdOucIOVec *readV, int n, overlap(block_idx, m_cfi.GetBufferSize(), readV[*chunkIt].offset, readV[*chunkIt].size, off, blk_off, size); memcpy(readV[*chunkIt].data + off, &(bi->block->m_buff[blk_off]), size); bytes_read += size; + m_stats.m_BytesRam += size; } } else { From eaa72a2fc64da5ddbf903b9cbdf8b5145fadd894 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Sat, 18 Jun 2016 21:33:38 -0700 Subject: [PATCH 170/171] Bugfix in hdfs healing mode. --- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 58309a2ef14..aee810f76b4 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -119,13 +119,9 @@ void IOFileBlock::RelinquishFile(File* f) { if (it->second == f) { - m_blocks.erase(it++); + m_blocks.erase(it); break; } - else - { - ++it; - } } } From 07f90411ee18248901b2108e7e0ca10e91affa98 Mon Sep 17 00:00:00 2001 From: Alja Mrak-Tadel Date: Tue, 21 Jun 2016 14:43:58 -0700 Subject: [PATCH 171/171] Implement defered open for hdfs healing mode. --- src/XrdFileCache/XrdFileCache.cc | 47 +++++----- src/XrdFileCache/XrdFileCacheIO.hh | 3 - src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 97 ++++++++++++++++++++- src/XrdFileCache/XrdFileCacheIOFileBlock.hh | 6 ++ 4 files changed, 124 insertions(+), 29 deletions(-) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 26fc24c17fa..708a003c8fe 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -399,7 +399,8 @@ Cache::Prepare(const char *url, int oflags, mode_t mode) { std::string curl(url); XrdCl::URL xx(curl); - const std::string& spath = xx.GetPath(); + std::string spath = xx.GetPath(); + spath += ".cinfo"; struct stat buf; int res = m_output_fs->Stat(spath.c_str(), &buf); @@ -422,34 +423,32 @@ Cache::Prepare(const char *url, int oflags, mode_t mode) int Cache::Stat(const char *curl, struct stat &sbuff) { - if (m_configuration.m_hdfsmode == false) - { - XrdCl::URL url(curl); - std::string name = url.GetPath(); + XrdCl::URL url(curl); + std::string name = url.GetPath(); + name += ".cinfo"; - if (m_output_fs->Stat(name.c_str(), &sbuff) == XrdOssOK) { - if ( S_ISDIR(sbuff.st_mode)) { - return 0; - } - else { - bool success = false; - XrdOssDF* infoFile = m_output_fs->newFile(m_configuration.m_username.c_str()); - XrdOucEnv myEnv; - name += ".cinfo"; - int res = infoFile->Open(name.c_str(), O_RDONLY, 0600, myEnv); - if (res >= 0) { - Info info(m_trace, 0); - if (info.Read(infoFile) > 0) { - sbuff.st_size = info.GetFileSize(); - success = true; - } + if (m_output_fs->Stat(name.c_str(), &sbuff) == XrdOssOK) { + if ( S_ISDIR(sbuff.st_mode)) { + return 0; + } + else { + bool success = false; + XrdOssDF* infoFile = m_output_fs->newFile(m_configuration.m_username.c_str()); + XrdOucEnv myEnv; + int res = infoFile->Open(name.c_str(), O_RDONLY, 0600, myEnv); + if (res >= 0) { + Info info(m_trace, 0); + if (info.Read(infoFile) > 0) { + sbuff.st_size = info.GetFileSize(); + success = true; } - infoFile->Close(); - delete infoFile; - return success ? 0 : 1; } + infoFile->Close(); + delete infoFile; + return success ? 0 : 1; } } + return 1; } diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index aadfa824231..15e21624065 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -21,9 +21,6 @@ namespace XrdFileCache //! Original data source. virtual XrdOucCacheIO *Base() { return m_io; } - //! Original data source URL. - virtual long long FSize() { return m_io->FSize(); } - //! Original data source URL. virtual const char *Path() { return m_io->Path(); } diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 110a65e353b..eba7243ddf6 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include "XrdFileCacheIOFileBlock.hh" #include "XrdFileCache.hh" @@ -30,15 +31,17 @@ #include "XrdSys/XrdSysError.hh" #include "XrdSfs/XrdSfsInterface.hh" +#include "XrdOuc/XrdOucEnv.hh" using namespace XrdFileCache; //______________________________________________________________________________ IOFileBlock::IOFileBlock(XrdOucCacheIO2 *io, XrdOucCacheStats &statsGlobal, Cache & cache) - : IO(io, statsGlobal, cache) + : IO(io, statsGlobal, cache), m_localStat(0) { m_blocksize = Cache::GetInstance().RefConfiguration().m_hdfsbsize; GetBlockSizeFromPath(); + initLocalStat(); } //______________________________________________________________________________ @@ -102,10 +105,100 @@ File* IOFileBlock::newBlockFile(long long off, int blocksize) file = new File(this, fname, off, blocksize); Cache::GetInstance().AddActive(this, file); } + else { + file->WakeUp(); + } return file; } + +//______________________________________________________________________________ +int IOFileBlock::FStat(struct stat &sbuff) +{ + // local stat is create in constructor. if file was on disk before + // attach that the only way stat was not successful is becuse there + // were info file read errors + if (!m_localStat) return -1; + + memcpy(&sbuff, m_localStat, sizeof(struct stat)); + return 0; +} + +//______________________________________________________________________________ +long long IOFileBlock::FSize() +{ + if (!m_localStat) return -1; + + return m_localStat->st_size; +} + +//______________________________________________________________________________ +int IOFileBlock::initLocalStat() +{ + XrdCl::URL url(GetPath()); + std::string path = url.GetPath(); + path += ".cinfo"; + + int res = -1; + struct stat tmpStat; + XrdOucEnv myEnv; + + // try to read from existing file + if (m_cache.GetOss()->Stat(path.c_str(), &tmpStat) == XrdOssOK) { + XrdOssDF* infoFile = m_cache.GetOss()->newFile(m_cache.RefConfiguration().m_username.c_str()); + if (infoFile->Open(path.c_str(), O_RDONLY, 0600, myEnv) == XrdOssOK) { + Info info(m_cache.GetTrace()); + if (info.Read(infoFile) > 0) { + tmpStat.st_size = info.GetFileSize(); + TRACEIO(Info, "IOFileBlock::initCachedStat successfuly read size from existing info file = " << tmpStat.st_size); + res = 0; + } + else { + // file exist but can't read it + TRACEIO(Error, "IOFileBlock::initCachedStat failed to read file size from info file"); + } + } + } + + // if there is no local info file, try to read from clinet and then save stat into a new *cinfo file + if (res) { + res = GetInput()->Fstat(tmpStat); + TRACEIO(Debug, "IOFileBlock::initCachedStat get stat from client res= " << res << "size = " << tmpStat.st_size); + if (res == 0) { + if (m_cache.GetOss()->Create(m_cache.RefConfiguration().m_username.c_str(), path.c_str(), 0600, myEnv, XRDOSS_mkpath) == XrdOssOK) { + XrdOssDF* infoFile = m_cache.GetOss()->newFile(m_cache.RefConfiguration().m_username.c_str()); + if (infoFile->Open(path.c_str(), O_RDWR, 0600, myEnv) == XrdOssOK) { + Info cfi(m_cache.GetTrace(), false); + cfi.SetBufferSize(m_cache.RefConfiguration().m_bufferSize); + cfi.SetFileSize(tmpStat.st_size); + cfi.WriteHeader(infoFile); + infoFile->Fsync(); + infoFile->Close(); + } + else { + TRACEIO(Error, "IOFileBlock::initCachedStat can't open info file path"); + } + delete infoFile; + } + else { + TRACEIO(Error, "IOFileBlock::initCachedStat can't create info file path"); + } + } + } + + + if (res == 0) + { + std::cerr << "local stat created \n"; + m_localStat = new struct stat; + memcpy(m_localStat, &tmpStat, sizeof(struct stat)); + } + + return res; +} + + //______________________________________________________________________________ void IOFileBlock::RelinquishFile(File* f) { @@ -141,7 +234,7 @@ int IOFileBlock::Read (char *buff, long long off, int size) { // protect from reads over the file size - long long fileSize = GetInput()->FSize(); + long long fileSize = FSize(); if (off >= fileSize) return 0; diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index dda7ff88f4d..9dbb7c093e4 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -64,14 +64,20 @@ namespace XrdFileCache //! Called to check if destruction needs to be done in a separate task. virtual bool ioActive(); + virtual int FStat(struct stat &sbuff); + + virtual long long FSize(); + virtual void RelinquishFile(File*); private: long long m_blocksize; //!< size of file-block std::map m_blocks; //!< map of created blocks XrdSysMutex m_mutex; //!< map mutex + struct stat *m_localStat; void GetBlockSizeFromPath(); + int initLocalStat(); File* newBlockFile(long long off, int blocksize); }; }