Skip to content

Commit

Permalink
move StorageFactory code to edm::storage namespace
Browse files Browse the repository at this point in the history
  • Loading branch information
Dr15Jones committed Sep 27, 2021
1 parent c1cb2f1 commit bf8f910
Show file tree
Hide file tree
Showing 76 changed files with 1,257 additions and 1,131 deletions.
1 change: 1 addition & 0 deletions FWCore/Services/plugins/CondorStatusUpdater.cc
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,7 @@ void CondorStatusService::updateImpl(time_t sinceLastUpdate) {
}
}

using namespace edm::storage;
// Update storage account information
auto const &stats = StorageAccount::summary();
uint64_t readOps = 0;
Expand Down
4 changes: 2 additions & 2 deletions IOPool/Common/bin/EdmCopyUtil.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,14 @@ static int copy_files(const boost::program_options::variables_map& vm) {
ofs.exceptions(std::ofstream::failbit | std::ofstream::badbit);
ofs.open(pathOut);

std::unique_ptr<Storage> s = StorageFactory::get()->open(filesIn[j]);
std::unique_ptr<edm::storage::Storage> s = edm::storage::StorageFactory::get()->open(filesIn[j]);
assert(s); // StorageFactory should throw if file open fails.

static unsigned int const COPYBUFSIZE = 10 * 1024 * 1024; // 10MB buffer
std::vector<char> buffer;
buffer.reserve(COPYBUFSIZE);

IOSize n;
edm::storage::IOSize n;
while ((n = s->read(&buffer[0], COPYBUFSIZE))) { // Note Storage throws on error
ofs.write(&buffer[0], n);
}
Expand Down
2 changes: 1 addition & 1 deletion IOPool/Input/src/RootPrimaryFileSequence.cc
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ namespace edm {

// Prestage the files
for (setAtFirstFile(); !noMoreFiles(); setAtNextFile()) {
StorageFactory::get()->stagein(fileNames()[0]);
storage::StorageFactory::get()->stagein(fileNames()[0]);
}
// Open the first file.
for (setAtFirstFile(); !noMoreFiles(); setAtNextFile()) {
Expand Down
2 changes: 1 addition & 1 deletion IOPool/Input/src/RootSecondaryFileSequence.cc
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ namespace edm {
// thousands of files and prestaging all those files can cause a site to fail.
// So, we stage in the first secondary file only.
setAtFirstFile();
StorageFactory::get()->stagein(fileNames()[0]);
storage::StorageFactory::get()->stagein(fileNames()[0]);

// Open the first file.
for (setAtFirstFile(); !noMoreFiles(); setAtNextFile()) {
Expand Down
6 changes: 3 additions & 3 deletions IOPool/Streamer/interface/StreamerInputFile.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ namespace edm {

private:
void openStreamerFile(std::string const& name, std::string const& LFN);
IOSize readBytes(char* buf, IOSize nBytes);
IOOffset skipBytes(IOSize nBytes);
storage::IOSize readBytes(char* buf, storage::IOSize nBytes);
storage::IOOffset skipBytes(storage::IOSize nBytes);

void readStartMessage();
int readEventMessage();
Expand Down Expand Up @@ -82,7 +82,7 @@ namespace edm {

bool newHeader_;

edm::propagate_const<std::unique_ptr<Storage>> storage_;
edm::propagate_const<std::unique_ptr<edm::storage::Storage>> storage_;

bool endOfFile_;
};
Expand Down
15 changes: 9 additions & 6 deletions IOPool/Streamer/src/StreamerInputFile.cc
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ namespace edm {

logFileAction(" Initiating request to open file ");

using namespace edm::storage;
IOOffset size = -1;
if (StorageFactory::get()->check(name, &size)) {
try {
Expand Down Expand Up @@ -110,8 +111,8 @@ namespace edm {
currentFileOpen_ = false;
}

IOSize StreamerInputFile::readBytes(char* buf, IOSize nBytes) {
IOSize n = 0;
storage::IOSize StreamerInputFile::readBytes(char* buf, storage::IOSize nBytes) {
storage::IOSize n = 0;
try {
n = storage_->read(buf, nBytes);
} catch (cms::Exception& ce) {
Expand All @@ -122,12 +123,12 @@ namespace edm {
return n;
}

IOOffset StreamerInputFile::skipBytes(IOSize nBytes) {
IOOffset n = 0;
storage::IOOffset StreamerInputFile::skipBytes(storage::IOSize nBytes) {
storage::IOOffset n = 0;
try {
// We wish to return the number of bytes skipped, not the final offset.
n = storage_->position(0, Storage::CURRENT);
n = storage_->position(nBytes, Storage::CURRENT) - n;
n = storage_->position(0, storage::Storage::CURRENT);
n = storage_->position(nBytes, storage::Storage::CURRENT) - n;
} catch (cms::Exception& ce) {
Exception ex(errors::FileReadError, "", ce);
ex.addContext("Calling StreamerInputFile::skipBytes()");
Expand All @@ -137,6 +138,7 @@ namespace edm {
}

void StreamerInputFile::readStartMessage() {
using namespace edm::storage;
IOSize nWant = sizeof(HeaderView);
IOSize nGot = readBytes(&headerBuf_[0], nWant);
if (nGot != nWant) {
Expand Down Expand Up @@ -224,6 +226,7 @@ namespace edm {
if (endOfFile_)
return 0;

using namespace edm::storage;
bool eventRead = false;
while (!eventRead) {
IOSize nWant = sizeof(EventHeader);
Expand Down
6 changes: 4 additions & 2 deletions IOPool/TFileAdaptor/interface/TStorageFactoryFile.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
#include "Utilities/StorageFactory/interface/IOPosBuffer.h"
#include "FWCore/Utilities/interface/get_underlying_safe.h"

class Storage;
namespace edm::storage {
class Storage;
}

/** TFile wrapper around #StorageFactory and #Storage. */
class TStorageFactoryFile : public TFile {
Expand Down Expand Up @@ -57,7 +59,7 @@ class TStorageFactoryFile : public TFile {

TStorageFactoryFile(void);

edm::propagate_const<std::unique_ptr<Storage>> storage_; //< Real underlying storage
edm::propagate_const<std::unique_ptr<edm::storage::Storage>> storage_; //< Real underlying storage
};

#endif // TFILE_ADAPTOR_TSTORAGE_FACTORY_FILE_H
2 changes: 1 addition & 1 deletion IOPool/TFileAdaptor/src/ReadRepacker.cc
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ int ReadRepacker::packInternal(long long int *pos, int *len, int nbuf, char *buf
break;
}

IOOffset extra_bytes_signed = (idx == 0) ? 0 : ((pos[idx] - iopb.offset()) - iopb.size());
edm::storage::IOOffset extra_bytes_signed = (idx == 0) ? 0 : ((pos[idx] - iopb.offset()) - iopb.size());
assert(extra_bytes_signed >= 0);
IOSize extra_bytes = static_cast<IOSize>(extra_bytes_signed);

Expand Down
8 changes: 5 additions & 3 deletions IOPool/TFileAdaptor/src/ReadRepacker.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@

class ReadRepacker {
public:
using IOSize = edm::storage::IOSize;
using IOPosBuffer = edm::storage::IOPosBuffer;
// Returns the number of input buffers it was able to pack into the IO operation.
int pack(long long int *pos, // An array of file offsets to read.
int *len, // An array of lengths to read.
Expand All @@ -55,13 +57,13 @@ class ReadRepacker {

// Two reads distanced by less than READ_COALESCE_SIZE will turn into one
// large read.
static const IOSize TEMPORARY_BUFFER_SIZE = 256 * 1024;
static constexpr IOSize TEMPORARY_BUFFER_SIZE = 256 * 1024;

// A read larger than BIG_READ_SIZE will not be coalesced.
static const IOSize READ_COALESCE_SIZE = 32 * 1024;
static constexpr IOSize READ_COALESCE_SIZE = 32 * 1024;

// The size of the temporary holding buffer for read-coalescing.
static const IOSize BIG_READ_SIZE = 256 * 1024;
static constexpr IOSize BIG_READ_SIZE = 256 * 1024;

private:
int packInternal(long long int *pos,
Expand Down
5 changes: 3 additions & 2 deletions IOPool/TFileAdaptor/src/TFileAdaptor.cc
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ TFileAdaptor::TFileAdaptor(edm::ParameterSet const& pset, edm::ActivityRegistry&
if (!(enabled_ = pset.getUntrackedParameter<bool>("enable", enabled_)))
return;

using namespace edm::storage;
StorageFactory* f = StorageFactory::getToModify();
doStats_ = pset.getUntrackedParameter<bool>("stats", doStats_);

Expand Down Expand Up @@ -229,7 +230,7 @@ void TFileAdaptor::stats(std::ostream& o) const {
<< " Prefetching:" << (enablePrefetching_ ? "true" : "false") << '\n'
<< " Cache hint:" << cacheHint_ << '\n'
<< " Read hint:" << readHint_ << '\n'
<< "Storage statistics: " << StorageAccount::summaryText() << "; tfile/read=?/?/"
<< "Storage statistics: " << edm::storage::StorageAccount::summaryText() << "; tfile/read=?/?/"
<< (TFile::GetFileBytesRead() / oneMeg) << "MB/?ms/?ms/?ms"
<< "; tfile/write=?/?/" << (TFile::GetFileBytesWritten() / oneMeg) << "MB/?ms/?ms/?ms";
}
Expand All @@ -244,7 +245,7 @@ void TFileAdaptor::statsXML(std::map<std::string, std::string>& data) const {
data.insert(std::make_pair("Parameter-untracked-bool-prefetching", (enablePrefetching_ ? "true" : "false")));
data.insert(std::make_pair("Parameter-untracked-string-cacheHint", cacheHint_));
data.insert(std::make_pair("Parameter-untracked-string-readHint", readHint_));
StorageAccount::fillSummary(data);
edm::storage::StorageAccount::fillSummary(data);
std::ostringstream r;
std::ostringstream w;
r << (TFile::GetFileBytesRead() / oneMeg);
Expand Down
2 changes: 2 additions & 0 deletions IOPool/TFileAdaptor/src/TStorageFactoryFile.cc
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ class TTreeCacheDebug : public TTreeCache {
};
#endif

using namespace edm::storage;

ClassImp(TStorageFactoryFile);
static StorageAccount::Counter *s_statsCtor = nullptr;
static StorageAccount::Counter *s_statsOpen = nullptr;
Expand Down
2 changes: 2 additions & 0 deletions IOPool/TFileAdaptor/src/TStorageFactorySystem.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

ClassImp(TStorageFactorySystem);

using namespace edm::storage;

TStorageFactorySystem::TStorageFactorySystem(const char *, Bool_t)
: TSystem("-StorageFactory", "Storage Factory System"), fDirp(nullptr) {
SetName("StorageFactory");
Expand Down
69 changes: 35 additions & 34 deletions Utilities/DCacheAdaptor/interface/DCacheFile.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,38 +5,39 @@
#include "Utilities/StorageFactory/interface/IOFlags.h"
#include <string>

class DCacheFile : public Storage {
public:
DCacheFile(void);
DCacheFile(IOFD fd);
DCacheFile(const char *name, int flags = IOFlags::OpenRead, int perms = 0666);
DCacheFile(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666);
~DCacheFile(void) override;

virtual void create(const char *name, bool exclusive = false, int perms = 0666);
virtual void create(const std::string &name, bool exclusive = false, int perms = 0666);
virtual void open(const char *name, int flags = IOFlags::OpenRead, int perms = 0666);
virtual void open(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666);

using Storage::position;
using Storage::read;
using Storage::write;

IOSize read(void *into, IOSize n) override;
IOSize readv(IOBuffer *into, IOSize buffers) override;
IOSize readv(IOPosBuffer *into, IOSize buffers) override;
IOSize write(const void *from, IOSize n) override;

IOOffset position(IOOffset offset, Relative whence = SET) override;
void resize(IOOffset size) override;

void close(void) override;
virtual void abort(void);

private:
IOFD m_fd;
bool m_close;
std::string m_name;
};

namespace edm::storage {
class DCacheFile : public Storage {
public:
DCacheFile(void);
DCacheFile(IOFD fd);
DCacheFile(const char *name, int flags = IOFlags::OpenRead, int perms = 0666);
DCacheFile(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666);
~DCacheFile(void) override;

virtual void create(const char *name, bool exclusive = false, int perms = 0666);
virtual void create(const std::string &name, bool exclusive = false, int perms = 0666);
virtual void open(const char *name, int flags = IOFlags::OpenRead, int perms = 0666);
virtual void open(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666);

using Storage::position;
using Storage::read;
using Storage::write;

IOSize read(void *into, IOSize n) override;
IOSize readv(IOBuffer *into, IOSize buffers) override;
IOSize readv(IOPosBuffer *into, IOSize buffers) override;
IOSize write(const void *from, IOSize n) override;

IOOffset position(IOOffset offset, Relative whence = SET) override;
void resize(IOOffset size) override;

void close(void) override;
virtual void abort(void);

private:
IOFD m_fd;
bool m_close;
std::string m_name;
};
} // namespace edm::storage
#endif // DCACHE_ADAPTOR_DCACHE_FILE_H
Loading

0 comments on commit bf8f910

Please sign in to comment.