Skip to content

Commit

Permalink
Merge pull request #133 from art-daq/rrivera/customJsonDocLoadSaveAnd…
Browse files Browse the repository at this point in the history 
…OtsMongodbBugFix

added feature for custom JSON doc save/load; also fixed mongodb bug i…
  • Loading branch information
@eflumerf
eflumerf committed Dec 11, 2023
2 parents e797d18 + a18c9cd commit 07541c1
Show file tree
Hide file tree
Showing 10 changed files with 407 additions and 25 deletions.
3 changes: 3 additions & 0 deletions otsdaq/ConfigurationInterface/ConfigurationInterface.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ class ConfigurationInterface
TableVersion /*version*/> getTableGroupMembers (std::string const& /*groupName*/, bool /*includeMetaDataTable*/ = false) const { __SS__; __THROW__(ss.str() + "ConfigurationInterface::... Must only call getTableGroupMembers in a mode with this functionality implemented (e.g. DatabaseConfigurationInterface)."); }
virtual void saveTableGroup (std::map<std::string /*name*/,TableVersion /*version*/> const& /*tableToVersionMap*/, std::string const& /*groupName*/) const { __SS__; __THROW__(ss.str() + "ConfigurationInterface::... Must only call saveTableGroup in a mode with this functionality implemented (e.g. DatabaseConfigurationInterface)."); }

virtual std::pair<std::string, TableVersion> saveCustomJSON (const std::string& JSON, const std::string& documentNameToSave) const { __COUTV__(JSON); __COUTV__(documentNameToSave); return std::make_pair("",TableVersion());}
virtual std::string loadCustomJSON (const std::string& documentNameToLoad, TableVersion documentVersionToLoad) const {__COUTV__(documentNameToLoad); __COUTV__(documentVersionToLoad); return "{}";};


protected:
virtual void fill (TableBase* configuration, TableVersion version) const = 0;
Expand Down
93 changes: 91 additions & 2 deletions otsdaq/ConfigurationInterface/Database_configInterface.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ try
__COUT_TYPE__(TLVL_DEBUG+20) << __COUT_HDR__ << "Getting cache for " << groupName << "(" << groupKey << ")" << __E__;


TableBase localGroupMemberCacheSaver("GroupCache_" + groupName);
TableBase localGroupMemberCacheSaver(TableBase::GROUP_CACHE_PREPEND + groupName);
TableVersion localVersion(atoi(groupKey.c_str()));
localGroupMemberCacheSaver.changeVersionAndActivateView(
localGroupMemberCacheSaver.createTemporaryView(),
Expand Down Expand Up @@ -464,7 +464,7 @@ try
std::string groupKey = tableGroup.substr(vi+2);
__COUT_TYPE__(TLVL_DEBUG+20) << __COUT_HDR__ << "Saving cache for " << groupName << "(" << groupKey << ")" << __E__;

TableBase localGroupMemberCacheSaver("GroupCache_" + groupName);
TableBase localGroupMemberCacheSaver(TableBase::GROUP_CACHE_PREPEND + groupName);
localGroupMemberCacheSaver.changeVersionAndActivateView(
localGroupMemberCacheSaver.createTemporaryView(),
TableVersion(atoi(groupKey.c_str())));
Expand Down Expand Up @@ -560,4 +560,93 @@ catch(...)
__SS_THROW__;
} //end saveTableGroup() catch




//==============================================================================
std::pair<std::string, TableVersion> DatabaseConfigurationInterface::saveCustomJSON(const std::string& json, const std::string& documentNameToSave) const
try
{
__COUT_TYPE__(TLVL_DEBUG+20) << __COUT_HDR__ << "Saving doc '" << documentNameToSave << "'" << __E__;

TableBase localDocSaver(TableBase::JSON_DOC_PREPEND + documentNameToSave);

std::set<TableVersion> versions = getVersions(&localDocSaver);
TableVersion version;
if(versions.size())
version = TableVersion::getNextVersion(*versions.rbegin());
else
version = TableVersion::DEFAULT;
__COUTV__(version);

localDocSaver.changeVersionAndActivateView(
localDocSaver.createTemporaryView(),
version);

localDocSaver.getViewP()->setCustomStorageData(json);

__COUT_TYPE__(TLVL_DEBUG+20) << __COUT_HDR__ << "Saving JSON string: " <<
localDocSaver.getViewP()->getCustomStorageData() << __E__;

__COUT_TYPE__(TLVL_DEBUG+20) << __COUT_HDR__ << "Saving JSON doc as " <<
localDocSaver.getView().getTableName() << "(" <<
localDocSaver.getView().getVersion().toString() << ")" << __E__;


// save to db, and do not allow overwrite
saveActiveVersion(&localDocSaver, false /* overwrite */);

return std::make_pair(localDocSaver.getTableName(),
localDocSaver.getView().getVersion());
} //end saveCustomJSON()
catch(std::exception const& e)
{
__SS__ << "DBI Exception saveCustomJSON for '" << documentNameToSave << "':\n\n" << e.what() << "\n";
__COUT_ERR__ << ss.str();
__SS_THROW__;
}
catch(...)
{
__SS__ << "DBI Unknown exception saveCustomJSON for '" << documentNameToSave << ".'\n";
__COUT_ERR__ << ss.str();
__SS_THROW__;
} //end saveCustomJSON() catch


//==============================================================================
std::string DatabaseConfigurationInterface::loadCustomJSON(const std::string& documentNameToLoad, TableVersion documentVersionToLoad) const
try
{

__COUT_TYPE__(TLVL_DEBUG+20) << __COUT_HDR__ << "Loading doc '" << documentNameToLoad << "-v" <<
documentVersionToLoad << "'" << __E__;

TableBase localDocLoader(TableBase::JSON_DOC_PREPEND + documentNameToLoad);

localDocLoader.changeVersionAndActivateView(
localDocLoader.createTemporaryView(),
documentVersionToLoad);

fill(&localDocLoader,documentVersionToLoad);

__COUT_TYPE__(TLVL_DEBUG+20) << __COUT_HDR__ << "Loaded JSON doc string " <<
localDocLoader.getViewP()->getCustomStorageData() << __E__;

return localDocLoader.getViewP()->getCustomStorageData();
} //end loadCustomJSON()
catch(std::exception const& e)
{
__SS__ << "DBI Exception saveCustomJSON for '" << documentNameToLoad << "-v" <<
documentVersionToLoad << "':\n\n" << e.what() << "\n";
__COUT_ERR__ << ss.str();
__SS_THROW__;
}
catch(...)
{
__SS__ << "DBI Unknown exception saveCustomJSON for '" << documentNameToLoad << "-v" <<
documentVersionToLoad << ".'\n";
__COUT_ERR__ << ss.str();
__SS_THROW__;
} //end loadCustomJSON() catch

DEFINE_OTS_CONFIGURATION_INTERFACE(DatabaseConfigurationInterface)
5 changes: 5 additions & 0 deletions otsdaq/ConfigurationInterface/Database_configInterface.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ class DatabaseConfigurationInterface : public ConfigurationInterface

// create a new table group from the contents map
void saveTableGroup (table_version_map_t const& memberMap, std::string const& tableGroup) const;


std::pair<std::string, TableVersion> saveCustomJSON (const std::string& JSON, const std::string& documentNameToSave) const override;
std::string loadCustomJSON (const std::string& documentNameToLoad, TableVersion documentVersionToLoad) const override;

private:
table_version_map_t getCachedTableGroupMembers (std::string const& tableGroup) const;
void saveTableGroupMemberCache (table_version_map_t const& memberMap, std::string const& tableGroup) const;
Expand Down
8 changes: 6 additions & 2 deletions otsdaq/TableCore/TableBase.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ using namespace ots;
#define __COUT_HDR__ ("TableBase-" + getTableName() + "\t<> ")

const std::string TableBase::GROUP_CACHE_PREPEND = "GroupCache_";
const std::string TableBase::JSON_DOC_PREPEND = "JSONDoc_";

//==============================================================================
// TableBase
Expand Down Expand Up @@ -50,7 +51,8 @@ TableBase::TableBase(const std::string& tableName,
}

//if special GROUP CACHE table, handle construction in a special way
if(tableName.substr(0,TableBase::GROUP_CACHE_PREPEND.length()) == TableBase::GROUP_CACHE_PREPEND)
if(tableName.substr(0,TableBase::GROUP_CACHE_PREPEND.length()) == TableBase::GROUP_CACHE_PREPEND ||
tableName.substr(0,TableBase::JSON_DOC_PREPEND.length()) == TableBase::JSON_DOC_PREPEND)
{
__COUT__ << "TableBase for '" << tableName << "' constructed." << __E__;
return;
Expand Down Expand Up @@ -1531,7 +1533,9 @@ std::string TableBase::convertToCaps(std::string& str, bool isTableName)
else // error! non-alpha
{
//allow underscores for group cache document name
if(str.substr(0,TableBase::GROUP_CACHE_PREPEND.length()) == TableBase::GROUP_CACHE_PREPEND && str[c] == '_')
if((str.substr(0,TableBase::GROUP_CACHE_PREPEND.length()) == TableBase::GROUP_CACHE_PREPEND ||
str.substr(0,TableBase::JSON_DOC_PREPEND.length()) == TableBase::JSON_DOC_PREPEND) &&
str[c] == '_')
{
capsStr += '-';
continue;
Expand Down
1 change: 1 addition & 0 deletions otsdaq/TableCore/TableBase.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ class TableBase

public:
static const std::string GROUP_CACHE_PREPEND;
static const std::string JSON_DOC_PREPEND;

protected:
std::string tableName_;
Expand Down
23 changes: 20 additions & 3 deletions otsdaq/TableCore/TableView.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,12 @@ TableView& TableView::copy(const TableView& src, TableVersion destinationVersion

std::string tmpCachePrepend = TableBase::GROUP_CACHE_PREPEND;
tmpCachePrepend = TableBase::convertToCaps(tmpCachePrepend);
std::string tmpJsonDocPrepend = TableBase::JSON_DOC_PREPEND;
tmpJsonDocPrepend = TableBase::convertToCaps(tmpJsonDocPrepend);

//if special GROUP CACHE table, handle construction in a special way
if(tableName_.substr(0,tmpCachePrepend.length()) == tmpCachePrepend)
if(tableName_.substr(0,tmpCachePrepend.length()) == tmpCachePrepend ||
tableName_.substr(0,tmpJsonDocPrepend.length()) == tmpJsonDocPrepend)
{
__COUT__ << "TableView copy for '" << tableName_ << "' done." << __E__;
return *this;
Expand Down Expand Up @@ -1930,9 +1933,13 @@ void TableView::printJSON(std::ostream& out) const
{ //handle special GROUP CACHE table
std::string tmpCachePrepend = TableBase::GROUP_CACHE_PREPEND;
tmpCachePrepend = TableBase::convertToCaps(tmpCachePrepend);
__COUT__ << " '" << tableName_ << "' vs " << tmpCachePrepend << __E__;
std::string tmpJsonDocPrepend = TableBase::JSON_DOC_PREPEND;
tmpJsonDocPrepend = TableBase::convertToCaps(tmpJsonDocPrepend);
__COUT__ << " '" << tableName_ << "' vs " << tmpCachePrepend <<
" or " << tmpJsonDocPrepend << __E__;
//if special GROUP CACHE table, handle construction in a special way
if(tableName_.substr(0,tmpCachePrepend.length()) == tmpCachePrepend)
if(tableName_.substr(0,tmpCachePrepend.length()) == tmpCachePrepend ||
tableName_.substr(0,tmpJsonDocPrepend.length()) == tmpJsonDocPrepend)
{
out << getCustomStorageData();
return;
Expand Down Expand Up @@ -2073,7 +2080,17 @@ int TableView::fillFromJSON(const std::string& json)
//handle special GROUP CACHE table
std::string tmpCachePrepend = TableBase::GROUP_CACHE_PREPEND;
tmpCachePrepend = TableBase::convertToCaps(tmpCachePrepend);
std::string tmpJsonDocPrepend = TableBase::JSON_DOC_PREPEND;
tmpJsonDocPrepend = TableBase::convertToCaps(tmpJsonDocPrepend);

//if special JSON DOC table, handle construction in a special way
if(tableName_.substr(0,tmpJsonDocPrepend.length()) == tmpJsonDocPrepend)
{
__COUT__ << "Special JSON doc: " << json << __E__;
setCustomStorageData(json);
return 0; //success
} //end special JSON DOC table construction

//if special GROUP CACHE table, handle construction in a special way
if(tableName_.substr(0,tmpCachePrepend.length()) == tmpCachePrepend)
{
Expand Down
20 changes: 19 additions & 1 deletion tools/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,25 @@ cet_make_exec(NAME otsdaq_fix_new_table_fields_tool LIBRARIES otsdaq::Configurat
cet_make_exec(NAME otsdaq_flatten_system_aliases LIBRARIES otsdaq::ConfigurationInterface)
cet_make_exec(NAME otsdaq_import_system_aliases LIBRARIES otsdaq::ConfigurationInterface)

cet_script(ALWAYS_COPY common.sh EclipseBuild.sh ots otsdaq_add_artdaq_tables otsdaq_beautify_artdaq_tables otsdaq_convert_artdaq_daq_tables otsdaq_convert_config_to_table otsweb_login.sh ots_remote_start redmine_login.sh reset_ots_snapshot.sh StartOTS.sh vless_ots.sh vtail_ots.sh)
cet_make_exec(NAME otsdaq_save_json_document LIBRARIES otsdaq::ConfigurationInterface)
cet_make_exec(NAME otsdaq_load_json_document LIBRARIES otsdaq::ConfigurationInterface)


cet_script(ALWAYS_COPY
common.sh
EclipseBuild.sh
ots
otsdaq_add_artdaq_tables
otsdaq_beautify_artdaq_tables
otsdaq_convert_artdaq_daq_tables
otsdaq_convert_config_to_table
otsweb_login.sh
ots_remote_start
redmine_login.sh
reset_ots_snapshot.sh
StartOTS.sh
vless_ots.sh
vtail_ots.sh)

install_fhicl(SUBDIRS fcl)
install_headers()
Expand Down
37 changes: 20 additions & 17 deletions tools/ots
View file
Original file line number Diff line number Diff line change
Expand Up @@ -652,11 +652,30 @@ if [ "x$ARTDAQ_DATABASE_TYPE" == "xfilesystemdb" ]; then

# As of 27-Oct-2023, no longer... check for somewhat-antiquated(pre-configuration-to-table) artdaq database


#check for somewhat-antiquated artdaq 'DAQ' tables and auto-fix
if [ ! -e ${ARTDAQ_DATABASE_FILESYSTEM_PATH}/artdaqDaqTableConversion ]; then
out "Converting artdaq 'daq' tables in databases..."

#backup because we are scared
cp -r ${ARTDAQ_DATABASE_FILESYSTEM_PATH} ${ARTDAQ_DATABASE_FILESYSTEM_PATH}bk12346
cp -r ${USER_DATA} ${USER_DATA}bk12346

otsdaq_convert_artdaq_daq_tables
echo "converted" > ${ARTDAQ_DATABASE_FILESYSTEM_PATH}/artdaqDaqTableConversion
out "Conversion complete. You must run wiz mode once now to activate the configuration changes."

if [ $ISCONFIG == 0 ]; then
exit
fi
fi
elif [ "x$ARTDAQ_DATABASE_TYPE" == "xmongodb" ]; then
out "mongodb identified."
else

out "${Red}${Bold}${Blink}Fatal Error${RstClr}."
out "ARTDAQ_DATABASE_URI=$ARTDAQ_DATABASE_URI"
out "Environment variable ${Cyan}${Bold}ARTDAQ_DATABASE_URI${RstClr} does not have a value!"
out "Environment variable ${Cyan}${Bold}ARTDAQ_DATABASE_URI${RstClr} does not have a valid type: ${ARTDAQ_DATABASE_TYPE}!"
out "To setup, use 'export ARTDAQ_DATABASE_URI=<value>'"


Expand All @@ -669,22 +688,6 @@ else
# mkdir $ARTDAQ_DATABASE_FILESYSTEM_PATH &>/dev/null 2>&1 #hide output
fi

#check for somewhat-antiquated artdaq 'DAQ' tables and auto-fix
if [ ! -e ${ARTDAQ_DATABASE_FILESYSTEM_PATH}/artdaqDaqTableConversion ]; then
out "Converting artdaq 'daq' tables in databases..."

#backup because we are scared
cp -r ${ARTDAQ_DATABASE_FILESYSTEM_PATH} ${ARTDAQ_DATABASE_FILESYSTEM_PATH}bk12346
cp -r ${USER_DATA} ${USER_DATA}bk12346

otsdaq_convert_artdaq_daq_tables
echo "converted" > ${ARTDAQ_DATABASE_FILESYSTEM_PATH}/artdaqDaqTableConversion
out "Conversion complete. You must run wiz mode once now to activate the configuration changes."

if [ $ISCONFIG == 0 ]; then
exit
fi
fi

export TABLE_INFO_PATH=${USER_DATA}/TableInfo
export SERVICE_DATA_PATH=${USER_DATA}/ServiceData #it is not recommended that the SERVICE_DATA_PATH environment variable be changed by the user, it may break some features.
Expand Down

0 comments on commit 07541c1

Please sign in to comment.