ConfigDataTree.cpp

Go to the documentation of this file.

#include <DatabaseUtils/DAQConfig/ConfigDataTree.h>
#include <DatabaseUtils/DAQConfig/DBColumnParams.h>
#include <DatabaseUtils/DAQConfig/DBColumnUtils.h>
#include <DatabaseUtils/DAQConfig/NamedConfigUtils.h>
#include <NovaDAQUtilities/EnvVarCache.h>
#include <boost/algorithm/string.hpp>
#include <iostream>

namespace NOVADB = nova::database;

namespace dbutils {
namespace daqconfig {

const std::string ConfigDataTree::KEY_FOR_SYSTEM_WIDE_PARAM("systemWideKey");
const std::string ConfigDataTree::KEY_SEPARATOR_STRING("/");
const std::string ConfigDataTree::NO_DEFAULT_KEY_STRING("NoDefault");

// The meaning of the bits in the CONFIG_DATA_TREE_DEBUG env var:
// - bit 0: prints the hash key and DB values for each database result
// - bit 1: prints selector and data column details
// - bit 2: prints information about values that are written to the DB
// - bit 3: enables tracing via line-based print statements
// - bit 4: sets the verbosity of the Table query to 1
// - bit 5: sets the verbosity of the Table write to 1

#define LINE_NUMBER_DEBUG if ((debugFlag & 0x08) != 0) { \
    std::cout << "ConfigDataTree line=" << __LINE__ << std::endl;}

bool sortingFunction(std::string leftString, std::string rightString) {
    try {
        int leftInt = boost::lexical_cast<int>(leftString);
        int rightInt = boost::lexical_cast<int>(rightString);
        return (leftInt < rightInt);
    }
    catch (...) {
        return (leftString < rightString);
    }
}

/**
 * Constructs a configuration data tree for the specified
 * input parameters.
 */
ConfigDataTree::
ConfigDataTree(const std::string& tableDefinitionFilename,
               const std::vector<int64_t>& subsystemConfigIdList,
               const ConfigDataSelector& selector,
               std::vector<std::string> dataColumnNameList)
{
    // seed failure
    _hasValidData = false;
    _tableDefinitionFilename = tableDefinitionFilename;

    // check if a debug flag has been specified
    int debugFlag = 0;
    std::string debugString =
        novadaq::EnvVarCache::getInstance().getEnvVar("CONFIG_DATA_TREE_DEBUG");
    if (debugString.length() > 0) {
        try {debugFlag = boost::lexical_cast<int>(debugString);}
        catch (...) {}
    }

    // set up the Table instance and its connection to the DB
    // (if something fails here, we simply return and leave the hasValidData
    // flag set to false)
    boost::shared_ptr<NOVADB::Table> dbTablePtr;
    dbTablePtr.reset(new NOVADB::Table(tableDefinitionFilename.c_str()));
    std::string detName = 
      novadaq::EnvVarCache::getInstance().getEnvVar("NOVADAQ_ENVIRONMENT");
    dbTablePtr->SetDetector(detName);
    dbTablePtr->SetTableType(NOVADB::kDAQTable);
    std::string dbUser =
        novadaq::EnvVarCache::getInstance().getEnvVar("CFGMGR_DB_USER");
    LINE_NUMBER_DEBUG;
    if (dbUser.length() > 0) {
        LINE_NUMBER_DEBUG;
        dbTablePtr->SetUser(dbUser);
    }
    LINE_NUMBER_DEBUG;
    if (! dbTablePtr->GetConnection()) {return;}
    LINE_NUMBER_DEBUG;
    if (! dbTablePtr->ExistsInDB()) {return;}
    LINE_NUMBER_DEBUG;
    _configSelector = selector;
    _dataColumnNameList = dataColumnNameList;
    _dataColumnTypeList.assign(_dataColumnNameList.size(), "unknown");

    _initialize(dbTablePtr, subsystemConfigIdList,
                selector, dataColumnNameList, debugFlag);
    LINE_NUMBER_DEBUG;
}

/**
 * Constructs a configuration data tree for the specified
 * table and subsystem configuration ID list.  Reasonable
 * assumptions are made for which selector columns are of
 * interest (e.g. dcmName, febId) and which data columns
 * are of interest.
 */
ConfigDataTree::
ConfigDataTree(const std::string& tableDefinitionFilename,
               const std::vector<int64_t>& subsystemConfigIdList)
{
    // seed failure
    _hasValidData = false;
    _tableDefinitionFilename = tableDefinitionFilename;

    // check if a debug flag has been specified
    int debugFlag = 0;
    std::string debugString =
        novadaq::EnvVarCache::getInstance().getEnvVar("CONFIG_DATA_TREE_DEBUG");
    if (debugString.length() > 0) {
        try {debugFlag = boost::lexical_cast<int>(debugString);}
        catch (...) {}
    }

    // set up the Table instance and its connection to the DB
    // (if something fails here, we simply return and leave the hasValidData
    // flag set to false)
    boost::shared_ptr<NOVADB::Table> dbTablePtr;
    dbTablePtr.reset(new NOVADB::Table(tableDefinitionFilename.c_str()));
    std::string detName = 
      novadaq::EnvVarCache::getInstance().getEnvVar("NOVADAQ_ENVIRONMENT");
    dbTablePtr->SetDetector(detName);
    dbTablePtr->SetTableType(NOVADB::kDAQTable);
    std::string dbUser =
        novadaq::EnvVarCache::getInstance().getEnvVar("CFGMGR_DB_USER");
    LINE_NUMBER_DEBUG;
    if (dbUser.length() > 0) {
        LINE_NUMBER_DEBUG;
        dbTablePtr->SetUser(dbUser);
    }
    LINE_NUMBER_DEBUG;
    if (! dbTablePtr->GetConnection()) {return;}
    LINE_NUMBER_DEBUG;
    if (! dbTablePtr->ExistsInDB()) {return;}
    LINE_NUMBER_DEBUG;

    // determine the selector and data columns of interest
    std::vector<std::string> colNames = dbTablePtr->GetColNames();

    // the order of the selectors matters, so we'll handle those carefully
    std::vector<std::string> tmpSelectionList;
    uint32_t indexForStartOfNormalHierarchy = 0;

    for (uint32_t idx = 0; idx < colNames.size(); ++idx) {
        std::string colName = boost::to_lower_copy(colNames[idx]);
        if (DBColumnUtils::isInfrastructureColumn(colName)) {continue;}
        if (DBColumnUtils::isSubsystemConfigReferenceColumn(colName)) {
            continue;
        }

        std::vector<std::string>::iterator iter = tmpSelectionList.begin();
        if (colName == DBColumnParams::DCMNAME_COLUMN_NAME) {
            if (tmpSelectionList.size() > indexForStartOfNormalHierarchy) {
                tmpSelectionList.insert(iter+indexForStartOfNormalHierarchy,
                                        colName);
            }
            else {
                tmpSelectionList.push_back(colName);
            }
        }
        else if (colName == DBColumnParams::FEBID_COLUMN_NAME) {
            if (tmpSelectionList.size() > indexForStartOfNormalHierarchy+1) {
                tmpSelectionList.insert(iter+indexForStartOfNormalHierarchy+1,
                                        colName);
            }
            else {
                tmpSelectionList.push_back(colName);
            }
        }
        else if (colName == DBColumnParams::PIXELID_COLUMN_NAME) {
            if (tmpSelectionList.size() > indexForStartOfNormalHierarchy+2) {
                tmpSelectionList.insert(iter+indexForStartOfNormalHierarchy+2,
                                        colName);
            }
            else {
                tmpSelectionList.push_back(colName);
            }
        }
        else if (colName == DBColumnParams::HOSTNAME_COLUMN_NAME) {
            if (tmpSelectionList.size() > indexForStartOfNormalHierarchy) {
                tmpSelectionList.insert(iter+indexForStartOfNormalHierarchy,
                                        colName);
            }
            else {
                tmpSelectionList.push_back(colName);
            }
        }
        else if (colName == "coolingstate") {
            if (tmpSelectionList.size() > 0) {
                tmpSelectionList.insert(iter, colName);
            }
            else {
                tmpSelectionList.push_back(colName);
            }
            ++indexForStartOfNormalHierarchy;
        }
        else if (colName == "pvname") {
            if (tmpSelectionList.size() > 0) {
                tmpSelectionList.insert(iter, colName);
            }
            else {
                tmpSelectionList.push_back(colName);
            }
            ++indexForStartOfNormalHierarchy;
        }
        else {
            _dataColumnNameList.push_back(colName);
            _dataColumnTypeList.push_back("unknown");
        }
    }
    LINE_NUMBER_DEBUG;

    for (uint32_t idx = 0; idx < tmpSelectionList.size(); ++idx) {
        std::string colName = tmpSelectionList[idx];
        _configSelector.addSelectionColumn(colName);
    }
    LINE_NUMBER_DEBUG;

    _initialize(dbTablePtr, subsystemConfigIdList,
                _configSelector, _dataColumnNameList, debugFlag);
    LINE_NUMBER_DEBUG;
}

bool ConfigDataTree::
saveValues(const std::vector<dbutils::SimpleParameterSet>& newDataList) const
{
    // check if a debug flag has been specified
    int debugFlag = 0;
    std::string debugString =
        novadaq::EnvVarCache::getInstance().getEnvVar("CONFIG_DATA_TREE_DEBUG");
    if (debugString.length() > 0) {
        try {debugFlag = boost::lexical_cast<int>(debugString);}
        catch (...) {}
    }

    boost::shared_ptr<NOVADB::Table> dbTablePtr;
    dbTablePtr.reset(new NOVADB::Table(_tableDefinitionFilename.c_str()));
    std::string detName = 
      novadaq::EnvVarCache::getInstance().getEnvVar("NOVADAQ_ENVIRONMENT");
    dbTablePtr->SetDetector(detName);
    dbTablePtr->SetTableType(NOVADB::kDAQTable);
    std::string dbUser =
        novadaq::EnvVarCache::getInstance().getEnvVar("CFGMGR_DB_USER");
    LINE_NUMBER_DEBUG;
    if (dbUser.length() > 0) {
        LINE_NUMBER_DEBUG;
        dbTablePtr->SetUser(dbUser);
    }
    LINE_NUMBER_DEBUG;
    if (! dbTablePtr->GetConnection()) {return false;}
    LINE_NUMBER_DEBUG;
    if (! dbTablePtr->ExistsInDB()) {return false;}
    LINE_NUMBER_DEBUG;

    if ((debugFlag & 0x20) != 0) {
        dbTablePtr->SetVerbosity(1);
    }

    if ((debugFlag & 0x04) != 0) {
        std::cout << "*** ConfigDataTree for " << dbTablePtr->Name()
                  << ", storing values:" << std::endl;
    }

    for (int idx = 0; idx < (int) newDataList.size(); ++idx) {
        dbutils::SimpleParameterSet paramSet = newDataList[idx];
        LINE_NUMBER_DEBUG;

        if ((debugFlag & 0x04) != 0) {
            std::cout << "  " << paramSet.toString() << std::endl;
        }

        boost::shared_ptr<NOVADB::Row> newRow(dbTablePtr->NewRow());
        if (newRow.get() == 0) {return false;}
        LINE_NUMBER_DEBUG;

        std::vector<std::string> keyList = paramSet.keys();
        for (int jdx = 0; jdx < (int) keyList.size(); ++jdx) {

            std::string key = keyList[jdx];
            std::string value;
            if (paramSet.get(key, value)) {
                newRow->Set(key, value);
            }
        }
        LINE_NUMBER_DEBUG;

        dbTablePtr->AddRow(newRow.get());
        LINE_NUMBER_DEBUG;
    }
    LINE_NUMBER_DEBUG;

    if (! dbTablePtr->WriteToDB()) {return false;}
    LINE_NUMBER_DEBUG;

    return true;
}

/**
 * Initializes the data in the tree by querying the database.
 */
void ConfigDataTree::
_initialize(boost::shared_ptr<nova::database::Table> dbTablePtr,
            const std::vector<int64_t>& subsystemConfigIdList,
            const ConfigDataSelector& selector,
            std::vector<std::string> dataColumnNameList,
            int debugFlag)
{
    _tableName = dbTablePtr->Name();
    LINE_NUMBER_DEBUG;

    // fetch information about keys to be used when selecting default values
    std::string keyForSystemWideDefault;
    DBColumnParams& colParams = DBColumnParams::getInstance();
    if (selector.getSelectionCount() > 0) {
        for (uint32_t sdx = 0; sdx < selector.getSelectionCount(); ++sdx) {
            std::string keyForDefault = "";
            std::string selectorName = selector.getSelectionColumn(sdx);
            if (! colParams.
                columnKeyToGetDefaultValue(selectorName, keyForDefault)) {
                keyForDefault = NO_DEFAULT_KEY_STRING;
                _noKeyElementForDefaults.push_back(true);
            }
            else {
                _noKeyElementForDefaults.push_back(false);
            }

            if (sdx > 0) {
                keyForSystemWideDefault.append(KEY_SEPARATOR_STRING);
            }
            keyForSystemWideDefault.append(keyForDefault);

            _keyElementsForDefaults.push_back(keyForDefault);

            _selectorDefaultValueMap[selectorName] = keyForDefault;
        }
    }
    else {
        keyForSystemWideDefault = KEY_FOR_SYSTEM_WIDE_PARAM;
    }
    LINE_NUMBER_DEBUG;

    // loop over each subsystem configuration ID that was provided
    for (uint32_t cfgIdx = 0; cfgIdx < subsystemConfigIdList.size(); ++cfgIdx) {
        dbTablePtr->Clear();
        LINE_NUMBER_DEBUG;

        // set the appropriate validity contexts
        for (uint32_t sdx = 0; sdx < selector.getSelectionCount(); ++sdx) {
            if (selector.hasSelectionValues(sdx)) {
                if (! dbTablePtr->
                    SetValidityRange(selector.getSelectionColumn(sdx),
                                     selector.getSelectionLowValue(sdx),
                                     selector.getSelectionHighValue(sdx))) {
                    LINE_NUMBER_DEBUG;
                    return;
                }
            }
        }
        if (! dbTablePtr->SetValidityRange("subsysCfgId",
                                            subsystemConfigIdList[cfgIdx])) {
            LINE_NUMBER_DEBUG;
            return;
        }

        // fetch the data from the DB
        if ((debugFlag & 0x10) != 0) {
            dbTablePtr->SetVerbosity(1);
        }
        if (! dbTablePtr->LoadFromDB()) {
            LINE_NUMBER_DEBUG;
            return;
        }
        LINE_NUMBER_DEBUG;

        // also, fetch all default values that we may need
        for (uint32_t sdx = selector.getSelectionCount(); sdx > 0; --sdx) {
            if (selector.hasSelectionValues(sdx-1)) {
                if (! dbTablePtr->
                    SetValidityRange(selector.getSelectionColumn(sdx-1),
                                     _keyElementsForDefaults[sdx-1],
                                     _keyElementsForDefaults[sdx-1])) {
                    LINE_NUMBER_DEBUG;
                    return;
                }
                if (! dbTablePtr->LoadFromDB()) {
                    LINE_NUMBER_DEBUG;
                    return;
                }
            }
        }
        LINE_NUMBER_DEBUG;

        // loop over each row that was selected and decide whether
        // to add it to the internal result set
        ConfigDataContainer tmpContainerForDefaults;
        for (int32_t idx = 0; idx < dbTablePtr->NRow(); ++idx) {
            NOVADB::Row* rowPtr = dbTablePtr->GetRow(idx);
            std::string valueString;
            NOVADB::Column* colPtr;

            // build the key that we'll use to store this entry in
            // the result set
            std::string key;
            std::vector<std::string> keyElements;
            bool thisKeyIsForADefault = false;
            if (selector.getSelectionCount() > 0) {
                for (uint32_t sdx = 0; sdx < selector.getSelectionCount();
                     ++sdx) {
                    std::string selectorName = selector.getSelectionColumn(sdx);
                    colPtr = rowPtr->Col(selectorName);
                    if (! colPtr->Get(valueString)) {
                        LINE_NUMBER_DEBUG;
                        return;
                    }

                    if (sdx > 0) {
                        key.append(KEY_SEPARATOR_STRING);
                    }
                    key.append(valueString);

                    keyElements.push_back(valueString);

                    if (valueString == _keyElementsForDefaults[sdx]) {
                        thisKeyIsForADefault = true;
                    }

                    // build up the list(s) of unique selector values
                    ConfigValueListMap::const_iterator iter1 =
                        _distinctSelectorValueMap.find(selectorName);
                    if (iter1 == _distinctSelectorValueMap.end()) {
                        ConfigValueListPtr valueListPtr(new ConfigValueList());
                        valueListPtr->push_back(valueString);
                        _distinctSelectorValueMap[selectorName] = valueListPtr;
                    }
                    else {
                        ConfigValueListPtr valueListPtr = iter1->second;
                        ConfigValueList::const_iterator iter2 =
                            std::find(valueListPtr->begin(),
                                      valueListPtr->end(), valueString);
                        if (iter2 == valueListPtr->end()) {
                            valueListPtr->push_back(valueString);
                        }
                    }
                }
            }
            else {
                key = KEY_FOR_SYSTEM_WIDE_PARAM;
            }

            // check if the result set already has an entry for this key
            ConfigDataContainer::const_iterator iter1 = _resultSet.find(key);
            if (iter1 != _resultSet.end()) {continue;}

            // check if a default value exists that makes this entry irrelevant
            bool foundDefault = false;
            for (uint32_t testIdx = keyElements.size(); testIdx > 0; --testIdx) {
                std::string tmpKey;
                for (uint32_t sdx = 0; sdx < keyElements.size(); ++sdx) {
                    std::string keyFragment = keyElements[sdx];
                    if (sdx >= (testIdx - 1)) {
                        keyFragment = _keyElementsForDefaults[sdx];
                    }

                    if (sdx > 0) {
                        tmpKey.append(KEY_SEPARATOR_STRING);
                    }
                    tmpKey.append(keyFragment);
                }

                iter1 = _resultSet.find(tmpKey);
                if (iter1 != _resultSet.end()) {
                    foundDefault = true;
                    break;
                }
            }
            if (foundDefault) {continue;}

            // if we made it this far, then we should store the entry in the
            // result set
            ConfigValueMapPtr mapPtr(new ConfigValueMap());

            // fetch the requested data values from the DB row
            for (uint32_t ndx = 0; ndx < dataColumnNameList.size(); ++ndx) {
                std::string columnName = dataColumnNameList[ndx];
                colPtr = rowPtr->Col(columnName);
                _dataColumnTypeList[ndx] = colPtr->Type();
                if (colPtr->IsNull()) {
                    (*mapPtr)[columnName] = "";
                }
                else {
                    if (! colPtr->Get(valueString)) {
                        LINE_NUMBER_DEBUG;
                        return;
                    }
                    (*mapPtr)[columnName] = valueString;
                }
            }

            // insert the entry into the result set or the temporary 
            // container for defaults
            if (thisKeyIsForADefault) {
                tmpContainerForDefaults[key] = mapPtr;
            }
            else {
                _resultSet[key] = mapPtr;
            }
        }
        LINE_NUMBER_DEBUG;

        // copy the defaults that were found with this configId to the
        // result set, now that they won't mess up the logic above
        ConfigDataContainer::const_iterator iter2;
        ConfigDataContainer::const_iterator iter2End =
            tmpContainerForDefaults.end();
        for (iter2 = tmpContainerForDefaults.begin();
             iter2 != iter2End; ++iter2) {
            _resultSet[iter2->first] = iter2->second;
        }

        // as soon as we find a system-wide default (or a system-wide
        // parameter value), we're done
        ConfigDataContainer::const_iterator iter3 = 
            _resultSet.find(keyForSystemWideDefault);
        if (iter3 != _resultSet.end()) {break;}
    }
    LINE_NUMBER_DEBUG;

    // sort the list(s) of distinct selector values
    ConfigValueListMap::const_iterator iter;
    ConfigValueListMap::const_iterator iterEnd =
        _distinctSelectorValueMap.end();
    for (iter = _distinctSelectorValueMap.begin(); iter != iterEnd; ++iter) {
        ConfigValueListPtr valueListPtr = iter->second;
        std::sort(valueListPtr->begin(), valueListPtr->end(), sortingFunction);
    }

    // print out the contents of the tree, if requested
    if ((debugFlag & 0x01) != 0) {
        std::cout << "*** ConfigDataTree for " << dbTablePtr->Name()
                  << std::endl;
        ConfigDataContainer::const_iterator resIterEnd = _resultSet.end();
        ConfigDataContainer::const_iterator resIter;
        for (resIter = _resultSet.begin(); resIter != resIterEnd; ++resIter) {
            ConfigValueMapPtr mapPtr = resIter->second;
            std::cout << "  Key=" << resIter->first;
            if (mapPtr.get() != 0) {
                ConfigValueMap::const_iterator mapIterEnd = mapPtr->end();
                ConfigValueMap::const_iterator mapIter;
                for (mapIter = mapPtr->begin(); mapIter != mapIterEnd; ++mapIter) {
                    std::cout << ", " << mapIter->first << "=" << mapIter->second;
                }
            }
            std::cout << std::endl;
        }
    }

    // print out the selector and data column details, if requested
    if ((debugFlag & 0x02) != 0) {
        std::cout << "*** ConfigDataTree for " << dbTablePtr->Name()
                  << std::endl;
        std::vector<std::string> selectorColumnNames = getSelectorColumnNames();
        for (uint32_t idx = 0; idx < selectorColumnNames.size(); ++idx) {
            std::string columnName = selectorColumnNames[idx];
            std::string defaultValue = getSelectorValueForDefault(columnName);
            std::vector<std::string> distinctValues =
                getDistinctSelectorColumnValues(columnName);
            std::cout << "  Selector = \"" << columnName << "\", default = \""
                      << defaultValue << "\"" << std::endl;
            std::cout << "    Distinct values = ";
            for (uint32_t jdx = 0; jdx < distinctValues.size(); ++jdx) {
                if (jdx > 0) {std::cout << ", ";}
                std::cout << distinctValues[jdx];
            }
            std::cout << std::endl;
        }
    }

    // indicate success
    _hasValidData = true;
    LINE_NUMBER_DEBUG;
}

} // end of namespace daqconfig
} // end of namespace dbutils