From f4feef85f4db14efd1369d477b40294e1044a86f Mon Sep 17 00:00:00 2001 From: Martin Eide <43970264+mrtineide@users.noreply.github.com> Date: Mon, 1 Sep 2025 14:54:39 +0200 Subject: [PATCH 1/4] Extend CCDB functions with optional headers --- CCDB/CMakeLists.txt | 6 + CCDB/include/CCDB/BasicCCDBManager.h | 47 +++- CCDB/include/CCDB/CcdbApi.h | 2 +- CCDB/src/CcdbApi.cxx | 5 +- CCDB/test/testCcdbApiHeaders.cxx | 396 +++++++++++++++++++++++++++ 5 files changed, 446 insertions(+), 10 deletions(-) create mode 100644 CCDB/test/testCcdbApiHeaders.cxx diff --git a/CCDB/CMakeLists.txt b/CCDB/CMakeLists.txt index 9436fa37de8e6..691c3311e117c 100644 --- a/CCDB/CMakeLists.txt +++ b/CCDB/CMakeLists.txt @@ -92,6 +92,12 @@ o2_add_test(CcdbDownloader PUBLIC_LINK_LIBRARIES O2::CCDB LABELS ccdb) +o2_add_test(CcdbApi-Headers + SOURCES test/testCcdbApiHeaders.cxx + COMPONENT_NAME ccdb + PUBLIC_LINK_LIBRARIES O2::CCDB + LABELS ccdb) + # extra CcdbApi test which dispatches to CCDBDownloader (tmp until full move done) #o2_add_test_command(NAME CcdbApi-MultiHandle # WORKING_DIRECTORY ${SIMTESTDIR} diff --git a/CCDB/include/CCDB/BasicCCDBManager.h b/CCDB/include/CCDB/BasicCCDBManager.h index 9668097c39473..f10a135c86728 100644 --- a/CCDB/include/CCDB/BasicCCDBManager.h +++ b/CCDB/include/CCDB/BasicCCDBManager.h @@ -23,9 +23,11 @@ #include #include #include +#include #include #include #include +#include class TGeoManager; // we need to forward-declare those classes which should not be cleaned up @@ -57,6 +59,7 @@ class CCDBManagerInstance int queries = 0; int fetches = 0; int failures = 0; + std::map cacheOfHeaders; bool isValid(long ts) { return ts < endvalidity && ts >= startvalidity; } bool isCacheValid(long ts) { @@ -70,6 +73,7 @@ class CCDBManagerInstance uuid = ""; startvalidity = 0; endvalidity = -1; + cacheOfHeaders.clear(); } }; @@ -98,9 +102,9 @@ class CCDBManagerInstance /// query timestamp long getTimestamp() const { return mTimestamp; } - /// retrieve an object of type T from CCDB as stored under path and timestamp + /// retrieve an object of type T from CCDB as stored under path and timestamp. Optional to get the headers. Can give a filter of headers to be saved in cache and returned (if present) template - T* getForTimeStamp(std::string const& path, long timestamp); + T* getForTimeStamp(std::string const& path, long timestamp, std::map* headers = nullptr, std::vector headerFilter = {}); /// retrieve an object of type T from CCDB as stored under path and using the timestamp in the middle of the run template @@ -112,10 +116,7 @@ class CCDBManagerInstance { // TODO: add some error info/handling when failing mMetaData = metaData; - auto obj = getForTimeStamp(path, timestamp); - if (headers) { - *headers = mHeaders; - } + auto obj = getForTimeStamp(path, timestamp, headers); return obj; } @@ -235,7 +236,7 @@ class CCDBManagerInstance }; template -T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) +T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp, std::map* headers, std::vector headerFilter) { mHeaders.clear(); // we clear at the beginning; to allow to retrieve the header information in a subsequent call T* ptr = nullptr; @@ -258,15 +259,45 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) mFetchedSize += s; } } + if (!headerFilter.empty()) { + LOGP(warn, "Header filter ignored when caching is disabled, giving back all headers"); + } + if (headers) { + *headers = mHeaders; + } } else { auto& cached = mCache[path]; cached.queries++; if ((!isOnline() && cached.isCacheValid(timestamp)) || (mCheckObjValidityEnabled && cached.isValid(timestamp))) { + // Give back the cached/saved headers + if (headers) { + *headers = cached.cacheOfHeaders; + } return reinterpret_cast(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get()); } ptr = mCCDBAccessor.retrieveFromTFileAny(path, mMetaData, timestamp, &mHeaders, cached.uuid, mCreatedNotAfter ? std::to_string(mCreatedNotAfter) : "", mCreatedNotBefore ? std::to_string(mCreatedNotBefore) : ""); + + // Cache the headers + if (headerFilter.empty()) { + // No filter, cache all headers + for (auto const& h : mHeaders) { + cached.cacheOfHeaders[h.first] = h.second; + } + } else { + // Cache only the asked for headers + for (auto const& k : headerFilter) { + auto it = mHeaders.find(std::string(k)); + if (it != mHeaders.end()) { + cached.cacheOfHeaders.insert_or_assign(it->first, it->second); // Only want to overwrite if the header exists in the source + } + } + } + if (headers) { + *headers = cached.cacheOfHeaders; + } + if (ptr) { // new object was shipped, old one (if any) is not valid anymore cached.fetches++; mFetches++; @@ -300,7 +331,7 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) size_t s = atol(sh->second.c_str()); mFetchedSize += s; cached.minSize = std::min(s, cached.minSize); - cached.maxSize = std::max(s, cached.minSize); + cached.maxSize = std::max(s, cached.minSize); // I think this should be maxSize, not minSize } } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set cached.failures++; diff --git a/CCDB/include/CCDB/CcdbApi.h b/CCDB/include/CCDB/CcdbApi.h index e41f58d5c6da9..4dab11d5972d8 100644 --- a/CCDB/include/CCDB/CcdbApi.h +++ b/CCDB/include/CCDB/CcdbApi.h @@ -281,7 +281,7 @@ class CcdbApi //: public DatabaseInterface * @return: True in case operation successful or false if there was a failure/problem. */ bool retrieveBlob(std::string const& path, std::string const& targetdir, std::map const& metadata, long timestamp, - bool preservePathStructure = true, std::string const& localFileName = "snapshot.root", std::string const& createdNotAfter = "", std::string const& createdNotBefore = "") const; + bool preservePathStructure = true, std::string const& localFileName = "snapshot.root", std::string const& createdNotAfter = "", std::string const& createdNotBefore = "", std::map* headers = nullptr) const; /** * Retrieve the headers of a CCDB entry, if it exists. diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index 27ad14cdf24fa..e8e58c1fe33eb 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -831,7 +831,7 @@ TObject* CcdbApi::retrieveFromTFile(std::string const& path, std::map const& metadata, - long timestamp, bool preservePath, std::string const& localFileName, std::string const& createdNotAfter, std::string const& createdNotBefore) const + long timestamp, bool preservePath, std::string const& localFileName, std::string const& createdNotAfter, std::string const& createdNotBefore, std::map* outHeaders) const { // we setup the target path for this blob @@ -879,6 +879,9 @@ bool CcdbApi::retrieveBlob(std::string const& path, std::string const& targetdir CCDBQuery querysummary(path, metadata, timestamp); updateMetaInformationInLocalFile(targetpath.c_str(), &headers, &querysummary); + if (outHeaders) { + *outHeaders = std::move(headers); // Re-use the same headers to give back to the callee + } return true; } diff --git a/CCDB/test/testCcdbApiHeaders.cxx b/CCDB/test/testCcdbApiHeaders.cxx new file mode 100644 index 0000000000000..1b8b75216d5ec --- /dev/null +++ b/CCDB/test/testCcdbApiHeaders.cxx @@ -0,0 +1,396 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file testCcdbApiHeaders.cxx +/// \brief Test BasicCCDBManager header/metadata information functionality with caching +/// \author martin.oines.eide@cern.ch + +#define BOOST_TEST_MODULE CCDB +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include +#include +#include +#include + +#include "CCDB/BasicCCDBManager.h" +#include "CCDB/CCDBTimeStampUtils.h" +#include "CCDB/CcdbApi.h" + +static std::string basePath; +// std::string ccdbUrl = "http://localhost:8080"; +std::string ccdbUrl = "http://ccdb-test.cern.ch:8080"; +bool hostReachable = false; + +/** + * Global fixture, ie general setup and teardown + * Copied from testBasicCCDBManager.cxx + */ +struct Fixture { + Fixture() + { + o2::ccdb::CcdbApi api; + if (std::getenv("ALICEO2_CCDB_HOST")) + ccdbUrl = std::string(std::getenv("ALICEO2_CCDB_HOST")); + api.init(ccdbUrl); + hostReachable = api.isHostReachable(); + char hostname[_POSIX_HOST_NAME_MAX]; + gethostname(hostname, _POSIX_HOST_NAME_MAX); + basePath = std::string("Users/m/meide/BasicCCDBManager/"); + + LOG(info) << "Path we will use in this test suite : " + basePath << std::endl; + LOG(info) << "ccdb url: " << ccdbUrl << std::endl; + LOG(info) << "Is host reachable ? --> " << hostReachable << std::endl; + } + ~Fixture() + { + if (hostReachable) { + o2::ccdb::CcdbApi api; + api.init(ccdbUrl); + api.truncate(basePath + "*"); // This deletes the data after test is run, disable if you want to inspect the data + LOG(info) << "Test data truncated/deleted (" << basePath << ")" << std::endl; + } + } +}; +BOOST_GLOBAL_FIXTURE(Fixture); +/** + * Just an accessor to the hostReachable variable to be used to determine whether tests can be ran or not. + * Copied from testCcdbApi.cxx + */ +struct if_reachable { + boost::test_tools::assertion_result operator()(boost::unit_test::test_unit_id) + { + return hostReachable; + } +}; +/** + * Fixture for the tests, i.e. code is ran in every test that uses it, i.e. it is like a setup and teardown for tests. + * Copied from testCcdbApi.cxx + */ +struct test_fixture { + test_fixture() + { + api.init(ccdbUrl); + metadata["Hello"] = "World"; + LOG(info) << "*** " << boost::unit_test::framework::current_test_case().p_name << " ***" << std::endl; + } + ~test_fixture() = default; + + o2::ccdb::CcdbApi api; + std::map metadata; +}; + +// Only compare known and stable keys (avoid volatile ones like Date) +static const std::set sStableKeys = { + "ETag", + "Valid-From", + "Valid-Until", + "Created", + "Last-Modified", + "Content-Disposition", + "Content-Location", + "path", + "partName", + "Content-MD5", + "Hello" // TODO find other headers to compare to +}; + +// Test that we get back the same header header keys as we put in (for stable keys) + +BOOST_AUTO_TEST_CASE(testCachedHeaders, *boost::unit_test::precondition(if_reachable())) +{ + /// ━━━━━━━ ARRANGE ━━━━━━━━━ + // First store objects to test with + test_fixture f; + + std::string pathA = basePath + "CachingA"; + std::string pathB = basePath + "CachingB"; + std::string pathC = basePath + "CachingC"; + std::string ccdbObjO = "testObjectO"; + std::string ccdbObjN = "testObjectN"; + std::string ccdbObjX = "testObjectX"; + std::map md = f.metadata; + long start = 1000, stop = 3000; + f.api.storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); + f.api.storeAsTFileAny(&ccdbObjN, pathB, md, start, stop); + f.api.storeAsTFileAny(&ccdbObjX, pathC, md, start, stop); + // initilize the BasicCCDBManager + o2::ccdb::BasicCCDBManager& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); + ccdbManager.clearCache(); + ccdbManager.setURL(ccdbUrl); + ccdbManager.setCaching(true); // This is what we want to test. + + /// ━━━━━━━━━━━ ACT ━━━━━━━━━━━━ + // Plan: get one object, then another, then the first again and check the headers are the same + std::map headers1, headers2, headers3; + + auto* obj1 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers1); + auto* obj2 = ccdbManager.getForTimeStamp(pathB, (start + stop) / 2, &headers2); + auto* obj3 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers3); // Should lead to a cache hit! + + /// ━━━━━━━━━━━ ASSERT ━━━━━━━━━━━━ + /// Check that we got something + BOOST_REQUIRE(obj1 != nullptr); + BOOST_REQUIRE(obj2 != nullptr); + BOOST_REQUIRE(obj3 != nullptr); + + LOG(debug) << "obj1: " << *obj1; + LOG(debug) << "obj2: " << *obj2; + LOG(debug) << "obj3: " << *obj3; + + // Sanity check + /// Check that the objects are correct + BOOST_TEST(*obj1 == ccdbObjO); + BOOST_TEST(*obj3 == ccdbObjO); + BOOST_TEST(obj3 == obj1); // should be the same object in memory since it is cached + + BOOST_TEST(obj2 != obj1); + + (*obj1) = "ModifiedObject"; + BOOST_TEST(*obj1 == "ModifiedObject"); + BOOST_TEST(*obj3 == "ModifiedObject"); // obj3 and obj1 are the same object in memory + + // Check that the headers are the same for the two retrievals of the same object + BOOST_REQUIRE(headers1.size() != 0); + BOOST_REQUIRE(headers3.size() != 0); + + LOG(debug) << "Headers1 size: " << headers1.size(); + for (const auto& h : headers1) { + LOG(debug) << " " << h.first << " -> " << h.second; + } + LOG(debug) << "Headers3 size: " << headers3.size(); + for (const auto& h : headers3) { + LOG(debug) << " " << h.first << " -> " << h.second; + } + + for (const auto& stableKey : sStableKeys) { + LOG(info) << "Checking key: " << stableKey; + + BOOST_REQUIRE(headers1.count(stableKey) > 0); + BOOST_REQUIRE(headers3.count(stableKey) > 0); + BOOST_TEST(headers1.at(stableKey) == headers3.at(stableKey)); + } + BOOST_TEST(headers1 != headers2, "The headers for different objects should be different"); + + // Test that we can change the map and the two headers are not affected + headers1["NewKey"] = "NewValue"; + headers3["NewKey"] = "DifferentValue"; + BOOST_TEST(headers1["NewKey"] != headers3["NewKey"]); // This tests that we have a deep copy of the headers +} + +BOOST_AUTO_TEST_CASE(testNonCachedHeaders, *boost::unit_test::precondition(if_reachable())) +{ + /// ━━━━━━━ ARRANGE ━━━━━━━━━ + // First store objects to test with + test_fixture f; + + std::string pathA = basePath + "NonCachingA"; + std::string pathB = basePath + "NonCachingB"; + std::string ccdbObjO = "testObjectO"; + std::string ccdbObjN = "testObjectN"; + std::map md = f.metadata; + long start = 1000, stop = 2000; + f.api.storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); + f.api.storeAsTFileAny(&ccdbObjN, pathB, md, start, stop); + // initilize the BasicCCDBManager + o2::ccdb::BasicCCDBManager& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); + ccdbManager.clearCache(); + ccdbManager.setURL(ccdbUrl); + ccdbManager.setCaching(false); // This is what we want to test, no caching + + /// ━━━━━━━━━━━ ACT ━━━━━━━━━━━━ + // Plan: get one object, then another, then the first again. Then check that the contents is the same but not the object in memory + std::map headers1, headers2, headers3; + + auto* obj1 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers1); + auto* obj2 = ccdbManager.getForTimeStamp(pathB, (start + stop) / 2, &headers2); + auto* obj3 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers3); // Should not be cached since explicitly disabled + + /// ━━━━━━━━━━━ ASSERT ━━━━━━━━━━━ + /// Check that we got something + BOOST_REQUIRE(obj1 != nullptr); + BOOST_REQUIRE(obj2 != nullptr); + BOOST_REQUIRE(obj3 != nullptr); + + LOG(debug) << "obj1: " << *obj1; + LOG(debug) << "obj2: " << *obj2; + LOG(debug) << "obj3: " << *obj3; + + // Sanity check + /// Check that the objects are correct + BOOST_TEST(*obj1 == ccdbObjO); + BOOST_TEST(*obj3 == ccdbObjO); + BOOST_TEST(obj2 != obj1); + BOOST_TEST(obj3 != obj1); // should NOT be the same object in memory + (*obj1) = "ModifiedObject"; + BOOST_TEST(*obj1 == "ModifiedObject"); + BOOST_TEST(*obj3 != "ModifiedObject"); // obj3 and obj1 are NOT the same object in memory + + BOOST_TEST(headers1.size() == headers3.size()); + + // Remove the date header since it may be different even for the same object since we might have asked in different seconds + headers1.erase("Date"); + headers3.erase("Date"); + BOOST_TEST(headers1 == headers3, "The headers for the same object should be the same even if not cached"); + + BOOST_TEST(headers1 != headers2, "The headers for different objects should be different"); + BOOST_TEST(headers1.size() != 0); + BOOST_TEST(headers3.size() != 0); + BOOST_TEST(headers2.size() != 0); + BOOST_TEST(headers1 != headers2, "The headers for different objects should be different"); +} + +BOOST_AUTO_TEST_CASE(test_header_filtering, *boost::unit_test::precondition(if_reachable())) +{ + /// ━━━━━━━ ARRANGE ━━━━━━━━━ + // First store objects to test with + test_fixture f; + + std::string pathA = basePath + "CachingA"; + std::string pathB = basePath + "CachingB"; + std::string ccdbObjO = "testObjectO"; + std::string ccdbObjN = "testObjectN"; + std::map md = f.metadata; + long start = 1000, stop = 3000; + f.api.storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); + f.api.storeAsTFileAny(&ccdbObjN, pathB, md, start + 1, stop + 1); + // initilize the BasicCCDBManager + o2::ccdb::BasicCCDBManager& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); + ccdbManager.clearCache(); + ccdbManager.setURL(ccdbUrl); + ccdbManager.setCaching(true); // This is what we want to test. + + // ━━━━━━━━━━━━ ACT ━━━━━━━━━━━━ + // Plan: get the objects but also include a string view to indicate which headers to keep + + // Convert the stable keys to a vector of string views + std::vector headerFilter; + for (const auto& k : sStableKeys) { + headerFilter.push_back(k); + } + + std::map headers1, headers2, headers3; + auto* obj1 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers1, headerFilter); + auto* obj2 = ccdbManager.getForTimeStamp(pathB, (start + stop) / 2, &headers2, headerFilter); + auto* obj3 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers3, headerFilter); // Should lead to a cache hit + + /// ━━━━━━━━━━━ ASSERT ━━━━━━━━━━━ + + /// Check that we got something + BOOST_REQUIRE(obj1 != nullptr); + BOOST_REQUIRE(obj2 != nullptr); + BOOST_REQUIRE(obj3 != nullptr); + + LOG(debug) << "obj1: " << *obj1; + LOG(debug) << "obj2: " << *obj2; + LOG(debug) << "obj3: " << *obj3; + /// Sanity check + // Check that the objects are correct + BOOST_TEST(*obj1 == ccdbObjO); + BOOST_TEST(*obj3 == ccdbObjO); + BOOST_TEST(obj3 == obj1); // should be the same object in memory since it is cached + BOOST_TEST(obj2 != obj1); + + // Modify one and check that the other is also modified + (*obj1) = "ModifiedObject"; + BOOST_TEST(*obj1 == "ModifiedObject"); + BOOST_TEST(*obj3 == "ModifiedObject"); + + BOOST_REQUIRE(headers1.size() != 0); + BOOST_REQUIRE(headers3.size() != 0); + + LOG(debug) << "Headers1 size: " << headers1.size(); + for (const auto& h : headers1) { + LOG(debug) << " " << h.first << " -> " << h.second; + } + LOG(debug) << "Headers3 size: " << headers3.size(); + for (const auto& h : headers3) { + LOG(debug) << " " << h.first << " -> " << h.second; + } + + /// Test the headers keys and values are the same for the two retrievals of the same object + for (const auto& stableKey : sStableKeys) { + LOG(debug) << "Checking header: " << stableKey; + + BOOST_TEST(headers1.count(stableKey) > 0, "stable key missing in headers1: " + stableKey); + BOOST_TEST(headers3.count(stableKey) > 0, "stable key missing in headers3: " + stableKey); + + BOOST_TEST(headers1.at(stableKey) == headers3.at(stableKey)); + } + + // Now test that the filtering worked, i.e. that we only have the stable keys + for (const auto& h : headers1) { + BOOST_TEST(sStableKeys.count(h.first) > 0, "Found non-stable key in headers1: " + h.first); + } + for (const auto& h : headers2) { + BOOST_TEST(sStableKeys.count(h.first) > 0, "Found non-stable key in headers2: " + h.first); + } + for (const auto& h : headers3) { + BOOST_TEST(sStableKeys.count(h.first) > 0, "Found non-stable key in headers3: " + h.first); + } + + // Now check that we just have the filtered keys + BOOST_TEST(headers1.size() == sStableKeys.size()); + BOOST_TEST(headers3.size() == sStableKeys.size()); + + // Extract the keys and values so that we can compare the sets + std::set header1Keys, header2Keys, header3Keys; + std::set header1Values, header2Values; + + for (const auto& h : headers1) { + header1Keys.insert(h.first); + } + for (const auto& h : headers2) { + header2Keys.insert(h.first); + } + for (const auto& h : headers3) { + header3Keys.insert(h.first); + } + + BOOST_TEST(header1Keys.size() == sStableKeys.size()); + BOOST_TEST(header2Keys.size() == sStableKeys.size()); + BOOST_TEST(header3Keys.size() == sStableKeys.size()); + + BOOST_TEST(header1Keys == header2Keys, boost::test_tools::per_element()); + BOOST_TEST(header2Keys == header3Keys, boost::test_tools::per_element()); + BOOST_TEST(header1Keys == header3Keys, boost::test_tools::per_element()); + + BOOST_TEST(header1Keys == sStableKeys, boost::test_tools::per_element()); + BOOST_TEST(header2Keys == sStableKeys, boost::test_tools::per_element()); + BOOST_TEST(header3Keys == sStableKeys, boost::test_tools::per_element()); + + /// Make sure that headers for different objects are different + BOOST_TEST(headers1 != headers2, "The headers for different objects should be different (mainly ETag)"); + + // Make a set of keys that we know beforehand is the same for different objects + std::set keysThatShouldBeDifferent = { + "Valid-From", + "Valid-Until", + "Created", + "ETag", + "Content-Disposition", + "Content-Location", + "path", + "Content-MD5", + }; + + for (const auto& stableKey : keysThatShouldBeDifferent) { + BOOST_TEST(headers2.count(stableKey) > 0, "stable key missing in headers2: " + stableKey); + header2Values.insert(headers2.at(stableKey)); + header1Values.insert(headers1.at(stableKey)); + } + + BOOST_TEST(header2Values.size() == keysThatShouldBeDifferent.size()); + + BOOST_TEST(header2Values != header1Values, boost::test_tools::per_element()); +} From 21ef6573e7063fc40e86ef18b570c75f0c77f660 Mon Sep 17 00:00:00 2001 From: Martin Eide <43970264+mrtineide@users.noreply.github.com> Date: Tue, 30 Sep 2025 17:42:59 +0200 Subject: [PATCH 2/4] Remove the header filtering & add more tests --- CCDB/include/CCDB/BasicCCDBManager.h | 32 +-- CCDB/src/CcdbApi.cxx | 2 +- CCDB/test/testCcdbApiHeaders.cxx | 292 +++++++++++++++------------ 3 files changed, 172 insertions(+), 154 deletions(-) diff --git a/CCDB/include/CCDB/BasicCCDBManager.h b/CCDB/include/CCDB/BasicCCDBManager.h index f10a135c86728..71287c2f07d76 100644 --- a/CCDB/include/CCDB/BasicCCDBManager.h +++ b/CCDB/include/CCDB/BasicCCDBManager.h @@ -27,7 +27,6 @@ #include #include #include -#include class TGeoManager; // we need to forward-declare those classes which should not be cleaned up @@ -102,9 +101,9 @@ class CCDBManagerInstance /// query timestamp long getTimestamp() const { return mTimestamp; } - /// retrieve an object of type T from CCDB as stored under path and timestamp. Optional to get the headers. Can give a filter of headers to be saved in cache and returned (if present) + /// retrieve an object of type T from CCDB as stored under path and timestamp. Optional to get the headers. template - T* getForTimeStamp(std::string const& path, long timestamp, std::map* headers = nullptr, std::vector headerFilter = {}); + T* getForTimeStamp(std::string const& path, long timestamp, std::map* headers = nullptr); /// retrieve an object of type T from CCDB as stored under path and using the timestamp in the middle of the run template @@ -236,7 +235,7 @@ class CCDBManagerInstance }; template -T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp, std::map* headers, std::vector headerFilter) +T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp, std::map* headers) { mHeaders.clear(); // we clear at the beginning; to allow to retrieve the header information in a subsequent call T* ptr = nullptr; @@ -259,9 +258,7 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp, mFetchedSize += s; } } - if (!headerFilter.empty()) { - LOGP(warn, "Header filter ignored when caching is disabled, giving back all headers"); - } + if (headers) { *headers = mHeaders; } @@ -278,22 +275,11 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp, ptr = mCCDBAccessor.retrieveFromTFileAny(path, mMetaData, timestamp, &mHeaders, cached.uuid, mCreatedNotAfter ? std::to_string(mCreatedNotAfter) : "", mCreatedNotBefore ? std::to_string(mCreatedNotBefore) : ""); - - // Cache the headers - if (headerFilter.empty()) { - // No filter, cache all headers - for (auto const& h : mHeaders) { - cached.cacheOfHeaders[h.first] = h.second; - } - } else { - // Cache only the asked for headers - for (auto const& k : headerFilter) { - auto it = mHeaders.find(std::string(k)); - if (it != mHeaders.end()) { - cached.cacheOfHeaders.insert_or_assign(it->first, it->second); // Only want to overwrite if the header exists in the source - } - } + // update the cached headers + for (auto const& h : mHeaders) { + cached.cacheOfHeaders[h.first] = h.second; } + // return the cached headers if (headers) { *headers = cached.cacheOfHeaders; } @@ -331,7 +317,7 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp, size_t s = atol(sh->second.c_str()); mFetchedSize += s; cached.minSize = std::min(s, cached.minSize); - cached.maxSize = std::max(s, cached.minSize); // I think this should be maxSize, not minSize + cached.maxSize = std::max(s, cached.minSize); } } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set cached.failures++; diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index e8e58c1fe33eb..763838b748cb0 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -880,7 +880,7 @@ bool CcdbApi::retrieveBlob(std::string const& path, std::string const& targetdir updateMetaInformationInLocalFile(targetpath.c_str(), &headers, &querysummary); if (outHeaders) { - *outHeaders = std::move(headers); // Re-use the same headers to give back to the callee + *outHeaders = std::move(headers); } return true; } diff --git a/CCDB/test/testCcdbApiHeaders.cxx b/CCDB/test/testCcdbApiHeaders.cxx index 1b8b75216d5ec..845f94c47ccd6 100644 --- a/CCDB/test/testCcdbApiHeaders.cxx +++ b/CCDB/test/testCcdbApiHeaders.cxx @@ -250,147 +250,179 @@ BOOST_AUTO_TEST_CASE(testNonCachedHeaders, *boost::unit_test::precondition(if_re BOOST_TEST(headers1 != headers2, "The headers for different objects should be different"); } -BOOST_AUTO_TEST_CASE(test_header_filtering, *boost::unit_test::precondition(if_reachable())) +BOOST_AUTO_TEST_CASE(CacheFirstRetrievalAndHeadersPersistence) { - /// ━━━━━━━ ARRANGE ━━━━━━━━━ - // First store objects to test with test_fixture f; + /// ━━━━━━━ ARRANGE ━━━━━━━━━ + // Prepare two validity slots for same path to test ETag change later + std::string path = basePath + "ObjA"; + std::string objV1 = "ObjectVersion1"; + std::string objV2 = "ObjectVersion2"; + std::map meta1{ + {"UserKey1", "UValue1"}, + {"UserKey2", "UValue2"}}; + long v1start = 10'000; + long v1stop = 20'000; + long v2start = v1stop; // contiguous slot + long v2stop = v2start + (v1stop - v1start); + long mid1 = (v1start + v1stop) / 2; + // Store 2 versions + f.api.storeAsTFileAny(&objV1, path, meta1, v1start, v1stop); + f.api.storeAsTFileAny(&objV2, path, meta1, v2start, v2stop); + + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + mgr.setURL(ccdbUrl); + mgr.clearCache(); + mgr.setCaching(true); + mgr.setFatalWhenNull(true); + mgr.setTimestamp(mid1); + + /// ━━━━━━━ACT━━━━━━━━━ + std::map headers1, headers2, headers4, headers5; + + // 1) First retrieval WITH headers inside 1st slot + auto* p1 = mgr.getForTimeStamp(path, mid1, &headers1); + size_t fetchedSizeAfterFirst = mgr.getFetchedSize(); + // 2) Second retrieval (cache hit) + auto* p2 = mgr.getForTimeStamp(path, mid1, &headers2); + size_t fetchedSizeAfterSecond = mgr.getFetchedSize(); + // 3) Third retrieval (cache hit) WITHOUT passing headers + auto* p3 = mgr.getForTimeStamp(path, mid1); + // 4) Fourth retrieval with headers again -> should still produce same headers + auto* p4 = mgr.getForTimeStamp(path, mid1, &headers4); + // 5) Fifth retrieval with headers again to check persistence + auto* p5 = mgr.getForTimeStamp(path, mid1, &headers5); + + /// ━━━━━━━ASSERT━━━━━━━━━ + + BOOST_TEST(p1 != nullptr); + BOOST_TEST(*p1 == objV1); + + BOOST_TEST(headers1.count("UserKey1") == 1); + BOOST_TEST(headers1.count("UserKey2") == 1); + BOOST_TEST(headers1["UserKey1"] == "UValue1"); + BOOST_TEST(headers1["UserKey2"] == "UValue2"); + BOOST_TEST(headers1.count("Valid-From") == 1); + BOOST_TEST(headers1.count("Valid-Until") == 1); + BOOST_TEST(headers1.count("ETag") == 1); + + /* Need to manually amend the headers1 to have cache valid until for comparison sake, + * the header is not set in the first request. + * It is only set if the internal cache of CCDB has seen this object before, apperently. + * This will never happen in this test since it was just created and not asked for before. + */ + headers1["Cache-Valid-Until"] = std::to_string(v1stop); + + /* In rare cases the header date might be different, if the second has ticked over between the requests + */ + headers1.erase("Date"); + headers2.erase("Date"); + headers4.erase("Date"); + headers5.erase("Date"); - std::string pathA = basePath + "CachingA"; - std::string pathB = basePath + "CachingB"; - std::string ccdbObjO = "testObjectO"; - std::string ccdbObjN = "testObjectN"; - std::map md = f.metadata; - long start = 1000, stop = 3000; - f.api.storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); - f.api.storeAsTFileAny(&ccdbObjN, pathB, md, start + 1, stop + 1); - // initilize the BasicCCDBManager - o2::ccdb::BasicCCDBManager& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); - ccdbManager.clearCache(); - ccdbManager.setURL(ccdbUrl); - ccdbManager.setCaching(true); // This is what we want to test. - - // ━━━━━━━━━━━━ ACT ━━━━━━━━━━━━ - // Plan: get the objects but also include a string view to indicate which headers to keep - - // Convert the stable keys to a vector of string views - std::vector headerFilter; - for (const auto& k : sStableKeys) { - headerFilter.push_back(k); - } - - std::map headers1, headers2, headers3; - auto* obj1 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers1, headerFilter); - auto* obj2 = ccdbManager.getForTimeStamp(pathB, (start + stop) / 2, &headers2, headerFilter); - auto* obj3 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers3, headerFilter); // Should lead to a cache hit - - /// ━━━━━━━━━━━ ASSERT ━━━━━━━━━━━ - - /// Check that we got something - BOOST_REQUIRE(obj1 != nullptr); - BOOST_REQUIRE(obj2 != nullptr); - BOOST_REQUIRE(obj3 != nullptr); - - LOG(debug) << "obj1: " << *obj1; - LOG(debug) << "obj2: " << *obj2; - LOG(debug) << "obj3: " << *obj3; - /// Sanity check - // Check that the objects are correct - BOOST_TEST(*obj1 == ccdbObjO); - BOOST_TEST(*obj3 == ccdbObjO); - BOOST_TEST(obj3 == obj1); // should be the same object in memory since it is cached - BOOST_TEST(obj2 != obj1); - - // Modify one and check that the other is also modified - (*obj1) = "ModifiedObject"; - BOOST_TEST(*obj1 == "ModifiedObject"); - BOOST_TEST(*obj3 == "ModifiedObject"); - - BOOST_REQUIRE(headers1.size() != 0); - BOOST_REQUIRE(headers3.size() != 0); + BOOST_TEST(p2 == p1); // same pointer for cached scenario + BOOST_TEST(headers2 == headers1); // identical header map + BOOST_TEST(fetchedSizeAfterSecond == fetchedSizeAfterFirst); // no new fetch - LOG(debug) << "Headers1 size: " << headers1.size(); - for (const auto& h : headers1) { - LOG(debug) << " " << h.first << " -> " << h.second; - } - LOG(debug) << "Headers3 size: " << headers3.size(); - for (const auto& h : headers3) { - LOG(debug) << " " << h.first << " -> " << h.second; - } + BOOST_TEST(p3 == p1); - /// Test the headers keys and values are the same for the two retrievals of the same object - for (const auto& stableKey : sStableKeys) { - LOG(debug) << "Checking header: " << stableKey; + BOOST_TEST(p4 == p1); + BOOST_TEST(headers4 == headers1); - BOOST_TEST(headers1.count(stableKey) > 0, "stable key missing in headers1: " + stableKey); - BOOST_TEST(headers3.count(stableKey) > 0, "stable key missing in headers3: " + stableKey); + // Mutate the returned header map locally and ensure it does not corrupt internal cache + headers4["UserKey1"] = "Tampered"; + BOOST_TEST(p5 == p1); + BOOST_TEST(headers5["UserKey1"] == "UValue1"); // internal unchanged +} - BOOST_TEST(headers1.at(stableKey) == headers3.at(stableKey)); - } +BOOST_AUTO_TEST_CASE(FailedFetchDoesNotGiveMetadata) +{ + test_fixture f; - // Now test that the filtering worked, i.e. that we only have the stable keys - for (const auto& h : headers1) { - BOOST_TEST(sStableKeys.count(h.first) > 0, "Found non-stable key in headers1: " + h.first); - } - for (const auto& h : headers2) { - BOOST_TEST(sStableKeys.count(h.first) > 0, "Found non-stable key in headers2: " + h.first); - } - for (const auto& h : headers3) { - BOOST_TEST(sStableKeys.count(h.first) > 0, "Found non-stable key in headers3: " + h.first); - } + /// ━━━━━━━ ARRANGE ━━━━━━━━━ + std::string path = basePath + "FailThenRecover"; + std::string content = "ContentX"; + std::map meta{{"Alpha", "Beta"}}; + long s = 300'000, e = 310'000; + f.api.storeAsTFileAny(&content, path, meta, s, e); + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + mgr.clearCache(); + mgr.setCaching(true); + mgr.setFatalWhenNull(false); + + /// ━━━━━━━ ACT ━━━━━━━━━ + // Intentionally pick a timestamp outside validity to fail first + long badTS = s - 1000; + long goodTS = (s + e) / 2; + std::map hFail, hGood; + auto* badObj = mgr.getForTimeStamp(path, badTS, &hFail); + auto* goodObj = mgr.getForTimeStamp(path, goodTS, &hGood); + + /// ━━━━━━━ ASSERT ━━━━━━━━━ + BOOST_TEST(!hFail.empty()); // Should have some headers + BOOST_TEST(hFail["Alpha"] != "Beta"); // But not the metadata + BOOST_TEST(hGood.count("Alpha") == 1); + BOOST_TEST(hGood["Alpha"] == "Beta"); + + mgr.setFatalWhenNull(true); +} - // Now check that we just have the filtered keys - BOOST_TEST(headers1.size() == sStableKeys.size()); - BOOST_TEST(headers3.size() == sStableKeys.size()); +BOOST_AUTO_TEST_CASE(FirstCallWithoutHeadersThenWithHeaders) +{ + test_fixture f; - // Extract the keys and values so that we can compare the sets - std::set header1Keys, header2Keys, header3Keys; - std::set header1Values, header2Values; + std::string path = basePath + "LateHeaders"; + std::string body = "Late"; + std::map meta{{"LateKey", "LateVal"}}; + long s = 400'000, e = 410'000; + f.api.storeAsTFileAny(&body, path, meta, s, e); + + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + mgr.clearCache(); + mgr.setCaching(true); + long ts = (s + e) / 2; + + // 1) First call with nullptr headers + auto* first = mgr.getForTimeStamp(path, ts); + BOOST_TEST(first != nullptr); + BOOST_TEST(*first == body); + + // 2) Second call asking for headers - should return the full set + std::map h2; + auto* second = mgr.getForTimeStamp(path, ts, &h2); + BOOST_TEST(second == first); + BOOST_TEST(h2.count("LateKey") == 1); + BOOST_TEST(h2["LateKey"] == "LateVal"); + BOOST_TEST(h2.count("Valid-From") == 1); + BOOST_TEST(h2.count("Valid-Until") == 1); +} - for (const auto& h : headers1) { - header1Keys.insert(h.first); - } - for (const auto& h : headers2) { - header2Keys.insert(h.first); - } - for (const auto& h : headers3) { - header3Keys.insert(h.first); - } +BOOST_AUTO_TEST_CASE(HeadersAreStableAcrossMultipleHits) +{ + test_fixture f; - BOOST_TEST(header1Keys.size() == sStableKeys.size()); - BOOST_TEST(header2Keys.size() == sStableKeys.size()); - BOOST_TEST(header3Keys.size() == sStableKeys.size()); - - BOOST_TEST(header1Keys == header2Keys, boost::test_tools::per_element()); - BOOST_TEST(header2Keys == header3Keys, boost::test_tools::per_element()); - BOOST_TEST(header1Keys == header3Keys, boost::test_tools::per_element()); - - BOOST_TEST(header1Keys == sStableKeys, boost::test_tools::per_element()); - BOOST_TEST(header2Keys == sStableKeys, boost::test_tools::per_element()); - BOOST_TEST(header3Keys == sStableKeys, boost::test_tools::per_element()); - - /// Make sure that headers for different objects are different - BOOST_TEST(headers1 != headers2, "The headers for different objects should be different (mainly ETag)"); - - // Make a set of keys that we know beforehand is the same for different objects - std::set keysThatShouldBeDifferent = { - "Valid-From", - "Valid-Until", - "Created", - "ETag", - "Content-Disposition", - "Content-Location", - "path", - "Content-MD5", - }; - - for (const auto& stableKey : keysThatShouldBeDifferent) { - BOOST_TEST(headers2.count(stableKey) > 0, "stable key missing in headers2: " + stableKey); - header2Values.insert(headers2.at(stableKey)); - header1Values.insert(headers1.at(stableKey)); + std::string path = basePath + "StableHeaders"; + std::string body = "Stable"; + std::map meta{{"HK", "HV"}}; + long s = 500'000, e = 510'000; + f.api.storeAsTFileAny(&body, path, meta, s, e); + + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + mgr.clearCache(); + mgr.setCaching(true); + long ts = (s + e) / 2; + + std::map h1; + auto* o1 = mgr.getForTimeStamp(path, ts, &h1); + BOOST_TEST(o1 != nullptr); + BOOST_TEST(h1.count("HK") == 1); + + std::string etag = h1["ETag"]; + for (int i = 0; i < 15; ++i) { + std::map hi; + auto* oi = mgr.getForTimeStamp(path, ts, &hi); + BOOST_TEST(oi == o1); + BOOST_TEST(hi.count("HK") == 1); + BOOST_TEST(hi["ETag"] == etag); } - - BOOST_TEST(header2Values.size() == keysThatShouldBeDifferent.size()); - - BOOST_TEST(header2Values != header1Values, boost::test_tools::per_element()); } From 2028c5bf2f6ad4cd2711d8c1c8e83b8b550e8123 Mon Sep 17 00:00:00 2001 From: Martin <43970264+mrtineide@users.noreply.github.com> Date: Tue, 30 Sep 2025 19:07:13 +0200 Subject: [PATCH 3/4] Fix code snippets and enhance metadata retrieval documentation Updated code snippets in README for header retrieval and added optional parameter documentation details. Also fixed some CI formatting errors. --- CCDB/README.md | 30 ++++++++++++++++++------------ CCDB/test/testCcdbApiHeaders.cxx | 3 ++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CCDB/README.md b/CCDB/README.md index e098617cf44e3..1ae5f29dcf0e2 100644 --- a/CCDB/README.md +++ b/CCDB/README.md @@ -13,7 +13,7 @@ in circumstances of reduced or no network connectivity. There are currently 2 different kinds of store/retrieve functions, which we expect to unify in the immediate future: 2. `storeAsTFile/retrieveFromTFile` API serializing a `TObject` in a ROOT `TFile`. -3. A strongly-typed `storeAsTFileAny/retrieveFromTFileAny` API allowing to handle any type T +3. A strongly-typed `storeAsTFileAny/retrieveFromTFileAny` API allowing to handle any type T having a ROOT dictionary. We encourage to use this API by default. ## Central and local instances of the CCDB @@ -37,12 +37,12 @@ api.init("http://ccdb-test.cern.ch:8080"); // or http://localhost:8080 for a loc auto deadpixels = new o2::FOO::DeadPixelMap(); api.storeAsTFileAny(deadpixels, "FOO/DeadPixels", metadata); // read like this (you have to specify the type) -auto deadpixelsback = api.retrieveFromTFileAny("FOO/DeadPixels", metadata); -// read like this to get the headers as well, and thus the metadata attached to the object +auto deadpixelsback = api.retrieveFromTFileAny("FOO/DeadPixels", metadata); +// read like this to get the headers as well, and thus the metadata attached to the object std::map headers; -auto deadpixelsback = api.retrieveFromTFileAny("FOO/DeadPixels", metadata /* constraint the objects retrieved to those matching the metadata */, -1 /* timestamp */, &headers /* the headers attached to the returned object */); +auto deadpixelsback = api.retrieveFromTFileAny("FOO/DeadPixels", metadata /* constraint the objects retrieved to those matching the metadata */, -1 /* timestamp */, &headers /* the headers attached to the returned object */); // finally, use this method to retrieve only the headers (and thus the metadata) -std::map headers = f.api.retrieveHeaders("FOO/DeadPixels", f.metadata); +std::map headers = api.retrieveHeaders("FOO/DeadPixels", metadata); ``` * creating a local snapshot and fetching objects therefrom @@ -85,7 +85,7 @@ user code. This class The class was written for the use-case of transport MC simulation. Typical usage should be like ```c++ -// setup manager once (at start of processing) +// setup manager once (at start of processing) auto& mgr = o2::ccdb::BasicCCDBManager::instance(); mgr.setURL("http://ourccdbserverver.cern.ch"); mgr.setTimestamp(timestamp_which_we_want_to_anchor_to); @@ -111,6 +111,12 @@ This feature is useful to avoid using newer objects if the CCDB is updated in pa In cached mode, the manager can check that local objects are still valid by requiring `mgr.setLocalObjectValidityChecking(true)`, in this case a CCDB query is performed only if the cached object is no longer valid. +If you want the headers/metadata for the object retrieved from the CCDB there is an optional paramater to `BasicCCDBManager::getForTimeStamp`. These headers are also cached (when caching is enabled) and is updated when a CCDB query is sent. +```c++ +std::map headers; +mgr.getForTimeStamp(path, timstamp, metadata, &headers); +``` + ## Future ideas / todo: - [ ] offer improved error handling / exceptions @@ -129,26 +135,26 @@ A few prototypic command line tools are offered. These can be used in scriptable and facilitate the following tasks: 1. Upload and annotate a generic C++ object serialized in a ROOT file - + ```bash o2-ccdb-upload -f myRootFile.root --key histogram1 --path /Detector1/QA/ --meta "Description=Foo;Author=Person1;Uploader=Person2" ``` This will upload the object serialized in `myRootFile.root` under the key `histogram1`. Object will be put to the CCDB path `/Detector1/QA`. For full list of options see `o2-ccdb-upload --help`. - + 2. Download a CCDB object to a local ROOT file (including its meta information) - + ```bash o2-ccdb-downloadccdbfile --path /Detector1/QA/ --dest /tmp/CCDB --timestamp xxx ``` This will download the CCDB object under path given by `--path` to a directory given by `--dest` on the disc. (The final filename will be `/tmp/CCDB/Detector1/QA/snapshot.root` for the moment). All meta-information as well as the information associated to this query will be appended to the file. - + For full list of options see `o2-ccdb-downloadccdbfile --help`. - + 3. Inspect the content of a ROOT file and print summary about type of contained (CCDB) objects and its meta information - + ```bash o2-ccdb-inspectccdbfile filename ``` diff --git a/CCDB/test/testCcdbApiHeaders.cxx b/CCDB/test/testCcdbApiHeaders.cxx index 845f94c47ccd6..4cad30b7099d5 100644 --- a/CCDB/test/testCcdbApiHeaders.cxx +++ b/CCDB/test/testCcdbApiHeaders.cxx @@ -40,8 +40,9 @@ struct Fixture { Fixture() { o2::ccdb::CcdbApi api; - if (std::getenv("ALICEO2_CCDB_HOST")) + if (std::getenv("ALICEO2_CCDB_HOST")) { ccdbUrl = std::string(std::getenv("ALICEO2_CCDB_HOST")); + } api.init(ccdbUrl); hostReachable = api.isHostReachable(); char hostname[_POSIX_HOST_NAME_MAX]; From 2246830d98e57d749247e155e2b2af66f5e29d3c Mon Sep 17 00:00:00 2001 From: Martin Eide <43970264+mrtineide@users.noreply.github.com> Date: Thu, 2 Oct 2025 15:54:09 +0200 Subject: [PATCH 4/4] Change unit test with to have correct includes This fixes some tests that are fine on my desktop but the subprocess aborts even when all tests are green in the CI. The error is this: malloc_consolidate(): invalid chunk size Was told this was a heap corruption. And changing the includes fixes the error, not sure why. Changed also to only use one instance of CCDB api so we do not call CURL global cleanup out of order with the BasicCCDBManger singelton. Just in case that matters when running the test(s). --- CCDB/test/testCcdbApiHeaders.cxx | 106 +++++++++++++------------------ 1 file changed, 45 insertions(+), 61 deletions(-) diff --git a/CCDB/test/testCcdbApiHeaders.cxx b/CCDB/test/testCcdbApiHeaders.cxx index 4cad30b7099d5..bcfa2a5b44bc2 100644 --- a/CCDB/test/testCcdbApiHeaders.cxx +++ b/CCDB/test/testCcdbApiHeaders.cxx @@ -18,14 +18,11 @@ #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK -#include -#include -#include -#include - +#include #include "CCDB/BasicCCDBManager.h" #include "CCDB/CCDBTimeStampUtils.h" #include "CCDB/CcdbApi.h" +#include static std::string basePath; // std::string ccdbUrl = "http://localhost:8080"; @@ -39,15 +36,15 @@ bool hostReachable = false; struct Fixture { Fixture() { - o2::ccdb::CcdbApi api; + auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); if (std::getenv("ALICEO2_CCDB_HOST")) { ccdbUrl = std::string(std::getenv("ALICEO2_CCDB_HOST")); } - api.init(ccdbUrl); - hostReachable = api.isHostReachable(); + ccdbManager.setURL(ccdbUrl); + hostReachable = ccdbManager.getCCDBAccessor().isHostReachable(); char hostname[_POSIX_HOST_NAME_MAX]; gethostname(hostname, _POSIX_HOST_NAME_MAX); - basePath = std::string("Users/m/meide/BasicCCDBManager/"); + basePath = std::string("Users/m/meide/Tests/") + hostname + "/pid-" + getpid() + "/BasicCCDBManager/"; LOG(info) << "Path we will use in this test suite : " + basePath << std::endl; LOG(info) << "ccdb url: " << ccdbUrl << std::endl; @@ -56,9 +53,7 @@ struct Fixture { ~Fixture() { if (hostReachable) { - o2::ccdb::CcdbApi api; - api.init(ccdbUrl); - api.truncate(basePath + "*"); // This deletes the data after test is run, disable if you want to inspect the data + o2::ccdb::BasicCCDBManager::instance().getCCDBAccessor().truncate(basePath + "*"); // This deletes the data after test is run, disable if you want to inspect the data LOG(info) << "Test data truncated/deleted (" << basePath << ")" << std::endl; } } @@ -74,22 +69,6 @@ struct if_reachable { return hostReachable; } }; -/** - * Fixture for the tests, i.e. code is ran in every test that uses it, i.e. it is like a setup and teardown for tests. - * Copied from testCcdbApi.cxx - */ -struct test_fixture { - test_fixture() - { - api.init(ccdbUrl); - metadata["Hello"] = "World"; - LOG(info) << "*** " << boost::unit_test::framework::current_test_case().p_name << " ***" << std::endl; - } - ~test_fixture() = default; - - o2::ccdb::CcdbApi api; - std::map metadata; -}; // Only compare known and stable keys (avoid volatile ones like Date) static const std::set sStableKeys = { @@ -112,23 +91,24 @@ BOOST_AUTO_TEST_CASE(testCachedHeaders, *boost::unit_test::precondition(if_reach { /// ━━━━━━━ ARRANGE ━━━━━━━━━ // First store objects to test with - test_fixture f; - + auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); std::string pathA = basePath + "CachingA"; std::string pathB = basePath + "CachingB"; std::string pathC = basePath + "CachingC"; std::string ccdbObjO = "testObjectO"; std::string ccdbObjN = "testObjectN"; std::string ccdbObjX = "testObjectX"; - std::map md = f.metadata; + std::map md = { + {"Hello", "World"}, + {"Key1", "Value1"}, + {"Key2", "Value2"}, + }; long start = 1000, stop = 3000; - f.api.storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); - f.api.storeAsTFileAny(&ccdbObjN, pathB, md, start, stop); - f.api.storeAsTFileAny(&ccdbObjX, pathC, md, start, stop); + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjN, pathB, md, start, stop); + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjX, pathC, md, start, stop); // initilize the BasicCCDBManager - o2::ccdb::BasicCCDBManager& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); ccdbManager.clearCache(); - ccdbManager.setURL(ccdbUrl); ccdbManager.setCaching(true); // This is what we want to test. /// ━━━━━━━━━━━ ACT ━━━━━━━━━━━━ @@ -193,20 +173,21 @@ BOOST_AUTO_TEST_CASE(testNonCachedHeaders, *boost::unit_test::precondition(if_re { /// ━━━━━━━ ARRANGE ━━━━━━━━━ // First store objects to test with - test_fixture f; - + auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); std::string pathA = basePath + "NonCachingA"; std::string pathB = basePath + "NonCachingB"; std::string ccdbObjO = "testObjectO"; std::string ccdbObjN = "testObjectN"; - std::map md = f.metadata; + std::map md = { + {"Hello", "World"}, + {"Key1", "Value1"}, + {"Key2", "Value2"}, + }; long start = 1000, stop = 2000; - f.api.storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); - f.api.storeAsTFileAny(&ccdbObjN, pathB, md, start, stop); + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjN, pathB, md, start, stop); // initilize the BasicCCDBManager - o2::ccdb::BasicCCDBManager& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); ccdbManager.clearCache(); - ccdbManager.setURL(ccdbUrl); ccdbManager.setCaching(false); // This is what we want to test, no caching /// ━━━━━━━━━━━ ACT ━━━━━━━━━━━━ @@ -217,6 +198,7 @@ BOOST_AUTO_TEST_CASE(testNonCachedHeaders, *boost::unit_test::precondition(if_re auto* obj2 = ccdbManager.getForTimeStamp(pathB, (start + stop) / 2, &headers2); auto* obj3 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers3); // Should not be cached since explicitly disabled + ccdbManager.setCaching(true); // Restore default state /// ━━━━━━━━━━━ ASSERT ━━━━━━━━━━━ /// Check that we got something BOOST_REQUIRE(obj1 != nullptr); @@ -249,12 +231,17 @@ BOOST_AUTO_TEST_CASE(testNonCachedHeaders, *boost::unit_test::precondition(if_re BOOST_TEST(headers3.size() != 0); BOOST_TEST(headers2.size() != 0); BOOST_TEST(headers1 != headers2, "The headers for different objects should be different"); + + // cleanup + delete obj1; + delete obj2; + delete obj3; } -BOOST_AUTO_TEST_CASE(CacheFirstRetrievalAndHeadersPersistence) +BOOST_AUTO_TEST_CASE(CacheFirstRetrievalAndHeadersPersistence, *boost::unit_test::precondition(if_reachable())) { - test_fixture f; /// ━━━━━━━ ARRANGE ━━━━━━━━━ + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); // Prepare two validity slots for same path to test ETag change later std::string path = basePath + "ObjA"; std::string objV1 = "ObjectVersion1"; @@ -268,11 +255,9 @@ BOOST_AUTO_TEST_CASE(CacheFirstRetrievalAndHeadersPersistence) long v2stop = v2start + (v1stop - v1start); long mid1 = (v1start + v1stop) / 2; // Store 2 versions - f.api.storeAsTFileAny(&objV1, path, meta1, v1start, v1stop); - f.api.storeAsTFileAny(&objV2, path, meta1, v2start, v2stop); + mgr.getCCDBAccessor().storeAsTFileAny(&objV1, path, meta1, v1start, v1stop); + mgr.getCCDBAccessor().storeAsTFileAny(&objV2, path, meta1, v2start, v2stop); - auto& mgr = o2::ccdb::BasicCCDBManager::instance(); - mgr.setURL(ccdbUrl); mgr.clearCache(); mgr.setCaching(true); mgr.setFatalWhenNull(true); @@ -294,6 +279,8 @@ BOOST_AUTO_TEST_CASE(CacheFirstRetrievalAndHeadersPersistence) // 5) Fifth retrieval with headers again to check persistence auto* p5 = mgr.getForTimeStamp(path, mid1, &headers5); + mgr.setFatalWhenNull(false); // restore default + /// ━━━━━━━ASSERT━━━━━━━━━ BOOST_TEST(p1 != nullptr); @@ -336,17 +323,16 @@ BOOST_AUTO_TEST_CASE(CacheFirstRetrievalAndHeadersPersistence) BOOST_TEST(headers5["UserKey1"] == "UValue1"); // internal unchanged } -BOOST_AUTO_TEST_CASE(FailedFetchDoesNotGiveMetadata) +BOOST_AUTO_TEST_CASE(FailedFetchDoesNotGiveMetadata, *boost::unit_test::precondition(if_reachable())) { - test_fixture f; /// ━━━━━━━ ARRANGE ━━━━━━━━━ + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); std::string path = basePath + "FailThenRecover"; std::string content = "ContentX"; std::map meta{{"Alpha", "Beta"}}; long s = 300'000, e = 310'000; - f.api.storeAsTFileAny(&content, path, meta, s, e); - auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + mgr.getCCDBAccessor().storeAsTFileAny(&content, path, meta, s, e); mgr.clearCache(); mgr.setCaching(true); mgr.setFatalWhenNull(false); @@ -368,17 +354,16 @@ BOOST_AUTO_TEST_CASE(FailedFetchDoesNotGiveMetadata) mgr.setFatalWhenNull(true); } -BOOST_AUTO_TEST_CASE(FirstCallWithoutHeadersThenWithHeaders) +BOOST_AUTO_TEST_CASE(FirstCallWithoutHeadersThenWithHeaders, *boost::unit_test::precondition(if_reachable())) { - test_fixture f; + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); std::string path = basePath + "LateHeaders"; std::string body = "Late"; std::map meta{{"LateKey", "LateVal"}}; long s = 400'000, e = 410'000; - f.api.storeAsTFileAny(&body, path, meta, s, e); + mgr.getCCDBAccessor().storeAsTFileAny(&body, path, meta, s, e); - auto& mgr = o2::ccdb::BasicCCDBManager::instance(); mgr.clearCache(); mgr.setCaching(true); long ts = (s + e) / 2; @@ -398,17 +383,16 @@ BOOST_AUTO_TEST_CASE(FirstCallWithoutHeadersThenWithHeaders) BOOST_TEST(h2.count("Valid-Until") == 1); } -BOOST_AUTO_TEST_CASE(HeadersAreStableAcrossMultipleHits) +BOOST_AUTO_TEST_CASE(HeadersAreStableAcrossMultipleHits, *boost::unit_test::precondition(if_reachable())) { - test_fixture f; + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); std::string path = basePath + "StableHeaders"; std::string body = "Stable"; std::map meta{{"HK", "HV"}}; long s = 500'000, e = 510'000; - f.api.storeAsTFileAny(&body, path, meta, s, e); + mgr.getCCDBAccessor().storeAsTFileAny(&body, path, meta, s, e); - auto& mgr = o2::ccdb::BasicCCDBManager::instance(); mgr.clearCache(); mgr.setCaching(true); long ts = (s + e) / 2;