Skip to content

DPL: use rapidjson to dump metrics #14825

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ktf merged 1 commit into from
Nov 20, 2025
Merged

DPL: use rapidjson to dump metrics #14825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions Framework/Core/include/Framework/DeviceMetricsInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,11 +154,11 @@ struct DeviceMetricsInfo {
std::vector<MetricsStorage<StringMetric>> stringMetrics; // We do not keep so many strings as metrics as history is less relevant.
std::vector<MetricsStorage<float>> floatMetrics;
std::vector<MetricsStorage<int8_t>> enumMetrics;
std::vector<std::array<size_t, metricStorageSize<int>()>> intTimestamps;
std::vector<std::array<size_t, metricStorageSize<uint64_t>()>> uint64Timestamps;
std::vector<std::array<size_t, metricStorageSize<float>()>> floatTimestamps;
std::vector<std::array<size_t, metricStorageSize<StringMetric>()>> stringTimestamps;
std::vector<std::array<size_t, metricStorageSize<int8_t>()>> enumTimestamps;
std::vector<TimestampsStorage<int>> intTimestamps;
std::vector<TimestampsStorage<uint64_t>> uint64Timestamps;
std::vector<TimestampsStorage<float>> floatTimestamps;
std::vector<TimestampsStorage<StringMetric>> stringTimestamps;
std::vector<TimestampsStorage<int8_t>> enumTimestamps;
std::vector<float> max;
std::vector<float> min;
std::vector<float> average;
Expand Down
107 changes: 54 additions & 53 deletions Framework/Core/src/ResourcesMonitoringHelper.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

#include "ResourcesMonitoringHelper.h"
#include "Framework/DeviceMetricsInfo.h"
#define BOOST_BIND_GLOBAL_PLACEHOLDERS
#include <boost/property_tree/json_parser.hpp>
#include <fstream>
#include <rapidjson/prettywriter.h>
#include <rapidjson/ostreamwrapper.h>

#include <ostream>
#include <string>
#include <string_view>
#include <algorithm>
#include <cassert>
Expand All @@ -22,34 +23,28 @@
using namespace o2::framework;

template <typename T>
inline static T retriveValue(T val)
void fillNodeWithValue(rapidjson::Writer<rapidjson::OStreamWrapper>& w,
size_t filledMetrics,
MetricsStorage<T> const& metricsStorage,
TimestampsStorage<T> const& timestampsStorage)
{
return val;
}

inline static std::string retriveValue(const std::reference_wrapper<const StringMetric> val)
{
return std::string(val.get().data);
}

template <typename T, typename TIMESTAMPS>
boost::property_tree::ptree fillNodeWithValue(const DeviceMetricsInfo& deviceMetrics,
const T& metricsStorage, const TIMESTAMPS& timestampsStorage, size_t labelIndex, size_t storeIndex)
{
unsigned int loopRange = std::min(deviceMetrics.metrics[labelIndex].filledMetrics, metricsStorage[storeIndex].size());
boost::property_tree::ptree metricNode;
unsigned int loopRange = std::min(filledMetrics, metricsStorage.size());

w.StartArray();
for (unsigned int idx = 0; idx < loopRange; ++idx) {
boost::property_tree::ptree values;
values.add("timestamp", timestampsStorage[storeIndex][idx]);
w.StartObject();
w.Key("timestamp");
std::string s = std::to_string(timestampsStorage[idx]);
w.String(s.c_str(), s.size());
w.Key("value");
if constexpr (std::is_arithmetic_v<T>) {
values.add("value", std::to_string(retriveValue(std::cref(metricsStorage[storeIndex][idx]))));
w.String(std::to_string(metricsStorage[idx]).c_str());
} else {
values.add("value", retriveValue(std::cref(metricsStorage[storeIndex][idx])));
w.String(metricsStorage[idx].data);
}
metricNode.push_back(std::make_pair("", values));
w.EndObject();
}
return metricNode;
w.EndArray();
}

bool ResourcesMonitoringHelper::dumpMetricsToJSON(const std::vector<DeviceMetricsInfo>& metrics,
Expand All @@ -58,19 +53,23 @@ bool ResourcesMonitoringHelper::dumpMetricsToJSON(const std::vector<DeviceMetric
std::vector<std::regex> const& performanceMetricsRegex,
std::ostream& out) noexcept
{

assert(metrics.size() == specs.size());

if (metrics.empty()) {
return false;
}

boost::property_tree::ptree root;
for (unsigned int idx = 0; idx < metrics.size(); ++idx) {
rapidjson::OStreamWrapper osw(out);
rapidjson::PrettyWriter<rapidjson::OStreamWrapper> w(osw);

// Top level obejct for all the metrics
w.StartObject();

for (unsigned int idx = 0; idx < metrics.size(); ++idx) {
w.Key(specs[idx].id.c_str());
const auto& deviceMetrics = metrics[idx];
boost::property_tree::ptree deviceRoot;

w.StartObject();
for (size_t mi = 0; mi < deviceMetrics.metricLabels.size(); mi++) {
std::string_view metricLabel{deviceMetrics.metricLabels[mi].label, deviceMetrics.metricLabels[mi].size};

Expand All @@ -83,40 +82,42 @@ bool ResourcesMonitoringHelper::dumpMetricsToJSON(const std::vector<DeviceMetric
}
auto storeIdx = deviceMetrics.metrics[mi].storeIdx;

size_t filledMetrics = deviceMetrics.metrics[mi].filledMetrics;
if (deviceMetrics.metrics[mi].filledMetrics == 0) {
continue;
}
// if so

boost::property_tree::ptree metricNode;

w.Key(metricLabel.data(), metricLabel.size());
switch (deviceMetrics.metrics[mi].type) {
case MetricType::Int:
metricNode = fillNodeWithValue(deviceMetrics, deviceMetrics.intMetrics, deviceMetrics.intTimestamps, mi, storeIdx);
fillNodeWithValue(w, filledMetrics, deviceMetrics.intMetrics[storeIdx],
deviceMetrics.intTimestamps[storeIdx]);
break;

case MetricType::Float:
metricNode = fillNodeWithValue(deviceMetrics, deviceMetrics.floatMetrics, deviceMetrics.floatTimestamps, mi, storeIdx);
fillNodeWithValue(w, filledMetrics, deviceMetrics.floatMetrics[storeIdx],
deviceMetrics.floatTimestamps[storeIdx]);
break;

case MetricType::String:
metricNode = fillNodeWithValue(deviceMetrics, deviceMetrics.stringMetrics, deviceMetrics.stringTimestamps, mi, storeIdx);
fillNodeWithValue(w, filledMetrics, deviceMetrics.stringMetrics[storeIdx],
deviceMetrics.stringTimestamps[storeIdx]);
break;

case MetricType::Uint64:
metricNode = fillNodeWithValue(deviceMetrics, deviceMetrics.uint64Metrics, deviceMetrics.uint64Timestamps, mi, storeIdx);
fillNodeWithValue(w, filledMetrics, deviceMetrics.uint64Metrics[storeIdx],
deviceMetrics.uint64Timestamps[storeIdx]);
break;

default:
continue;
}
deviceRoot.add_child(std::string(metricLabel), metricNode);
}

root.add_child(specs[idx].id, deviceRoot);
w.EndObject();
}

boost::property_tree::ptree driverRoot;
w.Key("driver");
w.StartObject();
for (size_t mi = 0; mi < driverMetrics.metricLabels.size(); mi++) {
std::string_view const metricLabel{driverMetrics.metricLabels[mi].label, driverMetrics.metricLabels[mi].size};
auto same = [metricLabel](std::regex const& matcher) -> bool {
Expand All @@ -130,39 +131,39 @@ bool ResourcesMonitoringHelper::dumpMetricsToJSON(const std::vector<DeviceMetric

auto storeIdx = driverMetrics.metrics[mi].storeIdx;
// and if data is there
if (driverMetrics.metrics[mi].filledMetrics == 0) {
size_t filledMetrics = driverMetrics.metrics[mi].filledMetrics;
if (filledMetrics == 0) {
continue;
}

// if so
boost::property_tree::ptree metricNode;

w.Key(metricLabel.data(), metricLabel.size());
switch (driverMetrics.metrics[mi].type) {
case MetricType::Int:
metricNode = fillNodeWithValue(driverMetrics, driverMetrics.intMetrics, driverMetrics.intTimestamps, mi, storeIdx);
fillNodeWithValue(w, filledMetrics, driverMetrics.intMetrics[storeIdx],
driverMetrics.intTimestamps[storeIdx]);
break;

case MetricType::Float:
metricNode = fillNodeWithValue(driverMetrics, driverMetrics.floatMetrics, driverMetrics.floatTimestamps, mi, storeIdx);
fillNodeWithValue(w, filledMetrics, driverMetrics.floatMetrics[storeIdx],
driverMetrics.floatTimestamps[storeIdx]);
break;

case MetricType::String:
metricNode = fillNodeWithValue(driverMetrics, driverMetrics.stringMetrics, driverMetrics.stringTimestamps, mi, storeIdx);
fillNodeWithValue(w, filledMetrics, driverMetrics.stringMetrics[storeIdx],
driverMetrics.stringTimestamps[storeIdx]);
break;

case MetricType::Uint64:
metricNode = fillNodeWithValue(driverMetrics, driverMetrics.uint64Metrics, driverMetrics.uint64Timestamps, mi, storeIdx);
fillNodeWithValue(w, filledMetrics, driverMetrics.uint64Metrics[storeIdx],
driverMetrics.uint64Timestamps[storeIdx]);
break;

default:
continue;
}
driverRoot.add_child(std::string{metricLabel}, metricNode);
}

root.add_child("driver", driverRoot);

boost::property_tree::json_parser::write_json(out, root);
w.EndObject();
w.EndObject();

return true;
}
6 changes: 4 additions & 2 deletions Framework/Core/test/test_ResourcesMonitoringHelpers.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,8 @@ TEST_CASE("StreamMetrics")
ResourcesMonitoringHelper::dumpMetricsToJSON(metrics,
driverMetrics, specs, performanceMetrics,
streamer);
REQUIRE(streamer.str() == R"JSON({
std::string streamed = streamer.str();
std::string expected = R"JSON({
"someDevice": {
"ckey": [
{
Expand Down Expand Up @@ -266,5 +267,6 @@ TEST_CASE("StreamMetrics")
]
}
}
)JSON");
)JSON";
REQUIRE(std::regex_replace(streamed, std::regex(R"(\s+)"), "") == std::regex_replace(expected, std::regex(R"(\s+)"), ""));
}