From 962b57957b92c8a7aacd98219be2a1848216e230 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 26 May 2025 22:40:56 +0200 Subject: [PATCH] Get ROOT v6-36-00 to work. --- Framework/AnalysisSupport/src/Plugin.cxx | 5 +- .../AnalysisSupport/src/RNTuplePlugin.cxx | 206 ++++++++++++------ 2 files changed, 144 insertions(+), 67 deletions(-) diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx index 68c4c1cb00d09..1363672a6518e 100644 --- a/Framework/AnalysisSupport/src/Plugin.cxx +++ b/Framework/AnalysisSupport/src/Plugin.cxx @@ -121,7 +121,10 @@ std::vector getListOfTables(std::unique_ptr& f) break; } - void* v = f->GetObjectChecked(key->GetName(), TClass::GetClass("ROOT::Experimental::RNTuple")); + void* v = f->GetObjectChecked(key->GetName(), TClass::GetClass("ROOT::RNTuple")); + if (!v) { + v = f->GetObjectChecked(key->GetName(), TClass::GetClass("ROOT::Experimental::RNTuple")); + } if (v) { std::string s = key->GetName(); size_t pos = s.find('-'); diff --git a/Framework/AnalysisSupport/src/RNTuplePlugin.cxx b/Framework/AnalysisSupport/src/RNTuplePlugin.cxx index a910964e6527c..2c6fffe872db3 100644 --- a/Framework/AnalysisSupport/src/RNTuplePlugin.cxx +++ b/Framework/AnalysisSupport/src/RNTuplePlugin.cxx @@ -32,8 +32,20 @@ #include #include +#if __has_include() +#include +namespace rns = ROOT; +using DPLFieldToken = rns::RFieldToken; +using DPLLocalIndex = rns::RNTupleLocalIndex; +#else +namespace rns = ROOT::Experimental; +using DPLFieldToken = rns::REntry::RFieldToken; +using DPLLocalIndex = rns::RClusterIndex; +#endif + + template class - std::unique_ptr; + std::unique_ptr; namespace o2::framework { @@ -53,13 +65,13 @@ class RNTupleFileSystem : public VirtualRootFileSystemBase public: ~RNTupleFileSystem() override; - virtual ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource source) = 0; + virtual rns::RNTuple* GetRNTuple(arrow::dataset::FileSource source) = 0; }; class SingleRNTupleFileSystem : public RNTupleFileSystem { public: - SingleRNTupleFileSystem(ROOT::Experimental::RNTuple* tuple) + SingleRNTupleFileSystem(rns::RNTuple* tuple) : RNTupleFileSystem(), mTuple(tuple) { @@ -72,14 +84,14 @@ class SingleRNTupleFileSystem : public RNTupleFileSystem return "rntuple"; } - ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource) override + rns::RNTuple* GetRNTuple(arrow::dataset::FileSource) override { // Simply return the only TTree we have return mTuple; } private: - ROOT::Experimental::RNTuple* mTuple; + rns::RNTuple* mTuple; }; arrow::Result SingleRNTupleFileSystem::GetFileInfo(std::string const& path) @@ -110,16 +122,16 @@ class RNTupleFileFragment : public arrow::dataset::FileFragment handler->format->type_name().c_str(), format->type_name().c_str()); } - mNTuple = handler->GetObjectAsOwner(); + mNTuple = handler->GetObjectAsOwner(); } - ROOT::Experimental::RNTuple* GetRNTuple() + rns::RNTuple* GetRNTuple() { return mNTuple.get(); } private: - std::unique_ptr mNTuple; + std::unique_ptr mNTuple; }; class RNTupleFileFormat : public arrow::dataset::FileFormat @@ -173,36 +185,77 @@ class RNTupleFileFormat : public arrow::dataset::FileFormat std::shared_ptr physical_schema) override; }; -struct RootNTupleVisitor : public ROOT::Experimental::Detail::RFieldVisitor { - void VisitArrayField(const ROOT::Experimental::RArrayField& field) override +template +requires requires (T&& f) { f.GetSubFields(); } +auto getSubfields(T const&field) { + return field.GetSubFields(); +} + +template +requires requires (T&& f) { f.GetConstSubfields(); } +auto getSubfields(T const&field) { + return field.GetConstSubfields(); +} + +struct RootNTupleVisitor : public rns::Detail::RFieldVisitor { + void VisitArrayField(const rns::RArrayField& field) override { int size = field.GetLength(); RootNTupleVisitor valueVisitor{}; - auto valueField = field.GetSubFields()[0]; + auto valueField = getSubfields(field)[0]; valueField->AcceptVisitor(valueVisitor); auto type = valueVisitor.datatype; this->datatype = arrow::fixed_size_list(type, size); } - void VisitRVecField(const ROOT::Experimental::RRVecField& field) override + void VisitRVecField(const rns::RRVecField& field) override { RootNTupleVisitor valueVisitor{}; - auto valueField = field.GetSubFields()[0]; + auto valueField = getSubfields(field)[0]; valueField->AcceptVisitor(valueVisitor); auto type = valueVisitor.datatype; this->datatype = arrow::list(type); } - void VisitField(const ROOT::Experimental::RFieldBase& field) override + void VisitField(const rns::RFieldBase& field) override { throw o2::framework::runtime_error_f("Unknown field %s with type %s", field.GetFieldName().c_str(), field.GetTypeName().c_str()); } - void VisitIntField(const ROOT::Experimental::RField& field) override +#if __has_include() + void VisitInt32Field(const rns::RIntegralField& field) override { this->datatype = arrow::int32(); } + void VisitInt8Field(const rns::RIntegralField& field) override + { + this->datatype = arrow::int8(); + } + + void VisitInt16Field(const rns::RIntegralField& field) override + { + this->datatype = arrow::int16(); + } + + void VisitUInt32Field(const rns::RIntegralField& field) override + { + this->datatype = arrow::uint32(); + } + void VisitUInt8Field(const rns::RIntegralField& field) override + { + this->datatype = arrow::uint8(); + } + + void VisitUInt16Field(const rns::RIntegralField& field) override + { + this->datatype = arrow::int16(); + } +#else + void VisitIntField(const rns::RField& field) override + { + this->datatype = arrow::int32(); + } void VisitInt8Field(const ROOT::Experimental::RField& field) override { this->datatype = arrow::int8(); @@ -227,18 +280,20 @@ struct RootNTupleVisitor : public ROOT::Experimental::Detail::RFieldVisitor { { this->datatype = arrow::int16(); } +#endif + - void VisitBoolField(const ROOT::Experimental::RField& field) override + void VisitBoolField(const rns::RField& field) override { this->datatype = arrow::boolean(); } - void VisitFloatField(const ROOT::Experimental::RField& field) override + void VisitFloatField(const rns::RField& field) override { this->datatype = arrow::float32(); } - void VisitDoubleField(const ROOT::Experimental::RField& field) override + void VisitDoubleField(const rns::RField& field) override { this->datatype = arrow::float64(); } @@ -246,7 +301,7 @@ struct RootNTupleVisitor : public ROOT::Experimental::Detail::RFieldVisitor { }; } // namespace o2::framework -auto arrowTypeFromRNTuple(ROOT::Experimental::RFieldBase const& field, int size) +auto arrowTypeFromRNTuple(rns::RFieldBase const& field, int size) { o2::framework::RootNTupleVisitor visitor; field.AcceptVisitor(visitor); @@ -255,34 +310,34 @@ auto arrowTypeFromRNTuple(ROOT::Experimental::RFieldBase const& field, int size) namespace o2::framework { -std::unique_ptr rootFieldFromArrow(std::shared_ptr field, std::string name) +std::unique_ptr rootFieldFromArrow(std::shared_ptr field, std::string name) { using namespace ROOT::Experimental; switch (field->type()->id()) { case arrow::Type::BOOL: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::UINT8: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::UINT16: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::UINT32: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::UINT64: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::INT8: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::INT16: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::INT32: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::INT64: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::FLOAT: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::DOUBLE: - return std::make_unique>(name); + return std::make_unique>(name); case arrow::Type::STRING: - return std::make_unique>(name); + return std::make_unique>(name); default: throw runtime_error("Unsupported arrow column type"); } @@ -290,7 +345,7 @@ std::unique_ptr rootFieldFromArrow(std::shared_p class RNTupleFileWriter : public arrow::dataset::FileWriter { - std::shared_ptr mWriter; + std::shared_ptr mWriter; bool firstBatch = true; std::vector> valueArrays; std::vector> valueTypes; @@ -304,7 +359,7 @@ class RNTupleFileWriter : public arrow::dataset::FileWriter { using namespace ROOT::Experimental; - auto model = RNTupleModel::CreateBare(); + auto model = rns::RNTupleModel::CreateBare(); // Let's create a model from the physical schema for (auto i = 0u; i < schema->fields().size(); ++i) { auto& field = schema->field(i); @@ -314,11 +369,11 @@ class RNTupleFileWriter : public arrow::dataset::FileWriter case arrow::Type::FIXED_SIZE_LIST: { auto list = std::static_pointer_cast(field->type()); auto valueField = field->type()->field(0); - model->AddField(std::make_unique(field->name(), rootFieldFromArrow(valueField, "_0"), list->list_size())); + model->AddField(std::make_unique(field->name(), rootFieldFromArrow(valueField, "_0"), list->list_size())); } break; case arrow::Type::LIST: { auto valueField = field->type()->field(0); - model->AddField(std::make_unique(field->name(), rootFieldFromArrow(valueField, "_0"))); + model->AddField(std::make_unique(field->name(), rootFieldFromArrow(valueField, "_0"))); } break; default: { model->AddField(rootFieldFromArrow(field, field->name())); @@ -327,7 +382,7 @@ class RNTupleFileWriter : public arrow::dataset::FileWriter } auto fileStream = std::dynamic_pointer_cast(destination_); auto* file = dynamic_cast(fileStream->GetDirectory()); - mWriter = RNTupleWriter::Append(std::move(model), destination_locator_.path, *file, {}); + mWriter = rns::RNTupleWriter::Append(std::move(model), destination_locator_.path, *file, {}); } arrow::Status Write(const std::shared_ptr& batch) override @@ -413,7 +468,7 @@ class RNTupleFileWriter : public arrow::dataset::FileWriter int64_t pos = 0; auto entry = mWriter->CreateEntry(); - std::vector tokens; + std::vector tokens; tokens.reserve(batch->num_columns()); std::vector typeIds; typeIds.reserve(batch->num_columns()); @@ -435,7 +490,7 @@ class RNTupleFileWriter : public arrow::dataset::FileWriter auto value_slice = list->value_slice(pos); valueCount[ci] = value_slice->length(); - auto bindValue = [&vc = valueCount, ci, token](auto array, std::unique_ptr& entry) -> void { + auto bindValue = [&vc = valueCount, ci, token](auto array, std::unique_ptr& entry) -> void { using value_type = std::decay_t::value_type; auto v = std::make_shared>((value_type*)array->raw_values(), vc[ci]); entry->BindValue(token, v); @@ -504,6 +559,19 @@ class RNTupleFileWriter : public arrow::dataset::FileWriter }; }; +template +requires requires (T const&m) { m.GetFieldZero(); } +auto &getFieldZero(T const &m) { + return m.GetFieldZero(); +} + +template +requires requires (T const&m) { m.GetConstFieldZero(); } +auto &getFieldZero(T const &m) { + return m.GetConstFieldZero(); +} + + arrow::Result> RNTupleFileFormat::Inspect(const arrow::dataset::FileSource& source) const { @@ -514,15 +582,17 @@ arrow::Result> RNTupleFileFormat::Inspect(const a throw runtime_error_f("Unexpected kind of filesystem %s to handle payload %s.\n", source.filesystem()->type_name().c_str(), source.path().c_str()); } // We know this is a RNTuple, so we can continue with the inspection. - auto rntuple = objectHandler->GetObjectAsOwner().release(); - - auto inspector = ROOT::Experimental::RNTupleInspector::Create(rntuple); + auto rntuple = objectHandler->GetObjectAsOwner().release(); - auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple); +#if __has_include() + auto reader = rns::RNTupleReader::Open(*rntuple); +#else + auto reader = rns::RNTupleReader::Open(rntuple); +#endif - auto& tupleField0 = reader->GetModel().GetFieldZero(); + auto& tupleField0 = getFieldZero(reader->GetModel()); std::vector> fields; - for (auto& tupleField : tupleField0.GetSubFields()) { + for (auto& tupleField : getSubfields(tupleField0)) { auto field = std::make_shared(tupleField->GetFieldName(), arrowTypeFromRNTuple(*tupleField, tupleField->GetValueSize())); fields.push_back(field); } @@ -544,8 +614,12 @@ arrow::Result RNTupleFileFormat::ScanBatchesAsync( std::vector> fields = dataset_schema->fields(); int64_t rows = -1; - ROOT::Experimental::RNTuple* rntuple = ntupleFragment->GetRNTuple(); - auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple); + rns::RNTuple* rntuple = ntupleFragment->GetRNTuple(); +#if __has_include() + auto reader = rns::RNTupleReader::Open(*rntuple); +#else + auto reader = rns::RNTupleReader::Open(rntuple); +#endif auto& model = reader->GetModel(); for (auto& physicalField : fields) { auto bulk = model.CreateBulk(physicalField->name()); @@ -583,11 +657,11 @@ arrow::Result RNTupleFileFormat::ScanBatchesAsync( auto clusterIt = descriptor.FindClusterId(0, 0); // No adoption for now... // bulk.AdoptBuffer(buffer, totalEntries) - while (clusterIt != kInvalidDescriptorId) { + while (clusterIt != rns::kInvalidDescriptorId) { auto& index = descriptor.GetClusterDescriptor(clusterIt); auto mask = std::make_unique(index.GetNEntries()); std::fill(mask.get(), mask.get() + index.GetNEntries(), true); - void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()); + void* ptr = bulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()); int readLast = index.GetNEntries(); readEntries += readLast; status &= static_cast(valueBuilder)->AppendValues(reinterpret_cast(ptr), readLast * listType->list_size()); @@ -614,11 +688,11 @@ arrow::Result RNTupleFileFormat::ScanBatchesAsync( throw runtime_error("Failed to reserve memory for array builder"); } auto clusterIt = descriptor.FindClusterId(0, 0); - while (clusterIt != kInvalidDescriptorId) { + while (clusterIt != rns::kInvalidDescriptorId) { auto& index = descriptor.GetClusterDescriptor(clusterIt); auto mask = std::make_unique(index.GetNEntries()); std::fill(mask.get(), mask.get() + index.GetNEntries(), true); - void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()); + void* ptr = bulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()); int readLast = index.GetNEntries(); readEntries += readLast; status &= valueBuilder->AppendValues(reinterpret_cast(ptr), readLast); @@ -659,11 +733,11 @@ arrow::Result RNTupleFileFormat::ScanBatchesAsync( listSize = fixedSizeList->list_size(); typeSize = fixedSizeList->field(0)->type()->byte_width(); auto clusterIt = descriptor.FindClusterId(0, 0); - while (clusterIt != kInvalidDescriptorId) { + while (clusterIt != rns::kInvalidDescriptorId) { auto& index = descriptor.GetClusterDescriptor(clusterIt); auto mask = std::make_unique(index.GetNEntries()); std::fill(mask.get(), mask.get() + index.GetNEntries(), true); - void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()); + void* inPtr = bulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()); int readLast = index.GetNEntries(); if (listSize == -1) { @@ -719,41 +793,41 @@ arrow::Result RNTupleFileFormat::ScanBatchesAsync( } }; - while (clusterIt != kInvalidDescriptorId) { + while (clusterIt != rns::kInvalidDescriptorId) { auto& index = descriptor.GetClusterDescriptor(clusterIt); auto mask = std::make_unique(index.GetNEntries()); std::fill(mask.get(), mask.get() + index.GetNEntries(), true); int readLast = index.GetNEntries(); switch (vlaListType->field(0)->type()->id()) { case arrow::Type::FLOAT: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; case arrow::Type::DOUBLE: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; case arrow::Type::INT8: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; case arrow::Type::INT16: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; case arrow::Type::INT32: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; case arrow::Type::INT64: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; case arrow::Type::UINT8: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; case arrow::Type::UINT16: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; case arrow::Type::UINT32: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; case arrow::Type::UINT64: { - copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); + copyOffsets((ROOT::Internal::VecOps::RVec*)offsetBulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast); } break; default: { throw runtime_error("Unsupported kind of VLA"); @@ -765,11 +839,11 @@ arrow::Result RNTupleFileFormat::ScanBatchesAsync( } } else { auto clusterIt = descriptor.FindClusterId(0, 0); - while (clusterIt != kInvalidDescriptorId) { + while (clusterIt != rns::kInvalidDescriptorId) { auto& index = descriptor.GetClusterDescriptor(clusterIt); auto mask = std::make_unique(index.GetNEntries()); std::fill(mask.get(), mask.get() + index.GetNEntries(), true); - void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()); + void* inPtr = bulk.ReadBulk(DPLLocalIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()); int readLast = index.GetNEntries(); if (listSize == -1) {