From 65ef111afc3795afbae54d2e3caddc267fa6ef00 Mon Sep 17 00:00:00 2001 From: Michael Lettrich Date: Thu, 23 Oct 2025 11:47:14 +0200 Subject: [PATCH] [Fix] Correct Reporting of CTF Compressed Payload in Metadata --- .../EncodedBlocks.h | 24 ++++++++++--------- .../DetectorsCommonDataFormats/Metadata.h | 17 +++++++++++-- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h index 6fb8825f7c395..a83c8f13deca4 100644 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h @@ -929,9 +929,11 @@ CTFIOSize EncodedBlocks::decode(D_IT dest, // it const auto& md = mMetadata[slot]; LOGP(debug, "Slot{} | NStored={} Ndict={} nData={}, MD: messageLength:{} opt:{} min:{} max:{} offs:{} width:{} ", slot, block.getNStored(), block.getNDict(), block.getNData(), md.messageLength, (int)md.opt, md.min, md.max, md.literalsPackingOffset, md.literalsPackingWidth); + constexpr size_t word_size = sizeof(W); + if (ansVersion == ANSVersionCompat) { if (!block.getNStored()) { - return {0, md.getUncompressedSize(), md.getCompressedSize()}; + return {0, md.getUncompressedSize(), md.getCompressedSize() * word_size}; } if (md.opt == Metadata::OptStore::EENCODE) { return decodeCompatImpl(dest, slot, decoderExt); @@ -943,7 +945,7 @@ CTFIOSize EncodedBlocks::decode(D_IT dest, // it return decodeUnpackImpl(dest, slot); } if (!block.getNStored()) { - return {0, md.getUncompressedSize(), md.getCompressedSize()}; + return {0, md.getUncompressedSize(), md.getCompressedSize() * word_size}; } if (md.opt == Metadata::OptStore::EENCODE) { return decodeRansV1Impl(dest, slot, decoderExt); @@ -991,7 +993,7 @@ CTFIOSize EncodedBlocks::decodeCompatImpl(dst_IT dstBegin, int slot, co } else { getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, NDecoderStreams); } - return {0, md.getUncompressedSize(), md.getCompressedSize()}; + return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)}; }; template @@ -1045,7 +1047,7 @@ CTFIOSize EncodedBlocks::decodeRansV1Impl(dst_IT dstBegin, int slot, co } else { getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, md.nStreams); } - return {0, md.getUncompressedSize(), md.getCompressedSize()}; + return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)}; }; template @@ -1079,7 +1081,7 @@ CTFIOSize EncodedBlocks::decodeUnpackImpl(dst_IT dest, int slot) const } else { rans::unpack(srcIt, messageLength, dest, packingWidth, offset); } - return {0, md.getUncompressedSize(), md.getCompressedSize()}; + return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)}; }; template @@ -1098,7 +1100,7 @@ CTFIOSize EncodedBlocks::decodeCopyImpl(dst_IT dest, int slot) const destPtr_t srcEnd = srcBegin + md.messageLength * sizeof(dest_t); std::copy(srcBegin, srcEnd, dest); - return {0, md.getUncompressedSize(), md.getCompressedSize()}; + return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)}; }; ///_____________________________________________________________________________ @@ -1268,7 +1270,7 @@ o2::ctf::CTFIOSize EncodedBlocks::entropyCodeRANSCompat(const input_IT dataSize, nLiteralWords); - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; } template @@ -1349,7 +1351,7 @@ CTFIOSize EncodedBlocks::encodeRANSV1External(const input_IT srcBegin, dataSize, literalsSize); - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; }; template @@ -1458,7 +1460,7 @@ CTFIOSize EncodedBlocks::encodeRANSV1Inplace(const input_IT srcBegin, c dataSize, literalsSize); - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; }; // namespace ctf template @@ -1491,7 +1493,7 @@ o2::ctf::CTFIOSize EncodedBlocks::pack(const input_IT srcBegin, const i } LOGP(debug, "StoreData {} bytes, offs: {}:{}", packedSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + packedSize * sizeof(storageBuffer_t)); - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; }; template @@ -1513,7 +1515,7 @@ o2::ctf::CTFIOSize EncodedBlocks::store(const input_IT srcBegin, const *thisMetadata = detail::makeMetadataStore(messageLength, opt, nBufferElems); - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; }; /// create a special EncodedBlocks containing only dictionaries made from provided vector of frequency tables diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h index abf7561eb25a9..975522767dce1 100644 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h @@ -37,7 +37,7 @@ struct Metadata { size_t nLiterals = 0; // Number of samples that were stored as literals. uint8_t messageWordSize = 0; // size in Bytes of a symbol in the encoded message. uint8_t coderType = 0; // what type of CTF Coder is used? (32 vs 64 bit coders). - uint8_t streamSize = 0; // how many Bytes is the rANS encoder emmiting during a stream-out step. + uint8_t streamSize = 0; // number of Bytes emmitted during rANS stream out (ransCompat) or lower renorming bound (ransV1). uint8_t probabilityBits = 0; // The encoder renormed the distribution of source symbols to sum up to 2^probabilityBits. OptStore opt = OptStore::EENCODE; // The type of storage operation that was conducted. int32_t min = 0; // min symbol of the source dataset. @@ -48,8 +48,21 @@ struct Metadata { int nDataWords = 0; // Amount of words used to store the actual data. int nLiteralWords = 0; // Amount of words used to store literal (incompressible) samples. + /** + * @brief Uncompressed size of stored data in bytes + * + * @return size_t Uncompressed size in bytes + */ size_t getUncompressedSize() const { return messageLength * messageWordSize; } - size_t getCompressedSize() const { return (nDictWords + nDataWords + nLiteralWords) * streamSize; } + + /** + * @brief Size of the stored, compressed data in multiples of the underlying buffer word size + * + * @return size_t The size in multiples of the underlying buffer word size + * @warning This size is in number of words of the underlying storage buffer. + * Multiply with the size of the storage buffer type to get the correct size in bytes. + */ + size_t getCompressedSize() const { return nDictWords + nDataWords + nLiteralWords; } void clear() { nStreams = 0;