Skip to content

[Fix] Correct Reporting of CTF Compressed Payload in Metadata #14761

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
davidrohr merged 1 commit into from
Oct 23, 2025
Merged

[Fix] Correct Reporting of CTF Compressed Payload in Metadata #14761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -929,9 +929,11 @@ CTFIOSize EncodedBlocks<H, N, W>::decode(D_IT dest, // it
const auto& md = mMetadata[slot];
LOGP(debug, "Slot{} | NStored={} Ndict={} nData={}, MD: messageLength:{} opt:{} min:{} max:{} offs:{} width:{} ", slot, block.getNStored(), block.getNDict(), block.getNData(), md.messageLength, (int)md.opt, md.min, md.max, md.literalsPackingOffset, md.literalsPackingWidth);

constexpr size_t word_size = sizeof(W);

if (ansVersion == ANSVersionCompat) {
if (!block.getNStored()) {
return {0, md.getUncompressedSize(), md.getCompressedSize()};
return {0, md.getUncompressedSize(), md.getCompressedSize() * word_size};
}
if (md.opt == Metadata::OptStore::EENCODE) {
return decodeCompatImpl(dest, slot, decoderExt);
Expand All @@ -943,7 +945,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decode(D_IT dest, // it
return decodeUnpackImpl(dest, slot);
}
if (!block.getNStored()) {
return {0, md.getUncompressedSize(), md.getCompressedSize()};
return {0, md.getUncompressedSize(), md.getCompressedSize() * word_size};
}
if (md.opt == Metadata::OptStore::EENCODE) {
return decodeRansV1Impl(dest, slot, decoderExt);
Expand Down Expand Up @@ -991,7 +993,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decodeCompatImpl(dst_IT dstBegin, int slot, co
} else {
getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, NDecoderStreams);
}
return {0, md.getUncompressedSize(), md.getCompressedSize()};
return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)};
};

template <typename H, int N, typename W>
Expand Down Expand Up @@ -1045,7 +1047,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decodeRansV1Impl(dst_IT dstBegin, int slot, co
} else {
getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, md.nStreams);
}
return {0, md.getUncompressedSize(), md.getCompressedSize()};
return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)};
};

template <typename H, int N, typename W>
Expand Down Expand Up @@ -1079,7 +1081,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decodeUnpackImpl(dst_IT dest, int slot) const
} else {
rans::unpack(srcIt, messageLength, dest, packingWidth, offset);
}
return {0, md.getUncompressedSize(), md.getCompressedSize()};
return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)};
};

template <typename H, int N, typename W>
Expand All @@ -1098,7 +1100,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decodeCopyImpl(dst_IT dest, int slot) const
destPtr_t srcEnd = srcBegin + md.messageLength * sizeof(dest_t);
std::copy(srcBegin, srcEnd, dest);

return {0, md.getUncompressedSize(), md.getCompressedSize()};
return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)};
};

///_____________________________________________________________________________
Expand Down Expand Up @@ -1268,7 +1270,7 @@ o2::ctf::CTFIOSize EncodedBlocks<H, N, W>::entropyCodeRANSCompat(const input_IT
dataSize,
nLiteralWords);

return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
}

template <typename H, int N, typename W>
Expand Down Expand Up @@ -1349,7 +1351,7 @@ CTFIOSize EncodedBlocks<H, N, W>::encodeRANSV1External(const input_IT srcBegin,
dataSize,
literalsSize);

return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
};

template <typename H, int N, typename W>
Expand Down Expand Up @@ -1458,7 +1460,7 @@ CTFIOSize EncodedBlocks<H, N, W>::encodeRANSV1Inplace(const input_IT srcBegin, c
dataSize,
literalsSize);

return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
}; // namespace ctf

template <typename H, int N, typename W>
Expand Down Expand Up @@ -1491,7 +1493,7 @@ o2::ctf::CTFIOSize EncodedBlocks<H, N, W>::pack(const input_IT srcBegin, const i
}

LOGP(debug, "StoreData {} bytes, offs: {}:{}", packedSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + packedSize * sizeof(storageBuffer_t));
return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
};

template <typename H, int N, typename W>
Expand All @@ -1513,7 +1515,7 @@ o2::ctf::CTFIOSize EncodedBlocks<H, N, W>::store(const input_IT srcBegin, const

*thisMetadata = detail::makeMetadataStore<input_t, storageBuffer_t>(messageLength, opt, nBufferElems);

return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
};

/// create a special EncodedBlocks containing only dictionaries made from provided vector of frequency tables
Expand Down
17 changes: 15 additions & 2 deletions DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ struct Metadata {
size_t nLiterals = 0; // Number of samples that were stored as literals.
uint8_t messageWordSize = 0; // size in Bytes of a symbol in the encoded message.
uint8_t coderType = 0; // what type of CTF Coder is used? (32 vs 64 bit coders).
uint8_t streamSize = 0; // how many Bytes is the rANS encoder emmiting during a stream-out step.
uint8_t streamSize = 0; // number of Bytes emmitted during rANS stream out (ransCompat) or lower renorming bound (ransV1).
uint8_t probabilityBits = 0; // The encoder renormed the distribution of source symbols to sum up to 2^probabilityBits.
OptStore opt = OptStore::EENCODE; // The type of storage operation that was conducted.
int32_t min = 0; // min symbol of the source dataset.
Expand All @@ -48,8 +48,21 @@ struct Metadata {
int nDataWords = 0; // Amount of words used to store the actual data.
int nLiteralWords = 0; // Amount of words used to store literal (incompressible) samples.

/**
* @brief Uncompressed size of stored data in bytes
*
* @return size_t Uncompressed size in bytes
*/
size_t getUncompressedSize() const { return messageLength * messageWordSize; }
size_t getCompressedSize() const { return (nDictWords + nDataWords + nLiteralWords) * streamSize; }

/**
* @brief Size of the stored, compressed data in multiples of the underlying buffer word size
*
* @return size_t The size in multiples of the underlying buffer word size
* @warning This size is in number of words of the underlying storage buffer.
* Multiply with the size of the storage buffer type to get the correct size in bytes.
*/
size_t getCompressedSize() const { return nDictWords + nDataWords + nLiteralWords; }
void clear()
{
nStreams = 0;
Expand Down