From c4d4de7e72286b7899c28a31f643b799f1d03bbd Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 21 May 2025 21:06:40 +0200 Subject: [PATCH] GPU: Add memoryScaling fuzzing debug option --- GPU/GPUTracking/Base/GPUReconstruction.cxx | 2 +- GPU/GPUTracking/Base/GPUReconstructionCPU.cxx | 8 ++++++ .../DataTypes/GPUMemorySizeScalers.cxx | 26 +++++++++++++++++++ .../DataTypes/GPUMemorySizeScalers.h | 11 +++++--- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Interface/GPUO2Interface.cxx | 2 +- 6 files changed, 45 insertions(+), 5 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstruction.cxx b/GPU/GPUTracking/Base/GPUReconstruction.cxx index a4e5d5e1189f5..ad7a31cbd7470 100644 --- a/GPU/GPUTracking/Base/GPUReconstruction.cxx +++ b/GPU/GPUTracking/Base/GPUReconstruction.cxx @@ -304,7 +304,7 @@ int32_t GPUReconstruction::InitPhaseBeforeDevice() mProcessingSettings->rtc.optConstexpr = false; } - mMemoryScalers->factor = GetProcessingSettings().memoryScalingFactor; + mMemoryScalers->scalingFactor = GetProcessingSettings().memoryScalingFactor; mMemoryScalers->conservative = GetProcessingSettings().conservativeMemoryEstimate; mMemoryScalers->returnMaxVal = GetProcessingSettings().forceMaxMemScalers != 0; if (GetProcessingSettings().forceMaxMemScalers > 1) { diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx index 5f80a56e9e64e..2d1061616d907 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx @@ -216,6 +216,14 @@ int32_t GPUReconstructionCPU::ExitDevice() int32_t GPUReconstructionCPU::RunChains() { mMemoryScalers->temporaryFactor = 1.; + if (GetProcessingSettings().memoryScalingFuzz) { + static std::mt19937 rng; + static std::uniform_int_distribution dist(0, 1000000); + uint64_t fuzzFactor = GetProcessingSettings().memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().memoryScalingFuzz; + GPUInfo("Fuzzing memory scaling factor with %lu", fuzzFactor); + mMemoryScalers->fuzzScalingFactor(fuzzFactor); + } + mStatNEvents++; mNEventsProcessed++; diff --git a/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.cxx b/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.cxx index 8b8fbc3ecae20..42ac2e8015f45 100644 --- a/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.cxx +++ b/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.cxx @@ -15,6 +15,8 @@ #include "GPUMemorySizeScalers.h" #include "GPULogging.h" +#include + using namespace o2::gpu; void GPUMemorySizeScalers::rescaleMaxMem(size_t newAvailableMemory) @@ -36,3 +38,27 @@ void GPUMemorySizeScalers::rescaleMaxMem(size_t newAvailableMemory) tpcMaxMergedTrackHits = (double)tmp.tpcMaxMergedTrackHits * scaleFactor; availableMemory = newAvailableMemory; } + +double GPUMemorySizeScalers::getScalingFactor() +{ + if (!doFuzzing) { + return scalingFactor; + } + static std::uniform_int_distribution dist(0, 1000000); + static std::mt19937 rng; + if (fuzzSeed) { + rng = std::mt19937(fuzzSeed); + fuzzLimit = dist(rng) / 10; + fuzzSeed = 0; + } + if (dist(rng) > fuzzLimit) { + return scalingFactor; + } + return scalingFactor * 0.000001 * dist(rng); +} + +void GPUMemorySizeScalers::fuzzScalingFactor(uint64_t seed) +{ + fuzzSeed = seed; + doFuzzing = true; +} diff --git a/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.h b/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.h index e5012d86742f8..164ecb32c26c7 100644 --- a/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.h +++ b/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.h @@ -28,7 +28,9 @@ struct GPUMemorySizeScalers { size_t nITSTracks = 0; // General scaling factor - double factor = 1; + double scalingFactor = 1; + uint64_t fuzzSeed = 0; + uint64_t fuzzLimit = 0; double temporaryFactor = 1; bool conservative = 0; @@ -64,11 +66,14 @@ struct GPUMemorySizeScalers { size_t tpcMaxMergedTrackHits = 200000000; size_t availableMemory = 20500000000; bool returnMaxVal = false; + bool doFuzzing = false; void rescaleMaxMem(size_t newAvailableMemory); + double getScalingFactor(); + void fuzzScalingFactor(uint64_t seed); inline size_t getValue(size_t maxVal, size_t val) { - return returnMaxVal ? maxVal : (std::min(maxVal, offset + val) * factor * temporaryFactor); + return returnMaxVal ? maxVal : (std::min(maxVal, offset + val) * (doFuzzing == 0 ? scalingFactor : getScalingFactor()) * temporaryFactor); } inline size_t NTPCPeaks(size_t tpcDigits, bool perSector = false) { return getValue(perSector ? tpcMaxPeaks : (GPUCA_NSECTORS * tpcMaxPeaks), hitOffset + tpcDigits * tpcPeaksPerDigit); } @@ -81,7 +86,7 @@ struct GPUMemorySizeScalers { inline size_t NTPCSectorTrackHits(size_t tpcHits, uint8_t withRejection = 0) { return getValue(tpcMaxSectorTrackHits, tpcHits * (withRejection ? tpcSectorTrackHitsPerHitWithRejection : tpcSectorTrackHitsPerHit)); } inline size_t NTPCMergedTracks(size_t tpcSectorTracks) { return getValue(tpcMaxMergedTracks, tpcSectorTracks * (conservative ? 1.0 : tpcMergedTrackPerSectorTrack)); } inline size_t NTPCMergedTrackHits(size_t tpcSectorTrackHitss) { return getValue(tpcMaxMergedTrackHits, tpcSectorTrackHitss * tpcMergedTrackHitPerSectorHit); } - inline size_t NTPCUnattachedHitsBase1024(int32_t type) { return (returnMaxVal || conservative) ? 1024 : std::min(1024, tpcCompressedUnattachedHitsBase1024[type] * factor * temporaryFactor); } + inline size_t NTPCUnattachedHitsBase1024(int32_t type) { return (returnMaxVal || conservative) ? 1024 : std::min(1024, tpcCompressedUnattachedHitsBase1024[type] * (doFuzzing == 0 ? scalingFactor : getScalingFactor()) * temporaryFactor); } }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 12f40cda4c398..238994ee53af5 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -312,6 +312,7 @@ AddOption(memoryAllocationStrategy, int8_t, 0, "", 0, "Memory Allocation Strageg AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory", min(0ul)) AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(0ul)) AddOption(memoryScalingFactor, float, 1.f, "", 0, "Factor to apply to all memory scalers") +AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) AddOption(conservativeMemoryEstimate, bool, false, "", 0, "Use some more conservative defaults for larger buffers during TPC processing") AddOption(tpcInputWithClusterRejection, uint8_t, 0, "", 0, "Indicate whether the TPC input is CTF data with cluster rejection, to tune buffer estimations") AddOption(forceMaxMemScalers, uint64_t, 0, "", 0, "Force using the maximum values for all buffers, Set a value n > 1 to rescale all maximums to a memory size of n") diff --git a/GPU/GPUTracking/Interface/GPUO2Interface.cxx b/GPU/GPUTracking/Interface/GPUO2Interface.cxx index 81eb2c285192b..f7e972315a739 100644 --- a/GPU/GPUTracking/Interface/GPUO2Interface.cxx +++ b/GPU/GPUTracking/Interface/GPUO2Interface.cxx @@ -110,7 +110,7 @@ int32_t GPUO2Interface::Initialize(const GPUO2InterfaceConfiguration& config) return (1); } if (!mCtx[i].mRec->IsGPU() && mCtx[i].mRec->GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) { - mCtx[i].mRec->MemoryScalers()->factor *= 2; + mCtx[i].mRec->MemoryScalers()->scalingFactor *= 2; } } if (mConfig->configProcessing.doublePipeline) {