Skip to content

aggregator-workflow.sh: update CMV workflow default parameters #15370

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ehellbar wants to merge 2 commits into
base: dev
Choose a base branch
from
+60 −44
Open

aggregator-workflow.sh: update CMV workflow default parameters #15370

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9414bb2
aggregator-workflow.sh: decouple IDC and CMV workflows
ehellbar May 6, 2026
27c8c05
aggregator-workflow.sh: update CMV workflow default parameters
ehellbar May 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 60 additions & 44 deletions prodtests/full-system-test/aggregator-workflow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,25 +155,35 @@ if workflow_has_parameter CALIB_PROXIES; then
if [[ -n ${CALIBDATASPEC_BARREL_SPORADIC:-} ]]; then
add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_BARREL_SPORADIC\" $(get_proxy_connection barrel_sp input sporadic)" "" 0
fi
elif [[ $AGGREGATOR_TASKS == TPC_IDCBOTH_SAC ]]; then
elif [[ $AGGREGATOR_TASKS == TPC_IDCBOTH_SAC || $AGGREGATOR_TASKS == TPC_CMV ]]; then
if [[ $EPNSYNCMODE != 1 ]]; then
echo "ERROR: TPC IDC / SAC calib workflow enabled without EPNSYNCMODE, please note that there will not be input data for it" 1>&2
echo "ERROR: TPC IDC / SAC / CMV calib workflow enabled without EPNSYNCMODE, please note that there will not be input data for it" 1>&2
fi
CHANNELS_LIST=
[[ $EPNSYNCMODE == 0 ]] && FLP_ADDRESS="tcp://localhost:29950"
if [[ -n ${CALIBDATASPEC_TPCIDC_A:-} ]] || [[ -n ${CALIBDATASPEC_TPCIDC_C:-} ]] || [[ -n ${CALIBDATASPEC_TPCCMV:-} ]]; then
# define port for FLP
: ${TPC_IDC_FLP_PORT:=29950}
# define port for FLP and channel prefix
TPC_FLP_PORT=
TPC_FLP_CHAN_PREFIX=
if [[ $AGGREGATOR_TASKS == TPC_CMV ]] && [[ -n ${CALIBDATASPEC_TPCCMV:-} ]]; then
TPC_FLP_PORT=29952
TPC_FLP_CHAN_PREFIX=tpccmv
elif [[ $AGGREGATOR_TASKS == TPC_IDCBOTH_SAC ]] && [[ -n ${CALIBDATASPEC_TPCIDC_A:-} || -n ${CALIBDATASPEC_TPCIDC_C:-} ]]; then
TPC_FLP_PORT=29950
TPC_FLP_CHAN_PREFIX=tpcidc
fi
# expand FLPs; TPC uses from 001 to 145, but 145 is reserved for SAC
if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then
FLP_ADDRESS="tcp://alio2-cr1-mvs03-ib:${TPC_IDC_FLP_PORT}"
CHANNELS_LIST+="type=pull,name=tpcidc_flp,transport=zeromq,address=$FLP_ADDRESS,method=connect,rateLogging=10;"
else
for flp in $(seq -f "%03g" 1 144); do
[[ ! $FLP_IDS =~ (^|,)"$flp"(,|$) ]] && continue
[[ $EPNSYNCMODE == 1 ]] && FLP_ADDRESS="tcp://alio2-cr1-flp${flp}-ib:${TPC_IDC_FLP_PORT}"
CHANNELS_LIST+="type=pull,name=tpcidc_flp${flp},transport=zeromq,address=$FLP_ADDRESS,method=connect,rateLogging=10;"
done
if [[ -n $TPC_FLP_PORT ]]; then
if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then
FLP_ADDRESS="tcp://alio2-cr1-mvs03-ib:${TPC_FLP_PORT}"
CHANNELS_LIST+="type=pull,name=${TPC_FLP_CHAN_PREFIX}_flp,transport=zeromq,address=$FLP_ADDRESS,method=connect,rateLogging=10;"
else
for flp in $(seq -f "%03g" 1 144); do
[[ ! $FLP_IDS =~ (^|,)"$flp"(,|$) ]] && continue
[[ $EPNSYNCMODE == 1 ]] && FLP_ADDRESS="tcp://alio2-cr1-flp${flp}-ib:${TPC_FLP_PORT}"
CHANNELS_LIST+="type=pull,name=${TPC_FLP_CHAN_PREFIX}_flp${flp},transport=zeromq,address=$FLP_ADDRESS,method=connect,rateLogging=10;"
done
fi
fi
fi
if [[ -n ${CALIBDATASPEC_TPCSAC:-} ]]; then
Expand All @@ -184,22 +194,25 @@ if workflow_has_parameter CALIB_PROXIES; then
fi
if [[ -n $CHANNELS_LIST ]]; then
DATASPEC_LIST=
if [[ -n ${CALIBDATASPEC_TPCIDC_A:-} ]]; then
add_semicolon_separated DATASPEC_LIST "\"$CALIBDATASPEC_TPCIDC_A\""
fi
if [[ -n ${CALIBDATASPEC_TPCIDC_C:-} ]]; then
add_semicolon_separated DATASPEC_LIST "\"$CALIBDATASPEC_TPCIDC_C\""
fi
if [[ -n ${CALIBDATASPEC_TPCCMV:-} ]]; then
add_semicolon_separated DATASPEC_LIST "\"$CALIBDATASPEC_TPCCMV\""
fi
if [[ -n ${CALIBDATASPEC_TPCSAC:-} ]]; then
add_semicolon_separated DATASPEC_LIST "\"$CALIBDATASPEC_TPCSAC\""
if [[ $AGGREGATOR_TASKS == TPC_CMV ]]; then
if [[ -n ${CALIBDATASPEC_TPCCMV:-} ]]; then
add_semicolon_separated DATASPEC_LIST "\"$CALIBDATASPEC_TPCCMV\""
fi
else
if [[ -n ${CALIBDATASPEC_TPCIDC_A:-} ]]; then
add_semicolon_separated DATASPEC_LIST "\"$CALIBDATASPEC_TPCIDC_A\""
fi
if [[ -n ${CALIBDATASPEC_TPCIDC_C:-} ]]; then
add_semicolon_separated DATASPEC_LIST "\"$CALIBDATASPEC_TPCIDC_C\""
fi
if [[ -n ${CALIBDATASPEC_TPCSAC:-} ]]; then
add_semicolon_separated DATASPEC_LIST "\"$CALIBDATASPEC_TPCSAC\""
fi
fi
if [[ -z ${O2_TPC_IDC_CMV_IO_THREADS:-} ]]; then
O2_TPC_IDC_CMV_IO_THREADS=4;
fi
add_W o2-dpl-raw-proxy "--proxy-name tpcidc --io-threads ${O2_TPC_IDC_CMV_IO_THREADS} --dataspec \"$DATASPEC_LIST\" --sporadic-outputs --channel-config \"$CHANNELS_LIST\" ${TIMEFRAME_SHM_LIMIT+--timeframes-shm-limit} $TIMEFRAME_SHM_LIMIT" "" 0
fi
add_W o2-dpl-raw-proxy "--proxy-name ${TPC_FLP_CHAN_PREFIX} --io-threads ${O2_TPC_IDC_CMV_IO_THREADS} --dataspec \"$DATASPEC_LIST\" --sporadic-outputs --channel-config \"$CHANNELS_LIST\" ${TIMEFRAME_SHM_LIMIT+--timeframes-shm-limit} $TIMEFRAME_SHM_LIMIT" "" 0
fi
elif [[ $AGGREGATOR_TASKS == CALO_TF ]]; then
if [[ -n ${CALIBDATASPEC_CALO_TF:-} ]]; then
Expand Down Expand Up @@ -308,32 +321,35 @@ nTFs=$((1000 * 128 / ${NHBPERTF}))
nTFs_SAC=$((10000 * 128 / ${NHBPERTF}))
nBuffer=$((100 * 128 / ${NHBPERTF}))
nBuffer_cmv=$((50 * 128 / ${NHBPERTF}))
lanesCMVaggregate=${O2_TPC_CMV_AGGREGATE_NLANES:-8}
lanesCMVaggregate=${O2_TPC_CMV_AGGREGATE_NLANES:-4}
cmvCompression=${O2_TPC_CMV_COMPRESSION:---use-sparse --cmv-zero-threshold 1.0 --cmv-dynamic-precision-mean 1.0 --cmv-dynamic-precision-sigma 8.0 --use-compression-huffman}
cmvTimeframes=${O2_TPC_CMV_TIMEFRAMES:-4000}
IDC_DELTA="--disable-IDCDelta true" # off by default
# deltas are on by default; you need to request explicitly to switch them off;
if [[ "${DISABLE_IDC_DELTA:-}" == "1" ]]; then IDC_DELTA=""; fi
if [[ "${ENABLE_IDC_DELTA_FILE:-}" == "1" ]]; then IDC_DELTA+=" --dump-IDCDelta-calib-data true --output-dir $CALIB_DIR --meta-output-dir $EPN2EOS_METAFILES_DIR "; fi

if [[ "${DISABLE_IDC_PAD_MAP_WRITING:-}" == 1 ]]; then TPC_WRITING_PAD_STATUS_MAP=""; else TPC_WRITING_PAD_STATUS_MAP="--enableWritingPadStatusMap true"; fi

if ! workflow_has_parameter CALIB_LOCAL_INTEGRATED_AGGREGATOR && [[ $AGGREGATOR_TASKS == TPC_IDCBOTH_SAC || $AGGREGATOR_TASKS == ALL ]]; then
if [[ $CALIB_TPC_IDC == 1 ]]; then
add_W o2-tpc-idc-distribute "--crus ${crus} --timeframes ${nTFs} --output-lanes ${lanesFactorize} --send-precise-timestamp true --condition-tf-per-query ${nTFs} --n-TFs-buffer ${nBuffer}"
add_W o2-tpc-idc-factorize "--n-TFs-buffer ${nBuffer} --input-lanes ${lanesFactorize} --crus ${crus} --timeframes ${nTFs} --nthreads-grouping ${threadFactorize} --nthreads-IDC-factorization ${threadFactorize} --sendOutputFFT true --enable-CCDB-output true --enablePadStatusMap true ${TPC_WRITING_PAD_STATUS_MAP} --use-precise-timestamp true $IDC_DELTA" "TPCIDCGroupParam.groupPadsSectorEdges=32211"
add_W o2-tpc-idc-ft-aggregator "--rangeIDC 200 --inputLanes ${lanesFactorize} --nFourierCoeff 40 --nthreads 8"
fi
if [[ $CALIB_TPC_CMV == 1 ]]; then
if [[ -z ${O2_TPC_CMV_COMPRESSION:-} ]]; then O2_TPC_CMV_COMPRESSION="--use-sparse --cmv-zero-threshold 1.0 --cmv-dynamic-precision-mean 1.0 --cmv-dynamic-precision-sigma 8.0 --use-compression-huffman"; fi
if [[ -z ${O2_TPC_CMV_TIMEFRAMES:-} ]]; then O2_TPC_CMV_TIMEFRAMES="2000"; fi
add_W o2-tpc-cmv-distribute "--crus ${crus} --lanes 1 --output-lanes ${lanesCMVaggregate} --n-TFs-buffer ${nBuffer_cmv} --timeframes ${O2_TPC_CMV_TIMEFRAMES} --send-precise-timestamp "
add_W o2-tpc-cmv-aggregate "--crus ${crus} --input-lanes ${lanesCMVaggregate} --n-TFs-buffer ${nBuffer_cmv} --nthreads-compression 4 --timeframes ${O2_TPC_CMV_TIMEFRAMES} --use-precise-timestamp ${O2_TPC_CMV_COMPRESSION} --output-dir $CALIB_DIR --meta-output-dir $EPN2EOS_METAFILES_DIR "
fi
if [[ $CALIB_TPC_SAC == 1 ]]; then
add_W o2-tpc-sac-distribute "--timeframes ${nTFs_SAC} --output-lanes 1 "
add_W o2-tpc-sac-factorize "--timeframes ${nTFs_SAC} --nthreads-SAC-factorization 4 --input-lanes 1 --compression 2"
add_W o2-tpc-idc-ft-aggregator "--rangeIDC 200 --nFourierCoeff 40 --process-SACs true --inputLanes 1"
if ! workflow_has_parameter CALIB_LOCAL_INTEGRATED_AGGREGATOR; then
if [[ $AGGREGATOR_TASKS == TPC_IDCBOTH_SAC || $AGGREGATOR_TASKS == ALL ]]; then
if [[ $CALIB_TPC_IDC == 1 ]]; then
add_W o2-tpc-idc-distribute "--crus ${crus} --timeframes ${nTFs} --output-lanes ${lanesFactorize} --send-precise-timestamp true --condition-tf-per-query ${nTFs} --n-TFs-buffer ${nBuffer}"
add_W o2-tpc-idc-factorize "--n-TFs-buffer ${nBuffer} --input-lanes ${lanesFactorize} --crus ${crus} --timeframes ${nTFs} --nthreads-grouping ${threadFactorize} --nthreads-IDC-factorization ${threadFactorize} --sendOutputFFT true --enable-CCDB-output true --enablePadStatusMap true ${TPC_WRITING_PAD_STATUS_MAP} --use-precise-timestamp true $IDC_DELTA" "TPCIDCGroupParam.groupPadsSectorEdges=32211"
add_W o2-tpc-idc-ft-aggregator "--rangeIDC 200 --inputLanes ${lanesFactorize} --nFourierCoeff 40 --nthreads 8"
fi
if [[ $CALIB_TPC_SAC == 1 ]]; then
add_W o2-tpc-sac-distribute "--timeframes ${nTFs_SAC} --output-lanes 1 "
add_W o2-tpc-sac-factorize "--timeframes ${nTFs_SAC} --nthreads-SAC-factorization 4 --input-lanes 1 --compression 2"
add_W o2-tpc-idc-ft-aggregator "--rangeIDC 200 --nFourierCoeff 40 --process-SACs true --inputLanes 1"
fi
elif [[ $AGGREGATOR_TASKS == TPC_CMV || $AGGREGATOR_TASKS == ALL ]]; then
if [[ $CALIB_TPC_CMV == 1 ]]; then
add_W o2-tpc-cmv-distribute "--crus ${crus} --lanes 1 --output-lanes ${lanesCMVaggregate} --n-TFs-buffer ${nBuffer_cmv} --timeframes ${cmvTimeframes} --send-precise-timestamp "
add_W o2-tpc-cmv-aggregate "--crus ${crus} --input-lanes ${lanesCMVaggregate} --n-TFs-buffer ${nBuffer_cmv} --nthreads-compression 8 --timeframes ${cmvTimeframes} --use-precise-timestamp ${cmvCompression} --output-dir $CALIB_DIR --meta-output-dir $EPN2EOS_METAFILES_DIR "
CCDB_POPULATOR_UPLOAD_PATH=none
fi
fi
[[ $AGGREGATOR_TASKS == TPC_IDCBOTH_SAC ]] && [[ $CALIB_TPC_IDC == 0 && $CALIB_TPC_SAC == 0 && $CALIB_TPC_CMV == 1 ]] && CCDB_POPULATOR_UPLOAD_PATH="none"
fi

# Calo cal
Expand Down