Skip to content

Commit 33c670e

Browse files
davidrohrdavidrohr
committed
GPU CMake: Write separate headers for GPU device and non-device parameters, use same headers for compilation and for parameter file generation
1 parent f655530 commit 33c670e

File tree

5 files changed

+72
-68
lines changed

5 files changed

+72
-68
lines changed

‎GPU/GPUTracking/Base/cuda/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ add_custom_command(
7474
COMMAND cat ${GPUDIR}/Base/GPUStdSystemHeaders.h >> ${GPU_RTC_BIN}.src
7575
COMMAND ${CMAKE_CUDA_COMPILER} ${GPU_RTC_DEFINES} ${GPU_RTC_INCLUDES} -std=c++${CMAKE_CUDA_STANDARD} -D__CUDA_ARCH__=${RTC_CUDA_ARCH} -Wno-deprecated-gpu-targets -D__CUDACC__ -x c++ -M -MD -MT ${GPU_RTC_BIN}.src -MF ${GPU_RTC_BIN}.src.d ${GPU_RTC_SRC}
7676
COMMAND ${CMAKE_CUDA_COMPILER} ${GPU_RTC_DEFINES} ${GPU_RTC_INCLUDES} -std=c++${CMAKE_CUDA_STANDARD} -D__CUDA_ARCH__=${RTC_CUDA_ARCH} -Wno-deprecated-gpu-targets -D__CUDACC__ -x c++ -E -Xcompiler "-nostdinc -P" ${GPU_RTC_SRC} >> ${GPU_RTC_BIN}.src
77-
DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/cuda/GPUReconstructionCUDAIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h GPU_PARAM_HEADER_AUTO_ALL
77+
DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/cuda/GPUReconstructionCUDAIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h GPU_PARAM_HEADER_TARGET
7878
DEPFILE ${GPU_RTC_BIN}.src.d
7979
COMMAND_EXPAND_LISTS
8080
COMMENT "Preparing CUDA RTC source file ${GPU_RTC_BIN}.src"

‎GPU/GPUTracking/Base/hip/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ add_custom_command(
125125
COMMAND cat ${GPUDIR}/Base/hip/GPUReconstructionHIPIncludesSystem.h | grep -v GPUStdSystemHeaders.h >> ${GPU_RTC_BIN}.src
126126
COMMAND cat ${GPUDIR}/Base/GPUStdSystemHeaders.h >> ${GPU_RTC_BIN}.src
127127
COMMAND ${CMAKE_HIP_COMPILER} ${GPU_RTC_DEFINES} ${GPU_RTC_INCLUDES} -std=c++${CMAKE_HIP_STANDARD} -D__HIPCC__ -D__HIP_DEVICE_COMPILE__ -x c++ -nostdinc -E -P ${GPU_RTC_SRC} -MD -MT ${GPU_RTC_BIN}.src -MF ${GPU_RTC_BIN}.src.d >> ${GPU_RTC_BIN}.src
128-
DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/hip/GPUReconstructionHIPIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h ${MODULE}_HIPIFIED GPU_PARAM_HEADER_AUTO_ALL
128+
DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/hip/GPUReconstructionHIPIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h ${MODULE}_HIPIFIED GPU_PARAM_HEADER_TARGET
129129
DEPFILE ${GPU_RTC_BIN}.src.d
130130
COMMAND_EXPAND_LISTS
131131
COMMENT "Preparing HIP RTC source file ${GPU_RTC_BIN}.src"

‎GPU/GPUTracking/CMakeLists.txt‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,8 @@ set(ON_THE_FLY_DIR ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly)
130130
file(MAKE_DIRECTORY ${ON_THE_FLY_DIR})
131131
include(cmake/gpu_param_header_generator.cmake)
132132
set(GPU_DEFAULT_PARAMS_HEADER ${ON_THE_FLY_DIR}/GPUDefParametersDefaults.h)
133-
generate_gpu_param_header("AUTO" ${GPU_DEFAULT_PARAMS_HEADER}) # generate header with default GPU parameters, arch selected by CMake variables
133+
set(GPU_DEFAULT_PARAMS_HEADER_DEVICE ${ON_THE_FLY_DIR}/GPUDefParametersDefaultsDevice.h)
134+
generate_gpu_param_header("ALL" ${GPU_DEFAULT_PARAMS_HEADER} ${GPU_DEFAULT_PARAMS_HEADER_DEVICE} GPU_CONST_PARAM_ARCHITECTUES) # generate header with default GPU parameters, arch selected by CMake variables
134135

135136
set(HDRS_INSTALL
136137
${HDRS_CINT_O2}
@@ -161,6 +162,7 @@ set(HDRS_INSTALL
161162
Debug/GPUROOTDump.h
162163
Definitions/GPUDefConstantsAndSettings.h
163164
${GPU_DEFAULT_PARAMS_HEADER}
165+
${GPU_DEFAULT_PARAMS_HEADER_DEVICE}
164166
Definitions/GPUDefParametersWrapper.h
165167
Definitions/GPUDefParametersConstants.h
166168
Definitions/GPUDef.h
@@ -449,20 +451,19 @@ if(CUDA_ENABLED OR OPENCL_ENABLED OR HIP_ENABLED)
449451
message(WARNING "GPU Tracking disabled on MacOS")
450452
else()
451453
make_directory(${CMAKE_CURRENT_BINARY_DIR}/genGPUArch)
452-
set(GPU_CONST_PARAM_FILES "")
453-
set(GPU_ARCH_PARAMS_HEADER ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/GPUDefParametersDefaults_OnTheFly.h)
454-
generate_gpu_param_header("ALL" ${GPU_ARCH_PARAMS_HEADER} "GPU_CONST_PARAM_ARCHITECTUES")
454+
set(GPU_CONST_PARAM_FILES)
455455
foreach(GPU_ARCH ${GPU_CONST_PARAM_ARCHITECTUES})
456456
set(PARAMFILE ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/gpu_const_param_${GPU_ARCH}.par)
457457
add_custom_command(
458458
OUTPUT ${PARAMFILE}
459459
COMMAND bash -c
460-
"echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"${GPU_ARCH_PARAMS_HEADER}\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'"
460+
"echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"GPUDefParametersDefaults.h\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'"
461461
| root -l -b > /dev/null
462462
VERBATIM
463463
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch
464464
MAIN_DEPENDENCY Standalone/tools/dumpGPUDefParam.C
465-
DEPENDS ${GPU_ARCH_PARAMS_HEADER}
465+
DEPENDS ${GPU_DEFAULT_PARAMS_HEADER}
466+
${GPU_DEFAULT_PARAMS_HEADER_DEVICE}
466467
${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h
467468
${ON_THE_FLY_DIR}/GPUDefParametersLoad.inc
468469
COMMENT "Generating GPU parameter set for architecture ${GPU_ARCH}")

‎GPU/GPUTracking/cmake/gpu_param_header_generator.cmake‎

Lines changed: 55 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,31 @@
1212
# file gpu_param_header_generator.cmake
1313
# author Gabriele Cimador
1414

15-
function(generate_macros json_content output types arch_list arch_list_output)
16-
foreach(arch IN LISTS arch_list)
17-
set(OUTPUT_TMP_${arch} "")
18-
endforeach()
19-
set(arch_list_output_tmp)
20-
list(FIND arch_list "ALL" do_all_architectures)
21-
foreach(TYPE IN LISTS types)
22-
string(JSON n_params LENGTH "${json_content}" "${TYPE}")
15+
function(generate_gpu_param_header ARCH_LIST OUT_HEADER OUT_HEADER_DEVICE)
16+
list(FIND ARCH_LIST "ALL" do_all_architectures)
17+
list(FIND ARCH_LIST "AUTO" do_auto_architectures)
18+
if(do_all_architectures GREATER -1 OR do_auto_architectures GREATER -1)
19+
if(do_auto_architectures GREATER -1)
20+
detect_gpu_arch("AUTO")
21+
list(REMOVE_ITEM ARCH_LIST "AUTO")
22+
else()
23+
detect_gpu_arch("ALL")
24+
endif()
25+
list(APPEND ARCH_LIST ${TARGET_ARCH})
26+
endif()
27+
file(READ "${GPU_PARAM_JSON}" JSON_CONTENT)
28+
29+
# Types
30+
set(TYPES CORE LB PAR)
31+
set(ARCH_LIST_EXT "${ARCH_LIST};default;default_cpu")
32+
# Per architecture definitions
33+
set(JSON_ARCHITECTURES)
34+
35+
foreach(TYPE IN LISTS TYPES)
36+
string(JSON n_params LENGTH "${JSON_CONTENT}" "${TYPE}")
2337
math(EXPR last "${n_params} - 1")
2438
foreach(i RANGE 0 ${last})
25-
string(JSON param_name MEMBER "${json_content}" "${TYPE}" "${i}")
39+
string(JSON param_name MEMBER "${JSON_CONTENT}" "${TYPE}" "${i}")
2640
string(JSON n_archs LENGTH "${JSON_CONTENT}" "${TYPE}" "${param_name}")
2741
math(EXPR last_arch "${n_archs} - 1")
2842
foreach(iArch RANGE 0 ${last_arch})
@@ -31,12 +45,12 @@ function(generate_macros json_content output types arch_list arch_list_output)
3145
message(FATAL_ERROR "Bogus entry ${param_name} for ${arch}")
3246
endif()
3347
if(do_all_architectures GREATER -1)
34-
if(arch_list_output AND NOT arch MATCHES ^default)
35-
list(APPEND arch_list_output_tmp "${arch}")
48+
if(NOT arch MATCHES ^default)
49+
list(APPEND JSON_ARCHITECTURES "${arch}")
3650
endif()
3751
set(list_idx 0)
3852
else()
39-
list(FIND arch_list "${arch}" list_idx)
53+
list(FIND ARCH_LIST_EXT "${arch}" list_idx)
4054
endif()
4155
if(list_idx GREATER -1)
4256
string(JSON param_values GET "${JSON_CONTENT}" "${TYPE}" "${param_name}" "${arch}")
@@ -54,68 +68,58 @@ function(generate_macros json_content output types arch_list arch_list_output)
5468
set(MACRO_DEFINITION "#define ${MACRO_NAME} ${vals}")
5569
if(arch MATCHES ^default)
5670
# fallback defaults are wrapped in #ifndef
57-
string(APPEND OUTPUT_TMP_${arch} "#ifndef ${MACRO_NAME}\n ${MACRO_DEFINITION}\n#endif\n\n")
71+
string(APPEND generate_gpu_param_header_OUTPUT_TMP_${arch} "#ifndef ${MACRO_NAME}\n ${MACRO_DEFINITION}\n#endif\n\n")
5872
else()
59-
string(APPEND OUTPUT_TMP_${arch} "${MACRO_DEFINITION}\n")
73+
string(APPEND generate_gpu_param_header_OUTPUT_TMP_${arch} "${MACRO_DEFINITION}\n")
6074
endif()
6175
endif()
6276
endforeach()
6377
endforeach()
6478
endforeach()
65-
foreach(arch IN LISTS arch_list)
66-
set(${output}_${arch} "${OUTPUT_TMP_${arch}}" PARENT_SCOPE)
67-
endforeach()
68-
if(arch_list_output)
69-
list(REMOVE_DUPLICATES arch_list_output_tmp)
70-
list(SORT arch_list_output_tmp)
71-
set(${arch_list_output} "${arch_list_output_tmp}" PARENT_SCOPE)
72-
endif()
73-
endfunction()
7479

75-
function(generate_gpu_param_header GPU_ARCH OUT_HEADER)
76-
set(TARGET_ARCH "UNKNOWN")
77-
if(GPU_ARCH STREQUAL "AUTO")
78-
detect_gpu_arch("ALL")
79-
else()
80-
set(TARGET_ARCH ${GPU_ARCH})
80+
list(REMOVE_DUPLICATES JSON_ARCHITECTURES)
81+
list(SORT JSON_ARCHITECTURES)
82+
if(ARGC GREATER 3)
83+
set(${ARGV3} "${JSON_ARCHITECTURES}" PARENT_SCOPE)
8184
endif()
82-
file(READ "${GPU_PARAM_JSON}" JSON_CONTENT)
85+
if(do_all_architectures GREATER -1)
86+
list(REMOVE_ITEM ARCH_LIST "ALL")
87+
list(APPEND ARCH_LIST ${JSON_ARCHITECTURES})
88+
endif()
89+
list(REMOVE_DUPLICATES ARCH_LIST)
90+
list(SORT ARCH_LIST)
91+
92+
get_filename_component(DEVICE_HEADER_FILE "${OUT_HEADER_DEVICE}" NAME)
93+
8394
set(TMP_HEADER "#ifndef GPUDEFPARAMETERSDEFAULTS_H\n#define GPUDEFPARAMETERSDEFAULTS_H\n\n")
95+
set(TMP_HEADER_DEVICE "#ifndef GPUDEFPARAMETERSDEFAULTSDEVICE_H\n#define GPUDEFPARAMETERSDEFAULTSDEVICE_H\n\n")
8496
string(APPEND TMP_HEADER "// This file is auto-generated from gpu_params.json. Do not edit directly.\n")
85-
string(REPLACE "," ";" ARCH_LIST "${TARGET_ARCH}")
86-
string(APPEND TMP_HEADER "// Architectures: ${TARGET_ARCH}\n\n")
97+
string(APPEND TMP_HEADER_DEVICE "// This file is auto-generated from gpu_params.json. Do not edit directly.\n")
98+
string(APPEND TMP_HEADER_DEVICE "// Architectures: ${TARGET_ARCH}\n\n")
8799
string(APPEND TMP_HEADER "#if defined(GPUCA_GPUCODE) && !defined(GPUCA_GPUCODE_GENRTC) && !defined(GPUCA_GPUCODE_NO_LAUNCH_BOUNDS) // Avoid including for RTC generation besides normal include protection.\n\n")
100+
string(APPEND TMP_HEADER "#include \"${DEVICE_HEADER_FILE}\"\n")
88101

89-
# Types
90-
set(TYPES CORE LB PAR)
91-
# Per architecture definitions
92-
generate_macros("${JSON_CONTENT}" TMP_OUTPUT "${TYPES}" "${ARCH_LIST};default;default_cpu" "JSON_ARCHITECTURES")
93-
list(FIND ARCH_LIST "ALL" do_all_architectures)
94-
if(ARGC GREATER 2)
95-
set(${ARGV2} "${JSON_ARCHITECTURES}" PARENT_SCOPE)
96-
endif()
97-
if(do_all_architectures GREATER -1)
98-
set(ARCH_LIST ${JSON_ARCHITECTURES})
99-
endif()
100-
string(APPEND TMP_HEADER "#if 0\n")
102+
string(APPEND TMP_HEADER_DEVICE "#if 0\n")
101103
foreach(ARCH IN LISTS ARCH_LIST)
102-
string(APPEND TMP_HEADER "\n#elif defined(GPUCA_GPUTYPE_${ARCH})\n")
103-
string(APPEND TMP_HEADER ${TMP_OUTPUT_${ARCH}})
104+
string(APPEND TMP_HEADER_DEVICE "\n#elif defined(GPUCA_GPUTYPE_${ARCH})\n")
105+
string(APPEND TMP_HEADER_DEVICE ${generate_gpu_param_header_OUTPUT_TMP_${ARCH}})
104106
endforeach()
105-
string(APPEND TMP_HEADER "#else\n#error GPU TYPE NOT SET\n#endif\n")
107+
string(APPEND TMP_HEADER_DEVICE "#else\n#error GPU TYPE NOT SET\n#endif\n")
106108

107109
# Default parameters
108110
string(APPEND TMP_HEADER "\n// Default parameters if not defined for the target architecture\n\n")
109-
string(APPEND TMP_HEADER ${TMP_OUTPUT_default})
111+
string(APPEND TMP_HEADER ${generate_gpu_param_header_OUTPUT_TMP_default})
110112
string(APPEND TMP_HEADER "#endif // defined(GPUCA_GPUCODE) && !defined(GPUCA_GPUCODE_GENRTC) && !defined(GPUCA_GPUCODE_NO_LAUNCH_BOUNDS)\n\n")
111113

112114
# CPU fallback
113115
string(APPEND TMP_HEADER "#ifndef GPUCA_GPUCODE_GENRTC // Defaults for non-LB parameters also for CPU fallback\n\n")
114-
string(APPEND TMP_HEADER ${TMP_OUTPUT_default_cpu})
116+
string(APPEND TMP_HEADER ${generate_gpu_param_header_OUTPUT_TMP_default_cpu})
115117
string(APPEND TMP_HEADER "\n#endif // GPUCA_GPUCODE_GENRTC\n")
116118

117119
string(APPEND TMP_HEADER "\n#endif // GPUDEFPARAMETERSDEFAULTS_H\n")
120+
string(APPEND TMP_HEADER_DEVICE "\n#endif // GPUDEFPARAMETERSDEFAULTSDEVICE_H\n")
118121
file(GENERATE OUTPUT "${OUT_HEADER}" CONTENT "${TMP_HEADER}")
119-
message(STATUS "Generated ${OUT_HEADER}")
120-
add_custom_target(GPU_PARAM_HEADER_${GPU_ARCH}_ALL ALL DEPENDS ${OUT_HEADER} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/gpu_param_header_generator.cmake ${GPU_PARAM_JSON})
122+
file(GENERATE OUTPUT "${OUT_HEADER_DEVICE}" CONTENT "${TMP_HEADER_DEVICE}")
123+
message(STATUS "Generated ${OUT_HEADER} and ${OUT_HEADER_DEVICE}")
124+
add_custom_target(GPU_PARAM_HEADER_TARGET ALL DEPENDS ${OUT_HEADER} ${OUT_HEADER_DEVICE} ${GPU_PARAM_JSON})
121125
endfunction()

‎dependencies/FindO2GPU.cmake‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -93,18 +93,17 @@ function(detect_gpu_arch backend) # Detect GPU architecture, optionally filterri
9393
set(TARGET_ARCH "${CUDA_TARGET}" PARENT_SCOPE)
9494
elseif(backend STREQUAL "HIP") # HIP filter
9595
set(TARGET_ARCH "${HIP_TARGET}" PARENT_SCOPE)
96-
elseif(backend STREQUAL "ALL") # Return enabled backends
97-
set(_archs "")
98-
if(CUDA_ENABLED)
99-
list(APPEND _archs "${CUDA_TARGET}")
96+
elseif(backend STREQUAL "ALL" OR backend STREQUAL "AUTO") # Return all / enabled backends
97+
set(TARGET_ARCH)
98+
if(CUDA_ENABLED OR backend STREQUAL "ALL")
99+
list(APPEND TARGET_ARCH "${CUDA_TARGET}")
100100
endif()
101-
if(HIP_ENABLED)
102-
list(APPEND _archs "${HIP_TARGET}")
101+
if(HIP_ENABLED OR backend STREQUAL "ALL")
102+
list(APPEND TARGET_ARCH "${HIP_TARGET}")
103103
endif()
104-
if(OPENCL_ENABLED)
105-
list(APPEND _archs "OPENCL")
104+
if(OPENCL_ENABLED OR backend STREQUAL "ALL")
105+
list(APPEND TARGET_ARCH "OPENCL")
106106
endif()
107-
list(JOIN _archs "," TARGET_ARCH)
108107
set(TARGET_ARCH "${TARGET_ARCH}" PARENT_SCOPE)
109108
else()
110109
message(FATAL_ERROR "Unknown backend provided: ${backend}")

0 commit comments

Comments
 (0)