Full kbuild support (atop of https://github.com/minipli-oss/open-gpu-kernel-modules/tree/fixes)

Makefile

@@ -6,6 +6,19 @@
 # To install the build kernel modules: run (as root) `make modules_install`
 ###########################################################################
 
+# Experimental use of the Linux kernel's kbuild system
+USE_KBUILD ?= 0
+export USE_KBUILD
+
+###########################################################################
+# rules
+###########################################################################
+
+.PHONY: all
+all: modules
+
+ifneq ($(USE_KBUILD),1)
+
 ###########################################################################
 # variables
 ###########################################################################
@@ -16,15 +29,8 @@ nv_kernel_o_binary         = kernel-open/nvidia/nv-kernel.o_binary
 nv_modeset_kernel_o        = src/nvidia-modeset/$(OUTPUTDIR)/nv-modeset-kernel.o
 nv_modeset_kernel_o_binary = kernel-open/nvidia-modeset/nv-modeset-kernel.o_binary
 
-###########################################################################
-# rules
-###########################################################################
-
 include utils.mk
 
-.PHONY: all
-all: modules
-
 ###########################################################################
 # nv-kernel.o is the OS agnostic portion of nvidia.ko
 ###########################################################################
@@ -48,22 +54,31 @@ $(nv_modeset_kernel_o):
 $(nv_modeset_kernel_o_binary): $(nv_modeset_kernel_o)
 	cd $(dir $@) && ln -sf ../../$^ $(notdir $@)
 
+# Make the OS agnostic binaries a prerequisite of the modules target
+modules: $(nv_kernel_o_binary) $(nv_modeset_kernel_o_binary)
+endif
 
 ###########################################################################
 # After the OS agnostic portions are built, descend into kernel-open/ and build
 # the kernel modules with kbuild.
 ###########################################################################
 
 .PHONY: modules
-modules: $(nv_kernel_o_binary) $(nv_modeset_kernel_o_binary)
+modules:
+ifeq ($(USE_KBUILD),1)
+# create nv-kernel.o and nv-modeset-kernel.o by building stub modules -- a
+# required hack to overcome kbuild limitations regarding the maximum number
+# of object files to link into a module.
+	$(MAKE) -C kernel-open modules NV_PREPARE_ONLY=1
+endif
 	$(MAKE) -C kernel-open modules
 
 ###########################################################################
 # Install the built kernel modules using kbuild.
 ###########################################################################
 
 .PHONY: modules_install
-modules_install:
+modules_install: modules
 	$(MAKE) -C kernel-open modules_install
 
 ###########################################################################
@@ -75,11 +90,15 @@ clean: nvidia.clean nvidia-modeset.clean kernel-open.clean
 
 .PHONY: nvidia.clean
 nvidia.clean:
+ifneq ($(USE_KBUILD),1)
 	$(MAKE) -C src/nvidia clean
+endif
 
 .PHONY: nvidia-modeset.clean
 nvidia-modeset.clean:
+ifneq ($(USE_KBUILD),1)
 	$(MAKE) -C src/nvidia-modeset clean
+endif
 
 .PHONY: kernel-open.clean
 kernel-open.clean:

README.md

@@ -57,6 +57,10 @@ DEBUG - Set this to "1" to build the kernel modules as debug.  By default, the
     build compiles without debugging information.  This also enables
     various debug log messages in the kernel modules.
 
+USE_KBUILD - Set this to "1" to build all sources using the linux kernel build
+    system.  This is needed to support features like RANDSTRUCT, kCFI or
+    grsecurity kernels.
+
 These variables can be set on the make command line.  E.g.,
 
     make modules -j$(nproc) NV_VERBOSE=1

kernel-open/Kbuild

@@ -39,9 +39,95 @@ NV_BUILD_TYPE ?= release
 #
 ASSIGN_PER_OBJ_CFLAGS = \
  $(foreach _cflags_variable, \
- $(notdir $(1)) $(1), \
+ $(sort $(notdir $(1)) $(1)), \
  $(eval $(addprefix CFLAGS_,$(_cflags_variable)) += $(2)))
 
+#
+# As ASSIGN_PER_OBJ_CFLAGS, just for per-object CFLAGS_REMOVE.
+#
+ASSIGN_PER_OBJ_CFLAGS_REMOVE = \
+ $(foreach _cflags_variable, \
+ $(sort $(notdir $(1)) $(1)), \
+ $(eval $(addprefix CFLAGS_REMOVE_,$(_cflags_variable)) += $(2)))
+
+#
+# As ASSIGN_PER_OBJ_CFLAGS, just for per-object LDFLAGS.
+#
+ASSIGN_PER_OBJ_LDFLAGS = \
+ $(foreach _ldflags_variable, \
+ $(sort $(notdir $(1)) $(1)), \
+ $(eval $(addprefix LDFLAGS_,$(_ldflags_variable)) += $(2)))
+
+#
+# As ASSIGN_PER_OBJ_CFLAGS, just for per-object OBJCOPYFLAGS.
+#
+ASSIGN_PER_OBJ_OBJCOPYFLAGS = \
+ $(foreach _objcopyflags_variable, \
+ $(sort $(notdir $(1)) $(1)i), \
+ $(eval $(addprefix OBJCOPYFLAGS_,$(_objcopyflags_variable)) += $(2)))
+
+
+#
+# Define rule for generating a source file containing identification information
+# for the build.
+#
+# $(1) string name
+# $(2) module name
+# $(3) prerequisite object files
+##############################################################################
+
+NV_BUILD_USER         ?= $(shell whoami)
+NV_BUILD_HOST         ?= $(shell hostname)
+
+ifndef TARGET_OS
+  TARGET_OS           := $(shell uname)
+endif
+
+ifndef TARGET_ARCH
+  ifneq ($(TARGET_OS),SunOS)
+    TARGET_ARCH       := $(shell uname -m)
+  else
+    TARGET_ARCH       := $(shell isainfo -n)
+  endif
+  TARGET_ARCH         := $(subst i386,x86,$(TARGET_ARCH))
+  TARGET_ARCH         := $(subst i486,x86,$(TARGET_ARCH))
+  TARGET_ARCH         := $(subst i586,x86,$(TARGET_ARCH))
+  TARGET_ARCH         := $(subst i686,x86,$(TARGET_ARCH))
+  TARGET_ARCH         := $(subst amd64,x86_64,$(TARGET_ARCH))
+endif
+
+DATE                  ?= date
+
+NVIDSTRING ?= g_nvid_string.c
+
+ifeq ($(NV_BUILD_TYPE),debug)
+  NVIDSTRING_BUILD_TYPE_STRING = Debug Build
+else ifeq ($(NV_BUILD_TYPE),develop)
+  NVIDSTRING_BUILD_TYPE_STRING = Develop Build
+else ifeq ($(NV_BUILD_TYPE),release)
+  NVIDSTRING_BUILD_TYPE_STRING = Release Build
+else
+  NVIDSTRING_BUILD_TYPE_STRING = Custom '$(NV_BUILD_TYPE)' Build
+endif
+
+# NV_VERSION_STRING is set as preprocessor define
+NVIDIA_NVID_VERSION := \"NV_VERSION_STRING\"
+
+define GENERATE_NVIDSTRING
+  $(1)_BUILD_NVID := NVIDIA $$(strip $(2)) for $$(TARGET_ARCH)  $$(NVIDIA_NVID_VERSION)
+  $(1)_BUILD_NVID := $$($$(strip $(1))_BUILD_NVID)  $$(NVIDSTRING_BUILD_TYPE_STRING)
+  ifneq ($$(NVIDIA_NVID_EXTRA),)
+    $(1)_BUILD_NVID := $$($$(strip $(1))_BUILD_NVID)  $$(NVIDIA_NVID_EXTRA)
+  endif
+  $(1)_BUILD_NVID := $$($$(strip $(1))_BUILD_NVID)  ($$(NV_BUILD_USER)@$$(NV_BUILD_HOST))
+  # g_nvid_string.c depends on all objects except g_nvid_string.o, and version.mk
+  # $(NVIDSTRING) evaluated early, to allow reusing it
+  $$(obj)/$(NVIDSTRING): $$(adprefix $$(obj)/,$$(filter-out $(NVIDSTRING:.c=.o), $(3))) $$(VERSION_MK)
+	@mkdir -p $$(dir $$@)
+	@echo "const char $(1)[] = \"nvidia id: $$($$(strip $(1))_BUILD_NVID)  `$$(DATE)`\";" > $$@
+	@echo "const char *const p$$(strip $(1)) = $(1) + 11;" >> $$@;
+endef
+
 
 #
 # Include the specifics of the individual NVIDIA kernel modules.
@@ -70,13 +156,42 @@ endif
 quiet_cmd_symlink = SYMLINK $@
  cmd_symlink = ln -sf $(abspath $<) $@
 
+#
+# Command to neuter .modinfo section for stub modules which are our hacky
+# attempt to make kbuild support nested compound objects.
+#
+# Intended usage:
+#
+#   # stub module
+#   obj-m += nv-kernel.stub.o
+#   nv-kernel.stub-y := ...
+#
+#   # real module wanting to make use of nv-kernel.o
+#   obj-m += ndidia.o
+#   nvidia-y += nv-kernel.clean.o ...
+#
 
-$(foreach _module, $(NV_KERNEL_MODULES), \
- $(eval include $(src)/$(_module)/$(_module).Kbuild))
+MOD_STUB := common/mod_stub.o
+
+# not really needed, but to have conftest be the very first
+NV_OBJECTS_DEPEND_ON_CONFTEST += $(MOD_STUB)
 
+$(obj)/%.clean.o: OBJCOPYFLAGS += --rename-section .modinfo=.discard.modinfo
+$(obj)/%.clean.o: $(obj)/%.stub.o FORCE
+	$(call if_changed,objcopy)
 
+# early flags, which may be filtered for USE_KBUILD=1 builds
 ccflags-y += -I$(src)/common/inc
 ccflags-y += -I$(src)
+
+ifeq ($(USE_KBUILD),1)
+ ccflags-y += -DNV_USE_KBUILD
+endif
+
+$(foreach _module, $(NV_KERNEL_MODULES), \
+ $(eval include $(src)/$(_module)/$(_module).Kbuild))
+
+
 ccflags-y += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
 ccflags-y += -D__KERNEL__ -DMODULE -DNVRM
 ccflags-y += -DNV_VERSION_STRING=\"595.71.05\"

kernel-open/Makefile

@@ -64,12 +64,14 @@ else
             from CONFIG_CC_VERSION_TEXT in the kernel configuration.)
       else
           CC=$(cc_version_text)
+          CXX=$(subst cc,++,$(cc_version_text))
       endif
     endif
   endif
 
   CC ?= cc
   LD ?= ld
+  CXX ?= c++
   OBJDUMP ?= objdump
 
   ifndef ARCH
@@ -107,12 +109,20 @@ else
   KBUILD_PARAMS += NV_KERNEL_MODULES="$(NV_KERNEL_MODULES)"
   KBUILD_PARAMS += INSTALL_MOD_DIR="$(INSTALL_MOD_DIR)"
   KBUILD_PARAMS += NV_SPECTRE_V2=$(SPECTRE_V2_RETPOLINE)
+  ifeq ($(USE_KBUILD),1)
+    KBUILD_PARAMS += NV_PREPARE_ONLY=$(NV_PREPARE_ONLY)
+    ifeq ($(NV_PREPARE_ONLY),1)
+    KBUILD_PARAMS += KBUILD_MODPOST_NOFINAL=1
+    KBUILD_PARAMS += MODPOST=/bin/true
+    KBUILD_PARAMS += objtool-enabled=
+    endif
+  endif
 
   export ORIG_PAHOLE := $(PAHOLE)
 
   .PHONY: modules module clean clean_conftest modules_install
   modules clean modules_install:
-	@$(MAKE) "LD=$(LD)" "CC=$(CC)" "OBJDUMP=$(OBJDUMP)" \
+	@$(MAKE) "LD=$(LD)" "CC=$(CC)" "CXX=$(CXX)" "OBJDUMP=$(OBJDUMP)" \
 	  PAHOLE=$(CURDIR)/pahole.sh $(KBUILD_PARAMS) $@
 	@if [ "$@" = "modules" ]; then \
 	  for module in $(NV_KERNEL_MODULES); do \

kernel-open/common/inc/nvkms-kapi.h

@@ -608,13 +608,16 @@ struct NvKmsKapiGpuInfo {
  * Linux kernel options CONFIG_RANDSTRUCT_* randomize structs that are composed
  * entirely of function pointers, but can only control struct layout for sources
  * built by kbuild. NvKmsKapiCallbacks is shared between kbuild-built
- * nvidia-drm.ko, and the "OS-agnostic" portions of nvidia-modeset.ko (not built
- * by kbuild). Add a _padding member to disable struct randomization.
+ * nvidia-drm.ko, and the "OS-agnostic" portions of nvidia-modeset.ko (which
+ * may or may be not built by kbuild). Add a _padding member to disable struct
+ * randomization if it's not.
  *
  * Refer to https://github.com/NVIDIA/open-gpu-kernel-modules/issues/1033
  */
 struct NvKmsKapiCallbacks {
+#ifndef NV_USE_KBUILD
     int  _padding;
+#endif
     void (*suspendResume)(NvBool suspend);
     void (*remove)(NvU32 gpuId);
     void (*probe)(const struct NvKmsKapiGpuInfo *gpu_info);

kernel-open/common/mod_stub.c

@@ -0,0 +1,4 @@
+/* Stub module fragment for emulated nested compound object kbuild support */
+#include <linux/module.h>
+
+MODULE_LICENSE("Dual MIT/GPL");

kernel-open/conftest.sh

@@ -5033,8 +5033,9 @@ compile_test() {
             CODE="
             #include <drm/drm_atomic_helper.h>
 
-            static int conftest_drm_connector_mode_valid(struct drm_connector *connector,
-                                                         const struct drm_display_mode *mode) {
+            static enum drm_mode_status
+            conftest_drm_connector_mode_valid(struct drm_connector *connector,
+                                              const struct drm_display_mode *mode) {
                 return 0;
             }
 

kernel-open/nvidia-drm/nvidia-drm-connector.c

@@ -359,11 +359,12 @@ static int nv_drm_connector_get_modes(struct drm_connector *connector)
     return count;
 }
 
-static int nv_drm_connector_mode_valid(struct drm_connector    *connector,
+static enum drm_mode_status
+nv_drm_connector_mode_valid(struct drm_connector    *connector,
 #if defined(NV_DRM_CONNECTOR_HELPER_FUNCS_MODE_VALID_HAS_CONST_MODE_ARG)
-                                       const struct drm_display_mode *mode)
+                            const struct drm_display_mode *mode)
 #else
-                                       struct drm_display_mode *mode)
+                            struct drm_display_mode *mode)
 #endif
 {
     struct drm_device *dev = connector->dev;

kernel-open/nvidia-drm/nvidia-drm.Kbuild

@@ -14,8 +14,10 @@ NVIDIA_DRM_SOURCES += nvidia-drm/nvidia-drm-linux.c
 
 NVIDIA_DRM_OBJECTS = $(patsubst %.c,%.o,$(NVIDIA_DRM_SOURCES))
 
+ifneq ($(NV_PREPARE_ONLY),1)
 obj-m += nvidia-drm.o
 nvidia-drm-y := $(NVIDIA_DRM_OBJECTS)
+endif
 
 NVIDIA_DRM_KO = nvidia-drm/nvidia-drm.ko
 

kernel-open/nvidia-modeset/nvidia-modeset.Kbuild

@@ -11,8 +11,21 @@ NVIDIA_MODESET_SOURCES += nvidia-modeset/nv-kthread-q.c
 
 NVIDIA_MODESET_OBJECTS = $(patsubst %.c,%.o,$(NVIDIA_MODESET_SOURCES))
 
+ifneq ($(NV_PREPARE_ONLY),1)
 obj-m += nvidia-modeset.o
 nvidia-modeset-y := $(NVIDIA_MODESET_OBJECTS)
+endif
+
+ifeq ($(USE_KBUILD),1)
+# build nv-modeset-kernel.o using Linux's kbuild system
+nvidia_modeset_src := ../src/nvidia-modeset
+include $(src)/$(nvidia_modeset_src)/nvidia-modeset.Kbuild
+
+nvidia-modeset-y += $(NV_MODESET_KERNEL_O)
+
+NV_OBJECTS_DEPEND_ON_CONFTEST += $(NV_MODESET_KERNEL_O_OBJS)
+
+else # !USE_KBUILD
 
 NVIDIA_MODESET_KO = nvidia-modeset/nvidia-modeset.ko
 
@@ -46,6 +59,7 @@ $(obj)/$(NVIDIA_MODESET_BINARY_OBJECT_O): $(NVIDIA_MODESET_BINARY_OBJECT) FORCE
 	$(call if_changed,symlink)
 
 nvidia-modeset-y += $(NVIDIA_MODESET_BINARY_OBJECT_O)
+endif
 
 
 #
@@ -82,8 +96,10 @@ NVIDIA_MODESET_INTERFACE := nvidia-modeset/nv-modeset-interface.o
 # before v5.6 looks at "always"; kernel versions between v5.12 and v5.6
 # look at both.
 
+ifneq ($(NV_PREPARE_ONLY),1)
 always += $(NVIDIA_MODESET_INTERFACE)
 always-y += $(NVIDIA_MODESET_INTERFACE)
+endif
 
 $(obj)/$(NVIDIA_MODESET_INTERFACE): $(addprefix $(obj)/,$(NVIDIA_MODESET_OBJECTS))
 	$(LD) -r -o $@ $^

kernel-open/nvidia-peermem/nvidia-peermem.Kbuild

@@ -11,8 +11,10 @@ NVIDIA_PEERMEM_SOURCES += nvidia-peermem/nvidia-peermem.c
 
 NVIDIA_PEERMEM_OBJECTS = $(patsubst %.c,%.o,$(NVIDIA_PEERMEM_SOURCES))
 
+ifneq ($(NV_PREPARE_ONLY),1)
 obj-m += nvidia-peermem.o
 nvidia-peermem-y := $(NVIDIA_PEERMEM_OBJECTS)
+endif
 
 NVIDIA_PEERMEM_KO = nvidia-peermem/nvidia-peermem.ko