Embed built-in headers in library binary

tools/clang/include/clang/Lex/HLSLEmbeddedHeaders.h

@@ -0,0 +1,36 @@
+//===- HLSLEmbeddedHeaders.h - Embedded HLSL header data ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Provides access to the contents of the HLSL header files that ship with
+// DXC (located under tools/clang/lib/Headers/hlsl) as compiled-in data.
+//
+// The data is generated at build time from those header files by
+// utils/embed_header.py and utils/generate_hlsl_embedded_headers.py.  The
+// resulting StringMap maps each header's path relative to the hlsl/
+// directory (with no leading separator, e.g. "dx/linalg.h") to the raw
+// bytes of that file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEX_HLSLEMBEDDEDHEADERS_H
+#define LLVM_CLANG_LEX_HLSLEMBEDDEDHEADERS_H
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace hlsl {
+
+/// Returns a map from each HLSL header's relative path under
+/// tools/clang/lib/Headers/hlsl (e.g. "dx/linalg.h") to a StringRef
+/// holding the file's raw contents.
+const llvm::StringMap<llvm::StringRef> &getEmbeddedHeaders();
+
+} // namespace hlsl
+
+#endif // LLVM_CLANG_LEX_HLSLEMBEDDEDHEADERS_H

tools/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp

@@ -407,6 +407,15 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
         const FileEntry *FE =
             PP->LookupFile(Pos, Filename, false, nullptr, nullptr, CurDir,
                            nullptr, nullptr, nullptr);
+        // HLSL Change Begin - if the quoted lookup did not find the file,
+        // retry as an angled lookup so that bundled HLSL headers (resolved
+        // via clang::hlsl::getEmbeddedHeaders()) are also reachable from
+        // expected-* directives by their relative path.
+        if (!FE) {
+          FE = PP->LookupFile(Pos, Filename, true, nullptr, nullptr, CurDir,
+                              nullptr, nullptr, nullptr);
+        }
+        // HLSL Change End
         if (!FE) {
           Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
                        diag::err_verify_missing_file) << Filename << KindStr;

tools/clang/lib/Headers/hlsl/vk/khr/cooperative_matrix.h

@@ -11,7 +11,7 @@
 #error "CooperativeMatrix requires a minimum of SPIR-V 1.6"
 #endif
 
-#include "vk/spirv.h"
+#include <vk/spirv.h>
 
 namespace vk {
 namespace khr {
@@ -271,5 +271,5 @@ cooperativeMatrixSaturatingMultiplyAdd(
 } // namespace khr
 } // namespace vk
 
-#include "cooperative_matrix.impl"
+#include <vk/khr/cooperative_matrix.impl>
 #endif // _HLSL_VK_KHR_COOPERATIVE_MATRIX_H_

tools/clang/lib/Headers/hlsl/vk/khr/cooperative_matrix.impl

@@ -4,7 +4,7 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "vk/opcode_selector.h"
+#include <vk/opcode_selector.h>
 
 template <typename ResultType, typename ComponentType>
 [[vk::ext_instruction(/* OpMatrixTimesScalar */ 143)]] ResultType

tools/clang/lib/Lex/CMakeLists.txt

@@ -2,9 +2,102 @@
 
 set(LLVM_LINK_COMPONENTS support)
 
+# ----------------------------------------------------------------------------
+# Embedded HLSL headers.
+#
+# Each file under tools/clang/lib/Headers/hlsl/ that we want to ship as
+# compiled-in data is converted at build time to a small C++ snippet (an
+# llvm::StringRef "Data" definition) by utils/embed_header.py.  A second
+# script (utils/generate_hlsl_embedded_headers.py) then aggregates those
+# snippets into two ``.inc`` files:
+#
+#   * HLSLEmbeddedHeadersDecls.inc -- one ``namespace <ns> { #include ... }``
+#     per header so each ``Data`` lives in its own namespace.
+#   * HLSLEmbeddedHeadersEntries.inc -- an X-macro list of (rel_path, ns)
+#     pairs.
+#
+# Those ``.inc`` files are textually included by the hand-written
+# HLSLEmbeddedHeaders.cpp in this directory, which exposes the whole
+# collection through clang::hlsl::getEmbeddedHeaders() as a
+# StringMap<StringRef>.  This compiled-in data lets the preprocessor
+# satisfy #includes of the bundled headers without consulting the
+# filesystem.
+#
+# The list of embedded inputs is intentionally enumerated explicitly
+# below: file(GLOB ...) is evaluated at configuration time, so a glob
+# would silently miss files added between configure and build.
+# Vulkan-specific headers under hlsl/vk/ are only embedded when SPIR-V
+# codegen is enabled, to keep the DXC-only build free of unused data.
+# ----------------------------------------------------------------------------
+
+set(hlsl_headers_src_dir
+    "${CMAKE_CURRENT_SOURCE_DIR}/../Headers/hlsl")
+set(hlsl_embedded_dir
+    "${CMAKE_CURRENT_BINARY_DIR}/HLSLEmbeddedHeaders")
+set(embed_header_script
+    "${LLVM_MAIN_SRC_DIR}/utils/embed_header.py")
+set(generate_embedded_headers_script
+    "${LLVM_MAIN_SRC_DIR}/utils/generate_hlsl_embedded_headers.py")
+
+set(hlsl_embedded_inputs
+    enable_if.h
+    vector_utils.h
+    dx/linalg.h
+    )
+
+if (ENABLE_SPIRV_CODEGEN)
+  list(APPEND hlsl_embedded_inputs
+       vk/khr/cooperative_matrix.h
+       vk/khr/cooperative_matrix.impl
+       vk/opcode_selector.h
+       vk/spirv.h
+       )
+endif()
+
+set(hlsl_embedded_inc_files)
+set(hlsl_embedded_entries)
+foreach(rel_path ${hlsl_embedded_inputs})
+  set(src "${hlsl_headers_src_dir}/${rel_path}")
+  set(inc "${hlsl_embedded_dir}/${rel_path}.inc")
+  add_custom_command(
+    OUTPUT "${inc}"
+    DEPENDS "${src}" "${embed_header_script}"
+    COMMAND "${Python3_EXECUTABLE}" "${embed_header_script}" "${src}" "${inc}"
+    COMMENT "Embedding HLSL header ${rel_path}")
+  list(APPEND hlsl_embedded_inc_files "${inc}")
+  list(APPEND hlsl_embedded_entries "--entry" "${rel_path}=${inc}")
+endforeach()
+
+set(hlsl_embedded_decls "${hlsl_embedded_dir}/HLSLEmbeddedHeadersDecls.inc")
+set(hlsl_embedded_entries_inc
+    "${hlsl_embedded_dir}/HLSLEmbeddedHeadersEntries.inc")
+add_custom_command(
+  OUTPUT "${hlsl_embedded_decls}" "${hlsl_embedded_entries_inc}"
+  DEPENDS ${hlsl_embedded_inc_files} "${generate_embedded_headers_script}"
+  COMMAND "${Python3_EXECUTABLE}" "${generate_embedded_headers_script}"
+          --decls-output "${hlsl_embedded_decls}"
+          --entries-output "${hlsl_embedded_entries_inc}"
+          ${hlsl_embedded_entries}
+  COMMENT "Generating HLSL embedded headers includes")
+
+set_source_files_properties("${hlsl_embedded_decls}"
+                            "${hlsl_embedded_entries_inc}"
+                            PROPERTIES GENERATED TRUE)
+
+# Make the generated .inc files visible to HLSLEmbeddedHeaders.cpp.
+include_directories("${hlsl_embedded_dir}")
+
+# Carry the generated .inc files as an explicit dependency of the
+# library target so they're produced before HLSLEmbeddedHeaders.cpp is
+# compiled.
+set(hlsl_embedded_generated_files
+    "${hlsl_embedded_decls}"
+    "${hlsl_embedded_entries_inc}")
+
 add_clang_library(clangLex
   HeaderMap.cpp
   HeaderSearch.cpp
+  HLSLEmbeddedHeaders.cpp
   HLSLMacroExpander.cpp
   Lexer.cpp
   LiteralSupport.cpp
@@ -26,6 +119,7 @@ add_clang_library(clangLex
   ScratchBuffer.cpp
   TokenConcatenation.cpp
   TokenLexer.cpp
+  ${hlsl_embedded_generated_files}
 
   LINK_LIBS
   clangBasic

tools/clang/lib/Lex/HLSLEmbeddedHeaders.cpp

@@ -0,0 +1,44 @@
+//===- HLSLEmbeddedHeaders.cpp - Embedded HLSL header data ----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements clang::hlsl::getEmbeddedHeaders() by composing two small
+// build-time generated ``.inc`` files (one with the per-header
+// ``Data`` StringRef declarations, one with an X-macro list of the
+// (rel_path, namespace) pairs) with hand-written boilerplate that
+// remains in source control.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/HLSLEmbeddedHeaders.h"
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+
+#include <utility>
+
+// Pulls in one ``namespace ns { #include "<path>.inc" }`` block per
+// embedded header.  The included ``.inc`` snippets define a
+// ``llvm::StringRef Data`` at namespace scope.
+#include "HLSLEmbeddedHeadersDecls.inc"
+
+namespace hlsl {
+
+const llvm::StringMap<llvm::StringRef> &getEmbeddedHeaders() {
+  static const llvm::StringMap<llvm::StringRef> Map = []() {
+    llvm::StringMap<llvm::StringRef> M;
+#define HLSL_EMBEDDED_HEADER(REL, NS)                                          \
+  M.insert(std::make_pair(llvm::StringRef(REL), NS::Data));
+#include "HLSLEmbeddedHeadersEntries.inc"
+#undef HLSL_EMBEDDED_HEADER
+    return M;
+  }();
+  return Map;
+}
+
+} // namespace hlsl

tools/clang/lib/Lex/PPDirectives.cpp

@@ -12,22 +12,25 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Lex/HLSLEmbeddedHeaders.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/ModuleLoader.h"
 #include "clang/Lex/Pragma.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h" // HLSL Change - ignore line directives.
 #include "llvm/ADT/APInt.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SaveAndRestore.h"
-#include "clang/Lex/PreprocessorOptions.h" // HLSL Change - ignore line directives.
+#include <algorithm> // HLSL Change - std::replace for path separator normalisation.
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -724,6 +727,49 @@ const FileEntry *Preprocessor::LookupFile(
     }
   }
 
+  // HLSL Change Begin - Resolve #include's from compiled-in HLSL headers.
+
+  // Doing this after the normal file lookup allows this to only trigger as a
+  // fallback so the user may override it.
+  if (isAngled && !FromDir && !FromFile) {
+    const llvm::StringMap<llvm::StringRef> &EmbeddedHeaders =
+        hlsl::getEmbeddedHeaders();
+    // Normalize the filename to POSIX-style separators so that the lookup is
+    // consistent regardless of how the user spelled the include.
+    SmallString<128> NormalizedFilename(Filename);
+    std::replace(NormalizedFilename.begin(), NormalizedFilename.end(), '\\',
+                 '/');
+    auto It = EmbeddedHeaders.find(NormalizedFilename);
+    if (It != EmbeddedHeaders.end()) {
+      llvm::StringRef Data = It->second;
+      SmallString<128> VirtualName("<built-in:hlsl>/");
+      VirtualName.append(NormalizedFilename.begin(), NormalizedFilename.end());
+      const FileEntry *EmbeddedFE =
+          FileMgr.getVirtualFile(VirtualName, Data.size(), /*ModTime=*/0);
+      if (EmbeddedFE) {
+        if (!SourceMgr.isFileOverridden(EmbeddedFE)) {
+          SourceMgr.overrideFileContents(
+              EmbeddedFE,
+              llvm::MemoryBuffer::getMemBuffer(Data, VirtualName,
+                                               /*RequiresNullTerminator=*/
+                                               false));
+        }
+        if (SearchPath)
+          SearchPath->clear();
+        if (RelativePath) {
+          RelativePath->clear();
+          RelativePath->append(NormalizedFilename.begin(),
+                               NormalizedFilename.end());
+        }
+        CurDir = nullptr;
+        if (SuggestedModule)
+          *SuggestedModule = ModuleMap::KnownHeader();
+        return EmbeddedFE;
+      }
+    }
+  }
+  // HLSL Change End
+
   // Otherwise, we really couldn't find the file.
   return nullptr;
 }

tools/clang/test/CMakeLists.txt

@@ -9,8 +9,6 @@ endif ()
 
 string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} CLANG_TOOLS_DIR ${LLVM_RUNTIME_OUTPUT_INTDIR})
 
-set(HLSL_HEADERS_DIR ${LLVM_SOURCE_DIR}/tools/clang/lib/Headers/hlsl) # HLSL Change
-
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg

tools/clang/test/CodeGenDXIL/hlsl/linalg/api/matrix-class.hlsl

@@ -1,5 +1,5 @@
 // REQUIRES: dxil-1-10
-// RUN: %dxc -I %hlsl_headers -T cs_6_10 %s | FileCheck %s
+// RUN: %dxc -T cs_6_10 %s | FileCheck %s
 
 #include <dx/linalg.h>
 using namespace dx::linalg;

tools/clang/test/CodeGenDXIL/hlsl/linalg/api/matrix-multiply.hlsl

@@ -1,5 +1,5 @@
 // REQUIRES: dxil-1-10
-// RUN: %dxc -I %hlsl_headers -T cs_6_10 %s | FileCheck %s
+// RUN: %dxc -T cs_6_10 %s | FileCheck %s
 
 #include <dx/linalg.h>
 using namespace dx::linalg;

tools/clang/test/CodeGenDXIL/hlsl/linalg/api/vectors.hlsl

@@ -1,5 +1,5 @@
 // REQUIRES: dxil-1-10
-// RUN: %dxc -I %hlsl_headers -enable-16bit-types -T cs_6_10 %s | FileCheck %s
+// RUN: %dxc -enable-16bit-types -T cs_6_10 %s | FileCheck %s
 
 #include <dx/linalg.h>
 using namespace dx::linalg;

tools/clang/test/CodeGenDXIL/hlsl/linalg/attr-matrix-type.hlsl

@@ -1,6 +1,6 @@
 // REQUIRES: dxil-1-10
-// RUN: %dxc -I %hlsl_headers -T lib_6_10 -enable-16bit-types -fcgl %s | FileCheck %s
-// RUN: %dxc -I %hlsl_headers -T lib_6_10 -enable-16bit-types %s | FileCheck %s --check-prefix=CHECKVAL
+// RUN: %dxc -T lib_6_10 -enable-16bit-types -fcgl %s | FileCheck %s
+// RUN: %dxc -T lib_6_10 -enable-16bit-types %s | FileCheck %s --check-prefix=CHECKVAL
 
 #include <dx/linalg.h>
 using namespace dx::linalg;

tools/clang/test/CodeGenDXIL/hlsl/linalg/linalg-matrix-type.hlsl

@@ -1,6 +1,6 @@
 // REQUIRES: dxil-1-10
-// RUN: %dxc -I %hlsl_headers -T lib_6_10 -fcgl %s | FileCheck %s
-// RUN: %dxc -I %hlsl_headers -T lib_6_10 %s | FileCheck %s --check-prefix=CHECKVAL
+// RUN: %dxc -T lib_6_10 -fcgl %s | FileCheck %s
+// RUN: %dxc -T lib_6_10 %s | FileCheck %s --check-prefix=CHECKVAL
 
 #include <dx/linalg.h>
 using namespace dx::linalg;