DeployQL · mtbarta · Sep 18, 2024 · Sep 18, 2024 · Sep 23, 2024 · Sep 30, 2024
diff --git a/.github/workflows/build_and_test_cmake.yaml b/.github/workflows/build_and_test_cmake.yaml
@@ -27,52 +27,30 @@ jobs:
         run: |
           git submodule update --init --recursive
       - uses: lukka/get-cmake@latest
-      - name: Setup vcpkg
-        uses: lukka/run-vcpkg@v11
-        id: runvcpkg
-        with:
-          # This one is not needed, as it is the default value anyway.
-          vcpkgDirectory: '${{ github.workspace }}/tools/vcpkg'
-          vcpkgJsonGlob: '**/cmakepresets/vcpkg.json'
-
-      - name: Prints output of run-vcpkg's action.
-        run: echo "root='${{ steps.runvcpkg.outputs.RUNVCPKG_VCPKG_ROOT_OUT }}', triplet='${{ steps.runvcpkg.outputs.RUNVCPKG_VCPKG_DEFAULT_TRIPLET_OUT }}' "
-      - name: Run CMake+vcpkg+Ninja
-        uses: lukka/run-cmake@v10
-        id: runcmake
-        env:
-          CC: clang-18
-          CXX: clang++-18
-          CMAKE_C_COMPILER: clang-18
-          CMAKE_CXX_COMPILER: clang++-18
-          MKLROOT: ${{ github.workspace }}/builds/debug/vcpkg_installed/x64-linux/lib/intel64
-        with:
-          cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
-          configurePresetAdditionalArgs: "['-DOpenMP_CXX_FLAGS=-fopenmp=libiomp5', '-DOpenMP_CXX_LIB_NAMES=libiomp5', '-DOpenMP_libiomp5_LIBRARY=${{ github.workspace }}/builds/debug/vcpkg_installed/x64-linux/lib/intel64/libiomp5.so']"
-          configurePreset: 'debug'
-          buildPreset: 'debug'
-          testPreset: 'debug'
-          testPresetCmdString:  "['lintdb-tests', '--test-dir', 'builds/debug', '--output-on-failure']"
-
       - uses: actions/setup-python@v4
         with:
           python-version: 3.10.6
+      - name: Install Conan
+        id: conan
+        uses: turtlebrowser/get-conan@main
 
       - name: Install Dependencies
-        env:
-          CC: clang-18
-          CXX: clang++-18
-          CMAKE_C_COMPILER: clang-18
-          CMAKE_CXX_COMPILER: clang++-18
         run: |
-          echo "CXX=${CXX}"
-          pip install pytest numpy
-
-          sudo apt-get remove clang-14 clang-15
-          sudo rm /usr/bin/clang++
-          sudo ln -s /usr/bin/clang++-18 /usr/bin/clang++
+          pip install pytest numpy conan
+          sudo apt install g++ gcc libopenblas libopenblas-openmp-dev liblapack-dev
           pip install .
+
+      - name: Build
+        run: |
+          mkdir build && cd build \
+          && conan profile new default --detect \
+          && conan install .. --build=missing -s build_type=Release -s build_tests=True \
+          && conan build ..
       - name: Run Tests
+        run: |
+          cd ./build/Release/tests \
+          && ctest --output-on-failure lintdb-tests
+      - name: Run Python Tests
         run: |
           pytest lintdb/python/tests  
 
diff --git a/.gitmodules b/.gitmodules
@@ -11,3 +11,6 @@
 [submodule "third_party/dkm"]
 	path = third_party/dkm
 	url = https://github.com/genbattle/dkm.git
+[submodule "third_party/faiss"]
+	path = third_party/faiss
+	url = https://github.com/facebookresearch/faiss.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,14 +1,10 @@
 cmake_minimum_required(VERSION 3.25)
 
 include(CMakeFindDependencyMacro)
-
+include(GNUInstallDirs)
+include(ExternalProject)
 # allow faiss to build on m1 mac even though it's listed as unsupported.
-set(VCPKG_INSTALL_OPTIONS "--allow-unsupported")
 
-# Setup vcpkg script with CMake (note: should be placed before project() call)
-set(CMAKE_TOOLCHAIN_FILE
-    ${CMAKE_CURRENT_SOURCE_DIR}/tools/vcpkg/scripts/buildsystems/vcpkg.cmake
-    CACHE STRING "Vcpkg toolchain file")
 
 file(READ "version.txt" version)
 
@@ -19,64 +15,155 @@ project(
   LANGUAGES CXX)
 set(LINTDB_VERSION ${version})
 
-include(GNUInstallDirs)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++17 -fPIC -Wall -pthread")
 
-set(CMAKE_CXX_STANDARD 17)
+if (__x86_64__)
+  set(CMAKE_CXX_FLAGS "-msse4.2 ${CMAKE_CXX_FLAGS}")
+endif()
+
+set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
 
-set(CMAKE_CXX_FLAGS
-    "${CMAKE_CXX_FLAGS} -std=c++17 -fPIC -O3 -D_LIBCPP_DISABLE_AVAILABILITY"
-)
 
 if(MSVC OR LINUX)
-  set(BLA_VENDOR "Intel10_64lp")
+#  set(BLA_VENDOR "Intel10_64lp")
+#  set(BLA_VENDOR_THREADING "intel")
+  set(BLA_VENDOR "OpenBLAS")
 else()
     set(BLA_VENDOR "OpenBLAS")
 endif()
 
+set(CMAKE_CXX_STANDARD 17)
 # the below is caused by github actions failing to build flatbuffers. therefore,
 # we set this value so that we use a higher sdk version to build it.
 set(CMAKE_OSX_DEPLOYMENT_TARGET 10.13)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-
-# https://conda-forge.org/docs/maintainer/knowledge_base/#newer-c-features-with-old-sdk
-# if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(CMAKE_CXX_FLAGS
-# "${CMAKE_CXX_FLAGS} ") endif()
 
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 
-if(SKBUILD)
-  message("Building with scikit-build")
-  cmake_path(GET CMAKE_CURRENT_BINARY_DIR PARENT_PATH BUILD_PARENT_DIR)
-  set(ENV{MKLROOT}
-      "${BUILD_PARENT_DIR}/vcpkg_installed/x64-linux/lib/intel64")
-  set(OpenMP_libiomp5_LIBRARY
-      "${BUILD_PARENT_DIR}/vcpkg_installed/x64-linux/lib/intel64/libiomp5.so"
-  )
-  set(CMAKE_BUILD_TYPE Release)
+# OpenMP is getting pulled in through a version of blas. This needs to get figured out.
+find_package(OpenMP REQUIRED)
+
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/libs/libfaiss.cmake)
+include_directories(third_party/faiss)
+
+find_package(LAPACK REQUIRED)
+find_package(BLAS REQUIRED)
+find_package(RocksDB REQUIRED)
+find_package(glog CONFIG REQUIRED)
+find_package(Microsoft.GSL CONFIG REQUIRED)
+find_package(jsoncpp CONFIG REQUIRED)
+find_package(Bitsery CONFIG REQUIRED)
+find_package(Arrow REQUIRED)
+find_package(Drogon REQUIRED)
+
+if(OPENMP_FOUND)
+  message(STATUS "OpenMP flags: ${OpenMP_CXX_FLAGS} -- ${OpenMP_EXE_LINKER_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  set(CMAKE_EXE_LINKER_FLAGS
+          "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
 endif()
 
-set(CMAKE_INSTALL_PREFIX ${PROJECT_SOURCE_DIR})
+include_directories(third_party/tantivy)
+
+include_directories(lintdb)
+
+# Collect source files
+file(GLOB LINT_DB_SRC
+        "lintdb/*.cpp"
+        "lintdb/datasources/*.cpp"
+        "lintdb/quantizers/*.cpp"
+        "lintdb/quantizers/impl/*.cpp"
+        "lintdb/invlists/*.cpp"
+        "lintdb/query/*.cpp"
+        "lintdb/query/physical/*.cpp"
+        "lintdb/schema/*.cpp"
+        "lintdb/scoring/*.cpp"
+)
 
-option(ENABLE_PYTHON "Build Python extension." ON)
-option(ENABLE_BENCHMARKS "Build benchmarks." ON)
-option(ENABLE_SERVER "Build the server." OFF)
+# Collect header files
+file(GLOB LINT_DB_HEADERS
+        "lintdb/*.h"
+        "lintdb/datasources/*.h"
+        "lintdb/quantizers/*.h"
+        "lintdb/quantizers/impl/*.h"
+        "lintdb/invlists/*.h"
+        "lintdb/query/*.h"
+        "lintdb/query/physical/*.h"
+        "lintdb/schema/*.h"
+        "lintdb/scoring/*.h"
+        "lintdb/utils/*.h"
+)
 
-add_subdirectory(lintdb)
+add_library(lintdb ${LINT_DB_SRC})
+
+target_compile_options(
+        lintdb
+        PRIVATE $<$<COMPILE_LANGUAGE:CXX>:
+        -mfma
+        -mf16c
+        -mavx2
+        -msse4.2
+        -mpopcnt
+        -Wno-sign-compare
+        -Wno-unused-variable
+        -Wno-reorder
+        -Wno-unused-local-typedefs
+        -Wno-unused-function
+        -Wno-strict-aliasing>)
+
+target_link_libraries(
+        lintdb PUBLIC OpenMP::OpenMP_CXX ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}
+)
 
-if(ENABLE_PYTHON)
+target_link_libraries(lintdb
+        PUBLIC
+        faiss
+        JsonCpp::JsonCpp
+        Bitsery::bitsery
+        glog::glog
+        Microsoft.GSL::GSL
+        RocksDB::rocksdb
+        arrow::arrow
+)
+
+target_include_directories(lintdb
+        PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>)
+
+
+
+option(BUILD_PYTHON "Build Python extension." OFF)
+option(BUILD_BENCHMARKS "Build benchmarks." ON)
+option(BUILD_TESTS "Build tests." ON)
+option(BUILD_SERVER "Build the server." OFF)
+
+
+if(BUILD_PYTHON)
+  message(STATUS "building python extension")
   add_subdirectory(lintdb/python)
 endif()
 
-IF(ENABLE_SERVER)
+IF(BUILD_SERVER)
   add_subdirectory(lintdb/server)
 endif()
 
 include(CTest)
-if(BUILD_TESTING)
+if(BUILD_TESTS)
+  enable_testing()
   add_subdirectory(tests)
-endif()
 
+  if(EXISTS "${PROJECT_SOURCE_DIR}/lintdb/query/test")
+    add_subdirectory(lintdb/query/test)
+  endif()
+endif()
 
-if(ENABLE_BENCHMARKS)
+if(BUILD_BENCHMARKS)
   add_subdirectory(benchmarks)
 endif()
+
+install(TARGETS lintdb
+  DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}
+)
+foreach(header ${LINT_DB_HEADERS})
+  get_filename_component(dir ${header} DIRECTORY)
+  install(FILES ${header} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lintdb/${dir})
+endforeach()
diff --git a/CMakePresets.json b/CMakePresets.json
@@ -23,8 +23,7 @@
                 "BUILD_TESTING": "ON",
                 "CMAKE_INSTALL_LIBDIR": "lib",
                 "ENABLE_PYTHON": "ON",
-                "CMAKE_VERBOSE_MAKEFILE":"ON",
-                "BLA_VENDOR":"Intel10_64lp"
+                "CMAKE_VERBOSE_MAKEFILE":"ON"
             }
         },
         {
@@ -42,8 +41,7 @@
                 "CMAKE_INSTALL_LIBDIR": "lib",
                 "CMAKE_VERBOSE_MAKEFILE":"ON",
                 "ENABLE_PYTHON": "OFF",
-                "ENABLE_BENCHMARKS": "ON",
-                "BLA_VENDOR":"Intel10_64lp"
+                "ENABLE_BENCHMARKS": "ON"
             }
         },
         {
@@ -61,8 +59,7 @@
                 "CMAKE_INSTALL_LIBDIR": "lib",
                 "CMAKE_VERBOSE_MAKEFILE":"ON",
                 "ENABLE_PYTHON": "ON",
-                "BUILD_TESTING": "OFF",
-                "BLA_VENDOR":"Intel10_64lp"
+                "BUILD_TESTING": "OFF"
             }
         },
         {
@@ -80,8 +77,7 @@
                 "CMAKE_INSTALL_LIBDIR": "lib",
                 "ENABLE_PYTHON": "OFF",
                 "BUILD_TESTING": "OFF",
-                "ENABLE_SERVER": "ON",
-                "BLA_VENDOR":"Intel10_64lp"
+                "ENABLE_SERVER": "ON"
             }
         },
         {
@@ -100,8 +96,7 @@
                 "ENABLE_BENCHMARKS": "ON",
                 "ENABLE_PYTHON": "OFF",
                 "BUILD_TESTING": "OFF",
-                "ENABLE_SERVER": "OFF",
-                "BLA_VENDOR":"Intel10_64lp"
+                "ENABLE_SERVER": "OFF"
             }
         }
     ],

diff --git a/CMakeUserPresets.json b/CMakeUserPresets.json
@@ -0,0 +1,9 @@
+{
+    "version": 4,
+    "vendor": {
+        "conan": {}
+    },
+    "include": [
+        "build/Release/generators/CMakePresets.json"
+    ]
+}