B&B CPU determinism

benchmarks/linear_programming/cuopt/run_mip.cpp

@@ -148,7 +148,9 @@ int run_single_file(std::string file_path,
                     bool write_log_file,
                     bool log_to_console,
                     int reliability_branching,
-                    double time_limit)
+                    double time_limit,
+                    double work_limit,
+                    bool deterministic)
 {
   const raft::handle_t handle_{};
   cuopt::linear_programming::mip_solver_settings_t<int, double> settings;
@@ -197,14 +199,17 @@ int run_single_file(std::string file_path,
       }
     }
   }
-  settings.time_limit                    = time_limit;
-  settings.heuristics_only               = heuristics_only;
-  settings.num_cpu_threads               = num_cpu_threads;
-  settings.log_to_console                = log_to_console;
+  settings.time_limit       = time_limit;
+  settings.work_limit       = work_limit;
+  settings.heuristics_only  = heuristics_only;
+  settings.num_cpu_threads  = num_cpu_threads;
+  settings.log_to_console   = log_to_console;
+  settings.determinism_mode = deterministic ? CUOPT_MODE_DETERMINISTIC : CUOPT_MODE_OPPORTUNISTIC;
   settings.tolerances.relative_tolerance = 1e-12;
   settings.tolerances.absolute_tolerance = 1e-6;
   settings.presolve                      = true;
   settings.reliability_branching         = reliability_branching;
+  settings.seed                          = 42;
   cuopt::linear_programming::benchmark_info_t benchmark_info;
   settings.benchmark_info_ptr = &benchmark_info;
   auto start_run_solver       = std::chrono::high_resolution_clock::now();
@@ -258,7 +263,9 @@ void run_single_file_mp(std::string file_path,
                         bool write_log_file,
                         bool log_to_console,
                         int reliability_branching,
-                        double time_limit)
+                        double time_limit,
+                        double work_limit,
+                        bool deterministic)
 {
   std::cout << "running file " << file_path << " on gpu : " << device << std::endl;
   auto memory_resource = make_async();
@@ -274,7 +281,9 @@ void run_single_file_mp(std::string file_path,
                                   write_log_file,
                                   log_to_console,
                                   reliability_branching,
-                                  time_limit);
+                                  time_limit,
+                                  work_limit,
+                                  deterministic);
   // this is a bad design to communicate the result but better than adding complexity of IPC or
   // pipes
   exit(sol_found);
@@ -344,7 +353,12 @@ int main(int argc, char* argv[])
     .default_value(std::string("t"));
 
   program.add_argument("--time-limit")
-    .help("time limit")
+    .help("time limit in seconds")
+    .scan<'g', double>()
+    .default_value(std::numeric_limits<double>::infinity());
+
+  program.add_argument("--work-limit")
+    .help("work unit limit (for deterministic mode)")
     .scan<'g', double>()
     .default_value(std::numeric_limits<double>::infinity());
 
@@ -362,6 +376,11 @@ int main(int argc, char* argv[])
     .scan<'i', int>()
     .default_value(-1);
 
+  program.add_argument("-d", "--determinism")
+    .help("enable deterministic mode")
+    .default_value(false)
+    .implicit_value(true);
+
   // Parse arguments
   try {
     program.parse_args(argc, argv);
@@ -376,6 +395,7 @@ int main(int argc, char* argv[])
   std::string run_dir_arg = program.get<std::string>("--run-dir");
   bool run_dir            = run_dir_arg[0] == 't';
   double time_limit       = program.get<double>("--time-limit");
+  double work_limit       = program.get<double>("--work-limit");
 
   bool run_selected = program.get<std::string>("--run-selected")[0] == 't';
   int n_gpus        = program.get<int>("--n-gpus");
@@ -391,6 +411,7 @@ int main(int argc, char* argv[])
   double memory_limit       = program.get<double>("--memory-limit");
   bool track_allocations    = program.get<std::string>("--track-allocations")[0] == 't';
   int reliability_branching = program.get<int>("--reliability-branching");
+  bool deterministic        = program.get<bool>("--determinism");
 
   if (num_cpu_threads < 0) { num_cpu_threads = omp_get_max_threads() / n_gpus; }
 
@@ -479,7 +500,9 @@ int main(int argc, char* argv[])
                                write_log_file,
                                log_to_console,
                                reliability_branching,
-                               time_limit);
+                               time_limit,
+                               work_limit,
+                               deterministic);
           } else if (sys_pid < 0) {
             std::cerr << "Fork failed!" << std::endl;
             exit(1);
@@ -520,7 +543,9 @@ int main(int argc, char* argv[])
                     write_log_file,
                     log_to_console,
                     reliability_branching,
-                    time_limit);
+                    time_limit,
+                    work_limit,
+                    deterministic);
   }
 
   return 0;

cpp/include/cuopt/linear_programming/constants.h

@@ -32,6 +32,7 @@
 #define CUOPT_DUAL_INFEASIBLE_TOLERANCE       "dual_infeasible_tolerance"
 #define CUOPT_ITERATION_LIMIT                 "iteration_limit"
 #define CUOPT_TIME_LIMIT                      "time_limit"
+#define CUOPT_WORK_LIMIT                      "work_limit"
 #define CUOPT_PDLP_SOLVER_MODE                "pdlp_solver_mode"
 #define CUOPT_METHOD                          "method"
 #define CUOPT_PER_CONSTRAINT_RESIDUAL         "per_constraint_residual"
@@ -49,6 +50,7 @@
 #define CUOPT_CUDSS_DETERMINISTIC             "cudss_deterministic"
 #define CUOPT_PRESOLVE                        "presolve"
 #define CUOPT_DUAL_POSTSOLVE                  "dual_postsolve"
+#define CUOPT_MIP_DETERMINISM_MODE            "mip_determinism_mode"
 #define CUOPT_MIP_ABSOLUTE_TOLERANCE          "mip_absolute_tolerance"
 #define CUOPT_MIP_RELATIVE_TOLERANCE          "mip_relative_tolerance"
 #define CUOPT_MIP_INTEGRALITY_TOLERANCE       "mip_integrality_tolerance"
@@ -57,6 +59,7 @@
 #define CUOPT_MIP_HEURISTICS_ONLY             "mip_heuristics_only"
 #define CUOPT_MIP_SCALING                     "mip_scaling"
 #define CUOPT_MIP_PRESOLVE                    "mip_presolve"
+#define CUOPT_MIP_SEED                        "mip_seed"
 #define CUOPT_MIP_RELIABILITY_BRANCHING       "mip_reliability_branching"
 #define CUOPT_MIP_CUT_PASSES                  "mip_cut_passes"
 #define CUOPT_MIP_MIXED_INTEGER_ROUNDING_CUTS "mip_mixed_integer_rounding_cuts"
@@ -72,6 +75,10 @@
 #define CUOPT_NUM_GPUS                        "num_gpus"
 #define CUOPT_USER_PROBLEM_FILE               "user_problem_file"
 
+/* @brief MIP determinism mode constants */
+#define CUOPT_MODE_OPPORTUNISTIC 0
+#define CUOPT_MODE_DETERMINISTIC 1
+
 /* @brief LP/MIP termination status constants */
 #define CUOPT_TERIMINATION_STATUS_NO_TERMINATION   0
 #define CUOPT_TERIMINATION_STATUS_OPTIMAL          1
@@ -83,6 +90,7 @@
 #define CUOPT_TERIMINATION_STATUS_PRIMAL_FEASIBLE  7
 #define CUOPT_TERIMINATION_STATUS_FEASIBLE_FOUND   8
 #define CUOPT_TERIMINATION_STATUS_CONCURRENT_LIMIT 9
+#define CUOPT_TERIMINATION_STATUS_WORK_LIMIT       10
 
 /* @brief The objective sense constants */
 #define CUOPT_MINIMIZE 1

cpp/include/cuopt/linear_programming/mip/solver_settings.hpp

@@ -84,6 +84,7 @@ class mip_solver_settings_t {
   tolerances_t tolerances;
 
   f_t time_limit                = std::numeric_limits<f_t>::infinity();
+  f_t work_limit                = std::numeric_limits<f_t>::infinity();
   i_t node_limit                = std::numeric_limits<i_t>::max();
   bool heuristics_only          = false;
   i_t reliability_branching     = -1;
@@ -108,6 +109,23 @@ class mip_solver_settings_t {
   std::vector<std::shared_ptr<rmm::device_uvector<f_t>>> initial_solutions;
   bool mip_scaling = false;
   bool presolve    = true;
+  /**
+   * @brief Determinism mode for MIP solver.
+   *
+   * Controls the determinism behavior of the MIP solver:
+   * - CUOPT_MODE_OPPORTUNISTIC (0): Default mode, allows non-deterministic
+   *   parallelism for better performance
+   * - CUOPT_MODE_DETERMINISTIC (1): Ensures deterministic results across runs
+   *   at potential cost of performance
+   */
+  int determinism_mode = CUOPT_MODE_OPPORTUNISTIC;
+  /**
+   * @brief Random seed for the MIP solver.
+   *
+   * Controls the initial seed for random number generation in the solver.
+   * Use -1 to generate a random seed.
+   */
+  i_t seed = -1;
   // this is for extracting info from different places of the solver during
   // benchmarks
   benchmark_info_t* benchmark_info_ptr = nullptr;

cpp/include/cuopt/linear_programming/mip/solver_solution.hpp

@@ -1,6 +1,6 @@
 /* clang-format off */
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
 /* clang-format on */
@@ -30,6 +30,7 @@ enum class mip_termination_status_t : int8_t {
   Infeasible    = CUOPT_TERIMINATION_STATUS_INFEASIBLE,
   Unbounded     = CUOPT_TERIMINATION_STATUS_UNBOUNDED,
   TimeLimit     = CUOPT_TERIMINATION_STATUS_TIME_LIMIT,
+  WorkLimit     = CUOPT_TERIMINATION_STATUS_WORK_LIMIT,
 };
 
 template <typename i_t, typename f_t>

cpp/src/CMakeLists.txt

@@ -1,12 +1,13 @@
 # cmake-format: off
-# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 # cmake-format: on
 
 set(UTIL_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/utilities/seed_generator.cu
                    ${CMAKE_CURRENT_SOURCE_DIR}/utilities/logger.cpp
                    ${CMAKE_CURRENT_SOURCE_DIR}/utilities/version_info.cpp
-                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/timestamp_utils.cpp)
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/timestamp_utils.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/work_unit_scheduler.cpp)
 
 add_subdirectory(linear_programming)
 add_subdirectory(math_optimization)

cpp/src/dual_simplex/barrier.cu

@@ -1388,7 +1388,7 @@ class iteration_data_t {
     // v = alpha * A * w + beta * v = alpha * A * Dinv * A^T * y + beta * v
     matrix_vector_multiply(A, alpha, w, beta, v);
     if (debug) {
-      printf("||A|| = %.16e\n", vector_norm2<i_t, f_t>(A.x));
+      printf("||A|| = %.16e\n", vector_norm2<i_t, f_t>(A.x.underlying()));
       printf("||w|| = %.16e\n", vector_norm2<i_t, f_t>(w));
       printf("||v|| = %.16e\n", vector_norm2<i_t, f_t>(v));
     }

cpp/src/dual_simplex/basis_solves.cpp

@@ -13,6 +13,8 @@
 #include <dual_simplex/tic_toc.hpp>
 #include <dual_simplex/triangle_solve.hpp>
 
+#include <raft/common/nvtx.hpp>
+
 namespace cuopt::linear_programming::dual_simplex {
 
 template <typename i_t>
@@ -57,14 +59,14 @@ void get_basis_from_vstatus(i_t m,
 
 namespace {
 
-template <typename i_t, typename f_t>
+template <typename i_t, typename f_t, typename VectorI>
 void write_singleton_info(i_t m,
                           i_t col_singletons,
                           i_t row_singletons,
                           const csc_matrix_t<i_t, f_t>& B,
-                          const std::vector<i_t>& row_perm,
-                          const std::vector<i_t>& row_perm_inv,
-                          const std::vector<i_t>& col_perm)
+                          const VectorI& row_perm,
+                          const VectorI& row_perm_inv,
+                          const VectorI& col_perm)
 {
   FILE* file = fopen("singleton_debug.m", "w");
   if (file != NULL) {
@@ -94,7 +96,7 @@ void write_singleton_info(i_t m,
   fclose(file);
 }
 
-template <typename i_t, typename f_t>
+template <typename i_t, typename f_t, typename VectorI>
 void write_factor_info(const char* filename,
                        i_t m,
                        i_t row_singletons,
@@ -104,8 +106,8 @@ void write_factor_info(const char* filename,
                        const csc_matrix_t<i_t, f_t>& D,
                        const csc_matrix_t<i_t, f_t>& L,
                        const csc_matrix_t<i_t, f_t>& U,
-                       const std::vector<i_t>& row_perm,
-                       const std::vector<i_t>& col_perm)
+                       const VectorI& row_perm,
+                       const VectorI& col_perm)
 {
   FILE* file = fopen(filename, "w");
   if (file != NULL) {
@@ -165,6 +167,7 @@ i_t factorize_basis(const csc_matrix_t<i_t, f_t>& A,
                     std::vector<i_t>& deficient,
                     std::vector<i_t>& slacks_needed)
 {
+  raft::common::nvtx::range scope("LU::factorize_basis");
   const i_t m              = basic_list.size();
   constexpr f_t medium_tol = 1e-12;
 
@@ -779,6 +782,8 @@ i_t b_transpose_solve(const csc_matrix_t<i_t, f_t>& L,
   // U'*r = c
   // L'*w = r
 
+  raft::common::nvtx::range scope("LU::b_transpose_solve");
+
   // Solve for r such that U'*r = c
   std::vector<f_t> r = rhs;
   upper_triangular_transpose_solve(U, r);

cpp/src/dual_simplex/basis_updates.cpp

@@ -9,6 +9,7 @@
 #include <dual_simplex/basis_updates.hpp>
 #include <dual_simplex/initial_basis.hpp>
 #include <dual_simplex/triangle_solve.hpp>
+#include <raft/common/nvtx.hpp>
 
 #include <cmath>
 #include <limits>
@@ -35,6 +36,7 @@ i_t basis_update_t<i_t, f_t>::b_solve(const std::vector<f_t>& rhs,
                                       std::vector<f_t>& solution,
                                       std::vector<f_t>& Lsol) const
 {
+  raft::common::nvtx::range scope("LU::b_solve");
   const i_t m = L0_.m;
   assert(row_permutation_.size() == m);
   assert(rhs.size() == m);
@@ -86,6 +88,7 @@ template <typename i_t, typename f_t>
 i_t basis_update_t<i_t, f_t>::b_transpose_solve(const std::vector<f_t>& rhs,
                                                 std::vector<f_t>& solution) const
 {
+  raft::common::nvtx::range scope("LU::b_transpose_solve");
   // Observe that
   // P*B = L*U
   // B'*P' = U'*L'
@@ -2263,6 +2266,7 @@ int basis_update_mpf_t<i_t, f_t>::refactor_basis(
   std::vector<i_t>& nonbasic_list,
   std::vector<variable_status_t>& vstatus)
 {
+  raft::common::nvtx::range scope("LU::refactor_basis");
   std::vector<i_t> deficient;
   std::vector<i_t> slacks_needed;
   std::vector<i_t> superbasic_list;  // Empty superbasic list