diff --git a/Benchmarks/Benchmark.cpp b/Benchmarks/Benchmark.cpp deleted file mode 100644 index d14bc59a..00000000 --- a/Benchmarks/Benchmark.cpp +++ /dev/null @@ -1,403 +0,0 @@ -// SPDX-License-Identifier: MIT - -#include "Lua/LuaLibrary.h" -#include "LuaBridge/LuaBridge.h" - -#include -#include -#include -#include -#include -#include - -#if LUABRIDGE_BENCHMARK_WITH_SOL2 -#include -#endif - -namespace { - -struct Counter -{ - int value = 0; - - void inc() - { - ++value; - } - - int add(int x) - { - value += x; - return value; - } - - int get() const - { - return value; - } - - void set(int v) - { - value = v; - } -}; - -using Clock = std::chrono::steady_clock; - -struct CaseResult -{ - std::string name; - double nsPerOp = 0.0; - double mops = 0.0; -}; - -template -CaseResult runCase(const std::string& name, std::int64_t iterations, Fn&& fn) -{ - fn(); // warm-up - - const auto start = Clock::now(); - fn(); - const auto end = Clock::now(); - - const auto elapsedNs = std::chrono::duration_cast(end - start).count(); - const double nsPerOp = static_cast(elapsedNs) / static_cast(iterations); - const double mops = 1'000.0 / nsPerOp; - - return { name, nsPerOp, mops }; -} - -void printResult(const char* family, const CaseResult& r) -{ - std::cout << std::left << std::setw(12) << family - << " " << std::setw(30) << r.name - << " " << std::fixed << std::setprecision(2) - << std::setw(12) << r.nsPerOp - << " " << std::setw(10) << r.mops - << '\n'; -} - -void runLuaBridgeBenchmarks(std::int64_t iterations) -{ - lua_State* L = luaL_newstate(); - luaL_openlibs(L); - luabridge::registerMainThread(L); - -#if LUABRIDGE_HAS_EXCEPTIONS - luabridge::enableExceptions(L); -#endif - - luabridge::getGlobalNamespace(L) - .beginClass("Counter") - .addConstructor() - .addFunction("inc", &Counter::inc) - .addFunction("add", &Counter::add) - .addProperty("value", &Counter::get, &Counter::set) - .endClass() - .addFunction("add", +[](int a, int b) { return a + b; }); - - const auto doLua = [L](const char* chunk) - { - if (luaL_dostring(L, chunk) != LUABRIDGE_LUA_OK) - { - const char* message = lua_tostring(L, -1); - std::cerr << "Lua error: " << (message ? message : "unknown") << '\n'; - std::exit(1); - } - }; - - CaseResult empty; - CaseResult freeCall; - CaseResult memberCall; - CaseResult property; - CaseResult propertySet; - CaseResult propertyGet; - CaseResult cppGlobalGet; - CaseResult cppGlobalSet; - CaseResult cppTableGet; - CaseResult cppTableSet; - CaseResult cppChainedGet; - CaseResult cppChainedSet; - CaseResult cppToLua; - CaseResult cppCFunctionThroughLua; - - { - doLua("obj = Counter()"); - doLua("tbl = { value = 0 }"); - doLua("chain = { inner = { value = 0 } }"); - luabridge::setGlobal(L, 1.0, "gvalue"); - - volatile double sink = 0.0; - - const std::string emptyLoop = "for i = 1, " + std::to_string(iterations) + " do end"; - const std::string freeCallLoop = "for i = 1, " + std::to_string(iterations) + " do add(i, i) end"; - const std::string memberCallLoop = "for i = 1, " + std::to_string(iterations) + " do obj:inc() end"; - const std::string propertyLoop = "for i = 1, " + std::to_string(iterations) + " do obj.value = i; local x = obj.value end"; - const std::string propertySetLoop = "for i = 1, " + std::to_string(iterations) + " do obj.value = i end"; - const std::string propertyGetLoop = "local s = 0; for i = 1, " + std::to_string(iterations) + " do s = s + obj.value end"; - - empty = runCase("lua_empty_loop", iterations, [&] { doLua(emptyLoop.c_str()); }); - freeCall = runCase("lua_to_cpp_free_fn", iterations, [&] { doLua(freeCallLoop.c_str()); }); - memberCall = runCase("lua_to_cpp_member", iterations, [&] { doLua(memberCallLoop.c_str()); }); - property = runCase("lua_to_cpp_property", iterations, [&] { doLua(propertyLoop.c_str()); }); - propertySet = runCase("lua_to_cpp_property_set", iterations, [&] { doLua(propertySetLoop.c_str()); }); - propertyGet = runCase("lua_to_cpp_property_get", iterations, [&] { doLua(propertyGetLoop.c_str()); }); - - cppGlobalGet = runCase("cpp_table_global_get", iterations, [&] - { - double x = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - x += *luabridge::getGlobal(L, "gvalue"); - sink = x; - }); - - cppGlobalSet = runCase("cpp_table_global_set", iterations, [&] - { - double v = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - { - v += 1.0; - luabridge::setGlobal(L, v, "gvalue"); - } - sink = static_cast(luabridge::getGlobal(L, "gvalue")); - }); - - const auto tableRef = luabridge::getGlobal(L, "tbl"); - cppTableGet = runCase("cpp_table_get", iterations, [&] - { - double x = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - x += tableRef.unsafeRawgetField("value"); - sink = x; - }); - - cppTableSet = runCase("cpp_table_set", iterations, [&] - { - double v = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - { - v += 1.0; - tableRef.unsafeRawsetField("value", v); - } - sink = tableRef.unsafeRawgetField("value"); - }); - - const auto chainRef = luabridge::getGlobal(L, "chain"); - cppChainedGet = runCase("cpp_table_chained_get", iterations, [&] - { - double x = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - x += chainRef["inner"].unsafeRawgetField("value"); - sink = x; - }); - - cppChainedSet = runCase("cpp_table_chained_set", iterations, [&] - { - double v = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - { - v += 1.0; - chainRef["inner"].unsafeRawsetField("value", v); - } - - sink = chainRef["inner"].unsafeRawgetField("value"); - }); - - doLua("function f(a, b) return a + b end"); - auto f = luabridge::getGlobal(L, "f"); - auto callable = f.callable(); - cppToLua = runCase("cpp_to_lua_call", iterations, [&] - { - for (std::int64_t i = 0; i < iterations; ++i) - (void) callable(static_cast(i), static_cast(i)); - }); - - auto addFn = luabridge::getGlobal(L, "add"); - cppCFunctionThroughLua = runCase("cpp_c_function_through_lua", iterations, [&] - { - int x = 0; - for (std::int64_t i = 0; i < iterations; ++i) - x += addFn.call(static_cast(i), static_cast(i)).valueOr(0); - sink = static_cast(x); - }); - } - - printResult("LuaBridge", empty); - printResult("LuaBridge", freeCall); - printResult("LuaBridge", memberCall); - printResult("LuaBridge", property); - printResult("LuaBridge", propertySet); - printResult("LuaBridge", propertyGet); - printResult("LuaBridge", cppGlobalGet); - printResult("LuaBridge", cppGlobalSet); - printResult("LuaBridge", cppTableGet); - printResult("LuaBridge", cppTableSet); - printResult("LuaBridge", cppChainedGet); - printResult("LuaBridge", cppChainedSet); - printResult("LuaBridge", cppToLua); - printResult("LuaBridge", cppCFunctionThroughLua); - - lua_close(L); -} - -#if LUABRIDGE_BENCHMARK_WITH_SOL2 -void runSol2Benchmarks(std::int64_t iterations) -{ - sol::state lua; - lua.open_libraries(sol::lib::base, sol::lib::math, sol::lib::table, sol::lib::string); - - Counter counter; - - lua.new_usertype("Counter", - sol::call_constructor, - sol::constructors(), - "inc", &Counter::inc, - "add", &Counter::add, - "value", sol::property(&Counter::get, &Counter::set)); - - lua.set_function("add", +[](int a, int b) { return a + b; }); - lua["counter"] = &counter; - lua.script("obj = Counter()"); - lua.script("tbl = { value = 0 }"); - lua.script("chain = { inner = { value = 0 } }"); - lua["gvalue"] = 1.0; - - volatile double sink = 0.0; - - const std::string emptyLoop = "for i = 1, " + std::to_string(iterations) + " do end"; - const std::string freeCallLoop = "for i = 1, " + std::to_string(iterations) + " do add(i, i) end"; - const std::string memberCallLoop = "for i = 1, " + std::to_string(iterations) + " do obj:inc() end"; - const std::string propertyLoop = "for i = 1, " + std::to_string(iterations) + " do obj.value = i; local x = obj.value end"; - const std::string propertySetLoop = "for i = 1, " + std::to_string(iterations) + " do obj.value = i end"; - const std::string propertyGetLoop = "local s = 0; for i = 1, " + std::to_string(iterations) + " do s = s + obj.value end"; - - const auto empty = runCase("lua_empty_loop", iterations, [&] { lua.script(emptyLoop); }); - const auto freeCall = runCase("lua_to_cpp_free_fn", iterations, [&] { lua.script(freeCallLoop); }); - const auto memberCall = runCase("lua_to_cpp_member", iterations, [&] { lua.script(memberCallLoop); }); - const auto property = runCase("lua_to_cpp_property", iterations, [&] { lua.script(propertyLoop); }); - const auto propertySet = runCase("lua_to_cpp_property_set", iterations, [&] { lua.script(propertySetLoop); }); - const auto propertyGet = runCase("lua_to_cpp_property_get", iterations, [&] { lua.script(propertyGetLoop); }); - - const auto cppGlobalGet = runCase("cpp_table_global_get", iterations, [&] - { - double x = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - x += lua["gvalue"].get(); - sink = x; - }); - - const auto cppGlobalSet = runCase("cpp_table_global_set", iterations, [&] - { - double v = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - { - v += 1.0; - lua["gvalue"] = v; - } - sink = lua["gvalue"].get(); - }); - - sol::table table = lua["tbl"]; - const auto cppTableGet = runCase("cpp_table_get", iterations, [&] - { - double x = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - x += table["value"].get(); - sink = x; - }); - - const auto cppTableSet = runCase("cpp_table_set", iterations, [&] - { - double v = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - { - v += 1.0; - table["value"] = v; - } - sink = table["value"].get(); - }); - - sol::table chain = lua["chain"]; - const auto cppChainedGet = runCase("cpp_table_chained_get", iterations, [&] - { - double x = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - x += chain["inner"]["value"].get(); - sink = x; - }); - - const auto cppChainedSet = runCase("cpp_table_chained_set", iterations, [&] - { - double v = 0.0; - for (std::int64_t i = 0; i < iterations; ++i) - { - v += 1.0; - chain["inner"]["value"] = v; - } - sink = chain["inner"]["value"].get(); - }); - - lua.script("function f(a, b) return a + b end"); - sol::protected_function f = lua["f"]; - const auto cppToLua = runCase("cpp_to_lua_call", iterations, [&] - { - for (std::int64_t i = 0; i < iterations; ++i) - (void) f(static_cast(i), static_cast(i)); - }); - - sol::protected_function addFn = lua["add"]; - const auto cppCFunctionThroughLua = runCase("cpp_c_function_through_lua", iterations, [&] - { - int x = 0; - for (std::int64_t i = 0; i < iterations; ++i) - x += addFn.call(static_cast(i), static_cast(i)); - sink = static_cast(x); - }); - - printResult("sol2", empty); - printResult("sol2", freeCall); - printResult("sol2", memberCall); - printResult("sol2", property); - printResult("sol2", propertySet); - printResult("sol2", propertyGet); - printResult("sol2", cppGlobalGet); - printResult("sol2", cppGlobalSet); - printResult("sol2", cppTableGet); - printResult("sol2", cppTableSet); - printResult("sol2", cppChainedGet); - printResult("sol2", cppChainedSet); - printResult("sol2", cppToLua); - printResult("sol2", cppCFunctionThroughLua); -} -#endif - -} // namespace - -int main(int argc, char** argv) -{ - std::int64_t iterations = 2'000'000; - - if (argc > 1) - { - const auto parsed = std::strtoll(argv[1], nullptr, 10); - if (parsed > 0) - iterations = parsed; - } - - std::cout << "Iterations: " << iterations << '\n'; - std::cout << std::left << std::setw(12) << "Family" - << " " << std::setw(30) << "Case" - << " " << std::setw(12) << "ns/op" - << " " << std::setw(10) << "Mop/s" - << '\n'; - - runLuaBridgeBenchmarks(iterations); - -#if LUABRIDGE_BENCHMARK_WITH_SOL2 - runSol2Benchmarks(iterations); -#else - std::cout << "sol2 not enabled (set LUABRIDGE_BENCHMARK_WITH_SOL2=ON and SOL2_INCLUDE_DIR)." << '\n'; -#endif - - return 0; -} diff --git a/Benchmarks/CMakeLists.txt b/Benchmarks/CMakeLists.txt index 0d2bc75c..efabe9d1 100644 --- a/Benchmarks/CMakeLists.txt +++ b/Benchmarks/CMakeLists.txt @@ -2,38 +2,103 @@ cmake_minimum_required(VERSION 3.10) include(FetchContent) -set(LUABRIDGE_BENCHMARK_WITH_SOL2 OFF CACHE BOOL "Whether to benchmark against sol2") +set(LUABRIDGE_BENCHMARK_WITH_SOL3 ON CACHE BOOL "Build Sol3 benchmark target") set(LUABRIDGE_SOL2_GIT_REPOSITORY "https://github.com/ThePhD/sol2.git" CACHE STRING "sol2 repository URL") set(LUABRIDGE_SOL2_GIT_TAG "v3.3.0" CACHE STRING "sol2 git tag or commit") -set(LUABRIDGE_BENCHMARK_SOURCES - Benchmark.cpp - ../Tests/Lua/LuaLibrary5.4.8.cpp) +set(LUABRIDGE_BENCHMARK_WITH_LUABRIDGE ON CACHE BOOL "Build LuaBridge benchmark target") +set(LUABRIDGE_VANILLA_GIT_REPOSITORY "https://github.com/vinniefalco/LuaBridge.git" CACHE STRING "LuaBridge vanilla repository URL") +set(LUABRIDGE_VANILLA_GIT_TAG "master" CACHE STRING "LuaBridge vanilla git tag or commit") -add_executable(LuaBridgeBenchmarks ${LUABRIDGE_BENCHMARK_SOURCES}) +set(LUABRIDGE_GOOGLE_BENCHMARK_GIT_REPOSITORY "https://github.com/google/benchmark.git" CACHE STRING "Google Benchmark repository URL") +set(LUABRIDGE_GOOGLE_BENCHMARK_GIT_TAG "v1.8.4" CACHE STRING "Google Benchmark git tag or commit") -target_include_directories(LuaBridgeBenchmarks PRIVATE - ${CMAKE_CURRENT_LIST_DIR}/.. - ${CMAKE_CURRENT_LIST_DIR}/../Source - ${CMAKE_CURRENT_LIST_DIR}/../Tests - ${CMAKE_CURRENT_LIST_DIR}/../Tests/Lua/Lua.5.4.8/src) +set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) +FetchContent_Declare( + googlebenchmark + GIT_REPOSITORY ${LUABRIDGE_GOOGLE_BENCHMARK_GIT_REPOSITORY} + GIT_TAG ${LUABRIDGE_GOOGLE_BENCHMARK_GIT_TAG}) +FetchContent_MakeAvailable(googlebenchmark) -target_compile_definitions(LuaBridgeBenchmarks PRIVATE - LUABRIDGE_BENCHMARK_LUA54=1 - LUABRIDGE_TEST_LUA_VERSION=504) - -if (LUABRIDGE_BENCHMARK_WITH_SOL2) +if (LUABRIDGE_BENCHMARK_WITH_SOL3) FetchContent_Declare( sol2 GIT_REPOSITORY ${LUABRIDGE_SOL2_GIT_REPOSITORY} GIT_TAG ${LUABRIDGE_SOL2_GIT_TAG}) - FetchContent_MakeAvailable(sol2) + FetchContent_GetProperties(sol2) + if (NOT sol2_POPULATED) + FetchContent_Populate(sol2) + endif() + + # Work around a sol2 optional::emplace bug on recent Apple Clang toolchains. + set(SOL2_OPTIONAL_IMPL "${sol2_SOURCE_DIR}/include/sol/optional_implementation.hpp") + if (EXISTS "${SOL2_OPTIONAL_IMPL}") + file(READ "${SOL2_OPTIONAL_IMPL}" SOL2_OPTIONAL_IMPL_CONTENT) + set(SOL2_OPTIONAL_REFBLOCK_OLD "\t\ttemplate \n\t\tT& emplace(Args&&... args) noexcept {\n\t\t\tstatic_assert(std::is_constructible::value, \"T must be constructible with Args\");\n\n\t\t\t*this = nullopt;\n\t\t\tthis->construct(std::forward(args)...);\n\t\t}\n") + string(FIND "${SOL2_OPTIONAL_IMPL_CONTENT}" "${SOL2_OPTIONAL_REFBLOCK_OLD}" SOL2_PATCH_NEEDLE_POS) + if (NOT SOL2_PATCH_NEEDLE_POS EQUAL -1) + set(SOL2_OPTIONAL_REFBLOCK_NEW "\t\ttemplate \n\t\tT& emplace(Args&&... args) noexcept {\n\t\t\tstatic_assert(std::is_constructible::value, \"T must be constructible with Args\");\n\n\t\t\t*this = nullopt;\n\t\t\tint emplace_workaround[] = { 0, ((*this = std::forward(args)), 0)... };\n\t\t\t(void) emplace_workaround;\n\t\t\treturn *m_value;\n\t\t}\n") + string(REPLACE + "${SOL2_OPTIONAL_REFBLOCK_OLD}" + "${SOL2_OPTIONAL_REFBLOCK_NEW}" + SOL2_OPTIONAL_IMPL_CONTENT + "${SOL2_OPTIONAL_IMPL_CONTENT}") + file(WRITE "${SOL2_OPTIONAL_IMPL}" "${SOL2_OPTIONAL_IMPL_CONTENT}") + endif() + endif() +endif() + +if (LUABRIDGE_BENCHMARK_WITH_LUABRIDGE) + FetchContent_Declare( + luabridge_vanilla + GIT_REPOSITORY ${LUABRIDGE_VANILLA_GIT_REPOSITORY} + GIT_TAG ${LUABRIDGE_VANILLA_GIT_TAG}) + FetchContent_GetProperties(luabridge_vanilla) + if (NOT luabridge_vanilla_POPULATED) + FetchContent_Populate(luabridge_vanilla) + endif() +endif() + +function(add_luabridge_benchmark_target target_name source_file) + add_executable(${target_name} + ${source_file} + benchmark_common.cpp + ../Tests/Lua/LuaLibrary5.4.8.cpp) + + target_include_directories(${target_name} PRIVATE + ${CMAKE_CURRENT_LIST_DIR} + ${CMAKE_CURRENT_LIST_DIR}/.. + ${CMAKE_CURRENT_LIST_DIR}/../Tests + ${CMAKE_CURRENT_LIST_DIR}/../Tests/Lua/Lua.5.4.8/src) + + target_compile_definitions(${target_name} PRIVATE + LUABRIDGE_BENCHMARK_LUA54=1 + LUABRIDGE_TEST_LUA_VERSION=504) + + target_link_libraries(${target_name} PRIVATE + benchmark::benchmark + benchmark::benchmark_main) +endfunction() + +add_luabridge_benchmark_target(LuaBridge3Benchmark benchmark_luabridge3.cpp) +target_include_directories(LuaBridge3Benchmark PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/../Source) + +if (LUABRIDGE_BENCHMARK_WITH_LUABRIDGE) + add_luabridge_benchmark_target(LuaBridgeVanillaBenchmark benchmark_luabridge.cpp) + target_include_directories(LuaBridgeVanillaBenchmark PRIVATE + ${luabridge_vanilla_SOURCE_DIR}/Source) +endif() - target_include_directories(LuaBridgeBenchmarks PRIVATE +if (LUABRIDGE_BENCHMARK_WITH_SOL3) + add_luabridge_benchmark_target(Sol3Benchmark benchmark_sol3.cpp) + target_include_directories(Sol3Benchmark PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/../Source ${sol2_SOURCE_DIR}/include) - target_compile_definitions(LuaBridgeBenchmarks PRIVATE - LUABRIDGE_BENCHMARK_WITH_SOL2=1 + target_compile_definitions(Sol3Benchmark PRIVATE SOL_ALL_SAFETIES_ON=1 + SOL_NO_EXCEPTIONS=1 SOL_LUA_VERSION=504) endif() diff --git a/Benchmarks/README.md b/Benchmarks/README.md index b684dceb..2bcd14e8 100644 --- a/Benchmarks/README.md +++ b/Benchmarks/README.md @@ -1,6 +1,12 @@ -# LuaBridge Benchmarks +# Lua Binding Benchmarks -This directory contains a standalone benchmark executable focused on LuaBridge call and property access overhead. The selected cases mirror the style of the sol2 benchmark categories (Lua -> C++ calls, C++ -> Lua calls, member calls, and property access). +This directory contains Google Benchmark based executables for: + +- LuaBridge3 (current workspace) +- LuaBridge vanilla (`https://github.com/vinniefalco/LuaBridge`) +- sol3 (`https://github.com/ThePhD/sol2`) + +All benchmark executables are built with the same embedded Lua 5.4.8 runtime source (`Tests/Lua/LuaLibrary5.4.8.cpp`) for fair comparisons. ## Build @@ -8,53 +14,71 @@ From project root: ```bash cmake -S . -B Build -DCMAKE_BUILD_TYPE=Release -DLUABRIDGE_BENCHMARKS=ON -cmake --build Build --target LuaBridgeBenchmarks --config Release +cmake --build Build --config Release --target LuaBridge3Benchmark LuaBridgeVanillaBenchmark ``` -## Run +To also build Sol3 benchmark target: ```bash -./Build/Benchmarks/LuaBridgeBenchmarks -# optional iterations -./Build/Benchmarks/LuaBridgeBenchmarks 5000000 +cmake -S . -B Build -DCMAKE_BUILD_TYPE=Release -DLUABRIDGE_BENCHMARKS=ON -DLUABRIDGE_BENCHMARK_WITH_SOL3=ON +cmake --build Build --config Release --target Sol3Benchmark ``` -The executable prints: -- `ns/op` (lower is better) -- `Mop/s` (higher is better) +## Dependency Sources (FetchContent) + +Defaults: -## Optional sol2 Side-by-Side +- Google Benchmark: `https://github.com/google/benchmark.git` (`v1.8.4`) +- sol3: `https://github.com/ThePhD/sol2.git` (`v3.5.0`) +- LuaBridge vanilla: `https://github.com/vinniefalco/LuaBridge.git` (`master`) -To compare with sol2 in the same executable, enable the option below. The build will download sol2 through CMake FetchContent: +You can override these at configure time: ```bash cmake -S . -B Build \ -DCMAKE_BUILD_TYPE=Release \ -DLUABRIDGE_BENCHMARKS=ON \ - -DLUABRIDGE_BENCHMARK_WITH_SOL2=ON -cmake --build Build --target LuaBridgeBenchmarks --config Release + -DLUABRIDGE_SOL2_GIT_REPOSITORY=https://github.com/ThePhD/sol2.git \ + -DLUABRIDGE_SOL2_GIT_TAG=v3.5.0 \ + -DLUABRIDGE_VANILLA_GIT_REPOSITORY=https://github.com/vinniefalco/LuaBridge.git \ + -DLUABRIDGE_VANILLA_GIT_TAG=master \ + -DLUABRIDGE_GOOGLE_BENCHMARK_GIT_REPOSITORY=https://github.com/google/benchmark.git \ + -DLUABRIDGE_GOOGLE_BENCHMARK_GIT_TAG=v1.8.4 ``` -By default, FetchContent uses: -- Repository: `https://github.com/ThePhD/sol2.git` -- Tag: `v3.3.0` +## Run Benchmarks -You can override these if needed: +Each executable supports standard Google Benchmark CLI flags. ```bash -cmake -S . -B Build \ - -DCMAKE_BUILD_TYPE=Release \ - -DLUABRIDGE_BENCHMARKS=ON \ - -DLUABRIDGE_BENCHMARK_WITH_SOL2=ON \ - -DLUABRIDGE_SOL2_GIT_REPOSITORY=https://github.com/ThePhD/sol2.git \ - -DLUABRIDGE_SOL2_GIT_TAG=v3.3.0 +./Build/Benchmarks/LuaBridge3Benchmark --benchmark_out=Build/Benchmarks/luabridge3.json --benchmark_out_format=json +./Build/Benchmarks/LuaBridgeVanillaBenchmark --benchmark_out=Build/Benchmarks/luabridge_vanilla.json --benchmark_out_format=json +./Build/Benchmarks/Sol3Benchmark --benchmark_out=Build/Benchmarks/sol3.json --benchmark_out_format=json # if enabled +``` + +Recommended consistency flags for fair comparison: + +```bash +--benchmark_min_time=0.1 --benchmark_repetitions=5 ``` -When enabled, the benchmark prints LuaBridge and sol2 rows for the same cases and iteration count. +## Plot Results + +The script `plot_benchmarks.py` merges one or more Google Benchmark JSON files and generates a grouped comparison chart. + +```bash +python3 Benchmarks/plot_benchmarks.py \ + --input Build/Benchmarks/luabridge3.json Build/Benchmarks/luabridge_vanilla.json Build/Benchmarks/sol3.json \ + --output Build/Benchmarks/lua_bindings_comparison.png +``` + +Outputs: + +- PNG chart (grouped bars, lower is better) +- Optional skipped/error report file next to the image (`*_skipped.txt`) -## Notes for Fair Comparison +## Notes -- Use Release mode and the same compiler for all libraries. -- Pin CPU frequency/governor where possible and avoid background load. -- Run each benchmark multiple times and compare medians. -- Keep Lua version identical between LuaBridge and sol2 runs. +- Some vanilla LuaBridge benchmarks are marked as skipped where the feature is unsupported. +- Sol3 target is optional (`LUABRIDGE_BENCHMARK_WITH_SOL3`) because current sol2 headers can fail to compile on some toolchains. +- If you need stricter reproducibility, pin all FetchContent dependencies to commits instead of branches. diff --git a/Benchmarks/benchmark_common.cpp b/Benchmarks/benchmark_common.cpp new file mode 100644 index 00000000..335642da --- /dev/null +++ b/Benchmarks/benchmark_common.cpp @@ -0,0 +1,31 @@ +// https://github.com/kunitoki/LuaBridge3 +// Copyright 2026, kunitoki +// Inspired from https://github.com/ThePhD/lua-bindings-shootout by ThePhD +// SPDX-License-Identifier: MIT + +#include "benchmark_common.hpp" + +#include + +namespace lbsbench { + +void luaCheckOrThrow(lua_State* L, int status, std::string_view where) +{ + if (status == LUA_OK) + return; + + const char* message = lua_tostring(L, -1); + std::string error(where); + error += ": "; + error += (message ? message : "unknown lua error"); + lua_pop(L, 1); + throw std::runtime_error(error); +} + +void luaDoStringOrThrow(lua_State* L, std::string_view code, std::string_view where) +{ + const int status = luaL_dostring(L, std::string(code).c_str()); + luaCheckOrThrow(L, status, where); +} + +} // namespace lbsbench diff --git a/Benchmarks/benchmark_common.hpp b/Benchmarks/benchmark_common.hpp new file mode 100644 index 00000000..464f3313 --- /dev/null +++ b/Benchmarks/benchmark_common.hpp @@ -0,0 +1,174 @@ +// https://github.com/kunitoki/LuaBridge3 +// Copyright 2026, kunitoki +// Inspired from https://github.com/ThePhD/lua-bindings-shootout by ThePhD +// SPDX-License-Identifier: MIT + +#pragma once + +#include "Lua/LuaLibrary.h" + +#include + +#include +#include +#include +#include + +namespace lbsbench { + +inline constexpr double kMagicValue = 24.0; + +struct Counter +{ + int value = 0; + + void inc() + { + ++value; + } + + int add(int x) + { + value += x; + return value; + } + + int get() const + { + return value; + } + + void set(int v) + { + value = v; + } +}; + +struct Basic +{ + double var = 0.0; + + double get() const + { + return var; + } + + void set(double v) + { + var = v; + } +}; + +struct BasicLarge +{ + std::int64_t var = 0; + std::int64_t var0 = 0; + std::int64_t var1 = 0; + std::int64_t var2 = 0; + std::int64_t var3 = 0; + std::int64_t var4 = 0; + std::int64_t var5 = 0; + std::int64_t var6 = 0; + std::int64_t var7 = 0; + std::int64_t var8 = 0; + std::int64_t var9 = 0; + std::int64_t var10 = 0; + std::int64_t var11 = 0; + std::int64_t var12 = 0; + std::int64_t var13 = 0; + std::int64_t var14 = 0; + std::int64_t var15 = 0; + std::int64_t var16 = 0; + std::int64_t var17 = 0; + std::int64_t var18 = 0; + std::int64_t var19 = 0; + std::int64_t var20 = 0; + std::int64_t var21 = 0; + std::int64_t var22 = 0; + std::int64_t var23 = 0; + std::int64_t var24 = 0; + std::int64_t var25 = 0; + std::int64_t var26 = 0; + std::int64_t var27 = 0; + std::int64_t var28 = 0; + std::int64_t var29 = 0; + std::int64_t var30 = 0; + std::int64_t var31 = 0; + std::int64_t var32 = 0; + std::int64_t var33 = 0; + std::int64_t var34 = 0; + std::int64_t var35 = 0; + std::int64_t var36 = 0; + std::int64_t var37 = 0; + std::int64_t var38 = 0; + std::int64_t var39 = 0; + std::int64_t var40 = 0; + std::int64_t var41 = 0; + std::int64_t var42 = 0; + std::int64_t var43 = 0; + std::int64_t var44 = 0; + std::int64_t var45 = 0; + std::int64_t var46 = 0; + std::int64_t var47 = 0; + std::int64_t var48 = 0; + std::int64_t var49 = 0; +}; + +struct ComplexBaseA +{ + double a = kMagicValue; + + double a_func() const + { + return a; + } +}; + +struct ComplexBaseB +{ + double b = kMagicValue; + + double b_func() const + { + return b; + } +}; + +struct ComplexAB : ComplexBaseA, ComplexBaseB +{ + double ab = kMagicValue; + + double ab_func() const + { + return ab; + } +}; + +struct StatefulFunction +{ + double operator()(double v) const + { + return v; + } +}; + +inline Basic* basic_return() +{ + static Basic value{}; + return &value; +} + +inline double basic_get_var(Basic* b) +{ + return b ? b->var : 0.0; +} + +void luaCheckOrThrow(lua_State* L, int status, std::string_view where); +void luaDoStringOrThrow(lua_State* L, std::string_view code, std::string_view where); + +inline void setSkipped(benchmark::State& state, std::string_view reason) +{ + state.SkipWithError(std::string(reason).c_str()); +} + +} // namespace lbsbench diff --git a/Benchmarks/benchmark_luabridge.cpp b/Benchmarks/benchmark_luabridge.cpp new file mode 100644 index 00000000..d1a7924f --- /dev/null +++ b/Benchmarks/benchmark_luabridge.cpp @@ -0,0 +1,373 @@ +// https://github.com/kunitoki/LuaBridge3 +// Copyright 2026, kunitoki +// Inspired from https://github.com/ThePhD/lua-bindings-shootout by ThePhD +// SPDX-License-Identifier: MIT + +#include "benchmark_common.hpp" + +#include + +#include + +namespace { + +using namespace lbsbench; + +int vanilla_multi_return(lua_State* L) +{ + const double i = lua_tonumber(L, 1); + luabridge::push(L, i); + luabridge::push(L, i * 2.0); + return 2; +} + +lua_State* makeLua() +{ + lua_State* L = luaL_newstate(); + luaL_openlibs(L); + return L; +} + +void registerBasic(lua_State* L) +{ + luabridge::getGlobalNamespace(L) + .beginClass("c") + .addConstructor() + .addFunction("set", &Basic::set) + .addFunction("get", &Basic::get) + .addData("var", &Basic::var) + .endClass(); +} + +void table_global_string_get_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::setGlobal(L, kMagicValue, "value"); + + double x = 0; + for (auto _ : state) + { + (void) _; + x += static_cast(luabridge::getGlobal(L, "value")); + } + + benchmark::DoNotOptimize(x); +} + +void table_global_string_set_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + + double v = 0; + for (auto _ : state) + { + (void) _; + v += kMagicValue; + luabridge::setGlobal(L, v, "value"); + } + + benchmark::DoNotOptimize(v); +} + +void table_get_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "warble = { value = 24.0 }", "vanilla table_get setup"); + luabridge::LuaRef t = luabridge::getGlobal(L, "warble"); + + double x = 0; + for (auto _ : state) + { + (void) _; + x += static_cast(t["value"]); + } + + benchmark::DoNotOptimize(x); +} + +void table_set_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "warble = { value = 24.0 }", "vanilla table_set setup"); + luabridge::LuaRef t = luabridge::getGlobal(L, "warble"); + + double v = 0; + for (auto _ : state) + { + (void) _; + v += kMagicValue; + t["value"] = v; + } + + benchmark::DoNotOptimize(v); +} + +void table_chained_get_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "ulahibe = { warble = { value = 24.0 } }", "vanilla chained_get setup"); + + double x = 0; + for (auto _ : state) + { + (void) _; + luabridge::LuaRef tw = luabridge::getGlobal(L, "ulahibe")["warble"]; + x += static_cast(tw["value"]); + } + + benchmark::DoNotOptimize(x); +} + +void table_chained_set_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "ulahibe = { warble = { value = 24.0 } }", "vanilla chained_set setup"); + + double v = 0; + for (auto _ : state) + { + (void) _; + v += kMagicValue; + luabridge::LuaRef tw = luabridge::getGlobal(L, "ulahibe")["warble"]; + tw["value"] = v; + } + + benchmark::DoNotOptimize(v); +} + +void c_function_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::getGlobalNamespace(L).addFunction("f", +[](double v) { return v; }); + luaDoStringOrThrow(L, "function invoke_f() return f(24.0) end", "vanilla c_function setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "invoke_f"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "vanilla invoke_f"); + lua_pop(L, 1); + } +} + +void lua_function_in_c_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "function f(i) return i end", "vanilla lua_function setup"); + + luabridge::LuaRef f = luabridge::getGlobal(L, "f"); + double x = 0; + for (auto _ : state) + { + (void) _; + x += static_cast(f(kMagicValue)); + } + + benchmark::DoNotOptimize(x); +} + +void c_function_through_lua_in_c_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::getGlobalNamespace(L).addFunction("f", +[](double v) { return v; }); + + luabridge::LuaRef f = luabridge::getGlobal(L, "f"); + double x = 0; + for (auto _ : state) + { + (void) _; + x += static_cast(f(kMagicValue)); + } + + benchmark::DoNotOptimize(x); +} + +void member_function_call_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + registerBasic(L); + luaDoStringOrThrow(L, "b = c()", "vanilla member setup"); + luaDoStringOrThrow(L, "function call_member() b:set(b:get() + 1.0) end", "vanilla member closure setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "call_member"); + luaCheckOrThrow(L, lua_pcall(L, 0, 0, 0), "vanilla call_member"); + } +} + +void userdata_variable_access_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + registerBasic(L); + luaDoStringOrThrow(L, "b = c()", "vanilla userdata setup"); + luaDoStringOrThrow(L, "function access_var() b.var = b.var + 1.0 return b.var end", "vanilla userdata closure setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "access_var"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "vanilla access_var"); + lua_pop(L, 1); + } +} + +void userdata_variable_access_large_measure(benchmark::State& state) +{ + setSkipped(state, "unsupported in LuaBridge vanilla benchmark parity mode"); +} + +void userdata_variable_access_last_measure(benchmark::State& state) +{ + setSkipped(state, "unsupported in LuaBridge vanilla benchmark parity mode"); +} + +void stateful_function_object_measure(benchmark::State& state) +{ + setSkipped(state, "unsupported in LuaBridge vanilla benchmark parity mode"); +} + +void multi_return_lua_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::getGlobalNamespace(L).addCFunction("f", vanilla_multi_return); + luaDoStringOrThrow(L, "function invoke_multi() local a,b=f(24.0) return a+b end", "vanilla multi_return_lua setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "invoke_multi"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "vanilla invoke_multi"); + lua_pop(L, 1); + } +} + +void multi_return_measure(benchmark::State& state) +{ + setSkipped(state, "unsupported conceptual multi-return conversion in LuaBridge vanilla"); +} + +void base_derived_measure(benchmark::State& state) +{ + setSkipped(state, "unsupported for multi inheritance in LuaBridge vanilla"); +} + +void return_userdata_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + registerBasic(L); + luabridge::getGlobalNamespace(L) + .addFunction("f", &basic_return) + .addFunction("h", &basic_get_var); + luaDoStringOrThrow(L, "function invoke_userdata() return h(f()) end", "vanilla return_userdata setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "invoke_userdata"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "vanilla invoke_userdata"); + lua_pop(L, 1); + } +} + +void optional_success_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "warble = { value = 24.0 }", "vanilla optional_success setup"); + + double x = 0; + for (auto _ : state) + { + (void) _; + luabridge::LuaRef tt = luabridge::getGlobal(L, "warble"); + if (tt.isTable()) + { + luabridge::LuaRef tv = tt["value"]; + x += tv.isNumber() ? static_cast(tv) : 1.0; + } + else + { + x += 1.0; + } + } + + benchmark::DoNotOptimize(x); +} + +void optional_half_failure_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "warble = { value = 'x' }", "vanilla optional_half_failure setup"); + + double x = 0; + for (auto _ : state) + { + (void) _; + luabridge::LuaRef tt = luabridge::getGlobal(L, "warble"); + if (tt.isTable()) + { + luabridge::LuaRef tv = tt["value"]; + x += tv.isNumber() ? static_cast(tv) : 1.0; + } + else + { + x += 1.0; + } + } + + benchmark::DoNotOptimize(x); +} + +void optional_failure_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + + double x = 0; + for (auto _ : state) + { + (void) _; + luabridge::LuaRef tt = luabridge::getGlobal(L, "warble"); + if (tt.isTable()) + { + luabridge::LuaRef tv = tt["value"]; + x += tv.isNumber() ? static_cast(tv) : 1.0; + } + else + { + x += 1.0; + } + } + + benchmark::DoNotOptimize(x); +} + +void implicit_inheritance_measure(benchmark::State& state) +{ + setSkipped(state, "unsupported for multi inheritance in LuaBridge vanilla"); +} + +} // namespace + +BENCHMARK(table_global_string_get_measure)->Name("table_global_string_get_measure"); +BENCHMARK(table_global_string_set_measure)->Name("table_global_string_set_measure"); +BENCHMARK(table_get_measure)->Name("table_get_measure"); +BENCHMARK(table_set_measure)->Name("table_set_measure"); +BENCHMARK(table_chained_get_measure)->Name("table_chained_get_measure"); +BENCHMARK(table_chained_set_measure)->Name("table_chained_set_measure"); +BENCHMARK(c_function_measure)->Name("c_function_measure"); +BENCHMARK(c_function_through_lua_in_c_measure)->Name("c_function_through_lua_in_c_measure"); +BENCHMARK(lua_function_in_c_measure)->Name("lua_function_in_c_measure"); +BENCHMARK(member_function_call_measure)->Name("member_function_call_measure"); +BENCHMARK(userdata_variable_access_measure)->Name("userdata_variable_access_measure"); +BENCHMARK(userdata_variable_access_large_measure)->Name("userdata_variable_access_large_measure"); +BENCHMARK(userdata_variable_access_last_measure)->Name("userdata_variable_access_last_measure"); +BENCHMARK(multi_return_lua_measure)->Name("multi_return_lua_measure"); +BENCHMARK(multi_return_measure)->Name("multi_return_measure"); +BENCHMARK(stateful_function_object_measure)->Name("stateful_function_object_measure"); +BENCHMARK(base_derived_measure)->Name("base_derived_measure"); +BENCHMARK(return_userdata_measure)->Name("return_userdata_measure"); +BENCHMARK(optional_success_measure)->Name("optional_success_measure"); +BENCHMARK(optional_half_failure_measure)->Name("optional_half_failure_measure"); +BENCHMARK(optional_failure_measure)->Name("optional_failure_measure"); +BENCHMARK(implicit_inheritance_measure)->Name("implicit_inheritance_measure"); diff --git a/Benchmarks/benchmark_luabridge3.cpp b/Benchmarks/benchmark_luabridge3.cpp new file mode 100644 index 00000000..b8f855da --- /dev/null +++ b/Benchmarks/benchmark_luabridge3.cpp @@ -0,0 +1,523 @@ +// https://github.com/kunitoki/LuaBridge3 +// Copyright 2026, kunitoki +// Inspired from https://github.com/ThePhD/lua-bindings-shootout by ThePhD +// SPDX-License-Identifier: MIT + +#include "benchmark_common.hpp" + +#include "LuaBridge/LuaBridge.h" + +#include + +#include +#include + +namespace { + +using namespace lbsbench; + +std::tuple lb3_multi_return(double value) +{ + return { value, value * 2.0 }; +} + +void registerBasic(lua_State* L) +{ + luabridge::getGlobalNamespace(L) + .beginClass("c") + .addConstructor() + .addFunction("set", &Basic::set) + .addFunction("get", &Basic::get) + .addProperty("var", &Basic::var) + .endClass(); +} + +void registerBasicLarge(lua_State* L) +{ + luabridge::getGlobalNamespace(L) + .beginClass("cl") + .addConstructor() + .addProperty("var", &BasicLarge::var) + .addProperty("var0", &BasicLarge::var0) + .addProperty("var1", &BasicLarge::var1) + .addProperty("var2", &BasicLarge::var2) + .addProperty("var3", &BasicLarge::var3) + .addProperty("var4", &BasicLarge::var4) + .addProperty("var5", &BasicLarge::var5) + .addProperty("var6", &BasicLarge::var6) + .addProperty("var7", &BasicLarge::var7) + .addProperty("var8", &BasicLarge::var8) + .addProperty("var9", &BasicLarge::var9) + .addProperty("var10", &BasicLarge::var10) + .addProperty("var11", &BasicLarge::var11) + .addProperty("var12", &BasicLarge::var12) + .addProperty("var13", &BasicLarge::var13) + .addProperty("var14", &BasicLarge::var14) + .addProperty("var15", &BasicLarge::var15) + .addProperty("var16", &BasicLarge::var16) + .addProperty("var17", &BasicLarge::var17) + .addProperty("var18", &BasicLarge::var18) + .addProperty("var19", &BasicLarge::var19) + .addProperty("var20", &BasicLarge::var20) + .addProperty("var21", &BasicLarge::var21) + .addProperty("var22", &BasicLarge::var22) + .addProperty("var23", &BasicLarge::var23) + .addProperty("var24", &BasicLarge::var24) + .addProperty("var25", &BasicLarge::var25) + .addProperty("var26", &BasicLarge::var26) + .addProperty("var27", &BasicLarge::var27) + .addProperty("var28", &BasicLarge::var28) + .addProperty("var29", &BasicLarge::var29) + .addProperty("var30", &BasicLarge::var30) + .addProperty("var31", &BasicLarge::var31) + .addProperty("var32", &BasicLarge::var32) + .addProperty("var33", &BasicLarge::var33) + .addProperty("var34", &BasicLarge::var34) + .addProperty("var35", &BasicLarge::var35) + .addProperty("var36", &BasicLarge::var36) + .addProperty("var37", &BasicLarge::var37) + .addProperty("var38", &BasicLarge::var38) + .addProperty("var39", &BasicLarge::var39) + .addProperty("var40", &BasicLarge::var40) + .addProperty("var41", &BasicLarge::var41) + .addProperty("var42", &BasicLarge::var42) + .addProperty("var43", &BasicLarge::var43) + .addProperty("var44", &BasicLarge::var44) + .addProperty("var45", &BasicLarge::var45) + .addProperty("var46", &BasicLarge::var46) + .addProperty("var47", &BasicLarge::var47) + .addProperty("var48", &BasicLarge::var48) + .addProperty("var49", &BasicLarge::var49) + .endClass(); +} + +lua_State* makeLua() +{ + lua_State* L = luaL_newstate(); + luaL_openlibs(L); + luabridge::registerMainThread(L); +#if LUABRIDGE_HAS_EXCEPTIONS + luabridge::enableExceptions(L); +#endif + return L; +} + +void table_global_string_get_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::setGlobal(L, kMagicValue, "value"); + + double x = 0; + for (auto _ : state) + { + (void) _; + x += static_cast(luabridge::getGlobal(L, "value")); + } + benchmark::DoNotOptimize(x); +} + +void table_global_string_set_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + + double v = 0; + for (auto _ : state) + { + (void) _; + v += kMagicValue; + luabridge::setGlobal(L, v, "value"); + } + + benchmark::DoNotOptimize(v); +} + +void table_get_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "warble = { value = 24.0 }", "table_get setup"); + luabridge::LuaRef t = luabridge::getGlobal(L, "warble"); + + double x = 0; + for (auto _ : state) + { + (void) _; + x += static_cast(t["value"]); + } + + benchmark::DoNotOptimize(x); +} + +void table_set_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "warble = { value = 24.0 }", "table_set setup"); + luabridge::LuaRef t = luabridge::getGlobal(L, "warble"); + + double v = 0; + for (auto _ : state) + { + (void) _; + v += kMagicValue; + t["value"] = v; + } + + benchmark::DoNotOptimize(v); +} + +void table_chained_get_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "ulahibe = { warble = { value = 24.0 } }", "table_chained_get setup"); + + double x = 0; + for (auto _ : state) + { + (void) _; + luabridge::LuaRef tw = luabridge::getGlobal(L, "ulahibe")["warble"]; + x += static_cast(tw["value"]); + } + + benchmark::DoNotOptimize(x); +} + +void table_chained_set_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "ulahibe = { warble = { value = 24.0 } }", "table_chained_set setup"); + + double v = 0; + for (auto _ : state) + { + (void) _; + v += kMagicValue; + luabridge::LuaRef tw = luabridge::getGlobal(L, "ulahibe")["warble"]; + tw["value"] = v; + } + + benchmark::DoNotOptimize(v); +} + +void c_function_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::getGlobalNamespace(L).addFunction("f", +[](double v) { return v; }); + + luaDoStringOrThrow(L, "function invoke_f() return f(24.0) end", "c_function setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "invoke_f"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "invoke_f"); + lua_pop(L, 1); + } +} + +void lua_function_in_c_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "function f(i) return i end", "lua_function setup"); + + luabridge::LuaRef f = luabridge::getGlobal(L, "f"); + double x = 0; + for (auto _ : state) + { + (void) _; + x += f.call(kMagicValue).valueOr(0.0); + } + + benchmark::DoNotOptimize(x); +} + +void c_function_through_lua_in_c_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::getGlobalNamespace(L).addFunction("f", +[](double v) { return v; }); + + luabridge::LuaRef f = luabridge::getGlobal(L, "f"); + double x = 0; + for (auto _ : state) + { + (void) _; + x += f.call(kMagicValue).valueOr(0.0); + } + + benchmark::DoNotOptimize(x); +} + +void member_function_call_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + registerBasic(L); + luaDoStringOrThrow(L, "b = c()", "member_function setup"); + luaDoStringOrThrow(L, "function call_member() b:set(b:get() + 1.0) end", "member_function closure setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "call_member"); + luaCheckOrThrow(L, lua_pcall(L, 0, 0, 0), "call_member"); + } +} + +void userdata_variable_access_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + registerBasic(L); + luaDoStringOrThrow(L, "b = c()", "userdata_variable_access setup"); + luaDoStringOrThrow(L, "function access_var() return b.var end", "userdata_variable_access closure setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "access_var"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "access_var"); + lua_pop(L, 1); + } +} + +void userdata_variable_access_large_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + registerBasicLarge(L); + luaDoStringOrThrow(L, "b = cl()", "userdata_variable_access_large setup"); + luaDoStringOrThrow(L, "function access_var_large() return b.var0 end", "userdata_variable_access_large closure setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "access_var_large"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "access_var_large"); + lua_pop(L, 1); + } +} + +void userdata_variable_access_last_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + registerBasicLarge(L); + luaDoStringOrThrow(L, "b = cl()", "userdata_variable_access_last setup"); + luaDoStringOrThrow(L, "function access_var_last() return b.var49 end", "userdata_variable_access_last closure setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "access_var_last"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "access_var_last"); + lua_pop(L, 1); + } +} + +void stateful_function_object_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::getGlobalNamespace(L).addFunction("f", StatefulFunction{}); + + luabridge::LuaRef f = luabridge::getGlobal(L, "f"); + double x = 0; + for (auto _ : state) + { + (void) _; + x += f.call(kMagicValue).valueOr(0.0); + } + + benchmark::DoNotOptimize(x); +} + +void multi_return_lua_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::getGlobalNamespace(L).addFunction("f", &lb3_multi_return); + luaDoStringOrThrow(L, "function invoke_multi() local a,b=f(24.0) return a+b end", "multi_return_lua setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "invoke_multi"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "invoke_multi"); + lua_pop(L, 1); + } +} + +void multi_return_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luabridge::getGlobalNamespace(L).addFunction("f", &lb3_multi_return); + luabridge::LuaRef f = luabridge::getGlobal(L, "f"); + + double x = 0; + for (auto _ : state) + { + (void) _; + auto result = f.call>(kMagicValue).valueOr(std::make_tuple(0.0, 0.0)); + x += std::get<0>(result); + x += std::get<1>(result); + } + + benchmark::DoNotOptimize(x); +} + +void base_derived_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + + luabridge::getGlobalNamespace(L) + .beginClass("ComplexBaseA") + .addFunction("a_func", &ComplexBaseA::a_func) + .addProperty("a", &ComplexBaseA::a) + .endClass() + .beginClass("ComplexBaseB") + .addFunction("b_func", &ComplexBaseB::b_func) + .addProperty("b", &ComplexBaseB::b) + .endClass() + .deriveClass("ComplexAB") + .addConstructor() + .addFunction("ab_func", &ComplexAB::ab_func) + .addProperty("ab", &ComplexAB::ab) + .endClass(); + + luaDoStringOrThrow(L, "obj = ComplexAB()", "base_derived setup"); + luaDoStringOrThrow(L, "function call_base() return obj:a_func() + obj:b_func() end", "base_derived closure setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "call_base"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "call_base"); + lua_pop(L, 1); + } +} + +void optional_success_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "warble = { value = 24.0 }", "optional_success setup"); + + double x = 0; + for (auto _ : state) + { + (void) _; + luabridge::LuaRef warble = luabridge::getGlobal(L, "warble"); + if (warble.isTable()) + { + auto result = warble.getField("value"); + x += result ? *result : 1.0; + } + else + x += 1.0; + } + + benchmark::DoNotOptimize(x); +} + +void optional_half_failure_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + luaDoStringOrThrow(L, "warble = { value = 'x' }", "optional_half_failure setup"); + + double x = 0; + for (auto _ : state) + { + (void) _; + luabridge::LuaRef warble = luabridge::getGlobal(L, "warble"); + if (warble.isTable()) + { + auto result = warble.getField("value"); + x += result ? *result : 1.0; + } + else + x += 1.0; + } + + benchmark::DoNotOptimize(x); +} + +void optional_failure_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + + double x = 0; + for (auto _ : state) + { + (void) _; + luabridge::LuaRef warble = luabridge::getGlobal(L, "warble"); + if (warble.isTable()) + { + auto result = warble.getField("value"); + x += result ? *result : 1.0; + } + else + x += 1.0; + } + + benchmark::DoNotOptimize(x); +} + +void return_userdata_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + registerBasic(L); + luabridge::getGlobalNamespace(L) + .addFunction("f", &basic_return) + .addFunction("h", &basic_get_var); + luaDoStringOrThrow(L, "function invoke_userdata() return h(f()) end", "return_userdata setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "invoke_userdata"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "invoke_userdata"); + lua_pop(L, 1); + } +} + +void implicit_inheritance_measure(benchmark::State& state) +{ + lua_State* L = makeLua(); + + luabridge::getGlobalNamespace(L) + .beginClass("ComplexBaseA") + .addFunction("a_func", &ComplexBaseA::a_func) + .endClass() + .deriveClass("ComplexAB") + .addConstructor() + .addFunction("ab_func", &ComplexAB::ab_func) + .endClass() + .addFunction("call_a", +[](ComplexBaseA* obj) -> double { return obj->a_func(); }); + + luaDoStringOrThrow(L, "obj = ComplexAB()", "implicit_inheritance setup"); + luaDoStringOrThrow(L, "function test_implicit() return call_a(obj) end", "implicit_inheritance closure setup"); + + for (auto _ : state) + { + (void) _; + lua_getglobal(L, "test_implicit"); + luaCheckOrThrow(L, lua_pcall(L, 0, 1, 0), "test_implicit"); + lua_pop(L, 1); + } +} + +} // namespace + +BENCHMARK(table_global_string_get_measure)->Name("table_global_string_get_measure"); +BENCHMARK(table_global_string_set_measure)->Name("table_global_string_set_measure"); +BENCHMARK(table_get_measure)->Name("table_get_measure"); +BENCHMARK(table_set_measure)->Name("table_set_measure"); +BENCHMARK(table_chained_get_measure)->Name("table_chained_get_measure"); +BENCHMARK(table_chained_set_measure)->Name("table_chained_set_measure"); +BENCHMARK(c_function_measure)->Name("c_function_measure"); +BENCHMARK(c_function_through_lua_in_c_measure)->Name("c_function_through_lua_in_c_measure"); +BENCHMARK(lua_function_in_c_measure)->Name("lua_function_in_c_measure"); +BENCHMARK(member_function_call_measure)->Name("member_function_call_measure"); +BENCHMARK(userdata_variable_access_measure)->Name("userdata_variable_access_measure"); +BENCHMARK(userdata_variable_access_large_measure)->Name("userdata_variable_access_large_measure"); +BENCHMARK(userdata_variable_access_last_measure)->Name("userdata_variable_access_last_measure"); +BENCHMARK(multi_return_lua_measure)->Name("multi_return_lua_measure"); +BENCHMARK(multi_return_measure)->Name("multi_return_measure"); +BENCHMARK(stateful_function_object_measure)->Name("stateful_function_object_measure"); +BENCHMARK(base_derived_measure)->Name("base_derived_measure"); +BENCHMARK(return_userdata_measure)->Name("return_userdata_measure"); +BENCHMARK(optional_success_measure)->Name("optional_success_measure"); +BENCHMARK(optional_half_failure_measure)->Name("optional_half_failure_measure"); +BENCHMARK(optional_failure_measure)->Name("optional_failure_measure"); +BENCHMARK(implicit_inheritance_measure)->Name("implicit_inheritance_measure"); diff --git a/Benchmarks/benchmark_sol3.cpp b/Benchmarks/benchmark_sol3.cpp new file mode 100644 index 00000000..9c0aaae2 --- /dev/null +++ b/Benchmarks/benchmark_sol3.cpp @@ -0,0 +1,464 @@ +// https://github.com/kunitoki/LuaBridge3 +// Copyright 2026, kunitoki +// Inspired from https://github.com/ThePhD/lua-bindings-shootout by ThePhD +// SPDX-License-Identifier: MIT + +#include "benchmark_common.hpp" + +#include + +#include + +#include + +namespace { + +using namespace lbsbench; + +std::tuple sol3_multi_return(double value) +{ + return { value, value * 2.0 }; +} + +void registerBasic(sol::state& lua) +{ + lua.new_usertype("c", + sol::constructors(), + "set", &Basic::set, + "get", &Basic::get, + "var", &Basic::var); +} + +void registerBasicLarge(sol::state& lua) +{ + lua.new_usertype("cl", + sol::constructors(), + "var", &BasicLarge::var, + "var0", &BasicLarge::var0, + "var1", &BasicLarge::var1, + "var2", &BasicLarge::var2, + "var3", &BasicLarge::var3, + "var4", &BasicLarge::var4, + "var5", &BasicLarge::var5, + "var6", &BasicLarge::var6, + "var7", &BasicLarge::var7, + "var8", &BasicLarge::var8, + "var9", &BasicLarge::var9, + "var10", &BasicLarge::var10, + "var11", &BasicLarge::var11, + "var12", &BasicLarge::var12, + "var13", &BasicLarge::var13, + "var14", &BasicLarge::var14, + "var15", &BasicLarge::var15, + "var16", &BasicLarge::var16, + "var17", &BasicLarge::var17, + "var18", &BasicLarge::var18, + "var19", &BasicLarge::var19, + "var20", &BasicLarge::var20, + "var21", &BasicLarge::var21, + "var22", &BasicLarge::var22, + "var23", &BasicLarge::var23, + "var24", &BasicLarge::var24, + "var25", &BasicLarge::var25, + "var26", &BasicLarge::var26, + "var27", &BasicLarge::var27, + "var28", &BasicLarge::var28, + "var29", &BasicLarge::var29, + "var30", &BasicLarge::var30, + "var31", &BasicLarge::var31, + "var32", &BasicLarge::var32, + "var33", &BasicLarge::var33, + "var34", &BasicLarge::var34, + "var35", &BasicLarge::var35, + "var36", &BasicLarge::var36, + "var37", &BasicLarge::var37, + "var38", &BasicLarge::var38, + "var39", &BasicLarge::var39, + "var40", &BasicLarge::var40, + "var41", &BasicLarge::var41, + "var42", &BasicLarge::var42, + "var43", &BasicLarge::var43, + "var44", &BasicLarge::var44, + "var45", &BasicLarge::var45, + "var46", &BasicLarge::var46, + "var47", &BasicLarge::var47, + "var48", &BasicLarge::var48, + "var49", &BasicLarge::var49); +} + +void table_global_string_get_measure(benchmark::State& state) +{ + sol::state lua; + lua["value"] = kMagicValue; + + double x = 0; + for (auto _ : state) + { + (void) _; + x += lua["value"].get(); + } + benchmark::DoNotOptimize(x); +} + +void table_global_string_set_measure(benchmark::State& state) +{ + sol::state lua; + double v = 0; + + for (auto _ : state) + { + (void) _; + v += kMagicValue; + lua["value"] = v; + } + + benchmark::DoNotOptimize(v); +} + +void table_get_measure(benchmark::State& state) +{ + sol::state lua; + lua.script("warble = { value = 24.0 }"); + sol::table t = lua["warble"]; + + double x = 0; + for (auto _ : state) + { + (void) _; + x += t["value"].get(); + } + + benchmark::DoNotOptimize(x); +} + +void table_set_measure(benchmark::State& state) +{ + sol::state lua; + lua.script("warble = { value = 24.0 }"); + sol::table t = lua["warble"]; + + double v = 0; + for (auto _ : state) + { + (void) _; + v += kMagicValue; + t["value"] = v; + } + + benchmark::DoNotOptimize(v); +} + +void table_chained_get_measure(benchmark::State& state) +{ + sol::state lua; + lua.script("ulahibe = { warble = { value = 24.0 } }"); + + double x = 0; + for (auto _ : state) + { + (void) _; + x += lua["ulahibe"]["warble"]["value"].get(); + } + + benchmark::DoNotOptimize(x); +} + +void table_chained_set_measure(benchmark::State& state) +{ + sol::state lua; + lua.script("ulahibe = { warble = { value = 24.0 } }"); + + double v = 0; + for (auto _ : state) + { + (void) _; + v += kMagicValue; + lua["ulahibe"]["warble"]["value"] = v; + } + + benchmark::DoNotOptimize(v); +} + +void c_function_measure(benchmark::State& state) +{ + sol::state lua; + lua.set_function("f", +[](double value) { return value; }); + lua.script("function invoke_f() return f(24.0) end"); + + for (auto _ : state) + { + (void) _; + lua["invoke_f"](); + } +} + +void lua_function_in_c_measure(benchmark::State& state) +{ + sol::state lua; + lua.script("function f(i) return i end"); + sol::function f = lua["f"]; + + double x = 0; + for (auto _ : state) + { + (void) _; + x += f.call(kMagicValue); + } + + benchmark::DoNotOptimize(x); +} + +void c_function_through_lua_in_c_measure(benchmark::State& state) +{ + sol::state lua; + lua.set_function("f", +[](double value) { return value; }); + sol::function f = lua["f"]; + + double x = 0; + for (auto _ : state) + { + (void) _; + x += f.call(kMagicValue); + } + + benchmark::DoNotOptimize(x); +} + +void member_function_call_measure(benchmark::State& state) +{ + sol::state lua; + registerBasic(lua); + lua.script("b = c.new()\nfunction call_member() b:set(b:get() + 1.0) end"); + + for (auto _ : state) + { + (void) _; + lua["call_member"](); + } +} + +void userdata_variable_access_measure(benchmark::State& state) +{ + sol::state lua; + registerBasic(lua); + lua.script("b = c.new()\nfunction access_var() b.var = b.var + 1.0 return b.var end"); + + for (auto _ : state) + { + (void) _; + lua["access_var"](); + } +} + +void userdata_variable_access_large_measure(benchmark::State& state) +{ + sol::state lua; + registerBasicLarge(lua); + lua.script("b = cl.new()\nfunction access_var_large() b.var0 = b.var0 + 1 return b.var0 end"); + + for (auto _ : state) + { + (void) _; + lua["access_var_large"](); + } +} + +void userdata_variable_access_last_measure(benchmark::State& state) +{ + sol::state lua; + registerBasicLarge(lua); + lua.script("b = cl.new()\nfunction access_var_last() b.var49 = b.var49 + 1 return b.var49 end"); + + for (auto _ : state) + { + (void) _; + lua["access_var_last"](); + } +} + +void stateful_function_object_measure(benchmark::State& state) +{ + sol::state lua; + lua.set_function("f", StatefulFunction{}); + sol::function f = lua["f"]; + + double x = 0; + for (auto _ : state) + { + (void) _; + x += f.call(kMagicValue); + } + + benchmark::DoNotOptimize(x); +} + +void multi_return_lua_measure(benchmark::State& state) +{ + sol::state lua; + lua.set_function("f", &sol3_multi_return); + lua.script("function invoke_multi() local a,b=f(24.0) return a+b end"); + + for (auto _ : state) + { + (void) _; + lua["invoke_multi"](); + } +} + +void multi_return_measure(benchmark::State& state) +{ + sol::state lua; + lua.set_function("f", &sol3_multi_return); + sol::function f = lua["f"]; + + double x = 0; + for (auto _ : state) + { + (void) _; + std::tuple values = f.call(kMagicValue); + x += std::get<0>(values); + x += std::get<1>(values); + } + + benchmark::DoNotOptimize(x); +} + +void base_derived_measure(benchmark::State& state) +{ + sol::state lua; + + lua.new_usertype("ComplexBaseA", + "a_func", &ComplexBaseA::a_func, + "a", &ComplexBaseA::a); + + lua.new_usertype("ComplexBaseB", + "b_func", &ComplexBaseB::b_func, + "b", &ComplexBaseB::b); + + lua.new_usertype("ComplexAB", + sol::base_classes, sol::bases(), + "ab_func", &ComplexAB::ab_func, + "ab", &ComplexAB::ab); + + ComplexAB ab; + lua["b"] = &ab; + + lua.script("function call_base() return b:a_func() + b:b_func() end"); + + for (auto _ : state) + { + (void) _; + lua["call_base"](); + } +} + +void optional_success_measure(benchmark::State& state) +{ + sol::state lua; + lua.script("warble = { value = 24.0 }"); + + double x = 0; + for (auto _ : state) + { + (void) _; + sol::optional value = lua["warble"]["value"]; + x += value.value_or(1.0); + } + + benchmark::DoNotOptimize(x); +} + +void optional_half_failure_measure(benchmark::State& state) +{ + sol::state lua; + lua.script("warble = { value = 'x' }"); + + double x = 0; + for (auto _ : state) + { + (void) _; + sol::optional value = lua["warble"]["value"]; + x += value.value_or(1.0); + } + + benchmark::DoNotOptimize(x); +} + +void optional_failure_measure(benchmark::State& state) +{ + sol::state lua; + + double x = 0; + for (auto _ : state) + { + (void) _; + sol::optional value = lua["warble"]["value"]; + x += value.value_or(1.0); + } + + benchmark::DoNotOptimize(x); +} + +void return_userdata_measure(benchmark::State& state) +{ + sol::state lua; + registerBasic(lua); + lua.set_function("f", &basic_return); + lua.set_function("h", &basic_get_var); + lua.script("function invoke_userdata() return h(f()) end"); + + for (auto _ : state) + { + (void) _; + lua["invoke_userdata"](); + } +} + +void implicit_inheritance_measure(benchmark::State& state) +{ + sol::state lua; + + lua.new_usertype("ComplexBaseA", + "a_func", &ComplexBaseA::a_func); + + lua.new_usertype("ComplexAB", + sol::constructors(), + sol::base_classes, sol::bases(), + "ab_func", &ComplexAB::ab_func); + + lua.set_function("call_a", +[](ComplexBaseA* obj) -> double { return obj->a_func(); }); + + lua.script("obj = ComplexAB.new()"); + lua.script("function test_implicit() return call_a(obj) end"); + + for (auto _ : state) + { + (void) _; + lua["test_implicit"](); + } +} + +} // namespace + +BENCHMARK(table_global_string_get_measure)->Name("table_global_string_get_measure"); +BENCHMARK(table_global_string_set_measure)->Name("table_global_string_set_measure"); +BENCHMARK(table_get_measure)->Name("table_get_measure"); +BENCHMARK(table_set_measure)->Name("table_set_measure"); +BENCHMARK(table_chained_get_measure)->Name("table_chained_get_measure"); +BENCHMARK(table_chained_set_measure)->Name("table_chained_set_measure"); +BENCHMARK(c_function_measure)->Name("c_function_measure"); +BENCHMARK(c_function_through_lua_in_c_measure)->Name("c_function_through_lua_in_c_measure"); +BENCHMARK(lua_function_in_c_measure)->Name("lua_function_in_c_measure"); +BENCHMARK(member_function_call_measure)->Name("member_function_call_measure"); +BENCHMARK(userdata_variable_access_measure)->Name("userdata_variable_access_measure"); +BENCHMARK(userdata_variable_access_large_measure)->Name("userdata_variable_access_large_measure"); +BENCHMARK(userdata_variable_access_last_measure)->Name("userdata_variable_access_last_measure"); +BENCHMARK(multi_return_lua_measure)->Name("multi_return_lua_measure"); +BENCHMARK(multi_return_measure)->Name("multi_return_measure"); +BENCHMARK(stateful_function_object_measure)->Name("stateful_function_object_measure"); +BENCHMARK(base_derived_measure)->Name("base_derived_measure"); +BENCHMARK(return_userdata_measure)->Name("return_userdata_measure"); +BENCHMARK(optional_success_measure)->Name("optional_success_measure"); +BENCHMARK(optional_half_failure_measure)->Name("optional_half_failure_measure"); +BENCHMARK(optional_failure_measure)->Name("optional_failure_measure"); +BENCHMARK(implicit_inheritance_measure)->Name("implicit_inheritance_measure"); diff --git a/Benchmarks/plot_benchmarks.py b/Benchmarks/plot_benchmarks.py new file mode 100644 index 00000000..c8880604 --- /dev/null +++ b/Benchmarks/plot_benchmarks.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: MIT + +import argparse +import json +import math +import os +from collections import defaultdict + +import matplotlib.pyplot as plt +import matplotlib.patches as mpatches +import numpy as np + + +# ── Label helpers ───────────────────────────────────────────────────────────── + +_SUFFIX = "_measure" + +def _clean_label(name: str) -> str: + """Convert a raw benchmark name to a human-readable label.""" + if name.endswith(_SUFFIX): + name = name[: -len(_SUFFIX)] + return name.replace("_", " ") + + +# ── JSON loading ────────────────────────────────────────────────────────────── + +def infer_library_name(path: str) -> str: + stem = os.path.splitext(os.path.basename(path))[0] + return stem.replace("benchmark_", "") + + +def load_google_benchmark_json(path: str, library_name: str) -> dict: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + + case_values: dict[str, float] = {} + case_errors: dict[str, str] = {} + + for entry in data.get("benchmarks", []): + name = entry.get("name", "") + run_type = entry.get("run_type", "") + + # Prefer aggregate mean when available + if run_type == "aggregate" and entry.get("aggregate_name") == "mean": + base_name = entry.get("run_name", name) + case_values[base_name] = entry.get("real_time", entry.get("cpu_time", 0.0)) + continue + + if run_type not in ("iteration", ""): + continue + + if entry.get("error_occurred"): + case_errors[name] = entry.get("error_message", "error") + continue + + if name not in case_values: + case_values[name] = entry.get("real_time", entry.get("cpu_time", 0.0)) + + return {"library": library_name, "values": case_values, "errors": case_errors} + + +# ── Merge ───────────────────────────────────────────────────────────────────── + +def merge_results(result_sets): + merged: dict[str, dict[str, float]] = defaultdict(dict) + errors: dict[str, dict[str, str]] = defaultdict(dict) + + for result in result_sets: + lib = result["library"] + for case_name, value in result["values"].items(): + merged[case_name][lib] = value + for case_name, error in result["errors"].items(): + errors[case_name][lib] = error + + return merged, errors + + +# ── Plotting ────────────────────────────────────────────────────────────────── + +# Dark theme colours +_BG = "#1E1E2E" # figure / axes background +_FG = "#CDD6F4" # text, ticks, labels +_GRID = "#313244" # grid lines +_SPINE = "#45475A" # axis spines +_UNSUP = "#585B70" # "unsupported" text + +# Bright palette suited for dark backgrounds +_PALETTE = [ + "#89B4FA", # blue + "#FAB387", # peach + "#A6E3A1", # green + "#F38BA8", # red + "#CBA6F7", # mauve + "#94E2D5", # teal + "#F9E2AF", # yellow + "#89DCEB", # sky +] + + +def plot_grouped_bars(merged: dict, errors: dict, output_file: str, log_scale: bool = False) -> None: + case_names = sorted(merged.keys()) + libraries = sorted({lib for cases in merged.values() for lib in cases}) + + if not case_names or not libraries: + raise RuntimeError("No benchmark samples found to plot") + + n_cases = len(case_names) + n_libs = len(libraries) + + # ── Layout ──────────────────────────────────────────────────────────────── + bar_h = 0.80 + group_h = bar_h / n_libs + fig_h = max(10, n_cases * bar_h + 2) + + plt.rcParams.update({ + "text.color": _FG, + "axes.labelcolor": _FG, + "xtick.color": _FG, + "ytick.color": _FG, + }) + + fig, ax = plt.subplots(figsize=(12, fig_h)) + fig.patch.set_facecolor(_BG) + ax.set_facecolor(_BG) + + colors = {lib: _PALETTE[i % len(_PALETTE)] for i, lib in enumerate(libraries)} + + y_positions = np.arange(n_cases, dtype=float) + + for i, library in enumerate(libraries): + values = [merged[cn].get(library, float("nan")) for cn in case_names] + offsets = (i - (n_libs - 1) / 2.0) * group_h + bar_y = y_positions + offsets + ax.barh( + bar_y, + values, + height=group_h * 0.85, + color=colors[library], + label=library, + zorder=4, + ) + + # "unsupported" label where no value exists + for y, val in zip(bar_y, values): + if math.isnan(val): + ax.text( + 0, y, + " unsupported", + va="center", ha="left", + fontsize=7, color=_UNSUP, style="italic", + zorder=5, + ) + + # ── Axes ────────────────────────────────────────────────────────────────── + ax.set_yticks(y_positions) + ax.set_yticklabels([_clean_label(cn) for cn in case_names], fontsize=9) + ax.invert_yaxis() + + if log_scale: + ax.set_xscale("log") + ax.set_xlabel("Time (ns, log scale)", fontsize=10) + else: + ax.set_xlabel("Time (ns)", fontsize=10) + + ax.xaxis.grid(True, color=_GRID, linestyle="--", alpha=1.0, zorder=2) + ax.set_axisbelow(True) + for spine in ax.spines.values(): + spine.set_edgecolor(_SPINE) + ax.spines["top"].set_visible(True) + ax.spines["right"].set_visible(False) + + # Tick marks and labels on both top and bottom x-axis + ax.xaxis.set_tick_params(which="both", top=True, bottom=True, labeltop=True, labelbottom=True) + ax.tick_params(axis="x", which="both", color=_SPINE) + + # ── Legend & title ──────────────────────────────────────────────────────── + legend_handles = [ + mpatches.Patch(color=colors[lib], label=lib) for lib in libraries + ] + legend = ax.legend( + handles=legend_handles, + loc="upper right", + fontsize=9, + framealpha=1.0, + facecolor=_SPINE, + edgecolor=_SPINE, + labelcolor=_FG, + ) + ax.set_title( + "Lua Binding Benchmarks — lower is better", + fontsize=13, pad=14, fontweight="bold", color=_FG, + ) + + fig.subplots_adjust(left=0.22, right=0.97, top=0.95, bottom=0.03) + plt.savefig(output_file, dpi=150, facecolor=fig.get_facecolor()) + plt.close() + + plt.rcParams.update({ + "text.color": "black", + "axes.labelcolor": "black", + "xtick.color": "black", + "ytick.color": "black", + }) + + # ── Text summary ────────────────────────────────────────────────────────── + txt_file = os.path.splitext(output_file)[0] + ".txt" + col_w = max(len(lib) for lib in libraries) + 2 + label_w = max(len(_clean_label(cn)) for cn in case_names) + 2 + + with open(txt_file, "w", encoding="utf-8") as f: + header = f"{'Benchmark':<{label_w}}" + "".join(f"{lib:>{col_w}}" for lib in libraries) + f.write(header + "\n") + f.write("-" * len(header) + "\n") + for cn in case_names: + row = f"{_clean_label(cn):<{label_w}}" + for lib in libraries: + val = merged[cn].get(lib) + cell = f"{val:>{col_w - 3}.1f} ns" if val is not None else f"{'n/a':>{col_w}}" + row += cell + f.write(row + "\n") + print(f"Saved: {txt_file}") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="Plot comparisons from Google Benchmark JSON files" + ) + parser.add_argument( + "--input", nargs="+", required=True, + help="One or more Google Benchmark JSON files" + ) + parser.add_argument( + "--output", default="Benchmarks/benchmark_comparison.png", + help="Output PNG file" + ) + parser.add_argument( + "--log", action="store_true", + help="Use a logarithmic x-axis" + ) + args = parser.parse_args() + + result_sets = [ + load_google_benchmark_json(path, infer_library_name(path)) + for path in args.input + ] + + merged, errors = merge_results(result_sets) + + output_dir = os.path.dirname(args.output) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + + plot_grouped_bars(merged, errors, args.output, log_scale=args.log) + print(f"Saved: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/Images/.gitignore b/Images/.gitignore new file mode 100644 index 00000000..d7a099a6 --- /dev/null +++ b/Images/.gitignore @@ -0,0 +1,2 @@ +!.gitignore +*.txt \ No newline at end of file diff --git a/Images/benchmarks.png b/Images/benchmarks.png new file mode 100644 index 00000000..1038bc1a Binary files /dev/null and b/Images/benchmarks.png differ diff --git a/README.md b/README.md index a4dff896..de6e820e 100644 --- a/README.md +++ b/README.md @@ -60,26 +60,9 @@ LuaBridge3 is usable from a compliant C++17 compiler and offers the following fe ## Performance -LuaBridge3 has been heavily optimized and now competes directly with [sol2](https://github.com/ThePhD/sol2) — one of the fastest C++/Lua binding libraries — across most workloads. In some cases (e.g. member function calls from Lua) LuaBridge3 is actually **faster** than sol2: - -| Case | LuaBridge3 (ns/op) | sol2 (ns/op) | -|------------------------------|--------------------|--------------| -| lua_empty_loop | 2.10 | 2.07 | -| lua_to_cpp_free_fn | 28.30 | 21.54 | -| lua_to_cpp_member | **70.23** | 101.13 | -| lua_to_cpp_property | 139.29 | 125.34 | -| lua_to_cpp_property_set | 70.70 | 62.58 | -| lua_to_cpp_property_get | 64.65 | 61.98 | -| cpp_table_global_get | 15.24 | 12.60 | -| cpp_table_global_set | **9.99** | 11.31 | -| cpp_table_get | 26.24 | 18.54 | -| cpp_table_set | 23.41 | 17.74 | -| cpp_table_chained_get | 70.55 | 27.87 | -| cpp_table_chained_set | 66.87 | 26.50 | -| cpp_to_lua_call | 33.41 | 30.92 | -| cpp_c_function_through_lua | 52.52 | 46.47 | - -Bold entries indicate cases where LuaBridge3 outperforms sol2. Lower is better (nanoseconds per operation). +LuaBridge3 has been heavily optimized and now competes directly with [sol2](https://github.com/ThePhD/sol2) — one of the fastest C++/Lua binding libraries — across most workloads. + +![Benchmarks](./Images/benchmarks.png) ## Improvements Over Vanilla LuaBridge diff --git a/Source/LuaBridge/detail/CFunctions.h b/Source/LuaBridge/detail/CFunctions.h index bdc357db..b8621190 100644 --- a/Source/LuaBridge/detail/CFunctions.h +++ b/Source/LuaBridge/detail/CFunctions.h @@ -104,6 +104,24 @@ auto pop_arguments(lua_State* L, std::tuple& t) return pop_arguments(L, t); } +//================================================================================================= +/** + * @brief Push a tuple on the stack. + */ +template +Result push_tuple_impl(lua_State* L, const Tuple& value, std::index_sequence) +{ + Result result; + (void)((result = Stack>>::push(L, std::get(value)), bool(result)) && ...); + return result; +} + +template +Result push_tuple(lua_State* L, const std::tuple& value) +{ + return push_tuple_impl(L, value, std::index_sequence_for{}); +} + //================================================================================================= /** * @brief Check if a method name is a metamethod. @@ -1507,13 +1525,21 @@ struct function static int call(lua_State* L, F&& func) { Result result; + int numResults = 1; #if LUABRIDGE_HAS_EXCEPTIONS try { #endif - result = Stack::push(L, invoke_callable_from_stack(L, std::forward(func))); - + if constexpr (detail::is_tuple_v) + { + numResults = static_cast(std::tuple_size_v); + result = detail::push_tuple(L, invoke_callable_from_stack(L, std::forward(func))); + } + else + { + result = Stack::push(L, invoke_callable_from_stack(L, std::forward(func))); + } #if LUABRIDGE_HAS_EXCEPTIONS } catch (const std::exception& e) @@ -1525,20 +1551,28 @@ struct function if (! result) raise_lua_error(L, "%s", result.error_cstr()); - return 1; + return numResults; } template static int call(lua_State* L, T* ptr, F&& func) { Result result; + int numResults = 1; #if LUABRIDGE_HAS_EXCEPTIONS try { #endif - result = Stack::push(L, invoke_member_callable_from_stack(L, ptr, std::forward(func))); - + if constexpr (detail::is_tuple_v) + { + numResults = static_cast(std::tuple_size_v); + result = detail::push_tuple(L, invoke_member_callable_from_stack(L, ptr, std::forward(func))); + } + else + { + result = Stack::push(L, invoke_member_callable_from_stack(L, ptr, std::forward(func))); + } #if LUABRIDGE_HAS_EXCEPTIONS } catch (const std::exception& e) @@ -1550,7 +1584,51 @@ struct function if (! result) raise_lua_error(L, "%s", result.error_cstr()); - return 1; + return numResults; + } +}; + +template +struct function, ArgsPack, Start> : function +{ + template + static int call(lua_State* L, F&& func) + { +#if LUABRIDGE_HAS_EXCEPTIONS + try + { +#endif + invoke_callable_from_stack(L, std::forward(func)); + +#if LUABRIDGE_HAS_EXCEPTIONS + } + catch (const std::exception& e) + { + raise_lua_error(L, "%s", e.what()); + } +#endif + + return 0; + } + + template + static int call(lua_State* L, T* ptr, F&& func) + { +#if LUABRIDGE_HAS_EXCEPTIONS + try + { +#endif + invoke_member_callable_from_stack(L, ptr, std::forward(func)); + +#if LUABRIDGE_HAS_EXCEPTIONS + } + catch (const std::exception& e) + { + raise_lua_error(L, "%s", e.what()); + } +#endif + + return 0; } }; diff --git a/Source/LuaBridge/detail/Invoke.h b/Source/LuaBridge/detail/Invoke.h index f0b64b56..fa52ee79 100644 --- a/Source/LuaBridge/detail/Invoke.h +++ b/Source/LuaBridge/detail/Invoke.h @@ -31,18 +31,8 @@ bool is_handler_valid(const F& f) noexcept return true; } -template -struct IsTuple : std::false_type -{ -}; - -template -struct IsTuple> : std::true_type -{ -}; - template -TypeResult decodeTupleResult(lua_State* L, int firstResultIndex, std::index_sequence) +TypeResult decode_tuple_result(lua_State* L, int first_result_index, std::index_sequence) { Tuple value; std::error_code ec; @@ -52,7 +42,7 @@ TypeResult decodeTupleResult(lua_State* L, int firstResultIndex, std::ind { using ElementType = std::tuple_element_t; - auto element = Stack::get(L, firstResultIndex + static_cast(Indices)); + auto element = Stack::get(L, first_result_index + static_cast(Indices)); if (! element) { ec = element.error(); @@ -71,30 +61,29 @@ TypeResult decodeTupleResult(lua_State* L, int firstResultIndex, std::ind } template -TypeResult decodeCallResult(lua_State* L, int firstResultIndex, int numReturnedValues) +TypeResult decode_call_result(lua_State* L, int first_result_index, int num_returned_values) { if constexpr (std::is_same_v || std::is_same_v>) { - if (numReturnedValues != 0) + if (num_returned_values != 0) return makeErrorCode(ErrorCode::InvalidTableSizeInCast); return {}; } - else - if constexpr (IsTuple::value) + else if constexpr (is_tuple_v) { - constexpr auto expectedSize = static_cast(std::tuple_size_v); - if (numReturnedValues != expectedSize) + constexpr auto expected_size = static_cast(std::tuple_size_v); + if (num_returned_values != expected_size) return makeErrorCode(ErrorCode::InvalidTableSizeInCast); - return decodeTupleResult(L, firstResultIndex, std::make_index_sequence>{}); + return decode_tuple_result(L, first_result_index, std::make_index_sequence{}); } else { - if (numReturnedValues < 1) + if (num_returned_values < 1) return makeErrorCode(ErrorCode::InvalidTypeCast); - return Stack::get(L, firstResultIndex); + return Stack::get(L, first_result_index); } } @@ -155,7 +144,7 @@ TypeResult callWithHandler(const Ref& object, F&& errorHandler, Args&&... arg const int firstResultIndex = initialTop + 1; const int numReturnedValues = lua_gettop(L) - initialTop; - return detail::decodeCallResult(L, firstResultIndex, numReturnedValues); + return detail::decode_call_result(L, firstResultIndex, numReturnedValues); } template diff --git a/Source/LuaBridge/detail/TypeTraits.h b/Source/LuaBridge/detail/TypeTraits.h index 8f8c8d6c..38b4a54d 100644 --- a/Source/LuaBridge/detail/TypeTraits.h +++ b/Source/LuaBridge/detail/TypeTraits.h @@ -9,6 +9,7 @@ #include "Config.h" #include +#include namespace luabridge { namespace detail { @@ -27,6 +28,18 @@ static inline constexpr bool is_base_of_template_v = is_base_of_template:: template constexpr bool dependent_false = false; +template +struct is_tuple : std::false_type +{ +}; + +template +struct is_tuple> : std::true_type +{ +}; + +template +constexpr bool is_tuple_v = is_tuple::value; } // namespace detail //================================================================================================= diff --git a/Source/LuaBridge/detail/Userdata.h b/Source/LuaBridge/detail/Userdata.h index a09d4e06..35e6aa3f 100644 --- a/Source/LuaBridge/detail/Userdata.h +++ b/Source/LuaBridge/detail/Userdata.h @@ -297,21 +297,33 @@ class Userdata const auto classId = detail::getClassRegistryKey(); const auto constId = detail::getConstRegistryKey(); - // Common-case fast path: exact class/const metatable match. - // This avoids parent-chain traversal and registry table lookups. - if (lua_getmetatable(L, absIndex) && lua_istable(L, -1)) + // Common-case fast path: compare the object's metatable directly against + // the registry class/const tables. This avoids an extra interior table + // lookup (getTypeIdentityKey) and the lua_istable guard (lua_getmetatable + // always pushes a table when it returns non-zero). Safe because scripts + // cannot set metatables on userdata (Lua security model, points 1-3). + if (lua_getmetatable(L, absIndex)) // Stack: ..., mt { - lua_rawgetp_x(L, -1, detail::getTypeIdentityKey()); - const void* identity = lua_touserdata(L, -1); - lua_pop(L, 1); - - if (identity == classId || (canBeConst && identity == constId)) + lua_rawgetp_x(L, LUA_REGISTRYINDEX, classId); // Stack: ..., mt, class_mt + if (lua_rawequal(L, -2, -1)) { - lua_pop(L, 1); + lua_pop(L, 2); return static_cast(static_cast(lua_touserdata(L, absIndex))->getPointer()); } + lua_pop(L, 1); // Stack: ..., mt + + if (canBeConst) + { + lua_rawgetp_x(L, LUA_REGISTRYINDEX, constId); // Stack: ..., mt, const_mt + if (lua_rawequal(L, -2, -1)) + { + lua_pop(L, 2); + return static_cast(static_cast(lua_touserdata(L, absIndex))->getPointer()); + } + lua_pop(L, 1); // Stack: ..., mt + } - lua_pop(L, 1); + lua_pop(L, 1); // Stack: ... } auto clazz = getClass(L, absIndex, constId, classId, canBeConst); diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index e7bca939..a4bef1f5 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -16,6 +16,9 @@ if (APPLE OR UNIX) GIT_REPOSITORY https://github.com/bombela/backward-cpp GIT_TAG master SYSTEM) + if (APPLE) + set(STACK_DETAILS_AUTO_DETECT OFF CACHE BOOL "Disable automatic detection of stack details for backward-cpp") + endif() FetchContent_MakeAvailable (backward) endif() diff --git a/Tests/Source/Tests.cpp b/Tests/Source/Tests.cpp index 0661c3d6..5f2a7457 100644 --- a/Tests/Source/Tests.cpp +++ b/Tests/Source/Tests.cpp @@ -311,9 +311,13 @@ TEST_F(LuaBridgeTest, TupleAsFunctionReturnValue) .addFunction("test", [x](Inner*) { return std::make_tuple(x, 42); }) .endClass(); - runLua("x = Inner () result = x:test ()"); - EXPECT_EQ(true, result().isTable()); - EXPECT_EQ(std::make_tuple(x, 42), (result>())); + runLua("x = Inner () result1, result2 = x:test ()"); + auto z1 = luabridge::getGlobal(L, "result1"); + auto z2 = luabridge::getGlobal(L, "result2"); + EXPECT_EQ(true, z1.isNumber()); + EXPECT_EQ(true, z2.isNumber()); + EXPECT_EQ(100, z1.unsafe_cast()); + EXPECT_EQ(42, z2.unsafe_cast()); } namespace { diff --git a/justfile b/justfile index a421aafe..34916b27 100644 --- a/justfile +++ b/justfile @@ -10,8 +10,16 @@ sanitize TYPE='address': benchmark: cmake -G Xcode -B Build -DLUABRIDGE_BENCHMARKS=ON . - cmake --build Build --config Release --target LuaBridgeBenchmarks -j8 - ./Build/Benchmarks/Release/LuaBridgeBenchmarks 1000000 + cmake --build Build --config Release --target LuaBridge3Benchmark -j8 + cmake --build Build --config Release --target LuaBridgeVanillaBenchmark -j8 + cmake --build Build --config Release --target Sol3Benchmark -j8 + ./Build/Benchmarks/Release/LuaBridge3Benchmark --benchmark_out_format=json --benchmark_out=Build/LuaBridge3Benchmark.json + ./Build/Benchmarks/Release/LuaBridgeVanillaBenchmark --benchmark_out_format=json --benchmark_out=Build/LuaBridgeVanillaBenchmark.json + ./Build/Benchmarks/Release/Sol3Benchmark --benchmark_out_format=json --benchmark_out=Build/Sol3Benchmark.json + @just plot + +plot: + python3.14 Benchmarks/plot_benchmarks.py --input Build/LuaBridge3Benchmark.json Build/LuaBridgeVanillaBenchmark.json Build/Sol3Benchmark.json --output Images/benchmarks.png clean: rm -rf Build