diff --git a/src/include/firebird/impl/msg/jrd.h b/src/include/firebird/impl/msg/jrd.h index 0595ee9da2e..bccd2a899d2 100644 --- a/src/include/firebird/impl/msg/jrd.h +++ b/src/include/firebird/impl/msg/jrd.h @@ -1020,3 +1020,4 @@ FB_IMPL_MSG(JRD, 1017, dsql_agg_non_agg_context, -104, "42", "000", "Aggregate f FB_IMPL_MSG(JRD, 1018, dsql_agg_param_not_accum, -204, "42", "000", "Aggregate function input parameters may be referenced only in ON ACCUMULATE DO") FB_IMPL_MSG(JRD, 1019, dsql_agg_exit_group, -204, "42", "000", "EXIT is not allowed in ON GROUP DO section of aggregate function") FB_IMPL_MSG(JRD, 1020, dsql_agg_return, -204, "42", "000", "RETURN is not allowed in ON START DO, ON ACCUMULATE DO or ON FINISH DO sections of aggregate function; use EXIT instead") +FB_IMPL_MSG(JRD, 1021, blob_out_of_length_write, -204, "42", "000", "Cannot write to blob. Position @1 is out of length @2") diff --git a/src/include/gen/Firebird.pas b/src/include/gen/Firebird.pas index 93e2512ee4e..b4e03a69234 100644 --- a/src/include/gen/Firebird.pas +++ b/src/include/gen/Firebird.pas @@ -6062,6 +6062,7 @@ IPerformanceStatsImpl = class(IPerformanceStats) isc_dsql_agg_param_not_accum = 335545338; isc_dsql_agg_exit_group = 335545339; isc_dsql_agg_return = 335545340; + isc_blob_out_of_length_write = 335545341; isc_gfix_db_name = 335740929; isc_gfix_invalid_sw = 335740930; isc_gfix_incmp_sw = 335740932; diff --git a/src/jrd/blb.cpp b/src/jrd/blb.cpp index c4fdb295d13..6a0c3e08b91 100644 --- a/src/jrd/blb.cpp +++ b/src/jrd/blb.cpp @@ -90,6 +90,78 @@ static void move_to_string(Jrd::thread_db*, dsc*, dsc*); static void slice_callback(array_slice*, ULONG, dsc*); static blb* store_array(thread_db*, jrd_tra*, bid*); +namespace { + +class ReplaceDataHelper +{ +public: + ReplaceDataHelper(thread_db* tdbb, const vcl* blb_pages, const offset_t position, const void* buffer, const ULONG length) : + m_newData(buffer), m_newLength(length), + m_level1Pages(*blb_pages), + m_pageDataLength(tdbb->getDatabase()->dbb_page_size - BLP_SIZE) + { + m_level1PageId = position / m_pageDataLength; // Number of used pages + m_offset = position % m_pageDataLength; // Position in the page + } + + inline void replaceInPage(blob_page* page) noexcept + { + UCHAR* data = reinterpret_cast(page->blp_page); + const ULONG dataLength = std::min(m_pageDataLength - m_offset, m_newLength - m_written); + fb_assert(dataLength <= m_pageDataLength); + + memcpy(data + m_offset, reinterpret_cast(m_newData) + m_written, dataLength); + m_written += dataLength; + m_offset = 0; // Offset only in the first page + }; + + // Move child page Id from level 1 to level 2 + inline ULONG setLevel2(const USHORT pagesOnRootPage) + { + const auto pageId = m_level1PageId; + m_level1PageId = pageId / pagesOnRootPage; // 100000 / 8000 = 12 // level1 page number + return pageId % pagesOnRootPage; // 100000 % 8000 = 4000 // level2 page number + } + + // Get level 1 or level 2 page + inline ULONG getNextLevel1PageId() noexcept + { + return m_level1Pages[m_level1PageId++]; + } + + // Pages are over, write to buffer + inline bool hasPages() const noexcept + { + return m_level1PageId < m_level1Pages.count(); + } + + inline bool needWrite() const noexcept + { + return m_written < m_newLength; + } + + [[maybe_unused]] + inline ULONG getWrittenLength() const noexcept + { + return m_written; + } + +private: + // Where to replace + offset_t m_offset = 0; + + // New Data + const void* m_newData; + const ULONG m_newLength; + + ULONG m_level1PageId = 0; + const vcl& m_level1Pages; + const USHORT m_pageDataLength; + + ULONG m_written = 0; +}; + +} void blb::BLB_cancel(thread_db* tdbb) { @@ -3033,3 +3105,103 @@ void blb::BLB_cancel() { BLB_cancel(JRD_get_thread_data()); } + +void blb::BLB_write(thread_db* tdbb, offset_t position, const void* buffer, const ULONG length) +{ + if (!(blb_flags & BLB_temporary) || (blb_flags & BLB_closed)) + ERR_post(Arg::Gds(isc_cannot_update_old_blob)); + + if (position > blb_length) + ERR_post(Arg::Gds(isc_blob_out_of_length_write) << Arg::Int64(position) << Arg::Int64(blb_length)); + + const offset_t end = position + length; + if (end > blb_length) + { + const offset_t middle = blb_length - position; + BLB_write(tdbb, position, buffer, middle); // Replace + BLB_put_segment(tdbb, (const UCHAR*)buffer + middle, length - middle); // Append + return; + } + + if (blb_level == 0) + { + blob_page* page = (blob_page*) getBuffer(); + memcpy(reinterpret_cast(page->blp_page) + position, buffer, length); + return; + } + + ReplaceDataHelper helper(tdbb, blb_pages, position, buffer, length); + blob_page* page = nullptr; + + WIN window(blb_pg_space_id, -1); + if (blb_flags & BLB_large_scan) + { + window.win_flags = WIN_large_scan; + window.win_scans = 1; + } + + auto releasePage = [&tdbb, &window](const bool mark) + { + if (mark) + CCH_MARK(tdbb, &window); + + if (window.win_flags & WIN_large_scan) + CCH_RELEASE_TAIL(tdbb, &window); + else + CCH_RELEASE(tdbb, &window); + }; + + // Level 1 blobs are much easier -- page number is in vector. + if (blb_level == 1) + { + while (helper.needWrite()) + { + if (!helper.hasPages()) // The last data is in the blb_buffer + { + page = reinterpret_cast(getBuffer()); + helper.replaceInPage(page); + fb_assert(helper.getWrittenLength() == length); + return; + } + + // Level 1 page constains data + window.win_page = helper.getNextLevel1PageId(); + page = reinterpret_cast(CCH_FETCH(tdbb, &window, LCK_write, pag_blob)); + helper.replaceInPage(page); + releasePage(true); + } + } + else + { + auto level2page = helper.setLevel2(blb_pointers); + while (helper.needWrite()) + { + if (!helper.hasPages()) // The last data is in the blb_buffer + { + helper.replaceInPage(page); + fb_assert(helper.getWrittenLength() == length); + return; + } + + // Level 1 page contains pointers + window.win_page = helper.getNextLevel1PageId(); + page = reinterpret_cast(CCH_FETCH(tdbb, &window, LCK_write, pag_blob)); + + // Level 2 pages contain data + const auto numberOfPagess = page->blp_length / sizeof(page->blp_page); + for (FB_SIZE_T i = level2page; i < numberOfPagess && helper.needWrite(); ++i) + { + auto level2Page = reinterpret_cast(CCH_HANDOFF(tdbb, &window, + page->blp_page[i], + LCK_write, pag_blob)); + + helper.replaceInPage(level2Page); + CCH_MARK(tdbb, &window); + } + releasePage(false); + + level2page = 0; // Offset only for the first pages + } + } + fb_assert(helper.getWrittenLength() == length); +} diff --git a/src/jrd/blb.h b/src/jrd/blb.h index 1a3ded79023..8aa0773df5f 100644 --- a/src/jrd/blb.h +++ b/src/jrd/blb.h @@ -133,6 +133,11 @@ class blb : public pool_alloc return destination; } + // Write data at any position in a temporally (new) blob + // The position of the new buffer must start inside the blob range, but its length may extend beyond it + // Existing data will be overwritten + void BLB_write(thread_db* tdbb, offset_t position, const void* buffer, const ULONG length); + private: static blb* allocate_blob(thread_db*, jrd_tra*); static blb* copy_blob(thread_db* tdbb, const bid* source, bid* destination, diff --git a/src/jrd/tests/BlobRandomWriteTest.cpp b/src/jrd/tests/BlobRandomWriteTest.cpp new file mode 100644 index 00000000000..15c9dd5c9c6 --- /dev/null +++ b/src/jrd/tests/BlobRandomWriteTest.cpp @@ -0,0 +1,287 @@ +#include "boost/test/unit_test.hpp" +#include "../../jrd/blb.h" + +#include "TestContext.h" + +#include + + +BOOST_AUTO_TEST_SUITE(EngineSuite) +BOOST_AUTO_TEST_SUITE(JrdClassesSuite) + + +BOOST_AUTO_TEST_SUITE(BlobTests) + +BOOST_AUTO_TEST_SUITE(BlobRandomWriteTests) + +namespace { + +static constexpr UCHAR STREAM_BLOB_BPB[] = { + isc_bpb_version1, + isc_bpb_type, 1, isc_bpb_type_stream, +}; + + +std::string getDefaultString(std::string_view header = "", int pageNumber = 0, std::optional dum = std::nullopt) +{ + std::string output; + output += header; + + // Bigger page size - level 1 + // Bigger 2 pages - level 2 + auto tdbb = JRD_get_thread_data(); + auto size = tdbb->getDatabase()->dbb_page_size; + + for (int i = 0 ; i < pageNumber; i++) + { + std::string dummy; + dummy.resize(size, dum.value_or('0' + i)); + output += dummy; + } + + return output; +} + +Jrd::blb* makeBlob(Jrd::bid& id, std::string_view testData = "") +{ + auto tdbb = JRD_get_thread_data(); + + Jrd::blb* blob = Jrd::blb::create2(tdbb, tdbb->getTransaction(), &id, sizeof(STREAM_BLOB_BPB), STREAM_BLOB_BPB); + BOOST_REQUIRE(blob != nullptr); + + blob->BLB_put_data(tdbb, (const UCHAR*)testData.data(), testData.length()); + + return blob; +} + + +std::string readBlob(Jrd::bid id) +{ + auto tdbb = JRD_get_thread_data(); + + auto blob = Jrd::blb::open(tdbb, tdbb->getTransaction(), &id); + + std::string buffer; + buffer.resize(blob->blb_length, '\0'); + const ULONG readLength = blob->BLB_get_data(JRD_get_thread_data(), (UCHAR*)buffer.data(), blob->blb_length, true); + return buffer; +} + +void replaceInBlob(Jrd::thread_db* tdbb, Jrd::blb*& blob, const ULONG pos, const std::string_view replacement) +{ + blob->BLB_write(tdbb, pos, replacement.data(), replacement.length()); + blob->BLB_close(tdbb); + blob = nullptr; +} + +std::string replaceInContent(std::string defaultData, ULONG posToInplace, std::string_view contentToInplace) +{ + auto sourceLength = defaultData.length(); + + const auto replacementEnd = posToInplace + contentToInplace.length(); + if (replacementEnd < sourceLength) + { + for (ULONG i = 0; i < contentToInplace.length(); ++i) + { + defaultData[posToInplace + i] = contentToInplace[i]; + } + } + else + { + defaultData.resize(posToInplace); + defaultData += contentToInplace; + } + return defaultData; +} + +} // anonymous namespace + +BOOST_FIXTURE_TEST_CASE(Level0Test, EngineHolder) +{ + Jrd::bid id; + + { // level 0 + const std::string_view testData = "Hello World, BLB_get_data, level=0"; + + // Full rewrite + auto blob = makeBlob(id, testData); + std::string buffer; + buffer.resize(blob->blb_length, '*'); + replaceInBlob(tdbb, blob, 0, buffer); + BOOST_TEST(readBlob(id) == buffer); + + // Middle write + blob = makeBlob(id, testData); + replaceInBlob(tdbb, blob, 12, " __BLB_write_,"); + BOOST_TEST(readBlob(id) == "Hello World, __BLB_write_, level=0"); + + // Ending is out of range - add to end + blob = makeBlob(id, testData); + replaceInBlob(tdbb, blob, 27, testData); + BOOST_TEST(readBlob(id) == "Hello World, BLB_get_data, Hello World, BLB_get_data, level=0"); + + // Beginning is out of range + blob = makeBlob(id, testData); + BOOST_CHECK_THROW(blob->BLB_write(tdbb, 40, (const void*)testData.data(), testData.length()), Firebird::Exception); + } +} + +BOOST_FIXTURE_TEST_CASE(Level1Test, EngineHolder) +{ + Jrd::bid id; + Jrd::blb* blob = nullptr; + + std::string result; + std::string expected; + + const std::string_view testData = "Hello World, BLB_get_data, level=1 | "; + std::string defaultData = getDefaultString(testData, 1); + { + // Full rewrite + auto blob = makeBlob(id, defaultData); + + replaceInBlob(tdbb, blob, 0, "new data"); + + result = readBlob(id); + expected = replaceInContent(defaultData, 0, "new data"); + BOOST_TEST(result == expected); + } + + std::string replacement; + + { + // Middle to end write + replacement.resize(200, '*'); + blob = makeBlob(id, defaultData); + replaceInBlob(tdbb, blob, blob->blb_length - 200, replacement); + + result = readBlob(id); + expected = replaceInContent(defaultData, defaultData.length() - 200, replacement); + BOOST_TEST(result.length() == expected.length()); + BOOST_REQUIRE(result.substr(0, 400) == expected.substr(0, 400)); + BOOST_REQUIRE(result.substr(result.length() - 300) == expected.substr(expected.length() - 300)); + BOOST_TEST(result == expected); + } + + { + // Middle + replacement.resize(200, '*'); + blob = makeBlob(id, defaultData); + replaceInBlob(tdbb, blob, blob->blb_length - 4000, replacement); + + result = readBlob(id); + expected = replaceInContent(defaultData, defaultData.length() - 4000, replacement); + BOOST_TEST(result.length() == expected.length()); + BOOST_REQUIRE(result.substr(result.length() - 4000, 300) == expected.substr(expected.length() - 4000, 300)); + BOOST_TEST(result == expected); + } + + { + // Ending is out of range - add to end + blob = makeBlob(id, defaultData); + replacement.clear(); + replacement.resize(blob->blb_length, '@'); + const auto insertPos = blob->blb_length - 1000; + replaceInBlob(tdbb, blob, insertPos, replacement); + + result = readBlob(id); + expected = replaceInContent(defaultData, insertPos, replacement); + BOOST_TEST(result.length() == expected.length()); + BOOST_REQUIRE(result.substr(0, 400) == expected.substr(0, 400)); + BOOST_REQUIRE(result.substr(result.length() - 300) == expected.substr(expected.length() - 300)); + BOOST_TEST(result == expected); + } + + { + defaultData = getDefaultString(testData, 8); + replacement = getDefaultString(testData, 3, '*'); + + // Big + blob = makeBlob(id, defaultData); + + const auto insertPos = blob->blb_length / 2; + blob->BLB_write(tdbb, insertPos, replacement.data(), replacement.length()); + blob->BLB_close(tdbb); + blob = nullptr; + + result = readBlob(id); + expected = replaceInContent(defaultData, insertPos, replacement); + BOOST_REQUIRE(result.length() == expected.length()); + + std::string_view resultView(result); + std::string_view expectedView(expected); + for (FB_SIZE_T i = 0; i < expected.length(); i += 1000) + { + auto left = std::min(1000, expected.length() - i); + BOOST_TEST(resultView.substr(i, left) == expected.substr(i, left)); + } + } +} + + +BOOST_FIXTURE_TEST_CASE(Level2Test, EngineHolder) +{ + // Takes some time + + Jrd::bid id; + Jrd::blb* blob = nullptr; + + std::string result; + std::string expected; + + const std::string_view testData = "Hello World, BLB_get_data, level=2 | "; + std::string defaultData; + std::string replacement; + + { + blob = makeBlob(id, defaultData); + + const auto insertPos = blob->blb_length / 2; + replaceInBlob(tdbb, blob, insertPos, replacement); + + result = readBlob(id); + expected = replaceInContent(defaultData, insertPos, replacement); + BOOST_REQUIRE(result.length() == expected.length()); + + std::string_view resultView(result); + std::string_view expectedView(expected); + for (FB_SIZE_T i = 0; i < expected.length(); i += 1000) + { + auto left = std::min(1000, expected.length() - i); + BOOST_TEST_INFO("Chunk position is " + std::to_string(i)); + BOOST_TEST(resultView.substr(i, left) == expected.substr(i, left)); + // if (resultView.substr(i, left) != expected.substr(i, left)) + // break; + } + } + + { + defaultData = getDefaultString(testData, 5050); + replacement = getDefaultString(testData, 150, '*'); + + blob = makeBlob(id, defaultData); + + const auto insertPos = 1998; + replaceInBlob(tdbb, blob, insertPos, replacement); + + result = readBlob(id); + expected = replaceInContent(defaultData, insertPos, replacement); + BOOST_REQUIRE(result.length() == expected.length()); + + std::string_view resultView(result); + std::string_view expectedView(expected); + for (FB_SIZE_T i = 0; i < expected.length(); i += 1000) + { + auto left = std::min(1000, expected.length() - i); + BOOST_TEST_INFO("Chunk position is " + std::to_string(i)); + BOOST_TEST(resultView.substr(i, left) == expected.substr(i, left)); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() // BlobRandomWriteTest + +BOOST_AUTO_TEST_SUITE_END() // BlobTests + +BOOST_AUTO_TEST_SUITE_END() // JrdClassesSuite +BOOST_AUTO_TEST_SUITE_END() // EngineSuite