diff --git a/README.md b/README.md index 5b48e275..77efe99d 100644 --- a/README.md +++ b/README.md @@ -320,6 +320,7 @@ sudo build/Debug/src/test/legacy_ublk/homeblk_ublk \ | `--vol_size_mb` / `--page_size` | Volume size and logical block size (default 4096) when creating. | | `--num_threads` | HomeBlocks iomgr reactor count. | | `--data_chunk_size_mb` / `--index_chunk_size_mb` | HomeStore chunk sizing (see the note below). | +| `--dynamic_chunk_allocation=` | Lazy (thin) data chunk allocation for new volumes: `true` starts a volume with a few chunks and grows on demand, `false` pre-allocates all of its chunks at create. Overrides the `dynamic_chunk_allocation` dynamic config (default `true`). | | `--device_id ` | ublk device id: `-1` to assign one, `>=0` to recover a kernel-preserved device. | In another terminal, drive I/O at the printed device. All raw I/O is `O_DIRECT` with the volume's page size as diff --git a/conanfile.py b/conanfile.py index ab53d813..1251fe3c 100644 --- a/conanfile.py +++ b/conanfile.py @@ -10,7 +10,7 @@ class HomeBlocksConan(ConanFile): name = "homeblocks" - version = "6.0.10" + version = "6.0.11" homepage = "https://github.com/eBay/HomeBlocks" description = "Block Store built on HomeStore" diff --git a/src/lib/home_blks_config.fbs b/src/lib/home_blks_config.fbs index d5550a08..aad51f83 100644 --- a/src/lib/home_blks_config.fbs +++ b/src/lib/home_blks_config.fbs @@ -22,6 +22,9 @@ table HomeBlksSettings{ // homestore dataservice chunk size; hs_data_chunk_size_mb: uint32 = 2048; + // lazy chunk allocator on/off + dynamic_chunk_allocation: bool = true; + // how often (in appended LSNs) the leader auto-proposes a SyncRSCommitLSN entry; also gates how // often a commit_lsn advance triggers a HomeStore checkpoint (SDSTOR-22888, CraftReplDev:: // checkpoint_lsn_interval_), tying checkpoint cadence to this same periodic cadence; diff --git a/src/lib/homeblks_impl.cpp b/src/lib/homeblks_impl.cpp index 60c9f851..e8f84593 100644 --- a/src/lib/homeblks_impl.cpp +++ b/src/lib/homeblks_impl.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "homeblks_impl.hpp" #include "listener.hpp" @@ -62,6 +63,10 @@ std::future< void > HomeBlocksImpl::shutdown_start() { LOGI("Setting shutdown start flag"); shutdown_started_.test_and_set(); + // Pending chunk releases skip write-back cache eviction from now on; can_shutdown() waits for them. + if (volume_chunk_selector_) { volume_chunk_selector_->start_shutdown(); } + if (index_chunk_selector_) { index_chunk_selector_->start_shutdown(); } + auto f = shutdown_promise_.get_future(); auto const nsecs = shutdown_timer_nsecs(); @@ -115,7 +120,9 @@ bool HomeBlocksImpl::no_outstanding_vols() const { bool HomeBlocksImpl::can_shutdown() const { // check if shutdown has started and no outstanding requests; - if (is_shutting_down() && no_outstanding_vols() && outstanding_reqs_.test_eq(0)) { + // A destroyed volume may still be releasing its chunks: that work touches homestore and its allocator resets must + // be persisted by homestore's final checkpoint, so homestore must not shut down before it is done. + if (is_shutting_down() && no_outstanding_vols() && outstanding_reqs_.test_eq(0) && !has_pending_chunk_releases()) { LOGI("Shutdown can proceed, outstanding requests: {}", outstanding_reqs_.get()); return true; } @@ -124,6 +131,15 @@ bool HomeBlocksImpl::can_shutdown() const { return false; } +bool HomeBlocksImpl::has_pending_chunk_releases() const { + if ((volume_chunk_selector_ && volume_chunk_selector_->has_pending_releases()) || + (index_chunk_selector_ && index_chunk_selector_->has_pending_releases())) { + LOGI("Chunk releases of destroyed volumes are still pending"); + return true; + } + return false; +} + void HomeBlocksImpl::do_shutdown() { LOGI("Shutdown timer triggered, checking for outstanding requests"); if (can_shutdown()) { @@ -275,6 +291,16 @@ hs_chunk_size_cfg_t HomeBlocksImpl::get_chunk_size() const { return hs_chunk_sz; } +bool HomeBlocksImpl::dynamic_chunk_allocation() const { + if (SISL_OPTIONS.count("dynamic_chunk_allocation")) { + auto const on = SISL_OPTIONS["dynamic_chunk_allocation"].as< bool >(); + LOGI("Using dynamic_chunk_allocation option value: {}", on); + return on; + } + + return HB_DYNAMIC_CONFIG(dynamic_chunk_allocation); +} + void HomeBlocksImpl::init_homestore() { LOGI("Starting iomgr with {} threads", config_.threads); ioenvironment.with_iomgr(iomgr::iomgr_params{.num_threads = config_.threads}).with_http_server(); @@ -297,6 +323,10 @@ void HomeBlocksImpl::init_homestore() { "index", [this](uint64_t volume_ordinal, const std::vector< chunk_num_t >& chunk_ids) { // Todo: decide it later whether needed for index }); + // Only index chunks hold btree nodes in the index write-back cache; data chunk blkids can never be in it. + index_chunk_selector_->set_evict_chunk_fn([](const auto& chunk) { + homestore::hs()->index_service().wb_cache().evict_chunk_blkids(*chunk->get_internal_chunk()); + }); using namespace homestore; // Note: timeline_consistency doesn't matter as we are using solo repl dev; diff --git a/src/lib/homeblks_impl.hpp b/src/lib/homeblks_impl.hpp index ec6f800d..ecd82995 100644 --- a/src/lib/homeblks_impl.hpp +++ b/src/lib/homeblks_impl.hpp @@ -153,6 +153,7 @@ class HomeBlocksImpl : public home_blocks, public std::enable_shared_from_this< bool is_restricted() const { return is_restricted_.load(); } bool is_shutting_down() const { return shutdown_started_.test(); } hs_chunk_size_cfg_t get_chunk_size() const; + bool dynamic_chunk_allocation() const; bool is_graceful_shutdown() const { return gracefully_shutdown_; } public: @@ -191,6 +192,7 @@ class HomeBlocksImpl : public home_blocks, public std::enable_shared_from_this< void inc_ref(uint64_t n = 1) { outstanding_reqs_.increment(n); } void dec_ref(uint64_t n = 1) { outstanding_reqs_.decrement(n); } bool can_shutdown() const; + bool has_pending_chunk_releases() const; bool no_outstanding_vols() const; diff --git a/src/lib/volume/tests/test_common.hpp b/src/lib/volume/tests/test_common.hpp index 2c57318e..d07f9fbb 100644 --- a/src/lib/volume/tests/test_common.hpp +++ b/src/lib/volume/tests/test_common.hpp @@ -59,6 +59,8 @@ SISL_OPTION_GROUP( "number"), (index_chunk_size_mb, "", "index_chunk_size_mb", "index_chunk_size_mb", ::cxxopts::value< uint32_t >()->default_value("128"), "number"), + (dynamic_chunk_allocation, "", "dynamic_chunk_allocation", "lazy chunk allocation on/off", + ::cxxopts::value< bool >()->default_value("true"), "true or false"), (data_chunk_size_mb, "", "data_chunk_size_mb", "data_chunk_size_mb", ::cxxopts::value< uint32_t >()->default_value("128"), "number")); diff --git a/src/lib/volume/tests/test_volume_chunk_selector.cpp b/src/lib/volume/tests/test_volume_chunk_selector.cpp index 9312be97..1e2f6542 100644 --- a/src/lib/volume/tests/test_volume_chunk_selector.cpp +++ b/src/lib/volume/tests/test_volume_chunk_selector.cpp @@ -9,6 +9,12 @@ #include #include "test_common.hpp" +#include +#include +#include +#include +#include + SISL_LOGGING_INIT(HOMEBLOCKS_LOG_MODS) SISL_OPTION_GROUP(test_volume_chunk_selector, (num_vols, "", "num_vols", "number of volumes", ::cxxopts::value< uint32_t >()->default_value("2"), @@ -78,11 +84,25 @@ uint16_t VChunk::get_chunk_id() const { return m_internal_chunk->get_chunk_id(); blk_num_t VChunk::get_total_blks() const { return m_internal_chunk->get_total_blks(); } uint64_t VChunk::size() const { return m_internal_chunk->size(); } + void VChunk::reset() {} + cshared< Chunk > VChunk::get_internal_chunk() const { return m_internal_chunk; } +// void VChunk::reset_block_allocator() {} + } // namespace homestore +template < typename Pred > +void wait_until(Pred&& pred, std::chrono::milliseconds timeout = std::chrono::milliseconds{5000}, + std::chrono::milliseconds poll = std::chrono::milliseconds{50}) { + auto deadline = std::chrono::steady_clock::now() + timeout; + while (!pred() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(poll); + } + ASSERT_TRUE(pred()); +} + class ChunkSelectorTest : public ::testing::Test { public: ChunkSelectorTest() {} @@ -148,6 +168,9 @@ TEST_F(ChunkSelectorTest, AllocateReleaseChunksTest) { chunk_sel->release_chunks(i /* ordinal */); } + wait_until([&] { return chunk_sel->num_free_chunks() == init_num_free_chunks; }, std::chrono::seconds{10}, + std::chrono::milliseconds{100}); + // All the chunks will be free. RELEASE_ASSERT_EQ(init_num_free_chunks, chunk_sel->num_free_chunks(), "num free chunks mismatch"); } @@ -232,6 +255,373 @@ TEST_F(ChunkSelectorTest, RecoverChunksTest) { RELEASE_ASSERT(chunk, "Chunk not available"); } +TEST_F(ChunkSelectorTest, ConcurrentAllocateSameVolumeTest) { + auto chunk_sel = + std::make_shared< VolumeChunkSelector >("test", [this](uint64_t, const std::vector< chunk_num_t >&) {}); + + add_chunks_per_pdev(chunk_sel, 2 /*pdevs*/, 10 /*chunks per pdev*/); + + std::vector< chunk_num_t > r1, r2; + uint32_t pdev1{UINT32_MAX}, pdev2{UINT32_MAX}; + std::atomic< bool > go{false}; + + auto worker = [&](std::vector< chunk_num_t >& out, uint32_t& pdev) { + while (!go.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + out = chunk_sel->allocate_init_chunks(0 /* same volume */, 64 * Ki, pdev); + }; + + std::thread t1(worker, std::ref(r1), std::ref(pdev1)); + std::thread t2(worker, std::ref(r2), std::ref(pdev2)); + + go.store(true, std::memory_order_release); + t1.join(); + t2.join(); + + ASSERT_FALSE(r1.empty()); + ASSERT_FALSE(r2.empty()); + + std::sort(r1.begin(), r1.end()); + std::sort(r2.begin(), r2.end()); + + EXPECT_EQ(r1, r2); + EXPECT_EQ(pdev1, pdev2); + + auto chunks = chunk_sel->get_chunks(0); + ASSERT_EQ(chunks.size(), r1.size()); + + std::unordered_set< chunk_num_t > uniq{r1.begin(), r1.end()}; + EXPECT_EQ(uniq.size(), r1.size()); +} + +TEST_F(ChunkSelectorTest, SelectChunkReturnsNullWhenOutOfSpaceTest) { + auto chunk_sel = + std::make_shared< VolumeChunkSelector >("test", [this](uint64_t, const std::vector< chunk_num_t >&) {}); + + // Single chunk volume, but no space in the chunk + auto chunk = std::make_shared< homestore::Chunk >(0 /*pdev*/, 0 /*chunk_id*/); + chunk->set_available_blks(0); + chunk_sel->add_chunk(chunk); + + uint32_t pdev_id{UINT32_MAX}; + auto ids = chunk_sel->allocate_init_chunks(0 /* ordinal */, 8 * Ki /* <= one chunk */, pdev_id, false); + ASSERT_FALSE(ids.empty()); + + homestore::blk_alloc_hints hints; + hints.application_hint = 0; + + auto start = std::chrono::steady_clock::now(); + auto out = chunk_sel->select_chunk(1 /* nblks */, hints); + auto dur = std::chrono::steady_clock::now() - start; + + EXPECT_EQ(out, nullptr); + EXPECT_LT(dur, std::chrono::seconds(2)); +} + +TEST_F(ChunkSelectorTest, ConcurrentSelectAndReleaseTest) { + auto chunk_sel = + std::make_shared< VolumeChunkSelector >("test", [this](uint64_t, const std::vector< chunk_num_t >&) {}); + + add_chunks_per_pdev(chunk_sel, 2 /*pdevs*/, 10 /*chunks per pdev*/); + const auto total_free_before_alloc = chunk_sel->num_free_chunks(); + + uint32_t pdev_id{UINT32_MAX}; + auto ids = chunk_sel->allocate_init_chunks(0 /* ordinal */, 64 * Ki, pdev_id); + ASSERT_FALSE(ids.empty()); + + homestore::blk_alloc_hints hints; + hints.application_hint = 0; + + std::atomic< bool > done{false}; + std::atomic< uint32_t > select_success{0}; + std::atomic< uint32_t > select_null{0}; + + std::thread selector([&] { + while (!done.load(std::memory_order_acquire)) { + auto chunk = chunk_sel->select_chunk(1 /* nblks */, hints); + if (chunk) { + ++select_success; + } else { + ++select_null; + break; + } + } + }); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::thread releaser([&] { + chunk_sel->release_chunks(0 /* ordinal */); + done.store(true, std::memory_order_release); + }); + + releaser.join(); + selector.join(); + + wait_until([&] { return chunk_sel->num_free_chunks() == total_free_before_alloc; }, + std::chrono::milliseconds{10000}); + + EXPECT_EQ(chunk_sel->num_free_chunks(), total_free_before_alloc); + EXPECT_TRUE(chunk_sel->get_chunks(0).empty()); + EXPECT_GT(select_success.load() + select_null.load(), 0u); +} + +TEST_F(ChunkSelectorTest, ResizeCallbackBlockedThenReleaseTest) { + std::promise< void > cb_entered_promise; + auto cb_entered = cb_entered_promise.get_future(); + + std::promise< void > allow_cb_exit_promise; + auto allow_cb_exit = allow_cb_exit_promise.get_future(); + + auto chunk_sel = std::make_shared< VolumeChunkSelector >( + "test", [&, this](uint64_t, const std::vector< chunk_num_t >& chunk_ids) { + EXPECT_FALSE(chunk_ids.empty()); + cb_entered_promise.set_value(); + allow_cb_exit.wait(); + }); + + add_chunks_per_pdev(chunk_sel, 1 /*pdevs*/, 10 /*chunks per pdev*/); + const auto total_free_before_alloc = chunk_sel->num_free_chunks(); + + uint32_t pdev_id{UINT32_MAX}; + auto ids = chunk_sel->allocate_init_chunks(0 /* ordinal */, 64 * Ki /* max_num_chunks > 1 */, pdev_id, true); + ASSERT_FALSE(ids.empty()); + + // Exhaust currently active chunks so select_chunk() must try resize + auto active_chunks = chunk_sel->get_chunks(0); + ASSERT_FALSE(active_chunks.empty()); + for (auto& c : active_chunks) { + c->get_internal_chunk()->set_available_blks(0); + } + + homestore::blk_alloc_hints hints; + hints.application_hint = 0; + + std::thread selector([&] { (void)chunk_sel->select_chunk(1 /* nblks */, hints); }); + + ASSERT_EQ(cb_entered.wait_for(std::chrono::seconds(2)), std::future_status::ready); + + std::thread releaser([&] { chunk_sel->release_chunks(0 /* ordinal */); }); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + allow_cb_exit_promise.set_value(); + + selector.join(); + releaser.join(); + + wait_until([&] { return chunk_sel->num_free_chunks() == total_free_before_alloc; }, + std::chrono::milliseconds{10000}); + + EXPECT_EQ(chunk_sel->num_free_chunks(), total_free_before_alloc); + EXPECT_TRUE(chunk_sel->get_chunks(0).empty()); +} + +TEST_F(ChunkSelectorTest, RecoverReleaseReallocateTest) { + auto chunk_sel = + std::make_shared< VolumeChunkSelector >("test", [this](uint64_t, const std::vector< chunk_num_t >&) {}); + + add_chunks_per_pdev(chunk_sel, 1 /*pdevs*/, 10 /*chunks per pdev*/); + const auto total_free = chunk_sel->num_free_chunks(); + + std::vector< homestore::chunk_num_t > recovered_ids{0, 1, 2}; + ASSERT_TRUE(chunk_sel->recover_chunks(0 /* ordinal */, 0 /* pdev */, 48 * Ki, recovered_ids)); + + auto recovered_chunks = chunk_sel->get_chunks(0); + ASSERT_EQ(recovered_chunks.size(), recovered_ids.size()); + + chunk_sel->release_chunks(0 /* ordinal */); + + wait_until([&] { return chunk_sel->num_free_chunks() == total_free; }, std::chrono::milliseconds{10000}); + + uint32_t pdev_id{UINT32_MAX}; + auto new_ids = chunk_sel->allocate_init_chunks(1 /* new ordinal */, 48 * Ki, pdev_id, true); + ASSERT_FALSE(new_ids.empty()); + EXPECT_EQ(pdev_id, 0u); +} + +TEST_F(ChunkSelectorTest, ResizeClampsToFreePoolTest) { + auto chunk_sel = + std::make_shared< VolumeChunkSelector >("test", [this](uint64_t, const std::vector< chunk_num_t >&) {}); + + // One pdev with 4 chunks of 16KB. vol0 may grow to 4 chunks and vol1 to 3, so the pool is overcommitted and a + // resize can ask for more chunks than are left + add_chunks_per_pdev(chunk_sel, 1 /*pdevs*/, 4 /*chunks per pdev*/); + + uint32_t pdev_id{UINT32_MAX}; + ASSERT_EQ(chunk_sel->allocate_init_chunks(0 /* ordinal */, 64 * Ki, pdev_id).size(), 1u); + ASSERT_EQ(chunk_sel->allocate_init_chunks(1 /* ordinal */, 48 * Ki, pdev_id).size(), 1u); + ASSERT_EQ(chunk_sel->num_free_chunks(), 2u); + + homestore::blk_alloc_hints hints; + + // vol0 wants 3 more chunks but only 2 are left; it must get those 2 instead of asserting + for (auto& c : chunk_sel->get_chunks(0)) { + c->get_internal_chunk()->set_available_blks(0); + } + hints.application_hint = 0; + EXPECT_NE(chunk_sel->select_chunk(1 /* nblks */, hints), nullptr); + EXPECT_EQ(chunk_sel->get_chunks(0).size(), 3u); + EXPECT_EQ(chunk_sel->num_free_chunks(), 0u); + + // vol1 is full and the pool is empty, so it is out of space. + for (auto& c : chunk_sel->get_chunks(1)) { + c->get_internal_chunk()->set_available_blks(0); + } + hints.application_hint = 1; + auto start = std::chrono::steady_clock::now(); + EXPECT_EQ(chunk_sel->select_chunk(1 /* nblks */, hints), nullptr); + EXPECT_LT(std::chrono::steady_clock::now() - start, std::chrono::seconds(2)); +} + +TEST_F(ChunkSelectorTest, ResizeAheadOfDemandTest) { + auto chunk_sel = + std::make_shared< VolumeChunkSelector >("test", [this](uint64_t, const std::vector< chunk_num_t >&) {}); + + add_chunks_per_pdev(chunk_sel, 1 /*pdevs*/, 10 /*chunks per pdev*/); + + uint32_t pdev_id{UINT32_MAX}; + ASSERT_EQ(chunk_sel->allocate_init_chunks(0 /* ordinal */, 64 * Ki, pdev_id).size(), 1u); + + // The only active chunk is less than half free but not full: select_chunk() must return it right away and grow + // the volume in the background + auto active = chunk_sel->get_chunks(0); + active[0]->get_internal_chunk()->set_available_blks(1); + + homestore::blk_alloc_hints hints; + hints.application_hint = 0; + EXPECT_EQ(chunk_sel->select_chunk(1 /* nblks */, hints), active[0]->get_internal_chunk()); + + wait_until([&] { return chunk_sel->get_chunks(0).size() > 1; }); +} + +TEST_F(ChunkSelectorTest, QuiesceWaitsForRunningResizeTest) { + std::promise< void > cb_entered_promise; + auto cb_entered = cb_entered_promise.get_future(); + std::promise< void > allow_cb_exit_promise; + auto allow_cb_exit = allow_cb_exit_promise.get_future(); + std::atomic< uint32_t > num_cb{0}; + + auto chunk_sel = + std::make_shared< VolumeChunkSelector >("test", [&, this](uint64_t, const std::vector< chunk_num_t >&) { + if (num_cb++ == 0) { cb_entered_promise.set_value(); } + allow_cb_exit.wait(); + }); + + add_chunks_per_pdev(chunk_sel, 1 /*pdevs*/, 10 /*chunks per pdev*/); + const auto total_free = chunk_sel->num_free_chunks(); + + uint32_t pdev_id{UINT32_MAX}; + ASSERT_EQ(chunk_sel->allocate_init_chunks(0 /* ordinal */, 64 * Ki, pdev_id).size(), 1u); + for (auto& c : chunk_sel->get_chunks(0)) { + c->get_internal_chunk()->set_available_blks(0); + } + + homestore::blk_alloc_hints hints; + hints.application_hint = 0; + std::thread selector([&] { (void)chunk_sel->select_chunk(1 /* nblks */, hints); }); + + // The resize worker is now persisting the new chunk list (volume superblock update) + ASSERT_EQ(cb_entered.wait_for(std::chrono::seconds(2)), std::future_status::ready); + + // quiesce_chunks() must not return while the worker can still touch the superblock. + auto quiesced = std::async(std::launch::async, [&] { chunk_sel->quiesce_chunks(0 /* ordinal */); }); + EXPECT_EQ(quiesced.wait_for(std::chrono::milliseconds(200)), std::future_status::timeout); + + allow_cb_exit_promise.set_value(); + ASSERT_EQ(quiesced.wait_for(std::chrono::seconds(2)), std::future_status::ready); + selector.join(); + + // After quiesce: no new resize, and the worker did not publish its chunks to the dying volume + EXPECT_EQ(chunk_sel->select_chunk(1 /* nblks */, hints), nullptr); + EXPECT_EQ(num_cb.load(), 1u); + EXPECT_EQ(chunk_sel->get_chunks(0).size(), 1u); + + chunk_sel->release_chunks(0 /* ordinal */); + wait_until([&] { return chunk_sel->num_free_chunks() == total_free; }); +} + +TEST_F(ChunkSelectorTest, AsyncReleaseWaitsForEvictionTest) { + auto chunk_sel = + std::make_shared< VolumeChunkSelector >("test", [this](uint64_t, const std::vector< chunk_num_t >&) {}); + + // Stand-in for the index write-back cache eviction; blocks until the test allows it + auto allow_evict_promise = std::make_shared< std::promise< void > >(); + std::shared_future< void > allow_evict = allow_evict_promise->get_future().share(); + auto num_evicted = std::make_shared< std::atomic< uint32_t > >(0); + chunk_sel->set_evict_chunk_fn([allow_evict, num_evicted](const auto&) { + allow_evict.wait(); + ++*num_evicted; + }); + + add_chunks_per_pdev(chunk_sel, 1 /*pdevs*/, 10 /*chunks per pdev*/); + const auto total_free = chunk_sel->num_free_chunks(); + + uint32_t pdev_id{UINT32_MAX}; + auto ids = chunk_sel->allocate_init_chunks(0 /* ordinal */, 48 * Ki, pdev_id, false /* lazy alloc */); + ASSERT_EQ(ids.size(), 3u); + + // release_chunks() returns without waiting for eviction, but the chunks must not be reusable until evicted + chunk_sel->release_chunks(0 /* ordinal */); + EXPECT_TRUE(chunk_sel->get_chunks(0).empty()); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(chunk_sel->num_free_chunks(), total_free - ids.size()); + EXPECT_EQ(num_evicted->load(), 0u); + + allow_evict_promise->set_value(); + wait_until([&] { return chunk_sel->num_free_chunks() == total_free; }); + EXPECT_EQ(num_evicted->load(), ids.size()); + + // The released chunks are reset and can be handed to a new volume (allocation asserts they are unowned). + auto new_ids = chunk_sel->allocate_init_chunks(1 /* ordinal */, 160 * Ki, pdev_id, false /* lazy alloc */); + ASSERT_EQ(new_ids.size(), total_free); + for (auto id : ids) { + EXPECT_NE(std::find(new_ids.begin(), new_ids.end(), id), new_ids.end()); + } +} + +TEST_F(ChunkSelectorTest, ShutdownSkipsEvictionOfPendingReleaseTest) { + auto chunk_sel = + std::make_shared< VolumeChunkSelector >("test", [this](uint64_t, const std::vector< chunk_num_t >&) {}); + + // Stand-in for the index write-back cache eviction; blocks until the test allows it + auto evict_entered_promise = std::make_shared< std::promise< void > >(); + auto evict_entered = evict_entered_promise->get_future(); + auto allow_evict_promise = std::make_shared< std::promise< void > >(); + std::shared_future< void > allow_evict = allow_evict_promise->get_future().share(); + auto num_evicted = std::make_shared< std::atomic< uint32_t > >(0); + chunk_sel->set_evict_chunk_fn([evict_entered_promise, allow_evict, num_evicted](const auto&) { + if (num_evicted->fetch_add(1) == 0) { evict_entered_promise->set_value(); } + allow_evict.wait(); + }); + + add_chunks_per_pdev(chunk_sel, 1 /*pdevs*/, 10 /*chunks per pdev*/); + const auto total_free = chunk_sel->num_free_chunks(); + + uint32_t pdev_id{UINT32_MAX}; + auto ids = chunk_sel->allocate_init_chunks(0 /* ordinal */, 48 * Ki, pdev_id, false /* lazy alloc */); + ASSERT_EQ(ids.size(), 3u); + EXPECT_FALSE(chunk_sel->has_pending_releases()); + + chunk_sel->release_chunks(0 /* ordinal */); + EXPECT_TRUE(chunk_sel->has_pending_releases()); + + // Shutdown starts while the release is evicting its first chunk + ASSERT_EQ(evict_entered.wait_for(std::chrono::seconds(2)), std::future_status::ready); + chunk_sel->start_shutdown(); + + // The release still runs, so shutdown has to keep waiting for it + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_TRUE(chunk_sel->has_pending_releases()); + EXPECT_EQ(chunk_sel->num_free_chunks(), total_free - ids.size()); + + allow_evict_promise->set_value(); + wait_until([&] { return !chunk_sel->has_pending_releases(); }); + + // Only the chunk already being evicted was evicted; the rest were just reset and returned to the free pool + EXPECT_EQ(num_evicted->load(), 1u); + EXPECT_EQ(chunk_sel->num_free_chunks(), total_free); +} + int main(int argc, char* argv[]) { int parsed_argc = argc; ::testing::InitGoogleTest(&parsed_argc, argv); diff --git a/src/lib/volume/volume.cpp b/src/lib/volume/volume.cpp index fe2a2a65..6caaf8c7 100644 --- a/src/lib/volume/volume.cpp +++ b/src/lib/volume/volume.cpp @@ -78,9 +78,10 @@ bool volume::init(bool is_recovery) { // first time creation of the volume, let's write the superblock; // Allocate initial set of chunks for the volume with thin provisioning. + shared< HomeBlocksImpl > hb = HomeBlocksImpl::instance(); uint32_t pdev_id; - auto chunk_ids = - volume_chunk_selector_->allocate_init_chunks(vol_info_->ordinal, vol_info_->size_bytes, pdev_id); + auto chunk_ids = volume_chunk_selector_->allocate_init_chunks(vol_info_->ordinal, vol_info_->size_bytes, + pdev_id, hb->dynamic_chunk_allocation()); if (chunk_ids.empty()) { LOGE("Failed to allocate chunks for volume: {}, uuid: {}", vol_info_->name, boost::uuids::to_string(vol_info_->id)); @@ -185,6 +186,10 @@ sisl::async::task< void > volume::destroy() { indx_tbl_ = nullptr; } + // Stop chunk selection and resize for this volume before its superblock goes away: a resize worker persists the + // new chunk list into the volume superblock. + volume_chunk_selector_->quiesce_chunks(vol_info_->ordinal); + // destroy the superblock which will remove sb from meta svc; sb_.destroy(); diff --git a/src/lib/volume/volume_chunk_selector.cpp b/src/lib/volume/volume_chunk_selector.cpp index 8846b994..fc68046f 100644 --- a/src/lib/volume/volume_chunk_selector.cpp +++ b/src/lib/volume/volume_chunk_selector.cpp @@ -29,17 +29,25 @@ void VolumeChunkSelector::add_chunk(homestore::cshared< Chunk >& chunk) { auto vol_chunk = std::make_shared< HBChunk >(chunk); auto chunk_id = homestore::VChunk(chunk).get_chunk_id(); auto pdev_id = homestore::VChunk(chunk).get_pdev_id(); + + LOGDEBUG("Adding chunk id {} to selector {}", chunk_id, m_module_name); + std::lock_guard lock(m_chunk_sel_mutex); m_all_chunks.emplace(chunk_id, vol_chunk); m_per_dev_chunks[pdev_id].emplace(chunk_id, vol_chunk); - LOGDEBUG("Adding chunk id {} to selector {}", chunk_id, m_module_name); } std::vector< chunk_num_t > VolumeChunkSelector::allocate_init_chunks(uint64_t volume_ordinal, uint64_t volume_size, uint32_t& pdev_id, bool lazy_alloc) { RELEASE_ASSERT(volume_ordinal < m_volume_chunks.size(), "Invalid ordinal for volume {}", volume_ordinal); + + std::unique_lock lock{m_chunk_sel_mutex}; + + // Fast path: volume already exists if (m_volume_chunks[volume_ordinal] != nullptr) { LOGW("Already allocated chunks for volume={}", volume_ordinal); auto volc = m_volume_chunks[volume_ordinal]; + pdev_id = volc->pdev; + std::vector< chunk_num_t > chunk_ids; for (auto& chunk : volc->m_chunks) { if (chunk) { chunk_ids.emplace_back(chunk->get_chunk_id()); } @@ -47,17 +55,13 @@ std::vector< chunk_num_t > VolumeChunkSelector::allocate_init_chunks(uint64_t vo return chunk_ids; } - uint64_t chunk_size{0}; - { - std::lock_guard lock(m_chunk_sel_mutex); - if (m_all_chunks.empty()) { - LOGE("No chunks available in system for volume={}", volume_ordinal); - return {}; - } - - chunk_size = m_all_chunks.begin()->second->size(); + if (m_all_chunks.empty()) { + LOGE("No chunks available in system for volume={}", volume_ordinal); + return {}; } + const uint64_t chunk_size = m_all_chunks.begin()->second->size(); + auto volc = std::make_shared< VolumeChunksInfo >(); volc->ordinal = volume_ordinal; volc->max_num_chunks = std::max(1UL, (volume_size + chunk_size - 1) / chunk_size); @@ -72,20 +76,20 @@ std::vector< chunk_num_t > VolumeChunkSelector::allocate_init_chunks(uint64_t vo // Initially we create num_chunks_per_vol_init active chunks. auto chunks = allocate_init_chunks_from_pdev(volc->num_active_chunks, volc->max_num_chunks); if (chunks.empty()) { - LOGE("Couldnt allocate chunks for volume={}", volume_ordinal); + LOGE("Couldn't allocate chunks for volume={}", volume_ordinal); return {}; } volc->pdev = (*chunks.begin())->get_pdev_id(); pdev_id = volc->pdev; volc->m_chunks.resize(volc->max_num_chunks); - m_volume_chunks[volume_ordinal] = volc; std::string str; uint64_t idx = 0; std::vector< chunk_num_t > chunk_ids; + chunk_ids.reserve(chunks.size()); + for (auto& chunk : chunks) { - // Add the chunks to the volume chunk list. RELEASE_ASSERT(chunk->m_vol_ordinal == INVALID_VOL_ORDINAL, "Chunk assigned to volume {}", chunk->m_vol_ordinal); chunk->m_vol_ordinal = volume_ordinal; @@ -94,6 +98,8 @@ std::vector< chunk_num_t > VolumeChunkSelector::allocate_init_chunks(uint64_t vo fmt::format_to(std::back_inserter(str), "{} ", chunk->get_chunk_id()); } + m_volume_chunks[volume_ordinal] = volc; + LOGI("Allocating initial module={} num_chunks={} for volume={} chunks={}", m_module_name, chunk_ids.size(), volume_ordinal, str); return chunk_ids; @@ -105,69 +111,102 @@ homestore::cshared< Chunk > VolumeChunkSelector::select_chunk(homestore::blk_cou if (!hints.application_hint) { return nullptr; } uint64_t volume_ordinal = hints.application_hint.value(); - // We dont take lock on volumes vector and volume chunks vector - // as they precreated and never changed - if (volume_ordinal >= m_volume_chunks.size() || !m_volume_chunks[volume_ordinal]) { return nullptr; } - auto volc = m_volume_chunks[volume_ordinal]; + shared< VolumeChunksInfo > volc; - // TODO to remove , keep trak of number of freed and alloc blks. - // Add chunk_selector interface to have additional functions on_alloc_blk, on_free_blk in homestore. - uint64_t total_blks = 0, available_blks = 0; - for (auto& chunk : volc->m_chunks) { - if (!chunk) continue; - total_blks += chunk->get_total_blks(); - available_blks += chunk->available_blks(); + { + std::shared_lock lock{m_chunk_sel_mutex}; + + if (volume_ordinal >= m_volume_chunks.size()) { return nullptr; } + + volc = m_volume_chunks[volume_ordinal]; + if (!volc || volc->releasing.load(std::memory_order_acquire)) { return nullptr; } + + volc->inflight_selects.fetch_add(1, std::memory_order_acq_rel); } - do { -#ifdef _PRERELEASE - if (iomgr_flip::instance()->test_flip("vol_num_chunks_force_resize_op")) { - // this is to simulate no blks available. - LOGI("volume resize op flip is set."); - resize_volume_num_chunks(nblks, volc); + // Hand-made RAII: decrement the counter on every exit path + struct PinGuard { + shared< VolumeChunksInfo > volc; + ~PinGuard() { + if (volc) { volc->inflight_selects.fetch_sub(1, std::memory_order_acq_rel); } } -#endif + } pin{volc}; + + // Recheck after pinning in case release started immediately after unlock. + if (volc->releasing.load(std::memory_order_acquire)) { return nullptr; } - // If the ratio of available_blks to total_blks is less than half or there is a request of nblks - // more than the available blks and there is room for more chunks then resize. - auto usage_ratio = (float)available_blks / total_blks; - if ((nblks > available_blks || usage_ratio < 0.5) && (volc->num_active_chunks.load() < volc->max_num_chunks)) { - // Check if number chunks needs to be increased. - resize_volume_num_chunks(nblks, volc); + constexpr auto k_sleep = std::chrono::milliseconds{100}; + constexpr auto k_max_wait = std::chrono::seconds{5}; + auto wait_deadline = std::chrono::steady_clock::time_point{}; + + do { + const auto num_active = volc->num_active_chunks.load(std::memory_order_acquire); + if (num_active == 0) { return nullptr; } + + // Grow ahead of demand: start a resize when free space runs low (or the force-resize flip is set), but don't + // wait for it while the active chunks still have free blks. + if (num_active < volc->max_num_chunks && volc->resize_op.load(std::memory_order_acquire) == ResizeOp::Idle) { + (void)resize_volume_num_chunks(nblks, volc); } - // This is the fastpath where we try to allocate the blks from the active chunks. - // Traverse through active chunks in the vector and find the first chunk - // which has some available blks. It may not satisfy all the nblks, in that case - // virtual_dev will call select_chunk again. - uint64_t num_active_chunks = volc->num_active_chunks; - for (uint64_t i = 0; i < num_active_chunks; i++) { - // Lock-free round-robin over the active chunks (shared atomic cursor). - auto const idx = volc->m_next_chunk_index.fetch_add(1, std::memory_order_relaxed) % num_active_chunks; + for (uint64_t i = 0; i < num_active; ++i) { + auto idx = volc->m_next_chunk_index.fetch_add(1, std::memory_order_relaxed) % num_active; auto chunk = volc->m_chunks[idx]; if (chunk && chunk->available_blks() > 0) { return chunk->get_internal_chunk(); } } + if (volc->releasing.load(std::memory_order_acquire)) { return nullptr; } + + if (num_active >= volc->max_num_chunks) { + LOGW("Volume {} is out of space: active={} max={}", volc->ordinal, num_active, volc->max_num_chunks); + return nullptr; + } + + auto rr = resize_volume_num_chunks(nblks, volc); + if (rr == ResizeResult::NoCapacity || rr == ResizeResult::Releasing) { return nullptr; } + + if (rr == ResizeResult::Busy || rr == ResizeResult::Started) { + if (wait_deadline == std::chrono::steady_clock::time_point{}) { + wait_deadline = std::chrono::steady_clock::now() + k_max_wait; + } + + if (std::chrono::steady_clock::now() >= wait_deadline) { + LOGW("Timed out waiting for volume {} resize progress, resize_op={}", volc->ordinal, + static_cast< int >(volc->resize_op.load(std::memory_order_acquire))); + return nullptr; + } + } else { + // reset bounded-wait tracking on any non-waiting outcome + wait_deadline = std::chrono::steady_clock::time_point{}; + } + LOGT("Waiting to allocate more chunks active={} total={}", volc->num_active_chunks.load(), volc->max_num_chunks); LOGT("{}", dump_chunks()); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + std::this_thread::sleep_for(k_sleep); + } while (true); return {}; } -void VolumeChunkSelector::resize_volume_num_chunks(homestore::blk_count_t nblks, shared< VolumeChunksInfo > volc) { - auto idle = ResizeOp::Idle, inprogress = ResizeOp::InProgress; - auto status = resize_op.compare_exchange_strong(idle, inprogress); - if (!status) { - // Some other thread is in process of adding the chunks. - return; - } +VolumeChunkSelector::ResizeResult VolumeChunkSelector::resize_volume_num_chunks(homestore::blk_count_t nblks, + shared< VolumeChunksInfo > volc) { + // Don't resize while releasing + if (!volc || volc->releasing.load(std::memory_order_acquire)) { return ResizeResult::Releasing; } + + // Some other thread is in process of adding the chunks + auto idle = ResizeOp::Idle; + if (!volc->resize_op.compare_exchange_strong(idle, ResizeOp::Queued)) { return ResizeResult::Busy; } // TODO chunk select will have on_alloc_blk, on_free_blk + // Only scan the published active prefix. Readers should treat + // [0, num_active_chunks) as the only visible chunk range uint64_t total_blks = 0, available_blks = 0; - for (auto& chunk : volc->m_chunks) { + const auto active = volc->num_active_chunks.load(std::memory_order_acquire); + for (uint64_t i = 0; i < active; ++i) { + auto const& chunk = volc->m_chunks[i]; if (!chunk) continue; total_blks += chunk->get_total_blks(); available_blks += chunk->available_blks(); @@ -182,52 +221,133 @@ void VolumeChunkSelector::resize_volume_num_chunks(homestore::blk_count_t nblks, } #endif if (!force_resize) { - auto usage_ratio = (float)available_blks / total_blks; - if ((nblks < available_blks && usage_ratio > 0.5)) { - // Check again if another thread already did the resize. - LOGI("Another thread already completed the resize op."); - return; + auto usage_ratio = total_blks ? (float)available_blks / total_blks : 0.0f; + if ((nblks < available_blks && usage_ratio > 0.5f)) { + // Enough free blks in the active chunks, no resize needed yet. + LOGT("No resize needed for volume={} available={} total={}", volc->ordinal, available_blks, total_blks); + volc->resize_op.store(ResizeOp::Idle, std::memory_order_release); + return ResizeResult::NotNeeded; } } - // Spawn background task to create new chunks. + { + std::shared_lock lock{m_chunk_sel_mutex}; + const auto active_now = volc->num_active_chunks.load(std::memory_order_acquire); + const auto remaining_slots = volc->max_num_chunks - active_now; + const auto free_on_pdev = m_per_dev_chunks.contains(volc->pdev) ? m_per_dev_chunks.at(volc->pdev).size() : 0; + + if (remaining_slots == 0 || free_on_pdev == 0) { + volc->resize_op.store(ResizeOp::Idle, std::memory_order_release); + return ResizeResult::NoCapacity; + } + } + + // Spawn background task to create new chunks LOGD("Initiating op to resize num chunks for module={} volume={} available={} total={}", m_module_name, volc->ordinal, available_blks, total_blks); + iomanager.run_on_forget(iomgr::reactor_regex::random_worker, [volc, this]() mutable { - std::string str; - auto num_chunks_to_alloc = std::min(static_cast< uint64_t >(num_chunks_per_resize), - (volc->max_num_chunks - volc->num_active_chunks.load())); - auto chunks = allocate_resize_chunks_from_pdev(volc->pdev, num_chunks_to_alloc); - RELEASE_ASSERT(!chunks.empty(), "No chunks available for resize volume={}", volc->ordinal) - - // Add the new chunks to the active chunks - auto indx = volc->num_active_chunks.load(); - for (auto chunk : chunks) { - volc->m_chunks[indx] = chunk; - indx++; - fmt::format_to(std::back_inserter(str), "{}({}) ", chunk->get_chunk_id(), chunk->get_pdev_id()); + // Publish InProgress before checking releasing: quiesce_chunks() sets releasing before checking for + // InProgress, so either it waits for this worker or this worker sees releasing and bails out. + auto queued = ResizeOp::Queued; + if (!volc->resize_op.compare_exchange_strong(queued, ResizeOp::InProgress)) { return; } + if (volc->releasing.load()) { + volc->resize_op.store(ResizeOp::Idle); + return; + } + + const auto num_chunks_to_alloc = + std::min(static_cast< uint64_t >(num_chunks_per_resize), + (volc->max_num_chunks - volc->num_active_chunks.load(std::memory_order_acquire))); + + if (num_chunks_to_alloc == 0) { + volc->resize_op.store(ResizeOp::Idle); + return; } + auto new_chunks = allocate_resize_chunks_from_pdev(volc->pdev, num_chunks_to_alloc); + if (new_chunks.empty()) { + LOGW("No chunks available for resize volume={}", volc->ordinal); + volc->resize_op.store(ResizeOp::Idle, std::memory_order_release); + return; + } + + // Build the metadata payload first, but do NOT publish new chunks yet. + // That way select_chunk() still only sees the old active prefix. std::vector< chunk_num_t > chunk_ids; - for (auto& chunk : volc->m_chunks) { - if (chunk) { chunk_ids.emplace_back(chunk->get_chunk_id()); } + bool release_started = false; + { + std::shared_lock lock{m_chunk_sel_mutex}; + + if (volc->releasing.load(std::memory_order_acquire)) { + release_started = true; + } else { + const auto active = volc->num_active_chunks.load(std::memory_order_acquire); + chunk_ids.reserve(active + new_chunks.size()); + + for (uint64_t i = 0; i < active; ++i) { + auto const& chunk = volc->m_chunks[i]; + if (chunk) { chunk_ids.emplace_back(chunk->get_chunk_id()); } + } + } + } + + // If release started after we grabbed chunks from the free pool, + // put them back instead of attaching them to the dying volume + if (release_started) { + std::unique_lock lock{m_chunk_sel_mutex}; + for (auto& chunk : new_chunks) { + m_per_dev_chunks[chunk->get_pdev_id()].emplace(chunk->get_chunk_id(), chunk); + } + volc->resize_op.store(ResizeOp::Idle); + return; + } + + for (auto& chunk : new_chunks) { + chunk_ids.emplace_back(chunk->get_chunk_id()); } - // Persist the new chunk ids to the metablk of volume - // before making them active chunks. Invoke the registered callback. + // Persist first, publish second m_update_vol_sb_cb(volc->ordinal, chunk_ids); - // Update the number of active chunks and compelete the resize operation. - volc->num_active_chunks = indx; - resize_op.store(ResizeOp::Idle); - LOGI("Resize op done. Allocated more chunks for volume={} total={} new={} new_chunks={}", m_module_name, - volc->ordinal, volc->num_active_chunks.load(), chunks.size(), str); + std::string str; + { + std::unique_lock lock{m_chunk_sel_mutex}; + + // Recheck after metadata update. Release may have started while + // callback was running; if so, do not publish these chunks. + if (volc->releasing.load(std::memory_order_acquire)) { + for (auto& chunk : new_chunks) { + m_per_dev_chunks[chunk->get_pdev_id()].emplace(chunk->get_chunk_id(), chunk); + } + volc->resize_op.store(ResizeOp::Idle); + return; + } + + // Publish by writing chunk pointers first, then bumping + // num_active_chunks. Readers only trust the active prefix. + auto idx = volc->num_active_chunks.load(std::memory_order_relaxed); + for (auto& chunk : new_chunks) { + RELEASE_ASSERT(chunk->m_vol_ordinal == INVALID_VOL_ORDINAL, "Chunk assigned to volume {}", + chunk->m_vol_ordinal); + chunk->m_vol_ordinal = volc->ordinal; + volc->m_chunks[idx++] = chunk; + fmt::format_to(std::back_inserter(str), "{}({}) ", chunk->get_chunk_id(), chunk->get_pdev_id()); + } + + volc->num_active_chunks.store(idx, std::memory_order_release); + } + + volc->resize_op.store(ResizeOp::Idle); + LOGI("Resize op done. Allocated more chunks for volume={} total={} new={} new_chunks={}", volc->ordinal, + volc->num_active_chunks.load(), new_chunks.size(), str); }); + + return ResizeResult::Started; } std::vector< shared< VolumeChunkSelector::HBChunk > > VolumeChunkSelector::allocate_init_chunks_from_pdev(uint64_t init_chunks, uint64_t total_chunks) { - std::lock_guard lock(m_chunk_sel_mutex); std::vector< shared< HBChunk > > result; RELEASE_ASSERT(init_chunks <= total_chunks, "Invalid chunks requested"); for (auto& [pdev, pdev_chunks] : m_per_dev_chunks) { @@ -252,7 +372,9 @@ VolumeChunkSelector::allocate_resize_chunks_from_pdev(uint32_t pdev_id, uint64_t std::lock_guard lock(m_chunk_sel_mutex); std::vector< shared< HBChunk > > result; auto& chunks = m_per_dev_chunks[pdev_id]; - RELEASE_ASSERT(num_chunks <= chunks.size(), "Not enough chunks for volume"); + // The capacity check in resize_volume_num_chunks() doesn't reserve chunks; other volumes on this pdev may have + // taken them since. Hand out what is left (possibly none). + num_chunks = std::min< uint64_t >(num_chunks, chunks.size()); // Allocate chunks from this pdev pool. uint32_t count = 0; @@ -266,10 +388,11 @@ VolumeChunkSelector::allocate_resize_chunks_from_pdev(uint32_t pdev_id, uint64_t bool VolumeChunkSelector::recover_chunks(uint64_t volume_ordinal, uint32_t pdev, uint64_t volume_size, const std::vector< chunk_num_t >& chunk_ids) { - std::lock_guard lock(m_chunk_sel_mutex); + std::unique_lock lock(m_chunk_sel_mutex); auto volc = m_volume_chunks[volume_ordinal]; RELEASE_ASSERT(!volc, "volume already exists"); + if (m_all_chunks.empty()) { return false; } auto chunk_size = m_all_chunks.begin()->second->size(); volc = std::make_shared< VolumeChunksInfo >(); volc->ordinal = volume_ordinal; @@ -306,26 +429,79 @@ bool VolumeChunkSelector::recover_chunks(uint64_t volume_ordinal, uint32_t pdev, return true; } +// Wait until no selector is still walking this volume and no resize worker is running. A worker that is only queued +// is not waited for: it may be queued on the very reactor we are blocking here, and once it runs it sees releasing and +// bails out before taking any chunks. +void VolumeChunkSelector::wait_for_quiesce(const VolumeChunksInfo& volc) const { + while (volc.inflight_selects.load() != 0 || volc.resize_op.load() == ResizeOp::InProgress) { + std::this_thread::yield(); + } +} + +void VolumeChunkSelector::quiesce_chunks(uint64_t volume_ordinal) { + shared< VolumeChunksInfo > volc; + { + std::unique_lock lock(m_chunk_sel_mutex); + volc = m_volume_chunks[volume_ordinal]; + if (!volc) { return; } + volc->releasing.store(true); + } + wait_for_quiesce(*volc); +} + +// Release the active chunks back to the per device chunk pool void VolumeChunkSelector::release_chunks(uint64_t volume_ordinal) { - // Release the active chunks back to the per device chunk pool. - std::lock_guard lock(m_chunk_sel_mutex); - std::string str; - uint64_t count = 0; - auto volc = m_volume_chunks[volume_ordinal]; - RELEASE_ASSERT(volc, "volume doesnt exists"); + shared< VolumeChunksInfo > volc; + + { + std::unique_lock lock(m_chunk_sel_mutex); + volc = std::exchange(m_volume_chunks[volume_ordinal], nullptr); + RELEASE_ASSERT(volc, "volume doesnt exists"); + volc->releasing.store(true); + } + + wait_for_quiesce(*volc); + + auto evict_fn = m_evict_chunk_fn; + + auto release_fn = [this, volume_ordinal, volc, evict_fn]() mutable { + std::string str; + uint64_t cnt{}; + + for (auto& chunk : volc->m_chunks) { + if (!chunk) { continue; } - for (auto chunk : volc->m_chunks) { - if (chunk) { - chunk->m_vol_ordinal = INVALID_VOL_ORDINAL; - m_per_dev_chunks[chunk->get_pdev_id()].emplace(chunk->get_chunk_id(), chunk); fmt::format_to(std::back_inserter(str), "{} ", chunk->get_chunk_id()); - count++; + ++cnt; + + // On shutdown only the allocator reset matters: the write-back cache doesn't survive a restart. + if (evict_fn && !m_stopping.load()) { evict_fn(chunk); } + + { + std::unique_lock lock{m_chunk_sel_mutex}; + chunk->reset(); + m_per_dev_chunks[chunk->get_pdev_id()].emplace(chunk->get_chunk_id(), chunk); + } } - } - m_volume_chunks[volume_ordinal] = nullptr; - LOGI("Released chunks for volume={} num_chunks={}", volume_ordinal, count); - LOGDEBUG("Released chunks={}", str); + volc->m_chunks.clear(); + volc->num_active_chunks.store(0, std::memory_order_release); + + LOGI("Released chunks for volume={} num_chunks={}", volume_ordinal, cnt); + LOGDEBUG("Released chunks={}", str); + }; + + if (evict_fn) { + // Clear wbc entries in a separate thread + ++m_pending_releases; + iomanager.run_on_forget(iomgr::reactor_regex::random_worker, + [this, release_fn = std::move(release_fn)]() mutable { + release_fn(); + --m_pending_releases; + }); + } else { + release_fn(); + } } void VolumeChunkSelector::foreach_chunks(std::function< void(homestore::cshared< Chunk >&) >&& cb) { @@ -335,7 +511,7 @@ void VolumeChunkSelector::foreach_chunks(std::function< void(homestore::cshared< } std::vector< shared< VolumeChunkSelector::HBChunk > > VolumeChunkSelector::get_chunks(uint64_t volume_ordinal) { - std::lock_guard lock(m_chunk_sel_mutex); + std::shared_lock lock(m_chunk_sel_mutex); std::vector< shared< VolumeChunkSelector::HBChunk > > chunks; RELEASE_ASSERT(volume_ordinal < m_volume_chunks.size(), "Invalid ordinal for volume {}", volume_ordinal); @@ -348,7 +524,7 @@ std::vector< shared< VolumeChunkSelector::HBChunk > > VolumeChunkSelector::get_c } uint64_t VolumeChunkSelector::num_free_chunks() const { - std::lock_guard lock(m_chunk_sel_mutex); + std::shared_lock lock(m_chunk_sel_mutex); uint64_t count = 0; for (const auto& [pdev, chunks] : m_per_dev_chunks) { count += chunks.size(); @@ -357,7 +533,7 @@ uint64_t VolumeChunkSelector::num_free_chunks() const { } void VolumeChunkSelector::dump_per_pdev_chunks() const { - std::lock_guard lock(m_chunk_sel_mutex); + std::shared_lock lock(m_chunk_sel_mutex); for (const auto& [pdev, chunks] : m_per_dev_chunks) { std::string str; for (const auto& [chunk_id, _] : chunks) { @@ -368,7 +544,7 @@ void VolumeChunkSelector::dump_per_pdev_chunks() const { } std::string VolumeChunkSelector::dump_chunks() const { - std::lock_guard lock(m_chunk_sel_mutex); + std::shared_lock lock(m_chunk_sel_mutex); std::string str; for (uint32_t i = 0; i < m_volume_chunks.size(); i++) { if (!m_volume_chunks[i]) { continue; } diff --git a/src/lib/volume/volume_chunk_selector.hpp b/src/lib/volume/volume_chunk_selector.hpp index 3ad2c3c9..1ef905c8 100644 --- a/src/lib/volume/volume_chunk_selector.hpp +++ b/src/lib/volume/volume_chunk_selector.hpp @@ -30,12 +30,30 @@ using Chunk = homestore::Chunk; class VolumeChunkSelector : public homestore::ChunkSelector { static constexpr homestore::chunk_num_t num_chunks_per_vol_init = 1; static constexpr homestore::chunk_num_t num_chunks_per_resize = 3; - static constexpr uint64_t INVALID_VOL_ORDINAL = UINT64_MAX; + static constexpr uint64_t INVALID_VOL_ORDINAL = UINT64_MAX; // not owned by any volume struct HBChunk : public homestore::VChunk { HBChunk(homestore::cshared< Chunk >& chunk) : homestore::VChunk(chunk) {} ~HBChunk() = default; uint64_t m_vol_ordinal{INVALID_VOL_ORDINAL}; + + void reset() { + m_vol_ordinal = INVALID_VOL_ORDINAL; + homestore::VChunk::reset(); + } + }; + +private: + // Queued: a resize worker has been scheduled but has not started running yet. + // InProgress: the worker is running (allocating chunks / persisting the volume superblock). + enum class ResizeOp { Idle, Queued, InProgress }; + + enum class ResizeResult { + Busy, // another thread is resizing + Started, // resize worker launched + NotNeeded, // enough free blocks already + NoCapacity, // cannot add more chunks + Releasing }; struct VolumeChunksInfo { @@ -54,6 +72,10 @@ class VolumeChunkSelector : public homestore::ChunkSelector { std::atomic< uint32_t > m_next_chunk_index{0}; uint64_t ordinal; uint32_t pdev; + + std::atomic< ResizeOp > resize_op{ResizeOp::Idle}; + std::atomic< uint32_t > inflight_selects{}; + std::atomic< bool > releasing{false}; }; public: @@ -65,9 +87,27 @@ class VolumeChunkSelector : public homestore::ChunkSelector { std::vector< chunk_num_t > allocate_init_chunks(uint64_t volume_ordinal, uint64_t volume_size, uint32_t& pdev_id, bool lazy_alloc = true); - // Called during destroy of volume or index. + // Called during destroy of volume, before its superblock is destroyed. Stops chunk selection and resize for the + // volume and waits for a running resize worker, so no resize can persist into the superblock afterwards. + void quiesce_chunks(uint64_t volume_ordinal); + + // Called during destroy of volume or index. Quiesces the volume if not done already and returns its chunks to the + // free pool, asynchronously when write-back cache entries have to be evicted first. void release_chunks(uint64_t volume_ordinal); + // How a released chunk's index write-back cache entries are evicted. Only the index selector sets this: data chunk + // blkids can never be in the index write-back cache. Without it chunks are released synchronously. + using EvictChunkFn = std::function< void(const shared< HBChunk >&) >; + void set_evict_chunk_fn(EvictChunkFn fn) { m_evict_chunk_fn = std::move(fn); } + + // Called when shutdown starts. Asynchronous releases skip write-back cache eviction from then on (the cache is + // memory only and gone after restart) and only reset the chunk allocators, so they finish quickly. + void start_shutdown() { m_stopping.store(true); } + + // Shutdown must wait for these: they still touch homestore, and their allocator resets have to be persisted by + // homestore's final checkpoint. + bool has_pending_releases() const { return m_pending_releases.load() != 0; } + // Called during recovery of volume or index . bool recover_chunks(uint64_t volume_ordinal, uint32_t pdev_id, uint64_t volume_size, const std::vector< chunk_num_t >& chunk_ids); @@ -88,16 +128,12 @@ class VolumeChunkSelector : public homestore::ChunkSelector { private: std::vector< shared< HBChunk > > allocate_init_chunks_from_pdev(uint64_t init_chunks, uint64_t total_chunks); std::vector< shared< HBChunk > > allocate_resize_chunks_from_pdev(uint32_t pdev, uint64_t num_chunks); - void resize_volume_num_chunks(homestore::blk_count_t nblks, shared< VolumeChunksInfo > volc); + ResizeResult resize_volume_num_chunks(homestore::blk_count_t nblks, shared< VolumeChunksInfo > volc); + void wait_for_quiesce(const VolumeChunksInfo& volc) const; void dump_per_pdev_chunks() const; std::string dump_chunks() const; private: - enum class ResizeOp { - Idle, - InProgress, - }; - // Store volume chunks details with index as volume ordinal. std::vector< shared< VolumeChunksInfo > > m_volume_chunks; @@ -111,9 +147,11 @@ class VolumeChunkSelector : public homestore::ChunkSelector { // for allocation. This pool is used for allocation of chunks to volume. // Chunks once allocated to volume are removed from this pool. std::unordered_map< uint64_t, ChunkMap > m_per_dev_chunks; - mutable std::mutex m_chunk_sel_mutex; + mutable std::shared_mutex m_chunk_sel_mutex; UpdateVolSbCb m_update_vol_sb_cb; - std::atomic< ResizeOp > resize_op{ResizeOp::Idle}; + EvictChunkFn m_evict_chunk_fn; + std::atomic< bool > m_stopping{false}; + std::atomic< uint32_t > m_pending_releases{0}; std::string m_module_name; }; diff --git a/src/lib/volume_mgr.cpp b/src/lib/volume_mgr.cpp index 9b2a2d76..31cce289 100644 --- a/src/lib/volume_mgr.cpp +++ b/src/lib/volume_mgr.cpp @@ -177,6 +177,12 @@ async_status HomeBlocksImpl::remove_volume(const volume_id_t& id) { co_return std::unexpected(std::errc::operation_not_supported); } + // The volume is marked DESTROYING only once do_remove_volume runs, so shutdown could miss a remove accepted now. + if (is_shutting_down()) { + LOGE("Can't serve volume remove, system is shutting down."); + co_return std::unexpected(std::errc::operation_not_supported); + } + auto vol = get_volume(id); if (!vol) { LOGE("volume with id {} not found, cannot remove", boost::uuids::to_string(id)); diff --git a/src/test/legacy_ublk/homeblk_ublk_cli.cpp b/src/test/legacy_ublk/homeblk_ublk_cli.cpp index 41d372d3..cee3cf10 100644 --- a/src/test/legacy_ublk/homeblk_ublk_cli.cpp +++ b/src/test/legacy_ublk/homeblk_ublk_cli.cpp @@ -63,6 +63,9 @@ SISL_OPTION_GROUP(homeblk_ublk, ::cxxopts::value< uint32_t >(), ""), (index_chunk_size_mb, "", "index_chunk_size_mb", "homeblocks index chunk size in MB (for small devices)", ::cxxopts::value< uint32_t >(), ""), + (dynamic_chunk_allocation, "", "dynamic_chunk_allocation", + "lazily allocate volume data chunks (overrides dynamic config)", ::cxxopts::value< bool >(), + "true or false"), (device_id, "", "device_id", "ublk device id: -1 to assign, >=0 to recover a preserved device", ::cxxopts::value< int32_t >()->default_value("-1"), ""))