From 14e30393ce402d2f5a09fc28f9d47945fab1ee4c Mon Sep 17 00:00:00 2001 From: Xuezhao Liu Date: Sun, 12 Jan 2025 11:10:41 +0000 Subject: [PATCH] DAOS-14010 rebuild: reint from stable epoch Signed-off-by: Xuezhao Liu --- src/include/daos_srv/vos.h | 14 ++++++- src/object/srv_internal.h | 6 ++- src/object/srv_obj_migrate.c | 72 ++++++++++++++++++++++++++++++++++-- src/vos/vos_container.c | 2 +- src/vos/vos_obj_index.c | 18 ++++++++- 5 files changed, 103 insertions(+), 9 deletions(-) diff --git a/src/include/daos_srv/vos.h b/src/include/daos_srv/vos.h index 9684e670e6b..0dbd4feb679 100644 --- a/src/include/daos_srv/vos.h +++ b/src/include/daos_srv/vos.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2015-2024 Intel Corporation. + * (C) Copyright 2015-2025 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1712,4 +1712,16 @@ vos_unpin_objects(daos_handle_t coh, struct vos_pin_handle *hdl); int vos_pin_objects(daos_handle_t coh, daos_unit_oid_t oids[], int count, struct vos_pin_handle **hdl); +/** + * Check if the oid exist in current vos. + * + * \param[in] coh container open handle. + * \param[in] oid oid to be checked. + * + * \return true exist. + * false does not exist. + */ +bool +vos_oi_exist(daos_handle_t coh, daos_unit_oid_t oid); + #endif /* __VOS_API_H */ diff --git a/src/object/srv_internal.h b/src/object/srv_internal.h index a24986247a5..bfd5044c4a1 100644 --- a/src/object/srv_internal.h +++ b/src/object/srv_internal.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2016-2025 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -101,7 +101,9 @@ struct migrate_pool_tls { /* migrate leader ULT */ unsigned int mpt_ult_running:1, mpt_init_tls:1, - mpt_fini:1; + mpt_fini:1, + mpt_reintegrating_check:1, + mpt_reintegrating:1; /* migration init error */ int mpt_init_err; diff --git a/src/object/srv_obj_migrate.c b/src/object/srv_obj_migrate.c index 61a82301367..941c736b96e 100644 --- a/src/object/srv_obj_migrate.c +++ b/src/object/srv_obj_migrate.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2019-2024 Intel Corporation. + * (C) Copyright 2019-2025 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -3241,6 +3241,7 @@ migrate_obj_ult(void *data) struct iter_obj_arg *arg = data; struct migrate_pool_tls *tls = NULL; daos_epoch_range_t epr; + daos_epoch_t stable_epoch = 0; int i; int rc = 0; @@ -3253,7 +3254,7 @@ migrate_obj_ult(void *data) /* Only reintegrating targets/pool needs to discard the object, * if sp_need_discard is 0, either the target does not need to * discard, or discard has been done. spc_discard_done means - * discarding has been done in the current VOS target. + * discarding has been done in the current VOS target. lxzlxz */ if (tls->mpt_pool->spc_pool->sp_need_discard) { while(!tls->mpt_pool->spc_discard_done) { @@ -3271,8 +3272,70 @@ migrate_obj_ult(void *data) } } + if (!tls->mpt_reintegrating_check) { + struct pool_target *target; + struct ds_pool *pool = tls->mpt_pool->spc_pool; + + ABT_rwlock_rdlock(pool->sp_lock); + rc = pool_map_find_target_by_rank_idx(pool->sp_map, dss_self_rank(), + dss_get_module_info()->dmi_tgt_id, + &target); + ABT_rwlock_unlock(pool->sp_lock); + D_ASSERT(rc == 1); + + if (target->ta_comp.co_status == PO_COMP_ST_UP && + target->ta_comp.co_in_ver <= tls->mpt_version) + tls->mpt_reintegrating = 1; + D_DEBUG(DB_REBUILD, "status %u in version %u, mpt_reintegrating %d\n", + target->ta_comp.co_status, target->ta_comp.co_in_ver, + tls->mpt_reintegrating); + + tls->mpt_reintegrating_check = 1; + } + + if (tls->mpt_reintegrating) { + struct ds_cont_child *cont_child = NULL; + + /* check again to see if the container is being destroyed. */ + migrate_get_cont_child(tls, arg->cont_uuid, &cont_child, false); + if (cont_child != NULL && !cont_child->sc_stopping) { + if (vos_oi_exist(cont_child->sc_hdl, arg->oid)) { + rc = vos_dtx_cmt_reindex(cont_child->sc_hdl); + if (rc < 0) { + ds_cont_child_put(cont_child); + D_GOTO(out, rc); + } + stable_epoch = vos_cont_get_global_stable_epoch(cont_child->sc_hdl); + } else { + /* If the object does not exist on the local target at all, + * it is either created after stable epoch or migrated to this + * target due to co-location. let's set stable epoch to 0, + * migrate the whole object. + */ + stable_epoch = 0; + } + } + + if (cont_child) + ds_cont_child_put(cont_child); + } + for (i = 0; i < arg->snap_cnt; i++) { - epr.epr_lo = i > 0 ? arg->snaps[i - 1] + 1 : 0; + daos_epoch_t lower_epoch = 0; + + if (arg->snaps[i] < stable_epoch) { + D_DEBUG(DB_REBUILD, DF_CONT " obj " DF_UOID " skip snap "DF_X64 + " < stable " DF_X64 "\n", DP_CONT(arg->pool_uuid, arg->cont_uuid), + DP_UOID(arg->oid), arg->snaps[i], stable_epoch); + continue; + } else { + if (i == 0) + lower_epoch = stable_epoch; + else + lower_epoch = max(stable_epoch, arg->snaps[i - 1]); + } + + epr.epr_lo = lower_epoch; epr.epr_hi = arg->snaps[i]; D_DEBUG(DB_REBUILD, DF_RB ": rebuild_snap %d " DF_X64 "-" DF_X64 "\n", DP_RB_MPT(tls), i, epr.epr_lo, epr.epr_hi); @@ -3281,13 +3344,14 @@ migrate_obj_ult(void *data) D_GOTO(free, rc); } - if (arg->snap_cnt > 0 && arg->punched_epoch != 0) { + if (arg->snap_cnt > 0 && arg->punched_epoch != 0 && arg->punched_epoch > stable_epoch) { rc = migrate_obj_punch(arg); if (rc) D_GOTO(free, rc); } epr.epr_lo = arg->snaps ? arg->snaps[arg->snap_cnt - 1] + 1 : 0; + epr.epr_lo = max(epr.epr_lo, stable_epoch); D_ASSERT(tls->mpt_max_eph != 0); epr.epr_hi = tls->mpt_max_eph; if (arg->epoch > 0) { diff --git a/src/vos/vos_container.c b/src/vos/vos_container.c index 94afcb0ea53..35157bf0ae4 100644 --- a/src/vos/vos_container.c +++ b/src/vos/vos_container.c @@ -925,7 +925,7 @@ vos_cont_get_local_stable_epoch(daos_handle_t coh) * epoch when reintegrate into the system. */ daos_epoch_t -vos_cont_get_globla_stable_epoch(daos_handle_t coh) +vos_cont_get_global_stable_epoch(daos_handle_t coh) { struct vos_container *cont; struct vos_cont_ext_df *cont_ext; diff --git a/src/vos/vos_obj_index.c b/src/vos/vos_obj_index.c index 772c1bf3e86..eb16ea81a03 100644 --- a/src/vos/vos_obj_index.c +++ b/src/vos/vos_obj_index.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2016-2025 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -227,6 +227,22 @@ static btr_ops_t oi_btr_ops = { .to_node_alloc = oi_node_alloc, }; +bool +vos_oi_exist(daos_handle_t coh, daos_unit_oid_t oid) +{ + struct vos_container *cont = vos_hdl2cont(coh); + d_iov_t key_iov; + d_iov_t val_iov; + int rc; + + d_iov_set(&key_iov, &oid, sizeof(oid)); + d_iov_set(&val_iov, NULL, 0); + + rc = dbtree_fetch(cont->vc_btr_hdl, BTR_PROBE_EQ, + DAOS_INTENT_DEFAULT, &key_iov, NULL, &val_iov); + return rc == 0; +} + /** * Locate a durable object in OI table. */