Message ID | 20210419085541.22310-8-eesposit@redhat.com |
---|---|
State | New |
Headers | show |
Series | Block layer thread-safety, continued | expand |
On Mon, Apr 19, 2021 at 10:55:40AM +0200, Emanuele Giuseppe Esposito wrote: > @@ -210,7 +212,7 @@ static int replication_return_value(BDRVReplicationState *s, int ret) > } > > if (ret < 0) { > - s->error = ret; > + qatomic_set(&s->error, ret); > ret = 0; > } > > @@ -307,6 +309,7 @@ out: > return ret; > } > > +/* Called with no I/O pending. */ It would be clearer to refer to the specific guarantee that no I/O is pending, like "Called from a drained section". There are more comments like this one below.
On 19/04/21 10:55, Emanuele Giuseppe Esposito wrote: > Replication functions are mostly called when the BDS is quiescent and > does not have any pending I/O. They do not need to synchronize on > anything since BDS and BB are now thread-safe. > > Signed-off-by: Paolo Bonzini <pbonzini@redhat.com> > Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com> This patch has the same issue that Stefan mentioned in patch 2. In particular, the following chain leads to the bdrv*drain family of functions and thus AIO_WAIT_WHILE: replication_start -> reopen_backing_file -> bdrv_subtree_drained_begin and also replication_stop -> commit_active_start -> bdrv_reopen_set_read_only The same is true of patch 8, where you have call sequences like bdrv_commit -> blk_flush -> bdrv_flush (in the generated file block/block-gen.c) -> bdrv_poll_co So patches 7 and 8 need to be shelved for now, as they can only go in with the overall removal of AioContext lock. Paolo > --- > block/replication.c | 54 ++++++++++----------------------------------- > 1 file changed, 12 insertions(+), 42 deletions(-) > > diff --git a/block/replication.c b/block/replication.c > index 97be7ef4de..25ee37b21b 100644 > --- a/block/replication.c > +++ b/block/replication.c > @@ -45,6 +45,8 @@ typedef struct BDRVReplicationState { > Error *blocker; > bool orig_hidden_read_only; > bool orig_secondary_read_only; > + > + /* This field is accessed asynchronously. */ > int error; > } BDRVReplicationState; > > @@ -210,7 +212,7 @@ static int replication_return_value(BDRVReplicationState *s, int ret) > } > > if (ret < 0) { > - s->error = ret; > + qatomic_set(&s->error, ret); > ret = 0; > } > > @@ -307,6 +309,7 @@ out: > return ret; > } > > +/* Called with no I/O pending. */ > static void secondary_do_checkpoint(BDRVReplicationState *s, Error **errp) > { > Error *local_err = NULL; > @@ -420,7 +423,7 @@ static void backup_job_completed(void *opaque, int ret) > > if (s->stage != BLOCK_REPLICATION_FAILOVER) { > /* The backup job is cancelled unexpectedly */ > - s->error = -EIO; > + qatomic_set(&s->error, -EIO); > } > > backup_job_cleanup(bs); > @@ -445,6 +448,7 @@ static bool check_top_bs(BlockDriverState *top_bs, BlockDriverState *bs) > return false; > } > > +/* Called with no I/O pending. */ > static void replication_start(ReplicationState *rs, ReplicationMode mode, > Error **errp) > { > @@ -452,12 +456,9 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, > BDRVReplicationState *s; > BlockDriverState *top_bs; > int64_t active_length, hidden_length, disk_length; > - AioContext *aio_context; > Error *local_err = NULL; > BackupPerf perf = { .use_copy_range = true, .max_workers = 1 }; > > - aio_context = bdrv_get_aio_context(bs); > - aio_context_acquire(aio_context); > s = bs->opaque; > > if (s->stage == BLOCK_REPLICATION_DONE || > @@ -467,20 +468,17 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, > * Ignore the request because the secondary side of replication > * doesn't have to do anything anymore. > */ > - aio_context_release(aio_context); > return; > } > > if (s->stage != BLOCK_REPLICATION_NONE) { > error_setg(errp, "Block replication is running or done"); > - aio_context_release(aio_context); > return; > } > > if (s->mode != mode) { > error_setg(errp, "The parameter mode's value is invalid, needs %d," > " but got %d", s->mode, mode); > - aio_context_release(aio_context); > return; > } > > @@ -492,21 +490,18 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, > if (!s->active_disk || !s->active_disk->bs || > !s->active_disk->bs->backing) { > error_setg(errp, "Active disk doesn't have backing file"); > - aio_context_release(aio_context); > return; > } > > s->hidden_disk = s->active_disk->bs->backing; > if (!s->hidden_disk->bs || !s->hidden_disk->bs->backing) { > error_setg(errp, "Hidden disk doesn't have backing file"); > - aio_context_release(aio_context); > return; > } > > s->secondary_disk = s->hidden_disk->bs->backing; > if (!s->secondary_disk->bs || !bdrv_has_blk(s->secondary_disk->bs)) { > error_setg(errp, "The secondary disk doesn't have block backend"); > - aio_context_release(aio_context); > return; > } > > @@ -518,7 +513,6 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, > active_length != hidden_length || hidden_length != disk_length) { > error_setg(errp, "Active disk, hidden disk, secondary disk's length" > " are not the same"); > - aio_context_release(aio_context); > return; > } > > @@ -529,7 +523,6 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, > !s->hidden_disk->bs->drv->bdrv_make_empty) { > error_setg(errp, > "Active disk or hidden disk doesn't support make_empty"); > - aio_context_release(aio_context); > return; > } > > @@ -537,7 +530,6 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, > reopen_backing_file(bs, true, &local_err); > if (local_err) { > error_propagate(errp, local_err); > - aio_context_release(aio_context); > return; > } > > @@ -550,7 +542,6 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, > !check_top_bs(top_bs, bs)) { > error_setg(errp, "No top_bs or it is invalid"); > reopen_backing_file(bs, false, NULL); > - aio_context_release(aio_context); > return; > } > bdrv_op_block_all(top_bs, s->blocker); > @@ -566,13 +557,11 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, > if (local_err) { > error_propagate(errp, local_err); > backup_job_cleanup(bs); > - aio_context_release(aio_context); > return; > } > job_start(&s->backup_job->job); > break; > default: > - aio_context_release(aio_context); > abort(); > } > > @@ -582,18 +571,15 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, > secondary_do_checkpoint(s, errp); > } > > - s->error = 0; > - aio_context_release(aio_context); > + qatomic_set(&s->error, 0); > } > > +/* Called with no I/O pending. */ > static void replication_do_checkpoint(ReplicationState *rs, Error **errp) > { > BlockDriverState *bs = rs->opaque; > BDRVReplicationState *s; > - AioContext *aio_context; > > - aio_context = bdrv_get_aio_context(bs); > - aio_context_acquire(aio_context); > s = bs->opaque; > > if (s->stage == BLOCK_REPLICATION_DONE || > @@ -603,38 +589,30 @@ static void replication_do_checkpoint(ReplicationState *rs, Error **errp) > * Ignore the request because the secondary side of replication > * doesn't have to do anything anymore. > */ > - aio_context_release(aio_context); > return; > } > > if (s->mode == REPLICATION_MODE_SECONDARY) { > secondary_do_checkpoint(s, errp); > } > - aio_context_release(aio_context); > } > > static void replication_get_error(ReplicationState *rs, Error **errp) > { > BlockDriverState *bs = rs->opaque; > BDRVReplicationState *s; > - AioContext *aio_context; > > - aio_context = bdrv_get_aio_context(bs); > - aio_context_acquire(aio_context); > s = bs->opaque; > > if (s->stage == BLOCK_REPLICATION_NONE) { > error_setg(errp, "Block replication is not running"); > - aio_context_release(aio_context); > return; > } > > - if (s->error) { > + if (qatomic_read(&s->error)) { > error_setg(errp, "I/O error occurred"); > - aio_context_release(aio_context); > return; > } > - aio_context_release(aio_context); > } > > static void replication_done(void *opaque, int ret) > @@ -648,10 +626,10 @@ static void replication_done(void *opaque, int ret) > s->active_disk = NULL; > s->secondary_disk = NULL; > s->hidden_disk = NULL; > - s->error = 0; > + qatomic_set(&s->error, 0); > } else { > s->stage = BLOCK_REPLICATION_FAILOVER_FAILED; > - s->error = -EIO; > + qatomic_set(&s->error, -EIO); > } > } > > @@ -659,10 +637,7 @@ static void replication_stop(ReplicationState *rs, bool failover, Error **errp) > { > BlockDriverState *bs = rs->opaque; > BDRVReplicationState *s; > - AioContext *aio_context; > > - aio_context = bdrv_get_aio_context(bs); > - aio_context_acquire(aio_context); > s = bs->opaque; > > if (s->stage == BLOCK_REPLICATION_DONE || > @@ -672,20 +647,18 @@ static void replication_stop(ReplicationState *rs, bool failover, Error **errp) > * Ignore the request because the secondary side of replication > * doesn't have to do anything anymore. > */ > - aio_context_release(aio_context); > return; > } > > if (s->stage != BLOCK_REPLICATION_RUNNING) { > error_setg(errp, "Block replication is not running"); > - aio_context_release(aio_context); > return; > } > > switch (s->mode) { > case REPLICATION_MODE_PRIMARY: > s->stage = BLOCK_REPLICATION_DONE; > - s->error = 0; > + qatomic_set(&s->error, 0); > break; > case REPLICATION_MODE_SECONDARY: > /* > @@ -700,7 +673,6 @@ static void replication_stop(ReplicationState *rs, bool failover, Error **errp) > if (!failover) { > secondary_do_checkpoint(s, errp); > s->stage = BLOCK_REPLICATION_DONE; > - aio_context_release(aio_context); > return; > } > > @@ -711,10 +683,8 @@ static void replication_stop(ReplicationState *rs, bool failover, Error **errp) > NULL, replication_done, bs, true, errp); > break; > default: > - aio_context_release(aio_context); > abort(); > } > - aio_context_release(aio_context); > } > > static const char *const replication_strong_runtime_opts[] = { >
diff --git a/block/replication.c b/block/replication.c index 97be7ef4de..25ee37b21b 100644 --- a/block/replication.c +++ b/block/replication.c @@ -45,6 +45,8 @@ typedef struct BDRVReplicationState { Error *blocker; bool orig_hidden_read_only; bool orig_secondary_read_only; + + /* This field is accessed asynchronously. */ int error; } BDRVReplicationState; @@ -210,7 +212,7 @@ static int replication_return_value(BDRVReplicationState *s, int ret) } if (ret < 0) { - s->error = ret; + qatomic_set(&s->error, ret); ret = 0; } @@ -307,6 +309,7 @@ out: return ret; } +/* Called with no I/O pending. */ static void secondary_do_checkpoint(BDRVReplicationState *s, Error **errp) { Error *local_err = NULL; @@ -420,7 +423,7 @@ static void backup_job_completed(void *opaque, int ret) if (s->stage != BLOCK_REPLICATION_FAILOVER) { /* The backup job is cancelled unexpectedly */ - s->error = -EIO; + qatomic_set(&s->error, -EIO); } backup_job_cleanup(bs); @@ -445,6 +448,7 @@ static bool check_top_bs(BlockDriverState *top_bs, BlockDriverState *bs) return false; } +/* Called with no I/O pending. */ static void replication_start(ReplicationState *rs, ReplicationMode mode, Error **errp) { @@ -452,12 +456,9 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, BDRVReplicationState *s; BlockDriverState *top_bs; int64_t active_length, hidden_length, disk_length; - AioContext *aio_context; Error *local_err = NULL; BackupPerf perf = { .use_copy_range = true, .max_workers = 1 }; - aio_context = bdrv_get_aio_context(bs); - aio_context_acquire(aio_context); s = bs->opaque; if (s->stage == BLOCK_REPLICATION_DONE || @@ -467,20 +468,17 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, * Ignore the request because the secondary side of replication * doesn't have to do anything anymore. */ - aio_context_release(aio_context); return; } if (s->stage != BLOCK_REPLICATION_NONE) { error_setg(errp, "Block replication is running or done"); - aio_context_release(aio_context); return; } if (s->mode != mode) { error_setg(errp, "The parameter mode's value is invalid, needs %d," " but got %d", s->mode, mode); - aio_context_release(aio_context); return; } @@ -492,21 +490,18 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, if (!s->active_disk || !s->active_disk->bs || !s->active_disk->bs->backing) { error_setg(errp, "Active disk doesn't have backing file"); - aio_context_release(aio_context); return; } s->hidden_disk = s->active_disk->bs->backing; if (!s->hidden_disk->bs || !s->hidden_disk->bs->backing) { error_setg(errp, "Hidden disk doesn't have backing file"); - aio_context_release(aio_context); return; } s->secondary_disk = s->hidden_disk->bs->backing; if (!s->secondary_disk->bs || !bdrv_has_blk(s->secondary_disk->bs)) { error_setg(errp, "The secondary disk doesn't have block backend"); - aio_context_release(aio_context); return; } @@ -518,7 +513,6 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, active_length != hidden_length || hidden_length != disk_length) { error_setg(errp, "Active disk, hidden disk, secondary disk's length" " are not the same"); - aio_context_release(aio_context); return; } @@ -529,7 +523,6 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, !s->hidden_disk->bs->drv->bdrv_make_empty) { error_setg(errp, "Active disk or hidden disk doesn't support make_empty"); - aio_context_release(aio_context); return; } @@ -537,7 +530,6 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, reopen_backing_file(bs, true, &local_err); if (local_err) { error_propagate(errp, local_err); - aio_context_release(aio_context); return; } @@ -550,7 +542,6 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, !check_top_bs(top_bs, bs)) { error_setg(errp, "No top_bs or it is invalid"); reopen_backing_file(bs, false, NULL); - aio_context_release(aio_context); return; } bdrv_op_block_all(top_bs, s->blocker); @@ -566,13 +557,11 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, if (local_err) { error_propagate(errp, local_err); backup_job_cleanup(bs); - aio_context_release(aio_context); return; } job_start(&s->backup_job->job); break; default: - aio_context_release(aio_context); abort(); } @@ -582,18 +571,15 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, secondary_do_checkpoint(s, errp); } - s->error = 0; - aio_context_release(aio_context); + qatomic_set(&s->error, 0); } +/* Called with no I/O pending. */ static void replication_do_checkpoint(ReplicationState *rs, Error **errp) { BlockDriverState *bs = rs->opaque; BDRVReplicationState *s; - AioContext *aio_context; - aio_context = bdrv_get_aio_context(bs); - aio_context_acquire(aio_context); s = bs->opaque; if (s->stage == BLOCK_REPLICATION_DONE || @@ -603,38 +589,30 @@ static void replication_do_checkpoint(ReplicationState *rs, Error **errp) * Ignore the request because the secondary side of replication * doesn't have to do anything anymore. */ - aio_context_release(aio_context); return; } if (s->mode == REPLICATION_MODE_SECONDARY) { secondary_do_checkpoint(s, errp); } - aio_context_release(aio_context); } static void replication_get_error(ReplicationState *rs, Error **errp) { BlockDriverState *bs = rs->opaque; BDRVReplicationState *s; - AioContext *aio_context; - aio_context = bdrv_get_aio_context(bs); - aio_context_acquire(aio_context); s = bs->opaque; if (s->stage == BLOCK_REPLICATION_NONE) { error_setg(errp, "Block replication is not running"); - aio_context_release(aio_context); return; } - if (s->error) { + if (qatomic_read(&s->error)) { error_setg(errp, "I/O error occurred"); - aio_context_release(aio_context); return; } - aio_context_release(aio_context); } static void replication_done(void *opaque, int ret) @@ -648,10 +626,10 @@ static void replication_done(void *opaque, int ret) s->active_disk = NULL; s->secondary_disk = NULL; s->hidden_disk = NULL; - s->error = 0; + qatomic_set(&s->error, 0); } else { s->stage = BLOCK_REPLICATION_FAILOVER_FAILED; - s->error = -EIO; + qatomic_set(&s->error, -EIO); } } @@ -659,10 +637,7 @@ static void replication_stop(ReplicationState *rs, bool failover, Error **errp) { BlockDriverState *bs = rs->opaque; BDRVReplicationState *s; - AioContext *aio_context; - aio_context = bdrv_get_aio_context(bs); - aio_context_acquire(aio_context); s = bs->opaque; if (s->stage == BLOCK_REPLICATION_DONE || @@ -672,20 +647,18 @@ static void replication_stop(ReplicationState *rs, bool failover, Error **errp) * Ignore the request because the secondary side of replication * doesn't have to do anything anymore. */ - aio_context_release(aio_context); return; } if (s->stage != BLOCK_REPLICATION_RUNNING) { error_setg(errp, "Block replication is not running"); - aio_context_release(aio_context); return; } switch (s->mode) { case REPLICATION_MODE_PRIMARY: s->stage = BLOCK_REPLICATION_DONE; - s->error = 0; + qatomic_set(&s->error, 0); break; case REPLICATION_MODE_SECONDARY: /* @@ -700,7 +673,6 @@ static void replication_stop(ReplicationState *rs, bool failover, Error **errp) if (!failover) { secondary_do_checkpoint(s, errp); s->stage = BLOCK_REPLICATION_DONE; - aio_context_release(aio_context); return; } @@ -711,10 +683,8 @@ static void replication_stop(ReplicationState *rs, bool failover, Error **errp) NULL, replication_done, bs, true, errp); break; default: - aio_context_release(aio_context); abort(); } - aio_context_release(aio_context); } static const char *const replication_strong_runtime_opts[] = {