diff --git a/module/zfs/dsl_scan.c b/module/zfs/dsl_scan.c
index eeec76f787..0e16002b37 100644
--- a/module/zfs/dsl_scan.c
+++ b/module/zfs/dsl_scan.c
@@ -1493,13 +1493,6 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 			    "traverse_dataset_destroyed()", err);
 		}
 
-		/*
-		 * If we didn't make progress, mark the async destroy as
-		 * stalled, so that we will not initiate a spa_sync() on
-		 * its behalf.
-		 */
-		scn->scn_async_stalled = (scn->scn_visited_this_txg == 0);
-
 		if (bptree_is_empty(dp->dp_meta_objset, dp->dp_bptree_obj)) {
 			/* finished; deactivate async destroy feature */
 			spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY, tx);
@@ -1512,6 +1505,14 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 			    dp->dp_bptree_obj, tx));
 			dp->dp_bptree_obj = 0;
 			scn->scn_async_destroying = B_FALSE;
+		} else {
+			/*
+			 * If we didn't make progress, mark the async destroy as
+			 * stalled, so that we will not initiate a spa_sync() on
+			 * its behalf.
+			 */
+			scn->scn_async_stalled =
+			    (scn->scn_visited_this_txg == 0);
 		}
 	}
 	if (scn->scn_visited_this_txg) {