Add ASSERT to debug encryption key mapping issues

This patch simply adds an ASSERT that confirms that the last
decrypting reference on a dataset waits until the dataset is
no longer dirty. This should help to debug issues where the
ZIO layer cannot find encryption keys after a dataset has been
disowned.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes #7637

include/sys/dmu.h

@@ -557,6 +557,7 @@ boolean_t dmu_buf_try_add_ref(dmu_buf_t *, objset_t *os, uint64_t object,
 
 void dmu_buf_rele(dmu_buf_t *db, void *tag);
 uint64_t dmu_buf_refcount(dmu_buf_t *db);
+uint64_t dmu_buf_user_refcount(dmu_buf_t *db);
 
 /*
  * dmu_buf_hold_array holds the DMU buffers which contain all bytes in a

module/zfs/dbuf.c

@@ -3306,6 +3306,20 @@ dbuf_refcount(dmu_buf_impl_t *db)
 	return (refcount_count(&db->db_holds));
 }
 
+uint64_t
+dmu_buf_user_refcount(dmu_buf_t *db_fake)
+{
+	uint64_t holds;
+	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+
+	mutex_enter(&db->db_mtx);
+	ASSERT3U(refcount_count(&db->db_holds), >=, db->db_dirtycnt);
+	holds = refcount_count(&db->db_holds) - db->db_dirtycnt;
+	mutex_exit(&db->db_mtx);
+
+	return (holds);
+}
+
 void *
 dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user,
     dmu_buf_user_t *new_user)

module/zfs/dsl_dataset.c

@@ -791,6 +791,15 @@ dsl_dataset_rele_flags(dsl_dataset_t *ds, ds_hold_flags_t flags, void *tag)
 	    (flags & DS_HOLD_FLAG_DECRYPT)) {
 		(void) spa_keystore_remove_mapping(ds->ds_dir->dd_pool->dp_spa,
 		    ds->ds_object, ds);
+
+		/*
+		 * Encrypted datasets require that users only release their
+		 * decrypting reference after the dirty data has actually
+		 * been written out. This ensures that the mapping exists
+		 * when it is needed to write out dirty data.
+		 */
+		ASSERT(dmu_buf_user_refcount(ds->ds_dbuf) != 0 ||
+		    !dsl_dataset_is_dirty(ds));
 	}
 
 	dmu_buf_rele(ds->ds_dbuf, tag);