Linux SPL module init: Handle memory allocation failures correctly

Upon inspection of our code, I noticed that we assume that
__alloc_percpu() cannot fail, and while it probably never has failed in
practice, technically, it can fail, so we should handle that.

Additionally, we incorrectly assume that `taskq_create()` in
spl_kmem_cache_init() cannot fail. The same remark applies to it.

Lastly, `spl-init()` failures should always return negative error
values, but in some places, we are returning positive 1, which is
incorrect. We change those values to their correct error codes.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Richard Yao <richard.yao@alumni.stonybrook.edu>
Closes #13847

module/os/linux/spl/spl-generic.c

@@ -705,7 +705,7 @@ spl_kvmem_init(void)
  * initialize each of the per-cpu seeds so that the sequences generated on each
  * CPU are guaranteed to never overlap in practice.
  */
-static void __init
+static int __init
 spl_random_init(void)
 {
 	uint64_t s[2];
@@ -714,6 +714,9 @@ spl_random_init(void)
 	spl_pseudo_entropy = __alloc_percpu(2 * sizeof (uint64_t),
 	    sizeof (uint64_t));
 
+	if (!spl_pseudo_entropy)
+		return (-ENOMEM);
+
 	get_random_bytes(s, sizeof (s));
 
 	if (s[0] == 0 && s[1] == 0) {
@@ -737,6 +740,8 @@ spl_random_init(void)
 		wordp[0] = s[0];
 		wordp[1] = s[1];
 	}
+
+	return (0);
 }
 
 static void
@@ -757,7 +762,8 @@ spl_init(void)
 {
 	int rc = 0;
 
-	spl_random_init();
+	if ((rc = spl_random_init()))
+		goto out0;
 
 	if ((rc = spl_kvmem_init()))
 		goto out1;
@@ -800,6 +806,8 @@ out3:
 out2:
 	spl_kvmem_fini();
 out1:
+	spl_random_fini();
+out0:
 	return (rc);
 }
 

module/os/linux/spl/spl-kmem-cache.c

@@ -1452,6 +1452,9 @@ spl_kmem_cache_init(void)
 	    spl_kmem_cache_kmem_threads * 8, INT_MAX,
 	    TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
 
+	if (spl_kmem_cache_taskq == NULL)
+		return (-ENOMEM);
+
 	return (0);
 }
 

module/os/linux/spl/spl-taskq.c

@@ -1379,7 +1379,7 @@ spl_taskq_init(void)
 	system_taskq = taskq_create("spl_system_taskq", MAX(boot_ncpus, 64),
 	    maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC);
 	if (system_taskq == NULL)
-		return (1);
+		return (-ENOMEM);
 
 	system_delay_taskq = taskq_create("spl_delay_taskq", MAX(boot_ncpus, 4),
 	    maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC);
@@ -1388,7 +1388,7 @@ spl_taskq_init(void)
 		cpuhp_remove_multi_state(spl_taskq_cpuhp_state);
 #endif
 		taskq_destroy(system_taskq);
-		return (1);
+		return (-ENOMEM);
 	}
 
 	dynamic_taskq = taskq_create("spl_dynamic_taskq", 1,
@@ -1399,7 +1399,7 @@ spl_taskq_init(void)
 #endif
 		taskq_destroy(system_taskq);
 		taskq_destroy(system_delay_taskq);
-		return (1);
+		return (-ENOMEM);
 	}
 
 	/*

module/os/linux/spl/spl-tsd.c

@@ -706,7 +706,7 @@ spl_tsd_init(void)
 {
 	tsd_hash_table = tsd_hash_table_init(TSD_HASH_TABLE_BITS_DEFAULT);
 	if (tsd_hash_table == NULL)
-		return (1);
+		return (-ENOMEM);
 
 	return (0);
 }

module/os/linux/spl/spl-zlib.c

@@ -204,7 +204,7 @@ spl_zlib_init(void)
 	    size, 0, NULL, NULL, NULL, NULL, NULL,
 	    KMC_KVMEM);
 	if (!zlib_workspace_cache)
-		return (1);
+		return (-ENOMEM);
 
 	return (0);
 }