Performance tests, fio enhancements

- Set fixed chunk pattern, for sane compression
- Adjust buffer to blocksize, for cross blocksize repeatability
- Use fixed seed, for improved repeatability
- Move comp-percent and comp-chunk to variables
- set variables (mostly) to old defaults

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Signed-off-by: Kjeld Schouten-Lebbing <kjeld@schouten-lebbing.nl>
Closes #9793

tests/zfs-tests/tests/perf/fio/mkfiles.fio

@@ -11,6 +11,7 @@
 
 #
 # Copyright (c) 2016 by Delphix. All rights reserved.
+# Copyright (c) 2020, Kjeld Schouten-Lebbing. All rights reserved.
 #
 
 [global]
@@ -24,7 +25,9 @@ thread=1
 directory=${DIRECTORY}
 numjobs=${NUMJOBS}
 filesize=${FILE_SIZE}
-buffer_compress_percentage=66
+randseed=${RANDSEED}
-buffer_compress_chunk=4096
+buffer_compress_percentage=${COMPPERCENT}
+buffer_pattern=0xdeadbeef
+buffer_compress_chunk=${COMPCHUNK}
 
 [job]

tests/zfs-tests/tests/perf/fio/random_readwrite.fio

@@ -10,7 +10,8 @@
 #
 
 #
-# Copyright (c) 2015, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2015, 2016, Delphix. All rights reserved.
+# Copyright (c) 2020, Kjeld Schouten-Lebbing. All rights reserved.
 #
 
 [global]
@@ -30,7 +31,9 @@ ioengine=psync
 sync=${SYNC_TYPE}
 direct=${DIRECT}
 numjobs=${NUMJOBS}
-buffer_compress_percentage=66
+randseed=${RANDSEED}
-buffer_compress_chunk=4096
+buffer_compress_percentage=${COMPPERCENT}
+buffer_pattern=0xdeadbeef
+buffer_compress_chunk=${COMPCHUNK}
 
 [job]

tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio

@@ -10,7 +10,8 @@
 #
 
 #
-# Copyright (c) 2017 by Delphix. All rights reserved.
+# Copyright (c) 2015, 2016, Delphix. All rights reserved.
+# Copyright (c) 2020, Kjeld Schouten-Lebbing. All rights reserved.
 #
 
 [global]
@@ -30,7 +31,9 @@ ioengine=psync
 sync=${SYNC_TYPE}
 direct=${DIRECT}
 numjobs=${NUMJOBS}
-buffer_compress_percentage=66
+randseed=${RANDSEED}
-buffer_compress_chunk=4096
+buffer_compress_percentage=${COMPPERCENT}
+buffer_pattern=0xdeadbeef
+buffer_compress_chunk=${COMPCHUNK}
 
 [job]

tests/zfs-tests/tests/perf/fio/random_writes.fio

@@ -10,7 +10,8 @@
 #
 
 #
-# Copyright (c) 2015, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2015, 2016, Delphix. All rights reserved.
+# Copyright (c) 2020, Kjeld Schouten-Lebbing. All rights reserved.
 #
 
 [global]
@@ -28,7 +29,9 @@ sync=${SYNC_TYPE}
 direct=${DIRECT}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
-buffer_compress_percentage=66
+randseed=${RANDSEED}
-buffer_compress_chunk=4096
+buffer_compress_percentage=${COMPPERCENT}
+buffer_pattern=0xdeadbeef
+buffer_compress_chunk=${COMPCHUNK}
 
 [job]

tests/zfs-tests/tests/perf/fio/sequential_readwrite.fio

@@ -0,0 +1,39 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015, 2016, Delphix. All rights reserved.
+# Copyright (c) 2020, Kjeld Schouten-Lebbing. All rights reserved.
+#
+
+[global]
+filename_format=file$jobnum
+nrfiles=16
+group_reporting=1
+fallocate=0
+overwrite=0
+thread=1
+rw=readwrite
+rwmixread=80
+time_based=1
+directory=${DIRECTORY}
+runtime=${RUNTIME}
+bssplit=4k/50:8k/30:128k/10:1m/10
+ioengine=psync
+sync=${SYNC_TYPE}
+direct=${DIRECT}
+numjobs=${NUMJOBS}
+randseed=${RANDSEED}
+buffer_compress_percentage=${COMPPERCENT}
+buffer_pattern=0xdeadbeef
+buffer_compress_chunk=${COMPCHUNK}
+
+[job]

tests/zfs-tests/tests/perf/fio/sequential_writes.fio

@@ -10,7 +10,8 @@
 #
 
 #
-# Copyright (c) 2015, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2015, 2016, Delphix. All rights reserved.
+# Copyright (c) 2020, Kjeld Schouten-Lebbing. All rights reserved.
 #
 
 [global]
@@ -28,7 +29,9 @@ sync=${SYNC_TYPE}
 direct=${DIRECT}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
-buffer_compress_percentage=66
+randseed=${RANDSEED}
-buffer_compress_chunk=4096
+buffer_compress_percentage=${COMPPERCENT}
+buffer_pattern=0xdeadbeef
+buffer_compress_chunk=${COMPCHUNK}
 
 [job]

tests/zfs-tests/tests/perf/perf.shlib

@@ -106,6 +106,9 @@ function do_fio_run_impl
 	log_note "DIRECTORY: " $DIRECTORY
 
 	export RUNTIME=$PERF_RUNTIME
+	export RANDSEED=$PERF_RANDSEED
+	export COMPPERCENT=$PERF_COMPPERCENT
+	export COMPCHUNK=$PERF_COMPCHUNK
 	export FILESIZE=$((TOTAL_SIZE / threads))
 	export NUMJOBS=$threads
 	export SYNC_TYPE=$sync

tests/zfs-tests/tests/perf/regression/random_reads.ksh

@@ -58,6 +58,9 @@ export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
 # Variables for use by fio.
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
@@ -65,6 +68,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}

tests/zfs-tests/tests/perf/regression/random_readwrite.ksh

@@ -58,6 +58,9 @@ export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
 # Variables for use by fio.
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'4 8 16 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
@@ -65,6 +68,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_IOSIZES=''		# bssplit used instead
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}

tests/zfs-tests/tests/perf/regression/random_readwrite_fixed.ksh

@@ -47,6 +47,9 @@ export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
 # Variables for use by fio.
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
@@ -54,6 +57,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_IOSIZES='8k 64k'
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}

tests/zfs-tests/tests/perf/regression/random_writes.ksh

@@ -57,6 +57,9 @@ export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
 # Variables for use by fio.
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
@@ -64,6 +67,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 256k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'32 128'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}

tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh

@@ -45,6 +45,9 @@ export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
 
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'1 2 4 8 16 32 64 128'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0 1'}
@@ -53,6 +56,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 16 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0 1'}

tests/zfs-tests/tests/perf/regression/sequential_reads.ksh

@@ -58,6 +58,9 @@ export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
 # Variables for use by fio.
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
@@ -65,6 +68,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}

tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh

@@ -48,6 +48,9 @@ export TOTAL_SIZE=$(($(get_max_arc_size) / 2))
 # Variables for use by fio.
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
@@ -55,6 +58,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}

tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh

@@ -54,6 +54,9 @@ export TOTAL_SIZE=$(($(get_max_arc_size) / 2))
 # Variables for use by fio.
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
@@ -61,6 +64,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}

tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh

@@ -52,6 +52,9 @@ export TOTAL_SIZE=$(($(get_max_dbuf_cache_size) * 3 / 4))
 # Variables for use by fio.
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
@@ -59,6 +62,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}

tests/zfs-tests/tests/perf/regression/sequential_writes.ksh

@@ -57,6 +57,9 @@ export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
 # Variables for use by fio.
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
@@ -64,6 +67,9 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 256k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export RANDSEED=${PERF_RANDSEED:-'1234'}
+	export COMPPERCENT=${PERF_COMPPERCENT:-'66'}
+	export COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
 	export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}