Add JSON output support to channel programs

The changes piggyback JSON output support on top of channel programs 
(#6558).  This way the JSON output support is targeted to scripting 
use cases and is easily maintainable since it really only touches 
one function (zfs_do_channel_program()).

This patch ports Joyent's JSON nvlist library from illumos to enable 
easy JSON printing of channel program output nvlist.  To keep the 
delta small I also took advantage of the fact that printing in
zfs_do_channel_program() was almost always done before exiting 
the program.

Reviewed by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: Richard Elling <Richard.Elling@RichardElling.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Alek Pinchuk <apinchuk@datto.com>
Closes #7281
This commit is contained in:
Alek P 2018-03-19 15:40:58 -04:00 committed by Brian Behlendorf
parent a76f3d0437
commit 272b5d730f
13 changed files with 639 additions and 11 deletions

View File

@ -27,6 +27,7 @@
* Copyright (c) 2013 Steven Hartland. All rights reserved.
* Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
* Copyright 2016 Nexenta Systems, Inc.
* Copyright (c) 2018 Datto Inc.
*/
#include <assert.h>
@ -358,7 +359,7 @@ get_usage(zfs_help_t idx)
case HELP_BOOKMARK:
return (gettext("\tbookmark <snapshot> <bookmark>\n"));
case HELP_CHANNEL_PROGRAM:
return (gettext("\tprogram [-n] [-t <instruction limit>] "
return (gettext("\tprogram [-jn] [-t <instruction limit>] "
"[-m <memory limit (b)>] <pool> <program file> "
"[lua args...]\n"));
case HELP_LOAD_KEY:
@ -7220,11 +7221,11 @@ zfs_do_channel_program(int argc, char **argv)
nvlist_t *outnvl;
uint64_t instrlimit = ZCP_DEFAULT_INSTRLIMIT;
uint64_t memlimit = ZCP_DEFAULT_MEMLIMIT;
boolean_t sync_flag = B_TRUE;
boolean_t sync_flag = B_TRUE, json_output = B_FALSE;
zpool_handle_t *zhp;
/* check options */
while ((c = getopt(argc, argv, "nt:m:")) != -1) {
while ((c = getopt(argc, argv, "nt:m:j")) != -1) {
switch (c) {
case 't':
case 'm': {
@ -7266,6 +7267,10 @@ zfs_do_channel_program(int argc, char **argv)
sync_flag = B_FALSE;
break;
}
case 'j': {
json_output = B_TRUE;
break;
}
case '?':
(void) fprintf(stderr, gettext("invalid option '%c'\n"),
optopt);
@ -7391,14 +7396,18 @@ zfs_do_channel_program(int argc, char **argv)
gettext("Channel program execution failed:\n%s\n"),
errstring);
if (ret == ETIME && instructions != 0)
(void) fprintf(stderr, "%llu Lua instructions\n",
(void) fprintf(stderr,
gettext("%llu Lua instructions\n"),
(u_longlong_t)instructions);
} else {
(void) printf("Channel program fully executed ");
if (nvlist_empty(outnvl)) {
(void) printf("with no return value.\n");
if (json_output) {
(void) nvlist_print_json(stdout, outnvl);
} else if (nvlist_empty(outnvl)) {
(void) fprintf(stdout, gettext("Channel program fully "
"executed and did not produce output.\n"));
} else {
(void) printf("with return value:\n");
(void) fprintf(stdout, gettext("Channel program fully "
"executed and produced output:\n"));
dump_nvlist(outnvl, 4);
}
}

View File

@ -205,6 +205,7 @@ AC_CONFIG_FILES([
tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile
tests/zfs-tests/tests/functional/cli_root/zfs/Makefile
tests/zfs-tests/tests/functional/cli_root/zfs_mount/Makefile
tests/zfs-tests/tests/functional/cli_root/zfs_program/Makefile
tests/zfs-tests/tests/functional/cli_root/zfs_promote/Makefile
tests/zfs-tests/tests/functional/cli_root/zfs_property/Makefile
tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile

View File

@ -20,6 +20,7 @@
*/
/*
* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, Joyent, Inc. All rights reserved.
*/
#ifndef _LIBNVPAIR_H
@ -46,6 +47,7 @@ extern int nvpair_value_match_regex(nvpair_t *, int, char *, regex_t *,
char **);
extern void nvlist_print(FILE *, nvlist_t *);
int nvlist_print_json(FILE *, nvlist_t *);
extern void dump_nvlist(nvlist_t *, int);
/*

View File

@ -18,6 +18,7 @@ lib_LTLIBRARIES = libnvpair.la
USER_C = \
libnvpair.c \
libnvpair_json.c \
nvpair_alloc_system.c
KERNEL_C = \

View File

@ -0,0 +1,403 @@
/*
* This file and its contents are supplied under the terms of the
* Common Development and Distribution License ("CDDL"), version 1.0.
* You may only use this file in accordance with the terms of version
* 1.0 of the CDDL.
*
* A full copy of the text of the CDDL should have accompanied this
* source. A copy of the CDDL is also available via the Internet at
* http://www.illumos.org/license/CDDL.
*/
/*
* Copyright (c) 2014, Joyent, Inc.
*/
#include <stdio.h>
#include <stdlib.h>
#include <strings.h>
#include <wchar.h>
#include <sys/debug.h>
#include "libnvpair.h"
#define FPRINTF(fp, ...) \
do { \
if (fprintf(fp, __VA_ARGS__) < 0) \
return (-1); \
} while (0)
/*
* When formatting a string for JSON output we must escape certain characters,
* as described in RFC4627. This applies to both member names and
* DATA_TYPE_STRING values.
*
* This function will only operate correctly if the following conditions are
* met:
*
* 1. The input String is encoded in the current locale.
*
* 2. The current locale includes the Basic Multilingual Plane (plane 0)
* as defined in the Unicode standard.
*
* The output will be entirely 7-bit ASCII (as a subset of UTF-8) with all
* representable Unicode characters included in their escaped numeric form.
*/
static int
nvlist_print_json_string(FILE *fp, const char *input)
{
mbstate_t mbr;
wchar_t c;
size_t sz;
bzero(&mbr, sizeof (mbr));
FPRINTF(fp, "\"");
while ((sz = mbrtowc(&c, input, MB_CUR_MAX, &mbr)) > 0) {
switch (c) {
case '"':
FPRINTF(fp, "\\\"");
break;
case '\n':
FPRINTF(fp, "\\n");
break;
case '\r':
FPRINTF(fp, "\\r");
break;
case '\\':
FPRINTF(fp, "\\\\");
break;
case '\f':
FPRINTF(fp, "\\f");
break;
case '\t':
FPRINTF(fp, "\\t");
break;
case '\b':
FPRINTF(fp, "\\b");
break;
default:
if ((c >= 0x00 && c <= 0x1f) ||
(c > 0x7f && c <= 0xffff)) {
/*
* Render both Control Characters and Unicode
* characters in the Basic Multilingual Plane
* as JSON-escaped multibyte characters.
*/
FPRINTF(fp, "\\u%04x", (int)(0xffff & c));
} else if (c >= 0x20 && c <= 0x7f) {
/*
* Render other 7-bit ASCII characters directly
* and drop other, unrepresentable characters.
*/
FPRINTF(fp, "%c", (int)(0xff & c));
}
break;
}
input += sz;
}
if (sz == (size_t)-1 || sz == (size_t)-2) {
/*
* We last read an invalid multibyte character sequence,
* so return an error.
*/
return (-1);
}
FPRINTF(fp, "\"");
return (0);
}
/*
* Dump a JSON-formatted representation of an nvlist to the provided FILE *.
* This routine does not output any new-lines or additional whitespace other
* than that contained in strings, nor does it call fflush(3C).
*/
int
nvlist_print_json(FILE *fp, nvlist_t *nvl)
{
nvpair_t *curr;
boolean_t first = B_TRUE;
FPRINTF(fp, "{");
for (curr = nvlist_next_nvpair(nvl, NULL); curr;
curr = nvlist_next_nvpair(nvl, curr)) {
data_type_t type = nvpair_type(curr);
if (!first)
FPRINTF(fp, ",");
else
first = B_FALSE;
if (nvlist_print_json_string(fp, nvpair_name(curr)) == -1)
return (-1);
FPRINTF(fp, ":");
switch (type) {
case DATA_TYPE_STRING: {
char *string = fnvpair_value_string(curr);
if (nvlist_print_json_string(fp, string) == -1)
return (-1);
break;
}
case DATA_TYPE_BOOLEAN: {
FPRINTF(fp, "true");
break;
}
case DATA_TYPE_BOOLEAN_VALUE: {
FPRINTF(fp, "%s", fnvpair_value_boolean_value(curr) ==
B_TRUE ? "true" : "false");
break;
}
case DATA_TYPE_BYTE: {
FPRINTF(fp, "%hhu", fnvpair_value_byte(curr));
break;
}
case DATA_TYPE_INT8: {
FPRINTF(fp, "%hhd", fnvpair_value_int8(curr));
break;
}
case DATA_TYPE_UINT8: {
FPRINTF(fp, "%hhu", fnvpair_value_uint8(curr));
break;
}
case DATA_TYPE_INT16: {
FPRINTF(fp, "%hd", fnvpair_value_int16(curr));
break;
}
case DATA_TYPE_UINT16: {
FPRINTF(fp, "%hu", fnvpair_value_uint16(curr));
break;
}
case DATA_TYPE_INT32: {
FPRINTF(fp, "%d", fnvpair_value_int32(curr));
break;
}
case DATA_TYPE_UINT32: {
FPRINTF(fp, "%u", fnvpair_value_uint32(curr));
break;
}
case DATA_TYPE_INT64: {
FPRINTF(fp, "%lld",
(long long)fnvpair_value_int64(curr));
break;
}
case DATA_TYPE_UINT64: {
FPRINTF(fp, "%llu",
(unsigned long long)fnvpair_value_uint64(curr));
break;
}
case DATA_TYPE_HRTIME: {
hrtime_t val;
VERIFY0(nvpair_value_hrtime(curr, &val));
FPRINTF(fp, "%llu", (unsigned long long)val);
break;
}
case DATA_TYPE_DOUBLE: {
double val;
VERIFY0(nvpair_value_double(curr, &val));
FPRINTF(fp, "%f", val);
break;
}
case DATA_TYPE_NVLIST: {
if (nvlist_print_json(fp,
fnvpair_value_nvlist(curr)) == -1)
return (-1);
break;
}
case DATA_TYPE_STRING_ARRAY: {
char **val;
uint_t valsz, i;
VERIFY0(nvpair_value_string_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
if (nvlist_print_json_string(fp, val[i]) == -1)
return (-1);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_NVLIST_ARRAY: {
nvlist_t **val;
uint_t valsz, i;
VERIFY0(nvpair_value_nvlist_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
if (nvlist_print_json(fp, val[i]) == -1)
return (-1);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_BOOLEAN_ARRAY: {
boolean_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_boolean_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, val[i] == B_TRUE ?
"true" : "false");
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_BYTE_ARRAY: {
uchar_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_byte_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, "%hhu", val[i]);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_UINT8_ARRAY: {
uint8_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_uint8_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, "%hhu", val[i]);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_INT8_ARRAY: {
int8_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_int8_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, "%hd", val[i]);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_UINT16_ARRAY: {
uint16_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_uint16_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, "%hu", val[i]);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_INT16_ARRAY: {
int16_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_int16_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, "%hd", val[i]);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_UINT32_ARRAY: {
uint32_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_uint32_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, "%u", val[i]);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_INT32_ARRAY: {
int32_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_int32_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, "%d", val[i]);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_UINT64_ARRAY: {
uint64_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_uint64_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, "%llu",
(unsigned long long)val[i]);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_INT64_ARRAY: {
int64_t *val;
uint_t valsz, i;
VERIFY0(nvpair_value_int64_array(curr, &val, &valsz));
FPRINTF(fp, "[");
for (i = 0; i < valsz; i++) {
if (i > 0)
FPRINTF(fp, ",");
FPRINTF(fp, "%lld", (long long)val[i]);
}
FPRINTF(fp, "]");
break;
}
case DATA_TYPE_UNKNOWN:
return (-1);
}
}
FPRINTF(fp, "}");
return (0);
}

View File

@ -18,7 +18,7 @@
.Nd executes ZFS channel programs
.Sh SYNOPSIS
.Cm "zfs program"
.Op Fl n
.Op Fl jn
.Op Fl t Ar instruction-limit
.Op Fl m Ar memory-limit
.Ar pool
@ -46,6 +46,10 @@ will be run on
and any attempts to access or modify other pools will cause an error.
.Sh OPTIONS
.Bl -tag -width "-t"
.It Fl j
Display channel program output in JSON format. When this flag is specified and
standard output is empty - channel program encountered an error. The details of
such an error will be printed to standard error in plain text.
.It Fl n
Executes a read-only channel program, which runs faster.
The program cannot change on-disk state by calling functions from the

View File

@ -300,7 +300,7 @@
.Ar snapshot Ar snapshot Ns | Ns Ar filesystem
.Nm
.Cm program
.Op Fl n
.Op Fl jn
.Op Fl t Ar timeout
.Op Fl m Ar memory_limit
.Ar pool script
@ -4264,7 +4264,7 @@ Display the path's inode change time as the first column of output.
.It Xo
.Nm
.Cm program
.Op Fl n
.Op Fl jn
.Op Fl t Ar timeout
.Op Fl m Ar memory_limit
.Ar pool script
@ -4286,6 +4286,10 @@ For full documentation of the ZFS channel program interface, see the manual
page for
.Xr zfs-program 8 .
.Bl -tag -width ""
.It Fl j
Display channel program output in JSON format. When this flag is specified and
standard output is empty - channel program encountered an error. The details of
such an error will be printed to standard error in plain text.
.It Fl n
Executes a read-only channel program, which runs faster.
The program cannot change on-disk state by calling functions from

View File

@ -177,6 +177,10 @@ tests = ['zfs_mount_001_pos', 'zfs_mount_002_pos', 'zfs_mount_003_pos',
'zfs_mount_encrypted', 'zfs_mount_remount']
tags = ['functional', 'cli_root', 'zfs_mount']
[tests/functional/cli_root/zfs_program]
tests = ['zfs_program_json']
tags = ['functional', 'cli_root', 'zfs_program']
[tests/functional/cli_root/zfs_promote]
tests = ['zfs_promote_001_pos', 'zfs_promote_002_pos', 'zfs_promote_003_pos',
'zfs_promote_004_pos', 'zfs_promote_005_pos', 'zfs_promote_006_neg',

View File

@ -16,6 +16,7 @@ SUBDIRS = \
zfs_inherit \
zfs_load-key \
zfs_mount \
zfs_program \
zfs_promote \
zfs_property \
zfs_receive \

View File

@ -0,0 +1,5 @@
pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/cli_root/zfs_program
dist_pkgdata_SCRIPTS = \
setup.ksh \
cleanup.ksh \
zfs_program_json.ksh

View File

@ -0,0 +1,30 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
# Use is subject to license terms.
#
. $STF_SUITE/include/libtest.shlib
default_cleanup

View File

@ -0,0 +1,32 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
# Use is subject to license terms.
#
. $STF_SUITE/include/libtest.shlib
DISK=${DISKS%% *}
default_setup $DISK

View File

@ -0,0 +1,132 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy is of the CDDL is also available via the Internet
# at http://www.illumos.org/license/CDDL.
#
# CDDL HEADER END
#
#
# Copyright (c) 2018 Datto Inc.
#
. $STF_SUITE/include/libtest.shlib
#
# DESCRIPTION:
#
# STRATEGY:
# 1. Compare JSON output formatting for a channel program to template
# 2. Using bad command line option (-Z) gives correct error output
#
verify_runnable "both"
function cleanup
{
log_must zfs destroy $TESTDS
return 0
}
log_onexit cleanup
log_assert "Channel programs output valid JSON"
TESTDS="$TESTPOOL/zcp-json"
log_must zfs create $TESTDS
TESTZCP="/$TESTDS/zfs_rlist.zcp"
cat > "$TESTZCP" << EOF
succeeded = {}
failed = {}
function list_recursive(root, prop)
for child in zfs.list.children(root) do
list_recursive(child, prop)
end
val, src = zfs.get_prop(root, prop)
if (val == nil) then
failed[root] = val
else
succeeded[root] = val
end
end
args = ...
argv = args["argv"]
list_recursive(argv[1], argv[2])
results = {}
results["succeeded"] = succeeded
results["failed"] = failed
return results
EOF
# 1. Compare JSON output formatting for a channel program to template
typeset -a pos_cmds=("recordsize" "type")
typeset -a pos_cmds_out=(
"{
\"return\": {
\"failed\": {},
\"succeeded\": {
\"$TESTDS\": 131072
}
}
}"
"{
\"return\": {
\"failed\": {},
\"succeeded\": {
\"$TESTDS\": \"filesystem\"
}
}
}")
typeset -i cnt=0
typeset cmd
for cmd in ${pos_cmds[@]}; do
log_must zfs program $TESTPOOL $TESTZCP $TESTDS $cmd 2>&1
log_must zfs program $TESTPOOL -j $TESTZCP $TESTDS $cmd 2>&1
# json.tool is needed to guarantee consistent ordering of fields
# sed is needed to trim trailing space in CentOS 6's json.tool output
OUTPUT=$(zfs program $TESTPOOL -j $TESTZCP $TESTDS $cmd 2>&1 | python -m json.tool | sed 's/[[:space:]]*$//')
if [ "$OUTPUT" != "${pos_cmds_out[$cnt]}" ]; then
log_note "Got :$OUTPUT"
log_note "Expected:${pos_cmds_out[$cnt]}"
log_fail "Unexpected channel program output";
fi
cnt=$((cnt + 1))
done
# 2. Using bad command line option (-Z) gives correct error output
typeset -a neg_cmds=("-Z")
typeset -a neg_cmds_out=(
"invalid option 'Z'
usage:
program [-jn] [-t <instruction limit>] [-m <memory limit (b)>] <pool> <program file> [lua args...]
For the property list, run: zfs set|get
For the delegated permission list, run: zfs allow|unallow")
cnt=0
for cmd in ${neg_cmds[@]}; do
log_mustnot zfs program $TESTPOOL $TESTZCP $TESTDS $cmd 2>&1
log_mustnot zfs program $TESTPOOL -j $TESTZCP $TESTDS $cmd 2>&1
OUTPUT=$(zfs program $TESTPOOL -j $TESTZCP $TESTDS $cmd 2>&1)
if [ "$OUTPUT" != "${neg_cmds_out[$cnt]}" ]; then
log_note "Got :$OUTPUT"
log_note "Expected:${neg_cmds_out[$cnt]}"
log_fail "Unexpected channel program error output";
fi
cnt=$((cnt + 1))
done
log_pass "Channel programs output valid JSON"