Fix coverity defects: zfs channel programs

CID 173243, 173245:  Memory - corruptions  (OVERRUN)
 Added size argument to lcompat_sprintf() to avoid use of INT_MAX

CID 173244:  Integer handling issues  (OVERFLOW_BEFORE_WIDEN)
 Added cast to uint64_t to avoid a 32 bit overflow warning

CID 173242:  Integer handling issues  (CONSTANT_EXPRESSION_RESULT)
 Conditionally removed unused luai_numisnan() floating point check

CID 173241:  Resource leaks  (RESOURCE_LEAK)
 Added missing close(fd) on error path

CID 173240:    (UNINIT)
Fixed uninitialized variable in get_special_prop()

CID 147560:  Null pointer dereferences  (NULL_RETURNS)
Cleaned up bad code merge in dsl_dataset_promote_check()

CID 28475:  Memory - illegal accesses  (OVERRUN)
Fixed lcompat_sprintf() to use a size paramater

CID 28418, 28422:  Error handling issues  (CHECKED_RETURN)
Added function result cast to (void) to avoid warning

CID 23935, 28411, 28412:  Memory - corruptions  (ARRAY_VS_SINGLETON)
Added casts to avoid exposing result as an array

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Don Brady <don.brady@delphix.com>
Closes #7181

cmd/zfs/zfs_main.c

@@ -7275,6 +7275,8 @@ zfs_do_channel_program(int argc, char **argv)
 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
 		(void) fprintf(stderr, gettext("cannot open pool '%s'"),
 		    poolname);
+		if (fd != 0)
+			(void) close(fd);
 		return (1);
 	}
 	zpool_close(zhp);

include/sys/lua/luaconf.h

@@ -12,7 +12,7 @@
 #include <sys/zfs_context.h>
 
 
-extern ssize_t lcompat_sprintf(char *, const char *, ...);
+extern ssize_t lcompat_sprintf(char *, size_t size, const char *, ...);
 extern int64_t lcompat_strtoll(const char *, char **);
 extern int64_t lcompat_pow(int64_t, int64_t);
 
@@ -402,9 +402,10 @@ extern int64_t lcompat_pow(int64_t, int64_t);
 #define	PRId64	"lld"
 #endif
 
-#define LUA_NUMBER_FMT		"%" PRId64
-#define lua_number2str(s,n)	lcompat_sprintf((s), LUA_NUMBER_FMT, (n))
 #define LUAI_MAXNUMBER2STR	32 /* 16 digits, sign, point, and \0 */
+#define LUA_NUMBER_FMT		"%" PRId64
+#define lua_number2str(s,n)	\
+	lcompat_sprintf((s), LUAI_MAXNUMBER2STR, LUA_NUMBER_FMT, (n))
 
 
 /*

module/lua/lapi.c

@@ -424,7 +424,7 @@ LUA_API lua_CFunction lua_tocfunction (lua_State *L, int idx) {
 LUA_API void *lua_touserdata (lua_State *L, int idx) {
   StkId o = index2addr(L, idx);
   switch (ttypenv(o)) {
-    case LUA_TUSERDATA: return (rawuvalue(o) + 1);
+    case LUA_TUSERDATA: return ((void *)(rawuvalue(o) + 1));
     case LUA_TLIGHTUSERDATA: return pvalue(o);
     default: return NULL;
   }

module/lua/lcompat.c

@@ -6,13 +6,13 @@
 
 
 ssize_t
-lcompat_sprintf(char *buf, const char *fmt, ...)
+lcompat_sprintf(char *buf, size_t size, const char *fmt, ...)
 {
 	ssize_t res;
 	va_list args;
 
 	va_start(args, fmt);
-	res = vsnprintf(buf, INT_MAX, fmt, args);
+	res = vsnprintf(buf, size, fmt, args);
 	va_end(args);
 
 	return (res);

module/lua/lctype.h

@@ -48,7 +48,7 @@
 /*
 ** add 1 to char to allow index -1 (EOZ)
 */
-#define testprop(c,p)	(luai_ctype_[(c)+1] & (p))
+#define testprop(c,p)	(luai_ctype_[(lu_byte)(c)+1] & (p))
 
 /*
 ** 'lalpha' (Lua alphabetic) and 'lalnum' (Lua alphanumeric) both include '_'

module/lua/llex.c

@@ -235,7 +235,7 @@ static void read_numeral (LexState *ls, SemInfo *seminfo) {
     expo = "Pp";
   for (;;) {
     if (check_next(ls, expo))  /* exponent part? */
-      check_next(ls, "+-");  /* optional exponent sign */
+      (void) check_next(ls, "+-");  /* optional exponent sign */
     if (lisxdigit(ls->current) || ls->current == '.')
       save_and_next(ls);
     else  break;

module/lua/lobject.c

@@ -201,7 +201,7 @@ const char *luaO_pushvfstring (lua_State *L, const char *fmt, va_list argp) {
       }
       case 'p': {
         char buff[4*sizeof(void *) + 8]; /* should be enough space for a `%p' */
-        int l = lcompat_sprintf(buff, "%p", va_arg(argp, void *));
+        int l = lcompat_sprintf(buff, sizeof(buff), "%p", va_arg(argp, void *));
         pushstr(L, buff, l);
         break;
       }

module/lua/lparser.c

@@ -1527,7 +1527,7 @@ static void retstat (LexState *ls) {
     }
   }
   luaK_ret(fs, first, nret);
-  testnext(ls, ';');  /* skip optional semicolon */
+  (void) testnext(ls, ';');  /* skip optional semicolon */
 }
 
 

module/lua/lstring.c

@@ -97,14 +97,16 @@ void luaS_resize (lua_State *L, int newsize) {
 static TString *createstrobj (lua_State *L, const char *str, size_t l,
                               int tag, unsigned int h, GCObject **list) {
   TString *ts;
+  char *sbuf;
   size_t totalsize;  /* total size of TString object */
   totalsize = sizeof(TString) + ((l + 1) * sizeof(char));
   ts = &luaC_newobj(L, tag, totalsize, list, 0)->ts;
   ts->tsv.len = l;
   ts->tsv.hash = h;
   ts->tsv.extra = 0;
-  memcpy(ts+1, str, l*sizeof(char));
+  sbuf = (char *)(TString *)(ts + 1);
-  ((char *)(ts+1))[l] = '\0';  /* ending 0 */
+  memcpy(sbuf, str, l*sizeof(char));
+  sbuf[l] = '\0';  /* ending 0 */
   return ts;
 }
 

module/lua/ltable.c

@@ -405,8 +405,10 @@ static Node *getfreepos (Table *t) {
 TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
   Node *mp;
   if (ttisnil(key)) luaG_runerror(L, "table index is nil");
+#if defined LUA_HAS_FLOAT_NUMBERS
   else if (ttisnumber(key) && luai_numisnan(L, nvalue(key)))
     luaG_runerror(L, "table index is NaN");
+#endif
   mp = mainposition(t, key);
   if (!ttisnil(gval(mp)) || isdummy(mp)) {  /* main position is taken? */
     Node *othern;

module/lua/ltable.h

@@ -11,7 +11,7 @@
 #include "lobject.h"
 
 
-#define gnode(t,i)	(&(t)->node[i])
+#define gnode(t,i)	((Node *)&(t)->node[i])
 #define gkey(n)		(&(n)->i_key.tvk)
 #define gval(n)		(&(n)->i_val)
 #define gnext(n)	((n)->i_key.nk.next)

module/zfs/dsl_dataset.c

@@ -2750,12 +2750,8 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 		return (err);
 
 	hds = ddpa->ddpa_clone;
-	snap = list_head(&ddpa->shared_snaps);
-	origin_ds = snap->ds;
 	max_snap_len = MAXNAMELEN - strlen(ddpa->ddpa_clonename) - 1;
 
-	snap = list_head(&ddpa->origin_snaps);
-
 	if (dsl_dataset_phys(hds)->ds_flags & DS_FLAG_NOPROMOTE) {
 		promote_rele(ddpa, FTAG);
 		return (SET_ERROR(EXDEV));
@@ -2789,6 +2785,7 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 
 	/* compute origin's new unique space */
 	snap = list_tail(&ddpa->clone_snaps);
+	ASSERT(snap != NULL);
 	ASSERT3U(dsl_dataset_phys(snap->ds)->ds_prev_snap_obj, ==,
 	    origin_ds->ds_object);
 	dsl_deadlist_space_range(&snap->ds->ds_deadlist,

module/zfs/zcp_get.c

@@ -303,7 +303,7 @@ get_special_prop(lua_State *state, dsl_dataset_t *ds, const char *dsname,
 {
 	int error = 0;
 	objset_t *os;
-	uint64_t numval;
+	uint64_t numval = 0;
 	char *strval = kmem_alloc(ZAP_MAXVALUELEN, KM_SLEEP);
 	char setpoint[ZFS_MAX_DATASET_NAME_LEN] =
 	    "Internal error - setpoint not determined";

module/zfs/zcp_synctask.c

@@ -297,7 +297,8 @@ zcp_synctask_wrapper(lua_State *state)
 	dsl_pool_t *dp = ri->zri_pool;
 
 	/* MOS space is triple-dittoed, so we multiply by 3. */
-	uint64_t funcspace = (info->blocks_modified << DST_AVG_BLKSHIFT) * 3;
+	uint64_t funcspace =
+	    ((uint64_t)info->blocks_modified << DST_AVG_BLKSHIFT) * 3;
 
 	zcp_parse_args(state, info->name, info->pargs, info->kwargs);