Fix coverity defects: zfs channel programs

CID 173243, 173245:  Memory - corruptions  (OVERRUN)
 Added size argument to lcompat_sprintf() to avoid use of INT_MAX

CID 173244:  Integer handling issues  (OVERFLOW_BEFORE_WIDEN)
 Added cast to uint64_t to avoid a 32 bit overflow warning

CID 173242:  Integer handling issues  (CONSTANT_EXPRESSION_RESULT)
 Conditionally removed unused luai_numisnan() floating point check

CID 173241:  Resource leaks  (RESOURCE_LEAK)
 Added missing close(fd) on error path

CID 173240:    (UNINIT)
Fixed uninitialized variable in get_special_prop()

CID 147560:  Null pointer dereferences  (NULL_RETURNS)
Cleaned up bad code merge in dsl_dataset_promote_check()

CID 28475:  Memory - illegal accesses  (OVERRUN)
Fixed lcompat_sprintf() to use a size paramater

CID 28418, 28422:  Error handling issues  (CHECKED_RETURN)
Added function result cast to (void) to avoid warning

CID 23935, 28411, 28412:  Memory - corruptions  (ARRAY_VS_SINGLETON)
Added casts to avoid exposing result as an array

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Don Brady <don.brady@delphix.com>
Closes #7181

module/lua/lapi.c

@@ -424,7 +424,7 @@ LUA_API lua_CFunction lua_tocfunction (lua_State *L, int idx) {
 LUA_API void *lua_touserdata (lua_State *L, int idx) {
   StkId o = index2addr(L, idx);
   switch (ttypenv(o)) {
-    case LUA_TUSERDATA: return (rawuvalue(o) + 1);
+    case LUA_TUSERDATA: return ((void *)(rawuvalue(o) + 1));
     case LUA_TLIGHTUSERDATA: return pvalue(o);
     default: return NULL;
   }

module/lua/lcompat.c

@@ -6,13 +6,13 @@
 
 
 ssize_t
-lcompat_sprintf(char *buf, const char *fmt, ...)
+lcompat_sprintf(char *buf, size_t size, const char *fmt, ...)
 {
 	ssize_t res;
 	va_list args;
 
 	va_start(args, fmt);
-	res = vsnprintf(buf, INT_MAX, fmt, args);
+	res = vsnprintf(buf, size, fmt, args);
 	va_end(args);
 
 	return (res);

module/lua/lctype.h

@@ -48,7 +48,7 @@
 /*
 ** add 1 to char to allow index -1 (EOZ)
 */
-#define testprop(c,p)	(luai_ctype_[(c)+1] & (p))
+#define testprop(c,p)	(luai_ctype_[(lu_byte)(c)+1] & (p))
 
 /*
 ** 'lalpha' (Lua alphabetic) and 'lalnum' (Lua alphanumeric) both include '_'

module/lua/llex.c

@@ -235,7 +235,7 @@ static void read_numeral (LexState *ls, SemInfo *seminfo) {
     expo = "Pp";
   for (;;) {
     if (check_next(ls, expo))  /* exponent part? */
-      check_next(ls, "+-");  /* optional exponent sign */
+      (void) check_next(ls, "+-");  /* optional exponent sign */
     if (lisxdigit(ls->current) || ls->current == '.')
       save_and_next(ls);
     else  break;

module/lua/lobject.c

@@ -201,7 +201,7 @@ const char *luaO_pushvfstring (lua_State *L, const char *fmt, va_list argp) {
       }
       case 'p': {
         char buff[4*sizeof(void *) + 8]; /* should be enough space for a `%p' */
-        int l = lcompat_sprintf(buff, "%p", va_arg(argp, void *));
+        int l = lcompat_sprintf(buff, sizeof(buff), "%p", va_arg(argp, void *));
         pushstr(L, buff, l);
         break;
       }

module/lua/lparser.c

@@ -1527,7 +1527,7 @@ static void retstat (LexState *ls) {
     }
   }
   luaK_ret(fs, first, nret);
-  testnext(ls, ';');  /* skip optional semicolon */
+  (void) testnext(ls, ';');  /* skip optional semicolon */
 }
 
 

module/lua/lstring.c

@@ -97,14 +97,16 @@ void luaS_resize (lua_State *L, int newsize) {
 static TString *createstrobj (lua_State *L, const char *str, size_t l,
                               int tag, unsigned int h, GCObject **list) {
   TString *ts;
+  char *sbuf;
   size_t totalsize;  /* total size of TString object */
   totalsize = sizeof(TString) + ((l + 1) * sizeof(char));
   ts = &luaC_newobj(L, tag, totalsize, list, 0)->ts;
   ts->tsv.len = l;
   ts->tsv.hash = h;
   ts->tsv.extra = 0;
-  memcpy(ts+1, str, l*sizeof(char));
-  ((char *)(ts+1))[l] = '\0';  /* ending 0 */
+  sbuf = (char *)(TString *)(ts + 1);
+  memcpy(sbuf, str, l*sizeof(char));
+  sbuf[l] = '\0';  /* ending 0 */
   return ts;
 }
 

module/lua/ltable.c

@@ -405,8 +405,10 @@ static Node *getfreepos (Table *t) {
 TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
   Node *mp;
   if (ttisnil(key)) luaG_runerror(L, "table index is nil");
+#if defined LUA_HAS_FLOAT_NUMBERS
   else if (ttisnumber(key) && luai_numisnan(L, nvalue(key)))
     luaG_runerror(L, "table index is NaN");
+#endif
   mp = mainposition(t, key);
   if (!ttisnil(gval(mp)) || isdummy(mp)) {  /* main position is taken? */
     Node *othern;

module/lua/ltable.h

@@ -11,7 +11,7 @@
 #include "lobject.h"
 
 
-#define gnode(t,i)	(&(t)->node[i])
+#define gnode(t,i)	((Node *)&(t)->node[i])
 #define gkey(n)		(&(n)->i_key.tvk)
 #define gval(n)		(&(n)->i_val)
 #define gnext(n)	((n)->i_key.nk.next)