updated the docker recipe in Makefile.am

Makefile.am

@@ -19,9 +19,20 @@ rpm: dist-gzip
 	cp @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz "@abs_builddir@/pkgs/RPM/SOURCES"
 	rpmbuild --define "_topdir @abs_builddir@/pkgs/RPM" -ba @abs_builddir@/pkgs/hio.spec --target=@build_cpu@
 
+## assumes --enable-static --disable-shared, so bin/hio-webs is the program
+## itself and libhio is linked into it. what is still missing from a FROM scratch
+## image is the system libraries it links against and the dynamic loader, so each
+## goes in at the path it is loaded from - there is no /etc/ld.so.cache in the
+## image to redirect the search.
 docker: all
-	mkdir -p data
+	rm -rf hio-webs-root hio-webs.tar
-	rm -rf data/*
+	mkdir -p hio-webs-root/bin hio-webs-root/data
-	tar -cvf hio-webs.tar bin/hio-webs data
+	cp bin/hio-webs hio-webs-root/bin/hio-webs
+	ldd bin/hio-webs | $(AWK) '$$2 == "=>" { print $$3 } $$1 ~ /^\// { print $$1 }' | \
+		while read -r so; do \
+			mkdir -p "hio-webs-root$${so%/*}"; \
+			cp -L "$$so" "hio-webs-root$$so" || exit 1; \
+		done
+	tar -C hio-webs-root -cvf hio-webs.tar .
 	docker build -t hio:webs -f Dockerfile .
-	rm -f hio-webs.tar
+	rm -rf hio-webs-root hio-webs.tar

Makefile.in

@@ -295,6 +295,8 @@ PTHREAD_LIBS = @PTHREAD_LIBS@
 QUADMATH_LIBS = @QUADMATH_LIBS@
 RANLIB = @RANLIB@
 SCTP_ENABLED = @SCTP_ENABLED@
+SCTP_MULTIHOMING_ENABLED = @SCTP_MULTIHOMING_ENABLED@
+SCTP_PEELOFF_ENABLED = @SCTP_PEELOFF_ENABLED@
 SED = @SED@
 SENDFILE_LIBS = @SENDFILE_LIBS@
 SET_MAKE = @SET_MAKE@
@@ -865,11 +867,17 @@ rpm: dist-gzip
 	rpmbuild --define "_topdir @abs_builddir@/pkgs/RPM" -ba @abs_builddir@/pkgs/hio.spec --target=@build_cpu@
 
 docker: all
-	mkdir -p data
+	rm -rf hio-webs-root hio-webs.tar
-	rm -rf data/*
+	mkdir -p hio-webs-root/bin hio-webs-root/data
-	tar -cvf hio-webs.tar bin/hio-webs data
+	cp bin/hio-webs hio-webs-root/bin/hio-webs
+	ldd bin/hio-webs | $(AWK) '$$2 == "=>" { print $$3 } $$1 ~ /^\// { print $$1 }' | \
+		while read -r so; do \
+			mkdir -p "hio-webs-root$${so%/*}"; \
+			cp -L "$$so" "hio-webs-root$$so" || exit 1; \
+		done
+	tar -C hio-webs-root -cvf hio-webs.tar .
 	docker build -t hio:webs -f Dockerfile .
-	rm -f hio-webs.tar
+	rm -rf hio-webs-root hio-webs.tar
 
 # Tell versions [3.59,3.63) of GNU make to not export all variables.
 # Otherwise a system limit (for SysV at least) may be exceeded.

bin/Makefile.in

@@ -329,6 +329,8 @@ PTHREAD_LIBS = @PTHREAD_LIBS@
 QUADMATH_LIBS = @QUADMATH_LIBS@
 RANLIB = @RANLIB@
 SCTP_ENABLED = @SCTP_ENABLED@
+SCTP_MULTIHOMING_ENABLED = @SCTP_MULTIHOMING_ENABLED@
+SCTP_PEELOFF_ENABLED = @SCTP_PEELOFF_ENABLED@
 SED = @SED@
 SENDFILE_LIBS = @SENDFILE_LIBS@
 SET_MAKE = @SET_MAKE@

configure

@@ -689,6 +689,8 @@ ax_pthread_config
 CPP
 SENDFILE_LIBS
 SOCKET_LIBS
+SCTP_MULTIHOMING_ENABLED
+SCTP_PEELOFF_ENABLED
 SCTP_ENABLED
 LIBM
 MACOSX_FALSE
@@ -1860,6 +1862,65 @@ printf "%s\n" "$ac_res" >&6; }
 
 } # ac_fn_check_decl
 
+# ac_fn_c_check_type LINENO TYPE VAR INCLUDES
+# -------------------------------------------
+# Tests whether TYPE exists after having included INCLUDES, setting cache
+# variable VAR accordingly.
+ac_fn_c_check_type ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
+printf %s "checking for $2... " >&6; }
+if eval test \${$3+y}
+then :
+  printf %s "(cached) " >&6
+else case e in #(
+  e) eval "$3=no"
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$4
+int
+main (void)
+{
+if (sizeof ($2))
+	 return 0;
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$4
+int
+main (void)
+{
+if (sizeof (($2)))
+	    return 0;
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+
+else case e in #(
+  e) eval "$3=yes" ;;
+esac
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;;
+esac
+fi
+eval ac_res=\$$3
+	       { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+printf "%s\n" "$ac_res" >&6; }
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+
+} # ac_fn_c_check_type
+
 # ac_fn_c_try_cpp LINENO
 # ----------------------
 # Try to preprocess conftest.$ac_ext, and return whether this succeeded.
@@ -1962,65 +2023,6 @@ printf "%s\n" "$ac_res" >&6; }
 
 } # ac_fn_c_check_member
 
-# ac_fn_c_check_type LINENO TYPE VAR INCLUDES
-# -------------------------------------------
-# Tests whether TYPE exists after having included INCLUDES, setting cache
-# variable VAR accordingly.
-ac_fn_c_check_type ()
-{
-  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
-  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
-printf %s "checking for $2... " >&6; }
-if eval test \${$3+y}
-then :
-  printf %s "(cached) " >&6
-else case e in #(
-  e) eval "$3=no"
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-$4
-int
-main (void)
-{
-if (sizeof ($2))
-	 return 0;
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"
-then :
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-$4
-int
-main (void)
-{
-if (sizeof (($2)))
-	    return 0;
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"
-then :
-
-else case e in #(
-  e) eval "$3=yes" ;;
-esac
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;;
-esac
-fi
-eval ac_res=\$$3
-	       { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
-printf "%s\n" "$ac_res" >&6; }
-  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
-
-} # ac_fn_c_check_type
-
 # ac_fn_c_try_run LINENO
 # ----------------------
 # Try to run conftest.$ac_ext, and return whether this succeeded. Assumes that
@@ -16298,7 +16300,159 @@ printf "%s\n" "#define HIO_ENABLE_SCTP 1" >>confdefs.h
 
 fi
 	fi
+
+	if test "x$SCTP_ENABLED" = "xyes"
+	then
+																						{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing sctp_bindx" >&5
+printf %s "checking for library containing sctp_bindx... " >&6; }
+if test ${ac_cv_search_sctp_bindx+y}
+then :
+  printf %s "(cached) " >&6
+else case e in #(
+  e) ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.
+   The 'extern "C"' is for builds by C++ compilers;
+   although this is not generally supported in C code supporting it here
+   has little cost and some practical benefit (sr 110532).  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char sctp_bindx (void);
+int
+main (void)
+{
+return sctp_bindx ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' sctp
+do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_search_sctp_bindx=$ac_res
 fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext
+  if test ${ac_cv_search_sctp_bindx+y}
+then :
+  break
+fi
+done
+if test ${ac_cv_search_sctp_bindx+y}
+then :
+
+else case e in #(
+  e) ac_cv_search_sctp_bindx=no ;;
+esac
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS ;;
+esac
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_sctp_bindx" >&5
+printf "%s\n" "$ac_cv_search_sctp_bindx" >&6; }
+ac_res=$ac_cv_search_sctp_bindx
+if test "$ac_res" != no
+then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+fi
+
+		ac_fn_c_check_func "$LINENO" "sctp_bindx" "ac_cv_func_sctp_bindx"
+if test "x$ac_cv_func_sctp_bindx" = xyes
+then :
+  printf "%s\n" "#define HAVE_SCTP_BINDX 1" >>confdefs.h
+
+fi
+ac_fn_c_check_func "$LINENO" "sctp_peeloff" "ac_cv_func_sctp_peeloff"
+if test "x$ac_cv_func_sctp_peeloff" = xyes
+then :
+  printf "%s\n" "#define HAVE_SCTP_PEELOFF 1" >>confdefs.h
+
+fi
+ac_fn_c_check_func "$LINENO" "sctp_getpaddrs" "ac_cv_func_sctp_getpaddrs"
+if test "x$ac_cv_func_sctp_getpaddrs" = xyes
+then :
+  printf "%s\n" "#define HAVE_SCTP_GETPADDRS 1" >>confdefs.h
+
+fi
+ac_fn_c_check_func "$LINENO" "sctp_getladdrs" "ac_cv_func_sctp_getladdrs"
+if test "x$ac_cv_func_sctp_getladdrs" = xyes
+then :
+  printf "%s\n" "#define HAVE_SCTP_GETLADDRS 1" >>confdefs.h
+
+fi
+ac_fn_c_check_func "$LINENO" "sctp_freepaddrs" "ac_cv_func_sctp_freepaddrs"
+if test "x$ac_cv_func_sctp_freepaddrs" = xyes
+then :
+  printf "%s\n" "#define HAVE_SCTP_FREEPADDRS 1" >>confdefs.h
+
+fi
+ac_fn_c_check_func "$LINENO" "sctp_freeladdrs" "ac_cv_func_sctp_freeladdrs"
+if test "x$ac_cv_func_sctp_freeladdrs" = xyes
+then :
+  printf "%s\n" "#define HAVE_SCTP_FREELADDRS 1" >>confdefs.h
+
+fi
+
+		if test "x$ac_cv_func_sctp_peeloff" = "xyes"
+		then
+			SCTP_PEELOFF_ENABLED="yes"
+		fi
+		if test "x$ac_cv_func_sctp_bindx" = "xyes" -a \
+		        "x$ac_cv_func_sctp_getpaddrs" = "xyes" -a \
+		        "x$ac_cv_func_sctp_getladdrs" = "xyes"
+		then
+			SCTP_MULTIHOMING_ENABLED="yes"
+		fi
+
+								ac_fn_c_check_type "$LINENO" "struct sctp_prim" "ac_cv_type_struct_sctp_prim" "
+			#include <sys/types.h>
+			#include <sys/socket.h>
+			#include <netinet/in.h>
+			#include <netinet/sctp.h>
+
+"
+if test "x$ac_cv_type_struct_sctp_prim" = xyes
+then :
+
+printf "%s\n" "#define HAVE_STRUCT_SCTP_PRIM 1" >>confdefs.h
+
+
+fi
+ac_fn_c_check_type "$LINENO" "struct sctp_setprim" "ac_cv_type_struct_sctp_setprim" "
+			#include <sys/types.h>
+			#include <sys/socket.h>
+			#include <netinet/in.h>
+			#include <netinet/sctp.h>
+
+"
+if test "x$ac_cv_type_struct_sctp_setprim" = xyes
+then :
+
+printf "%s\n" "#define HAVE_STRUCT_SCTP_SETPRIM 1" >>confdefs.h
+
+
+fi
+
+	fi
+fi
+
+test -z "$SCTP_PEELOFF_ENABLED" && SCTP_PEELOFF_ENABLED="no"
+
+test -z "$SCTP_MULTIHOMING_ENABLED" && SCTP_MULTIHOMING_ENABLED="no"
 
 ac_fn_c_check_header_compile "$LINENO" "net/if.h" "ac_cv_header_net_if_h" "
 	#include <sys/types.h>
@@ -24904,6 +25058,8 @@ echo "  Socket library: ${SOCKET_LIBS}"
 echo "  Sendfile library: ${SENDFILE_LIBS}"
 echo "  SSL library: ${SSL_LIBS}"
 echo "  SCTP support: ${SCTP_ENABLED}"
+echo "  SCTP peel-off: ${SCTP_PEELOFF_ENABLED}"
+echo "  SCTP multi-homing: ${SCTP_MULTIHOMING_ENABLED}"
 echo "-------------------------------------------------------------------------"
 
 

configure.ac

@@ -142,8 +142,48 @@ then
 			#include <netinet/in.h>
 		]])
 	fi
+
+	if test "x$SCTP_ENABLED" = "xyes"
+	then
+		dnl these library calls are a separate matter from the socket type. on
+		dnl the bsds they are in libc; on linux they live in libsctp
+		dnl (lksctp-tools), which may well be absent on a machine whose kernel
+		dnl and headers have sctp. so they are probed on their own and the basic
+		dnl one-to-one and one-to-many support stands either way.
+		dnl
+		dnl peel-off and multi-homing are reported apart because they are
+		dnl separate features sharing nothing but the library they come from:
+		dnl peel-off needs sctp_peeloff() and multi-homing needs the address
+		dnl calls. a system missing one must not lose both.
+		AC_SEARCH_LIBS([sctp_bindx], [sctp])
+		AC_CHECK_FUNCS([sctp_bindx sctp_peeloff sctp_getpaddrs sctp_getladdrs sctp_freepaddrs sctp_freeladdrs])
+		if test "x$ac_cv_func_sctp_peeloff" = "xyes"
+		then
+			SCTP_PEELOFF_ENABLED="yes"
+		fi
+		if test "x$ac_cv_func_sctp_bindx" = "xyes" -a \
+		        "x$ac_cv_func_sctp_getpaddrs" = "xyes" -a \
+		        "x$ac_cv_func_sctp_getladdrs" = "xyes"
+		then
+			SCTP_MULTIHOMING_ENABLED="yes"
+		fi
+
+		dnl SCTP_PRIMARY_ADDR takes a structure the two families spell
+		dnl differently - struct sctp_prim on linux, struct sctp_setprim on
+		dnl the bsds - with identical members.
+		AC_CHECK_TYPES([struct sctp_prim, struct sctp_setprim], [], [], [[
+			#include <sys/types.h>
+			#include <sys/socket.h>
+			#include <netinet/in.h>
+			#include <netinet/sctp.h>
+		]])
+	fi
 fi
 AC_SUBST(SCTP_ENABLED)
+test -z "$SCTP_PEELOFF_ENABLED" && SCTP_PEELOFF_ENABLED="no"
+AC_SUBST(SCTP_PEELOFF_ENABLED)
+test -z "$SCTP_MULTIHOMING_ENABLED" && SCTP_MULTIHOMING_ENABLED="no"
+AC_SUBST(SCTP_MULTIHOMING_ENABLED)
 AC_CHECK_HEADERS([net/if.h net/if_dl.h netinet/if_ether.h netpacket/packet.h net/bpf.h], [], [], [
 	#include <sys/types.h>
 	#include <sys/socket.h>])
@@ -947,5 +987,7 @@ echo "  Socket library: ${SOCKET_LIBS}"
 echo "  Sendfile library: ${SENDFILE_LIBS}"
 echo "  SSL library: ${SSL_LIBS}"
 echo "  SCTP support: ${SCTP_ENABLED}"
+echo "  SCTP peel-off: ${SCTP_PEELOFF_ENABLED}"
+echo "  SCTP multi-homing: ${SCTP_MULTIHOMING_ENABLED}"
 echo "-------------------------------------------------------------------------"
 ]

lib/Makefile.in

@@ -374,6 +374,8 @@ PTHREAD_LIBS = @PTHREAD_LIBS@
 QUADMATH_LIBS = @QUADMATH_LIBS@
 RANLIB = @RANLIB@
 SCTP_ENABLED = @SCTP_ENABLED@
+SCTP_MULTIHOMING_ENABLED = @SCTP_MULTIHOMING_ENABLED@
+SCTP_PEELOFF_ENABLED = @SCTP_PEELOFF_ENABLED@
 SED = @SED@
 SENDFILE_LIBS = @SENDFILE_LIBS@
 SET_MAKE = @SET_MAKE@

lib/dns-cli.c

@@ -161,7 +161,7 @@ HIO_DEBUG1 (hio, "DNC - releasing dns message - msgid:%d\n", (int)hio_ntoh16(hio
 
 	if (msgxtn->rtmridx != HIO_TMRIDX_INVALID)
 	{
-		hio_deltmrjob (hio, msgxtn->rtmridx);
+		hio_deltmrjob(hio, msgxtn->rtmridx);
 		HIO_ASSERT(hio, msgxtn->rtmridx == HIO_TMRIDX_INVALID);
 	}
 
@@ -574,7 +574,7 @@ static int on_udp_read (hio_dev_sck_t* dev, const void* data, hio_iolen_t dlen,
 			if (reqmsgxtn->rtmridx != HIO_TMRIDX_INVALID)
 			{
 				/* unschedule a timer job if any */
-				hio_deltmrjob (hio, reqmsgxtn->rtmridx);
+				hio_deltmrjob(hio, reqmsgxtn->rtmridx);
 				HIO_ASSERT(hio, reqmsgxtn->rtmridx == HIO_TMRIDX_INVALID);
 			}
 

lib/hio-cfg.h.in

@@ -295,6 +295,24 @@
 /* Define to 1 if you have the 'recvmsg' function. */
 #undef HAVE_RECVMSG
 
+/* Define to 1 if you have the 'sctp_bindx' function. */
+#undef HAVE_SCTP_BINDX
+
+/* Define to 1 if you have the 'sctp_freeladdrs' function. */
+#undef HAVE_SCTP_FREELADDRS
+
+/* Define to 1 if you have the 'sctp_freepaddrs' function. */
+#undef HAVE_SCTP_FREEPADDRS
+
+/* Define to 1 if you have the 'sctp_getladdrs' function. */
+#undef HAVE_SCTP_GETLADDRS
+
+/* Define to 1 if you have the 'sctp_getpaddrs' function. */
+#undef HAVE_SCTP_GETPADDRS
+
+/* Define to 1 if you have the 'sctp_peeloff' function. */
+#undef HAVE_SCTP_PEELOFF
+
 /* Define to 1 if you have the 'select' function. */
 #undef HAVE_SELECT
 
@@ -382,6 +400,12 @@
 /* Define to 1 if the system has the type 'struct lifreq'. */
 #undef HAVE_STRUCT_LIFREQ
 
+/* Define to 1 if the system has the type 'struct sctp_prim'. */
+#undef HAVE_STRUCT_SCTP_PRIM
+
+/* Define to 1 if the system has the type 'struct sctp_setprim'. */
+#undef HAVE_STRUCT_SCTP_SETPRIM
+
 /* Define to 1 if 'sin6_scope_id' is a member of 'struct sockaddr_in6'. */
 #undef HAVE_STRUCT_SOCKADDR_IN6_SIN6_SCOPE_ID
 

lib/hio-sck.h

@@ -262,6 +262,38 @@ typedef void (*hio_dev_sck_on_connect_t) (
  *
  * Leave it unset to have notifications discarded.
  */
+/**
+ * A decoded SCTP_ASSOC_CHANGE notification, as filled in by
+ * hio_dev_sck_parse_assoc_event(). The negotiated stream counts are worth
+ * having on their own: they are the outcome of the SCTP_INITMSG request from
+ * both ends, and are not knowable any other way.
+ */
+typedef struct hio_sctp_assoc_event_t hio_sctp_assoc_event_t;
+struct hio_sctp_assoc_event_t
+{
+	int          state;      /**< #hio_sctp_assoc_state_t */
+	int          error;      /**< the notification's error field, 0 if none */
+	hio_int32_t  assoc_id;   /**< pass this to hio_dev_sck_peeloff() */
+	hio_uint16_t ostreams;   /**< outbound streams negotiated */
+	hio_uint16_t instreams;  /**< inbound streams negotiated */
+};
+
+/**
+ * The states an association can report. The names mirror the SCTP_COMM_UP
+ * family so a reader of RFC 6458 recognises them, without the header's types
+ * appearing in this interface.
+ */
+enum hio_sctp_assoc_state_t
+{
+	HIO_SCTP_ASSOC_COMM_UP = 0,     /**< a new association is usable */
+	HIO_SCTP_ASSOC_COMM_LOST,       /**< it failed */
+	HIO_SCTP_ASSOC_RESTART,         /**< the peer restarted it */
+	HIO_SCTP_ASSOC_SHUTDOWN_COMP,   /**< it closed in an orderly way */
+	HIO_SCTP_ASSOC_CANT_STR_ASSOC,  /**< setup never completed */
+	HIO_SCTP_ASSOC_STATE_UNKNOWN    /**< something this build does not name */
+};
+typedef enum hio_sctp_assoc_state_t hio_sctp_assoc_state_t;
+
 typedef void (*hio_dev_sck_on_notification_t) (
 	hio_dev_sck_t*       dev,
 	const void*          data,
@@ -297,8 +329,8 @@ enum hio_dev_sck_type_t
 	HIO_DEV_SCK_SCTP4, /*  one-to-one sctp stream */
 	HIO_DEV_SCK_SCTP6, /*  one-to-one sctp stream */
 
-	HIO_DEV_SCK_SCTP4_SP, /*  one-to-one sctp seqpacket */
+	HIO_DEV_SCK_SCTP4_SEQPKT, /*  one-to-one sctp seqpacket */
-	HIO_DEV_SCK_SCTP6_SP, /*  one-to-one sctp seqpacket */
+	HIO_DEV_SCK_SCTP6_SEQPKT, /*  one-to-one sctp seqpacket */
 
 	/* ICMP at the IPv4 layer */
 	HIO_DEV_SCK_ICMP4,
@@ -388,12 +420,13 @@ struct hio_dev_sck_connect_t
 	hio_ntime_t connect_tmout;
 };
 
-#if 0
+/** flags for hio_dev_sck_bindx() */
-enum hio_dev_sck_listen_option_t
+enum hio_dev_sck_bindx_flag_t
 {
+	HIO_DEV_SCK_BINDX_ADD = 0, /**< add the addresses to the endpoint */
+	HIO_DEV_SCK_BINDX_REM = 1  /**< take them away from it */
 };
-typedef enum hio_dev_sck_listen_option_t hio_dev_sck_listen_option_t;
+typedef enum hio_dev_sck_bindx_flag_t hio_dev_sck_bindx_flag_t;
-#endif
 
 typedef struct hio_dev_sck_listen_t hio_dev_sck_listen_t;
 struct hio_dev_sck_listen_t
@@ -647,6 +680,108 @@ HIO_EXPORT int hio_dev_sck_shutdown (
 	int            how  /* bitwise-ORed of hio_dev_sck_shutdown_how_t enumerators */
 );
 
+/**
+ * The hio_dev_sck_peeloff() function takes one association off a one-to-many
+ * SCTP socket and gives it a socket, and a device, of its own. The new device
+ * arrives through on_connect() in the #HIO_DEV_SCK_ACCEPTED state carrying the
+ * one-to-one methods - a write queue of its own, a read-enable bit of its own,
+ * and addresses of its own, which is what per-association multi-homing needs.
+ * It is the same door an accepted tcp connection comes through.
+ *
+ * It may be called at any point in an association's life, and where it is
+ * called from is the whole point:
+ *
+ * - from on_notification(), on an SCTP_COMM_UP whose peer address you like -
+ *   use hio_dev_sck_parse_assoc_event() to get the association id;
+ * - from on_read(), when a message tells you this peer deserves a session -
+ *   the association id is already there, on the source address, via
+ *   hio_skad_get_assoc().
+ *
+ * Peeling mid-stream leaves a clean seam: messages already handed to on_read()
+ * are yours, whatever is still queued in the kernel moves to the new socket,
+ * and everything after arrives on the new device. Nothing is lost or
+ * duplicated. Call it synchronously from inside the callback though - defer it
+ * and this socket may read another message for that association first, which
+ * widens the seam by however long you waited.
+ *
+ * Only for the one-to-many SCTP types; anything else fails with #HIO_EINVAL.
+ * A build whose system lacks sctp_peeloff() fails with #HIO_ENOIMPL.
+ */
+HIO_EXPORT int hio_dev_sck_peeloff (
+	hio_dev_sck_t*    dev,
+	hio_int32_t       assoc_id
+);
+
+/**
+ * The hio_dev_sck_parse_assoc_event() function decodes what on_notification()
+ * hands over, when what it hands over is an association change. It exists so
+ * that deciding what to do about a new association does not require picking
+ * apart a kernel structure by hand.
+ *
+ * Returns 0 and fills 'ev' if the notification is an association change, or -1
+ * if it is not - some other kind of event, or a buffer too short to be one.
+ *
+ * -1 is an answer, not a failure: no error number is set, and none can be, as
+ * this is a pure decoder with no hio_t to set one on. A caller walking
+ * notifications simply ignores the ones it gets -1 for.
+ */
+HIO_EXPORT int hio_dev_sck_parse_assoc_event (
+	const void*                data,
+	hio_iolen_t                dlen,
+	hio_sctp_assoc_event_t*    ev
+);
+
+/**
+ * The hio_dev_sck_bindx() function adds or removes local addresses on an
+ * already bound SCTP endpoint - which is what local multi-homing is. The
+ * addresses belong to the endpoint rather than to any one association, so this
+ * is a socket operation and works the same whether associations are peeled off
+ * or kept on the one socket.
+ *
+ * hio_dev_sck_bind() must have been called first: sctp_bindx() adds to an
+ * existing binding, it does not replace it.
+ *
+ * Available only for the SCTP socket types; anything else fails with
+ * #HIO_ENOIMPL.
+ */
+HIO_EXPORT int hio_dev_sck_bindx (
+	hio_dev_sck_t*             dev,
+	const hio_skad_t*          addrs,
+	hio_oow_t                  naddrs,
+	hio_dev_sck_bindx_flag_t   flags
+);
+
+/**
+ * The hio_dev_sck_getladdrs() function reports the local addresses of the
+ * endpoint, and hio_dev_sck_getpaddrs() the peer addresses of the association -
+ * the two halves of what multi-homing established.
+ *
+ * On entry *naddrs is the room available in 'addrs'; on return it is how many
+ * were written. #HIO_EBUFFULL is reported if there were more, with *naddrs set
+ * to the number needed.
+ */
+HIO_EXPORT int hio_dev_sck_getladdrs (
+	hio_dev_sck_t*    dev,
+	hio_skad_t*       addrs,
+	hio_oow_t*        naddrs
+);
+
+HIO_EXPORT int hio_dev_sck_getpaddrs (
+	hio_dev_sck_t*    dev,
+	hio_skad_t*       addrs,
+	hio_oow_t*        naddrs
+);
+
+/**
+ * The hio_dev_sck_setprimaryaddr() function asks the peer to prefer one of its
+ * addresses as the primary path. It must be one of the addresses
+ * hio_dev_sck_getpaddrs() reports.
+ */
+HIO_EXPORT int hio_dev_sck_setprimaryaddr (
+	hio_dev_sck_t*      dev,
+	const hio_skad_t*   addr
+);
+
 HIO_EXPORT int hio_dev_sck_sendfileok (
 	hio_dev_sck_t* dev
 );

lib/hio.c

@@ -850,7 +850,7 @@ static HIO_INLINE void handle_event (hio_t* hio, hio_dev_t* dev, int events, int
 				tmrjob.handler = on_read_timeout;
 				tmrjob.idxptr = &dev->rtmridx;
 
-				hio_updtmrjob (hio, dev->rtmridx, &tmrjob);
+				hio_updtmrjob(hio, dev->rtmridx, &tmrjob);
 
 				/*hio_deltmrjob(hio, dev->rtmridx);
 				dev->rtmridx = HIO_TMRIDX_INVALID;*/
@@ -1427,7 +1427,7 @@ kill_device:
 	if (kill_and_free_device(dev, 0) <= -1)
 	{
 		HIO_ASSERT(hio, dev->dev_cap & HIO_DEV_CAP_ZOMBIE);
-		if (schedule_kill_zombie_job (dev) <= -1)
+		if (schedule_kill_zombie_job(dev) <= -1)
 		{
 			/* i have no choice but to free up the devide by force */
 			while (kill_and_free_device(dev, 1) <= -1)

lib/http-file.c

@@ -710,7 +710,7 @@ static void unbind_task_from_peer (file_t* file, int rcdown)
 
 	if (file->peer_tmridx != HIO_TMRIDX_INVALID)
 	{
-		hio_deltmrjob (hio, file->peer_tmridx);
+		hio_deltmrjob(hio, file->peer_tmridx);
 		HIO_ASSERT(hio, file->peer_tmridx == HIO_TMRIDX_INVALID);
 	}
 

lib/nwif.c

@@ -97,13 +97,13 @@ static int get_sco_ifconf (hio_t* hio, struct ifconf* ifc)
 		hio_seterrwithsyserr(hio, 0, errno);
 		goto oops;
 	}
-	close (h); h = -1;
+	close(h); h = -1;
 
 	return 0;
 
 oops:
 	if (ifc->ifc_buf) hio_freemem(hio, ifc->ifc_buf);
-	if (h >= 0) close (h);
+	if (h >= 0) close(h);
 	return -1;
 }
 
@@ -188,13 +188,13 @@ int hio_bcstrtoifindex (hio_t* hio, const hio_bch_t* ptr, unsigned int* index)
 	{
 		if (hio_comp_bcstr(ptr, ifc.ifc_req[i].ifr_name, 0) == 0)
 		{
-			free_sco_ifconf (hio, &ifc);
+			free_sco_ifconf(hio, &ifc);
 			*index = i + 1;
 			return 0;
 		}
 	}
 
-	free_sco_ifconf (hio, &ifc);
+	free_sco_ifconf(hio, &ifc);
 	return -1;
 
 #else
@@ -232,7 +232,7 @@ int hio_bcharstoifindex (hio_t* hio, const hio_bch_t* ptr, hio_oow_t len, unsign
 	if (hio_copy_bchars_to_bcstr(ifr.ifr_name, HIO_COUNTOF(ifr.ifr_name), ptr, len) < len) return -1; /* name too long */
 
 	x = ioctl(h, SIOCGIFINDEX, &ifr);
-	close (h);
+	close(h);
 
 	if (x >= 0)
 	{
@@ -271,13 +271,13 @@ int hio_bcharstoifindex (hio_t* hio, const hio_bch_t* ptr, hio_oow_t len, unsign
 	{
 		if (hio_comp_bchars_bcstr(ptr, len, ifc.ifc_req[i].ifr_name) == 0)
 		{
-			free_sco_ifconf (hio, &ifc);
+			free_sco_ifconf(hio, &ifc);
 			*index = i + 1;
 			return 0;
 		}
 	}
 
-	free_sco_ifconf (hio, &ifc);
+	free_sco_ifconf(hio, &ifc);
 	return -1;
 
 #else
@@ -316,7 +316,7 @@ int hio_ucstrtoifindex (hio_t* hio, const hio_uch_t* ptr, unsigned int* index)
 	if (hio_convutobcstr(hio, ptr, &wl, ifr.ifr_name, &ml) <= -1) return -1;
 
 	x = ioctl(h, SIOCGIFINDEX, &ifr);
-	close (h);
+	close(h);
 
 	if (x >= 0)
 	{
@@ -363,13 +363,13 @@ int hio_ucstrtoifindex (hio_t* hio, const hio_uch_t* ptr, unsigned int* index)
 	{
 		if (hio_comp_bcstr(tmp, ifc.ifc_req[i].ifr_name, 0) == 0)
 		{
-			free_sco_ifconf (hio, &ifc);
+			free_sco_ifconf(hio, &ifc);
 			*index = i + 1;
 			return 0;
 		}
 	}
 
-	free_sco_ifconf (hio, &ifc);
+	free_sco_ifconf(hio, &ifc);
 	return -1;
 
 #else
@@ -409,7 +409,7 @@ int hio_ucharstoifindex (hio_t* hio, const hio_uch_t* ptr, hio_oow_t len, unsign
 	ifr.ifr_name[ml] = '\0';
 
 	x = ioctl(h, SIOCGIFINDEX, &ifr);
-	close (h);
+	close(h);
 
 	if (x >= 0)
 	{
@@ -457,13 +457,13 @@ int hio_ucharstoifindex (hio_t* hio, const hio_uch_t* ptr, hio_oow_t len, unsign
 	{
 		if (hio_comp_bcstr(tmp, ifc.ifc_req[i].ifr_name, 0) == 0)
 		{
-			free_sco_ifconf (hio, &ifc);
+			free_sco_ifconf(hio, &ifc);
 			*index = i + 1;
 			return 0;
 		}
 	}
 
-	free_sco_ifconf (hio, &ifc);
+	free_sco_ifconf(hio, &ifc);
 	return -1;
 #else
 	return -1;
@@ -519,7 +519,7 @@ int hio_ifindextobcstr (hio_t* hio, unsigned int index, hio_bch_t* buf, hio_oow_
 
 #elif defined(HAVE_IF_INDEXTONAME)
 	hio_bch_t tmp[IF_NAMESIZE + 1];
-	if (if_indextoname (index, tmp) == HIO_NULL)
+	if (if_indextoname(index, tmp) == HIO_NULL)
 	{
 		hio_seterrwithsyserr(hio, 0, errno);
 		return -1;
@@ -539,12 +539,12 @@ int hio_ifindextobcstr (hio_t* hio, unsigned int index, hio_bch_t* buf, hio_oow_
 	if (index > num)
 	{
 		hio_seterrnum(hio, HIO_ENOENT);
-		free_sco_ifconf (hio, &ifc);
+		free_sco_ifconf(hio, &ifc);
 		return -1;
 	}
 
 	ml = hio_copy_bcstr(buf, len, ifc.ifc_req[index - 1].ifr_name);
-	free_sco_ifconf (hio, &ifc);
+	free_sco_ifconf(hio, &ifc);
 	return ml;
 
 #else
@@ -588,7 +588,7 @@ int hio_ifindextoucstr (hio_t* hio, unsigned int index, hio_uch_t* buf, hio_oow_
 	#endif
 
 	x = ioctl(h, SIOCGIFNAME, &ifr);
-	close (h);
+	close(h);
 
 	if (x <= -1)
 	{
@@ -630,13 +630,13 @@ int hio_ifindextoucstr (hio_t* hio, unsigned int index, hio_uch_t* buf, hio_oow_
 	num = ifc.ifc_len / HIO_SIZEOF(struct ifreq);
 	if (index > num)
 	{
-		free_sco_ifconf (hio, &ifc);
+		free_sco_ifconf(hio, &ifc);
 		return -1;
 	}
 
 	wl = len;
 	x = hio_convbtoucstr(ifc.ifc_req[index - 1].ifr_name, &ml, buf, &wl, 0);
-	free_sco_ifconf (hio, &ifc);
+	free_sco_ifconf(hio, &ifc);
 
 	if (x == -2 && wl > 1) buf[wl - 1] = '\0';
 	else if (x != 0) return -1;

lib/pro.c

@@ -160,20 +160,20 @@ static pid_t standard_fork_and_exec (hio_dev_pro_t* dev, int pfds[], hio_dev_pro
 		if (mi->flags & HIO_DEV_PRO_WRITEIN)
 		{
 			/* slave should read */
-			close (pfds[1]);
+			close(pfds[1]);
 			pfds[1] = HIO_SYSHND_INVALID;
 
 			/* let the pipe be standard input */
 			if (dup2(pfds[0], 0) <= -1) goto slave_oops;
 
-			close (pfds[0]);
+			close(pfds[0]);
 			pfds[0] = HIO_SYSHND_INVALID;
 		}
 
 		if (mi->flags & HIO_DEV_PRO_READOUT)
 		{
 			/* slave should write */
-			close (pfds[2]);
+			close(pfds[2]);
 			pfds[2] = HIO_SYSHND_INVALID;
 
 			if (dup2(pfds[3], 1) == -1) goto slave_oops;
@@ -183,13 +183,13 @@ static pid_t standard_fork_and_exec (hio_dev_pro_t* dev, int pfds[], hio_dev_pro
 				if (dup2(pfds[3], 2) == -1) goto slave_oops;
 			}
 
-			close (pfds[3]);
+			close(pfds[3]);
 			pfds[3] = HIO_SYSHND_INVALID;
 		}
 
 		if (mi->flags & HIO_DEV_PRO_READERR)
 		{
-			close (pfds[4]);
+			close(pfds[4]);
 			pfds[4] = HIO_SYSHND_INVALID;
 
 			if (dup2(pfds[5], 2) == -1) goto slave_oops;
@@ -199,7 +199,7 @@ static pid_t standard_fork_and_exec (hio_dev_pro_t* dev, int pfds[], hio_dev_pro
 				if (dup2(pfds[5], 1) == -1) goto slave_oops;
 			}
 
-			close (pfds[5]);
+			close(pfds[5]);
 			pfds[5] = HIO_SYSHND_INVALID;
 		}
 
@@ -218,18 +218,18 @@ static pid_t standard_fork_and_exec (hio_dev_pro_t* dev, int pfds[], hio_dev_pro
 			if ((mi->flags & HIO_DEV_PRO_OUTTONUL) && dup2(devnull, 1) == -1) goto slave_oops;
 			if ((mi->flags & HIO_DEV_PRO_ERRTONUL) && dup2(devnull, 2) == -1) goto slave_oops;
 
-			close (devnull);
+			close(devnull);
 			devnull = HIO_SYSHND_INVALID;
 		}
 
-		if (mi->flags & HIO_DEV_PRO_DROPIN) close (0);
+		if (mi->flags & HIO_DEV_PRO_DROPIN) close(0);
-		if (mi->flags & HIO_DEV_PRO_DROPOUT) close (1);
+		if (mi->flags & HIO_DEV_PRO_DROPOUT) close(1);
-		if (mi->flags & HIO_DEV_PRO_DROPERR) close (2);
+		if (mi->flags & HIO_DEV_PRO_DROPERR) close(2);
 
 		execv (param->argv[0], param->argv);
 
 		/* if exec fails, free 'param' parameter which is an inherited pointer */
-		free_param (hio, param);
+		free_param(hio, param);
 
 	slave_oops:
 		if (devnull != HIO_SYSHND_INVALID) close(devnull);
@@ -291,7 +291,7 @@ static int dev_pro_make_master (hio_dev_t* dev, void* ctx)
 	if (make_param(hio, info->cmd, info->flags, &param) <= -1) goto oops;
 /* TODO: more advanced fork and exec .. */
 	pid = standard_fork_and_exec(rdev, pfds, info, &param);
-	free_param (hio, &param);
+	free_param(hio, &param);
 	if (pid <= -1) goto oops;
 
 	rdev->child_pid = pid;
@@ -305,7 +305,7 @@ static int dev_pro_make_master (hio_dev_t* dev, void* ctx)
 		 * X
 		 * WRITE => 1
 		 */
-		close (pfds[0]);
+		close(pfds[0]);
 		pfds[0] = HIO_SYSHND_INVALID;
 
 		if (hio_makesyshndasync(hio, pfds[1]) <= -1) goto oops;
@@ -319,7 +319,7 @@ static int dev_pro_make_master (hio_dev_t* dev, void* ctx)
 		 *    X
 		 * READ => 2
 		 */
-		close (pfds[3]);
+		close(pfds[3]);
 		pfds[3] = HIO_SYSHND_INVALID;
 
 		if (hio_makesyshndasync(hio, pfds[2]) <= -1) goto oops;
@@ -333,7 +333,7 @@ static int dev_pro_make_master (hio_dev_t* dev, void* ctx)
 		 *      X
 		 * READ => 4
 		 */
-		close (pfds[5]);
+		close(pfds[5]);
 		pfds[5] = HIO_SYSHND_INVALID;
 
 		if (hio_makesyshndasync(hio, pfds[4]) <= -1) goto oops;
@@ -401,22 +401,25 @@ static int dev_pro_make_master (hio_dev_t* dev, void* ctx)
 	rdev->on_write = info->on_write;
 	rdev->on_close = info->on_close;
 
-	HIO_DEBUG7 (hio, "PRO(%p) -  slave[%d] %p slave[%d] %p slave[%d] %p\n", dev,
+	HIO_DEBUG7(hio, "PRO(%p) -  slave[%d] %p slave[%d] %p slave[%d] %p\n", dev,
 		HIO_DEV_PRO_IN, rdev->slave[HIO_DEV_PRO_IN],
 		HIO_DEV_PRO_OUT, rdev->slave[HIO_DEV_PRO_OUT],
 		HIO_DEV_PRO_ERR, rdev->slave[HIO_DEV_PRO_ERR]);
 	return 0;
 
 oops:
-	for (i = minidx; i <= maxidx; i++)
+	if (minidx >= 0 && maxidx >= 0)
 	{
-		if (pfds[i] != HIO_SYSHND_INVALID) close (pfds[i]);
+		for (i = minidx; i <= maxidx; i++)
+		{
+			if (pfds[i] != HIO_SYSHND_INVALID) close(pfds[i]);
+		}
 	}
 
 	if (rdev->mcmd)
 	{
 		hio_freemem(hio, rdev->mcmd);
-		free_param (hio, &param);
+		free_param(hio, &param);
 	}
 
 	for (i = HIO_COUNTOF(rdev->slave); i > 0; )
@@ -424,7 +427,7 @@ oops:
 		i--;
 		if (rdev->slave[i])
 		{
-			hio_dev_kill ((hio_dev_t*)rdev->slave[i]);
+			hio_dev_kill((hio_dev_t*)rdev->slave[i]);
 			rdev->slave[i] = HIO_NULL;
 		}
 	}
@@ -454,7 +457,7 @@ static int dev_pro_kill_master (hio_dev_t* dev, int force)
 				 * self-initiated termination or master-driven termination */
 				rdev->slave[i] = HIO_NULL;
 
-				hio_dev_kill ((hio_dev_t*)sdev);
+				hio_dev_kill((hio_dev_t*)sdev);
 			}
 		}
 	}
@@ -475,7 +478,7 @@ static int dev_pro_kill_master (hio_dev_t* dev, int force)
 				{
 					if (!(rdev->flags & HIO_DEV_PRO_FORGET_DIEHARD_CHILD))
 					{
-						kill (rdev->child_pid, SIGKILL);
+						kill(rdev->child_pid, SIGKILL);
 						killed = 1;
 						goto await_child;
 					}
@@ -495,11 +498,11 @@ static int dev_pro_kill_master (hio_dev_t* dev, int force)
 			 */
 		}
 
-		HIO_DEBUG2 (hio, "PRO(%p) -  REAPED CHILD %d\n", dev, (int)rdev->child_pid);
+		HIO_DEBUG2(hio, "PRO(%p) -  REAPED CHILD %d\n", dev, (int)rdev->child_pid);
 		rdev->child_pid = -1;
 	}
 
-	if (rdev->on_close) rdev->on_close (rdev, HIO_DEV_PRO_MASTER);
+	if (rdev->on_close) rdev->on_close(rdev, HIO_DEV_PRO_MASTER);
 	return 0;
 }
 
@@ -530,7 +533,7 @@ static int dev_pro_kill_slave (hio_dev_t* dev, int force)
 		rdev->master = HIO_NULL;
 
 		/* indicate EOF */
-		if (master->on_close) master->on_close (master, rdev->id);
+		if (master->on_close) master->on_close(master, rdev->id);
 
 		HIO_ASSERT(hio, master->slave_count > 0);
 		master->slave_count--;
@@ -541,7 +544,7 @@ static int dev_pro_kill_slave (hio_dev_t* dev, int force)
 			if (master->slave_count <= 0)
 			{
 				/* if this is the last slave, kill the master also */
-				hio_dev_kill ((hio_dev_t*)master);
+				hio_dev_kill((hio_dev_t*)master);
 				/* the master pointer is not valid from this point onwards
 				 * as the actual master device object is freed in hio_dev_kill() */
 			}
@@ -557,7 +560,7 @@ static int dev_pro_kill_slave (hio_dev_t* dev, int force)
 
 	if (rdev->pfd != HIO_SYSHND_INVALID)
 	{
-		close (rdev->pfd);
+		close(rdev->pfd);
 		rdev->pfd = HIO_SYSHND_INVALID;
 	}
 
@@ -567,7 +570,7 @@ static int dev_pro_kill_slave (hio_dev_t* dev, int force)
 static void dev_pro_fail_before_make_slave (void* ctx)
 {
 	slave_info_t* si = (slave_info_t*)ctx;
-	close (si->pfd);
+	close(si->pfd);
 }
 
 static int dev_pro_read_slave (hio_dev_t* dev, void* buf, hio_iolen_t* len, hio_devaddr_t* srcaddr)
@@ -620,7 +623,7 @@ static int dev_pro_write_slave (hio_dev_t* dev, const void* data, hio_iolen_t* l
 		if (HIO_LIKELY(pro->pfd != HIO_SYSHND_INVALID))
 		{
 			hio_dev_watch (dev, HIO_DEV_WATCH_STOP, 0);
-			close (pro->pfd);
+			close(pro->pfd);
 			pro->pfd = HIO_SYSHND_INVALID;
 		}
 		return 1; /* indicate that the operation got successful. the core will execute on_write() with the write length of 0. */
@@ -662,7 +665,7 @@ static int dev_pro_writev_slave (hio_dev_t* dev, const hio_iovec_t* iov, hio_iol
 		if (HIO_LIKELY(pro->pfd != HIO_SYSHND_INVALID))
 		{
 			hio_dev_watch (dev, HIO_DEV_WATCH_STOP, 0);
-			close (pro->pfd);
+			close(pro->pfd);
 			pro->pfd = HIO_SYSHND_INVALID;
 		}
 		return 1; /* indicate that the operation got successful. the core will execute on_write() with 0. */
@@ -714,7 +717,7 @@ static int dev_pro_ioctl (hio_dev_t* dev, int cmd, void* arg)
 				/* unlike dev_pro_kill_master(), i don't nullify rdev->slave[sid].
 				 * so i treat the closing ioctl as if it's a kill request
 				 * initiated by the slave device itself. */
-				hio_dev_kill ((hio_dev_t*)rdev->slave[sid]);
+				hio_dev_kill((hio_dev_t*)rdev->slave[sid]);
 
 				/* if this is the last slave, the master is destroyed as well.
 				 * therefore, using rdev is unsafe in the assertion below is unsafe.

lib/pty.c

@@ -158,7 +158,7 @@ static pid_t standard_fork_and_exec (hio_dev_pty_t* dev, int pfds[], hio_dev_pty
 	{
 		/* slave process */
 		/* child */
-		close (pfds[0]);  /* close the pty master */
+		close(pfds[0]);  /* close the pty master */
 		pfds[0] = HIO_SYSHND_INVALID;
 
 /*TODO: close all open file descriptors */
@@ -170,13 +170,13 @@ static pid_t standard_fork_and_exec (hio_dev_pty_t* dev, int pfds[], hio_dev_pty
 
 		if (dup2(pfds[1], 0) == -1 || dup2(pfds[1], 1) == -1 || dup2(pfds[1], 2) == -1) goto slave_oops;
 
-		close (pfds[1]);
+		close(pfds[1]);
 		pfds[1] = HIO_SYSHND_INVALID;
 
 		execve (param->argv[0], param->argv, param->fixed_env);
 
 		/* if exec fails, free 'param' parameter which is an inherited pointer */
-		free_param (hio, param);
+		free_param(hio, param);
 
 	slave_oops:
 		if (pfds[1] != HIO_SYSHND_INVALID) close(pfds[1]);
@@ -276,10 +276,10 @@ static int dev_pty_make (hio_dev_t* dev, void* ctx)
 
 	if (make_param(hio, info->cmd, info->flags, &param) <= -1) goto oops;
 	pid = standard_fork_and_exec(rdev, pfds, info, &param);
-	free_param (hio, &param);
+	free_param(hio, &param);
 	if (pid <= -1) goto oops;
 
-	close (pfds[1]); /* close the pty slave */
+	close(pfds[1]); /* close the pty slave */
 	pfds[1] = HIO_SYSHND_INVALID;
 
 	if (hio_makesyshndasync(hio, pfds[0]) <= -1) goto oops;
@@ -294,8 +294,8 @@ static int dev_pty_make (hio_dev_t* dev, void* ctx)
 	return 0;
 
 oops:
-	if (pfds[0] != HIO_SYSHND_INVALID) close (pfds[0]);
+	if (pfds[0] != HIO_SYSHND_INVALID) close(pfds[0]);
-	if (pfds[1] != HIO_SYSHND_INVALID) close (pfds[1]);
+	if (pfds[1] != HIO_SYSHND_INVALID) close(pfds[1]);
 	return -1;
 }
 
@@ -344,11 +344,11 @@ static int dev_pty_kill (hio_dev_t* dev, int force)
 		rdev->child_pid = -1;
 	}
 
-	if (rdev->on_close) rdev->on_close (rdev);
+	if (rdev->on_close) rdev->on_close(rdev);
 
 	if (rdev->hnd != HIO_SYSHND_INVALID)
 	{
-		close (rdev->hnd);
+		close(rdev->hnd);
 		rdev->hnd = HIO_SYSHND_INVALID;
 	}
 	return 0;
@@ -396,7 +396,7 @@ static int dev_pty_write (hio_dev_t* dev, const void* data, hio_iolen_t* len, co
 		if (HIO_LIKELY(pty->hnd != HIO_SYSHND_INVALID)) /* halt() doesn't close the pty immediately. so close the underlying pty */
 		{
 			hio_dev_watch (dev, HIO_DEV_WATCH_STOP, 0);
-			close (pty->hnd);
+			close(pty->hnd);
 			pty->hnd = HIO_SYSHND_INVALID;
 		}
 		return 1; /* indicate that the operation got successful. the core will execute on_write() with 0. */
@@ -433,7 +433,7 @@ static int dev_pty_writev (hio_dev_t* dev, const hio_iovec_t* iov, hio_iolen_t*
 		if (HIO_LIKELY(pty->hnd != HIO_SYSHND_INVALID)) /* halt() doesn't close the pty immediately. so close the underlying pty */
 		{
 			hio_dev_watch (dev, HIO_DEV_WATCH_STOP, 0);
-			close (pty->hnd);
+			close(pty->hnd);
 			pty->hnd = HIO_SYSHND_INVALID;
 		}
 		return 1; /* indicate that the operation got successful. the core will execute on_write() with 0. */

lib/sck.c

@@ -51,6 +51,24 @@
 #	include <netinet/sctp.h>
 #	if defined(IPPROTO_SCTP)
 #		define ENABLE_SCTP
+		/* these library calls are a separate question from the socket type. on
+		 * linux they live in libsctp, which a machine with sctp headers and an
+		 * sctp kernel may not have installed. the basic one-to-one and
+		 * one-to-many transports need none of them, so they stand either way.
+		 *
+		 * peel-off and multi-homing are probed apart because they are separate
+		 * features that share nothing but the library they come from: peel-off
+		 * needs sctp_peeloff() and multi-homing needs the address calls, and
+		 * neither calls into the other. bundling them would mean a system
+		 * missing one silently loses both - the same "they always come
+		 * together" assumption that let the old header-derived ENABLE_SCTP
+		 * stay wrong for so long. */
+#		if defined(HAVE_SCTP_PEELOFF)
+#			define ENABLE_SCTP_PEELOFF
+#		endif
+#		if defined(HAVE_SCTP_BINDX) && defined(HAVE_SCTP_GETPADDRS) && defined(HAVE_SCTP_GETLADDRS)
+#			define ENABLE_SCTP_MH
+#		endif
 #	endif
 #endif
 
@@ -322,13 +340,23 @@ static struct sck_type_map_t sck_type_map[] =
 	/* HIO_DEV_SCK_SCTP6 */
 	{ AF_INET6,  SOCK_STREAM,     IPPROTO_SCTP,      1, 1, HIO_DEV_CAP_STREAM },
 
-	/* HIO_DEV_SCK_SCTP4_SP - one-to-many. listen() is called but hio never
+	/* HIO_DEV_SCK_SCTP4_SEQPKT - one-to-many. listen() is called but hio never
 	 * accepts: associations are not devices in this model, they are told apart
-	 * by the source address on each message. */
+	 * by the source address on each message.
-	{ AF_INET,   SOCK_SEQPACKET,  IPPROTO_SCTP,      1, 1, 0 },
+	 *
+	 * marked unconnectable, as udp is. the socket api does allow connect() on a
+	 * one-to-many socket - it forms an association and makes it the default
+	 * destination - but nothing here can finish the job: a device with no
+	 * HIO_DEV_CAP_STREAM gets dev_evcb_sck_ready_stateless(), which looks only
+	 * at ERR and HUP and never at the progress bits, so on_connect() would
+	 * never fire. the device would sit in HIO_DEV_SCK_CONNECTING for good -
+	 * writes failing, a second connect() refused as already in progress, and
+	 * with a connect timeout set, silently halted a few seconds later.
+	 * refusing outright beats handing back a bricked device. */
+	{ AF_INET,   SOCK_SEQPACKET,  IPPROTO_SCTP,      0, 1, 0 },
 
-	/* HIO_DEV_SCK_SCTP6_SP */
+	/* HIO_DEV_SCK_SCTP6_SEQPKT */
-	{ AF_INET6,  SOCK_SEQPACKET,  IPPROTO_SCTP,      1, 1, 0 },
+	{ AF_INET6,  SOCK_SEQPACKET,  IPPROTO_SCTP,      0, 1, 0 },
 #else
 	{ -1,        0,               0,                 0, 0, 0 },
 	{ -1,        0,               0,                 0, 0, 0 },
@@ -605,10 +633,18 @@ oops:
 	return -1;
 }
 
+/* hio_dev_make() hands this back exactly what was passed to it, for the case
+ * where it fails before the make() method could take ownership of the handle.
+ *
+ * [NOTE] the ctx used to be a bare hio_syshnd_t and is now a struct whose first
+ * member is one. the old cast still read the right value - a pointer to a
+ * struct points to its first member - but only by accident of layout: put any
+ * field ahead of 'hnd' and this would close (int)type instead, which is a small
+ * integer, which is to say some other part of the program's descriptor. */
 static void dev_sck_fail_before_make_client (void* ctx)
 {
-	hio_syshnd_t* clisckhnd = (hio_syshnd_t*)ctx;
+	sck_make_client_ctx_t* mc = (sck_make_client_ctx_t*)ctx;
-	close(*clisckhnd);
+	close(mc->hnd);
 }
 
 static int dev_sck_kill (hio_dev_t* dev, int force)
@@ -884,7 +920,14 @@ static int sendmsg_sctp(
 	 * addressing by msg_name. it is ignored on a one-to-one socket. */
 	sinfo->sinfo_assoc_id = (sctp_assoc_t)assoc_id;
 
+	/* the plain stream and stateless send paths all ask for this; without it a
+	 * failing sctp write raises SIGPIPE and kills the process where the
+	 * equivalent tcp write merely returns EPIPE. */
+#if defined(MSG_NOSIGNAL)
+	return sendmsg(s, &msg, MSG_NOSIGNAL);
+#else
 	return sendmsg(s, &msg, 0);
+#endif
 }
 
 /* the core passes no destination for a connected socket - hio_dev_sck_write()
@@ -915,9 +958,8 @@ static HIO_INLINE hio_int32_t dstaddr_assoc (const hio_devaddr_t* dstaddr)
 	return (dstaddr && dstaddr->ptr)? hio_skad_get_assoc((const hio_skad_t*)dstaddr->ptr): 0;
 }
 
-static int dev_sck_read_sctp_sp (hio_dev_t* dev, void* buf, hio_iolen_t* len, hio_devaddr_t* srcaddr)
+static int dev_sck_read_sctp_seqpkt (hio_dev_t* dev, void* buf, hio_iolen_t* len, hio_devaddr_t* srcaddr)
 {
-/* NOTE: sctp support is far away from complete */
 	hio_t* hio = dev->hio;
 	hio_dev_sck_t* rdev = (hio_dev_sck_t*)dev;
 	hio_scklen_t srcaddrlen;
@@ -944,7 +986,12 @@ static int dev_sck_read_sctp_sp (hio_dev_t* dev, void* buf, hio_iolen_t* len, hi
 	if (msg_flags & MSG_NOTIFICATION)
 	{
 		/* an association or path event, not payload. handing this to
-		 * on_read() would splice an event record into the data stream. */
+		 * on_read() would splice an event record into the data stream.
+		 *
+		 * every one of these is passed on, association changes included. this
+		 * socket takes no view on which associations deserve a device of their
+		 * own - hio_dev_sck_peeloff() is a call the application makes, from
+		 * here or from on_read() later, for the associations it picks. */
 		if (rdev->on_notification) rdev->on_notification(rdev, buf, x);
 		return 0; /* nothing readable for the caller this time round */
 	}
@@ -970,7 +1017,7 @@ static int dev_sck_read_sctp_sp (hio_dev_t* dev, void* buf, hio_iolen_t* len, hi
 		if (!rdev->sctp_discarding)
 		{
 			rdev->sctp_discarding = 1;
-			HIO_INFO2 (hio, "SCK(%p) - discarding an sctp message too large for the read buffer of %zu octets\n", rdev, hio->bigbuf.capa);
+			HIO_INFO2(hio, "SCK(%p) - discarding an sctp message too large for the read buffer of %zu octets\n", rdev, hio->bigbuf.capa);
 		}
 		/* keep swallowing fragments until the one that ends the message */
 		if (msg_flags & MSG_EOR) rdev->sctp_discarding = 0;
@@ -1260,9 +1307,8 @@ static int dev_sck_writev_bpf (hio_dev_t* dev, const hio_iovec_t* iov, hio_iolen
 
 /* ------------------------------------------------------------------------------ */
 #if defined(ENABLE_SCTP)
-static int dev_sck_write_sctp_sp (hio_dev_t* dev, const void* data, hio_iolen_t* len, const hio_devaddr_t* dstaddr)
+static int dev_sck_write_sctp_seqpkt (hio_dev_t* dev, const void* data, hio_iolen_t* len, const hio_devaddr_t* dstaddr)
 {
-/* NOTE: sctp support is far away from complete */
 	hio_t* hio = dev->hio;
 	hio_dev_sck_t* rdev = (hio_dev_sck_t*)dev;
 	ssize_t x;
@@ -1292,7 +1338,7 @@ static int dev_sck_write_sctp_sp (hio_dev_t* dev, const void* data, hio_iolen_t*
 
 /* the one-to-one sctp types carry HIO_DEV_CAP_STREAM, so the core treats them
  * like a tcp socket: a zero-length write is the writing-end shutdown, and a
- * zero-length read is EOF. the seqpacket methods above answer neither of those
+ * zero-length read is EOF. the seqpkt methods above answer neither of those
  * conventions - a zero-length sendmsg() puts an empty message on the wire
  * rather than closing anything - so the stream variants need their own pair.
  *
@@ -1329,9 +1375,9 @@ static int dev_sck_read_sctp_stream (hio_dev_t* dev, void* buf, hio_iolen_t* len
 
 	/* x of 0 falls through as a zero length, which is what the core reads as
 	 * EOF on a stream device */
-	hio_skad_set_chan (&rdev->remoteaddr, sri.sinfo_stream);
+	hio_skad_set_chan(&rdev->remoteaddr, sri.sinfo_stream);
-	hio_skad_set_ppid (&rdev->remoteaddr, sri.sinfo_ppid);
+	hio_skad_set_ppid(&rdev->remoteaddr, sri.sinfo_ppid);
-	hio_skad_set_assoc (&rdev->remoteaddr, (hio_int32_t)sri.sinfo_assoc_id);
+	hio_skad_set_assoc(&rdev->remoteaddr, (hio_int32_t)sri.sinfo_assoc_id);
 
 	*len = x;
 	return 1;
@@ -1403,7 +1449,7 @@ static int dev_sck_writev_sctp_stream (hio_dev_t* dev, const hio_iovec_t* iov, h
 	return 1;
 }
 
-static int dev_sck_writev_sctp_sp (hio_dev_t* dev, const hio_iovec_t* iov, hio_iolen_t* iovcnt, const hio_devaddr_t* dstaddr)
+static int dev_sck_writev_sctp_seqpkt (hio_dev_t* dev, const hio_iovec_t* iov, hio_iolen_t* iovcnt, const hio_devaddr_t* dstaddr)
 {
 	hio_t* hio = dev->hio;
 	hio_dev_sck_t* rdev = (hio_dev_sck_t*)dev;
@@ -2118,7 +2164,7 @@ static hio_dev_mth_t dev_mth_clisck_sctp_stream =
 {
 	dev_sck_make_client,
 	dev_sck_kill,
-	HIO_NULL,
+	dev_sck_fail_before_make_client,
 	dev_sck_getsyshnd,
 	HIO_NULL,
 	dev_sck_ioctl,
@@ -2153,7 +2199,7 @@ static hio_dev_mth_t dev_mth_sck_stream =
 };
 
 #if defined(ENABLE_SCTP)
-static hio_dev_mth_t dev_mth_sck_sctp_sp =
+static hio_dev_mth_t dev_mth_sck_sctp_seqpkt =
 {
 	dev_sck_make,
 	dev_sck_kill,
@@ -2162,9 +2208,9 @@ static hio_dev_mth_t dev_mth_sck_sctp_sp =
 	HIO_NULL,
 	dev_sck_ioctl,     /* ioctl */
 
-	dev_sck_read_sctp_sp,
+	dev_sck_read_sctp_seqpkt,
-	dev_sck_write_sctp_sp,
+	dev_sck_write_sctp_seqpkt,
-	dev_sck_writev_sctp_sp,
+	dev_sck_writev_sctp_seqpkt,
 	HIO_NULL,          /* sendfile */
 
 	HIO_NULL,          /* readpending */
@@ -2210,7 +2256,7 @@ static hio_dev_mth_t dev_mth_clisck_stream =
 };
 
 #if defined(ENABLE_SCTP)
-static hio_dev_mth_t dev_mth_clisck_sctp_sp =
+static hio_dev_mth_t dev_mth_clisck_sctp_seqpkt =
 {
 	dev_sck_make_client,
 	dev_sck_kill,
@@ -2219,12 +2265,12 @@ static hio_dev_mth_t dev_mth_clisck_sctp_sp =
 	HIO_NULL,
 	dev_sck_ioctl,
 
-	dev_sck_read_sctp_sp,
+	dev_sck_read_sctp_seqpkt,
-	dev_sck_write_sctp_sp,
+	dev_sck_write_sctp_seqpkt,
-	dev_sck_writev_sctp_sp,
+	dev_sck_writev_sctp_seqpkt,
 	HIO_NULL,  /* sendfile */
 
-	HIO_NULL   /* readpending - no ssl on a seqpacket socket */
+	HIO_NULL   /* readpending - no ssl on a seqpkt socket */
 };
 #endif
 
@@ -2344,11 +2390,11 @@ static int harvest_outgoing_connection (hio_dev_sck_t* rdev)
 
 static int make_accepted_client_connection (hio_dev_sck_t* rdev, hio_syshnd_t clisck, hio_skad_t* remoteaddr, hio_dev_sck_type_t clisck_type)
 {
-	sck_make_client_ctx_t mc;
 	hio_t* hio = rdev->hio;
 	hio_dev_sck_t* clidev;
 	hio_scklen_t addrlen;
 	hio_dev_mth_t* dev_mth;
+	sck_make_client_ctx_t mc;
 
 	if (rdev->on_raw_accept)
 	{
@@ -2370,8 +2416,8 @@ static int make_accepted_client_connection (hio_dev_sck_t* rdev, hio_syshnd_t cl
 	 */
 #if defined(ENABLE_SCTP)
 	dev_mth = (sck_type_map[clisck_type].extra_dev_cap & HIO_DEV_CAP_STREAM)?
-	              ((sck_type_map[clisck_type].proto == IPPROTO_SCTP)? &dev_mth_clisck_sctp_stream: &dev_mth_clisck_stream):
+		((sck_type_map[clisck_type].proto == IPPROTO_SCTP)? &dev_mth_clisck_sctp_stream: &dev_mth_clisck_stream):
-	          (sck_type_map[clisck_type].proto == IPPROTO_SCTP)? &dev_mth_clisck_sctp_sp: &dev_mth_clisck_stateless;
+		((sck_type_map[clisck_type].proto == IPPROTO_SCTP)? &dev_mth_clisck_sctp_seqpkt: &dev_mth_clisck_stateless);
 #else
 	dev_mth = (sck_type_map[clisck_type].extra_dev_cap & HIO_DEV_CAP_STREAM)? &dev_mth_clisck_stream: &dev_mth_clisck_stateless;
 #endif
@@ -2380,7 +2426,7 @@ static int make_accepted_client_connection (hio_dev_sck_t* rdev, hio_syshnd_t cl
 	clidev = (hio_dev_sck_t*)hio_dev_make(hio, rdev->dev_size, dev_mth, rdev->dev_evcb, &mc);
 	if (HIO_UNLIKELY(!clidev))
 	{
-		/* [NOTE] 'clisck' is closed by callback methods called by hio_dev_make() upon failure */
+		/* [NOTE] 'clisck' is closed by callback(fail_before_make) methods called by hio_dev_make() upon failure */
 		HIO_DEBUG3(hio, "SCK(%p) - unable to make a new accepted device for %d - %js\n", rdev, (int)clisck, hio_geterrmsg(hio));
 		return -1;
 	}
@@ -2529,6 +2575,10 @@ static int accept_incoming_connection (hio_dev_sck_t* rdev)
 #if defined(SOCK_NONBLOCK) && defined(SOCK_CLOEXEC) && defined(HAVE_ACCEPT4)
 accept_done:
 #endif
+	/* no separate error handling to close clisck becuase it's supposed to be
+	 * handled by hio_dev_make() via the fail_before_make callback inside
+	 * make_accepted_client_connection(). if it fails even before hio_dev_make()
+	 * inside make_accepted_client_connection(), it should close the socket explicitly */
 	return make_accepted_client_connection(rdev, clisck, &remoteaddr, rdev->type);
 }
 
@@ -2634,7 +2684,6 @@ static int dev_evcb_sck_ready_stream (hio_dev_t* dev, int events)
 		#endif
 
 		case HIO_DEV_SCK_LISTENING:
-
 			if (events & HIO_DEV_EVENT_HUP)
 			{
 				/* device hang-up */
@@ -2798,7 +2847,7 @@ static hio_dev_evcb_t dev_sck_event_callbacks_stateless =
 	dev_evcb_sck_on_write_stateless
 };
 
-static hio_dev_evcb_t dev_sck_event_callbacks_sctp_sp =
+static hio_dev_evcb_t dev_sck_event_callbacks_sctp_seqpkt =
 {
 	dev_evcb_sck_ready_stateless,
 	dev_evcb_sck_on_read_stateless,
@@ -2972,7 +3021,7 @@ hio_dev_sck_t* hio_dev_sck_make (hio_t* hio, hio_oow_t xtnsize, const hio_dev_sc
 	{
 		rdev = (hio_dev_sck_t*)hio_dev_make(
 			hio, HIO_SIZEOF(hio_dev_sck_t) + xtnsize,
-			&dev_mth_sck_sctp_sp, &dev_sck_event_callbacks_sctp_sp, (void*)info);
+			&dev_mth_sck_sctp_seqpkt, &dev_sck_event_callbacks_sctp_seqpkt, (void*)info);
 	}
 #endif
 	else
@@ -3161,6 +3210,391 @@ int hio_dev_sck_shutdown (hio_dev_sck_t* dev, int how)
 	return 0;
 }
 
+/* ------------------------------------------------------------------------- */
+/* sctp peel-off                                                             */
+/* ------------------------------------------------------------------------- */
+
+#if defined(ENABLE_SCTP_PEELOFF)
+/* the one-to-one type a peeled-off association becomes. sctp_peeloff() hands
+ * back a SOCK_STREAM socket carrying one association, which is precisely what
+ * the one-to-one types already describe - so the peeled device gets the
+ * existing one-to-one methods, write queue and all. */
+static int sctp_one_to_one_type (hio_dev_sck_type_t sp_type, hio_dev_sck_type_t* one_to_one)
+{
+	switch (sp_type)
+	{
+		case HIO_DEV_SCK_SCTP4_SEQPKT:
+			*one_to_one = HIO_DEV_SCK_SCTP4;
+			return 0;
+
+		case HIO_DEV_SCK_SCTP6_SEQPKT:
+			*one_to_one = HIO_DEV_SCK_SCTP6;
+			return 0;
+
+		default:
+			return -1;
+	}
+}
+
+int hio_dev_sck_peeloff (hio_dev_sck_t* rdev, hio_int32_t assoc_id)
+{
+	hio_t* hio = rdev->hio;
+	hio_dev_sck_type_t clitype;
+	hio_syshnd_t clisck;
+	hio_skad_t remoteaddr;
+	hio_scklen_t addrlen;
+
+	if (sctp_one_to_one_type(rdev->type, &clitype) <= -1)
+	{
+		hio_seterrbfmt(hio, HIO_EINVAL, "not a one-to-many sctp socket");
+		return -1;
+	}
+
+	/* move the association onto a socket of its own. anything still queued for
+	 * it in the kernel moves with it, which is what makes a mid-stream peel
+	 * safe: messages already handed to on_read() are the caller's, the rest
+	 * arrive on the new device, and none are lost or seen twice. */
+	clisck = sctp_peeloff(rdev->hnd, (sctp_assoc_t)assoc_id);
+	if (clisck <= -1)
+	{
+		hio_seterrwithsyserr(hio, 0, errno);
+		return -1;
+	}
+
+	/* the extra area past the sockaddr is not written by getpeername(), so it
+	 * is cleared here rather than left holding the stream and ppid of whatever
+	 * message last used this device's remoteaddr. */
+	HIO_MEMSET(&remoteaddr, 0, HIO_SIZEOF(remoteaddr));
+	addrlen = HIO_SIZEOF(remoteaddr);
+	if (getpeername(clisck, (struct sockaddr*)&remoteaddr, &addrlen) <= -1)
+	{
+		/* fall back on the address the notification came from - the primary
+		 * path of the same association. the remaining addresses of a
+		 * multi-homed peer are reachable with hio_dev_sck_getpaddrs(). */
+		remoteaddr = rdev->remoteaddr;
+		hio_skad_set_chan(&remoteaddr, 0);
+		hio_skad_set_ppid(&remoteaddr, 0);
+		hio_skad_set_assoc(&remoteaddr, 0);
+	}
+
+	/* no separate error handling to close clisck becuase it's supposed to be
+	 * handled by hio_dev_make() via the fail_before_make callback inside
+	 * make_accepted_client_connection(). if it fails even before hio_dev_make()
+	 * inside make_accepted_client_connection(), it should close the socket explicitly */
+	return make_accepted_client_connection(rdev, clisck, &remoteaddr, clitype);
+}
+
+#else /* ENABLE_SCTP_PEELOFF */
+
+int hio_dev_sck_peeloff (hio_dev_sck_t* rdev, hio_int32_t assoc_id)
+{
+	hio_seterrbfmt(rdev->hio, HIO_ENOIMPL, "sctp peel-off not supported");
+	return -1;
+}
+
+#endif /* ENABLE_SCTP_PEELOFF */
+
+/* ------------------------------------------------------------------------- */
+
+#if defined(ENABLE_SCTP)
+
+int hio_dev_sck_parse_assoc_event (const void* data, hio_iolen_t dlen, hio_sctp_assoc_event_t* ev)
+{
+	struct sctp_assoc_change ac;
+
+	/* copied out rather than read in place: a caller may hand over a buffer
+	 * whose alignment is not this structure's to assume. */
+	if (dlen < (hio_iolen_t)HIO_SIZEOF(ac)) return -1; /* too short to be one */
+	HIO_MEMCPY(&ac, data, HIO_SIZEOF(ac));
+
+	if (ac.sac_type != SCTP_ASSOC_CHANGE) return -1; /* a different event, not an error */
+
+	switch (ac.sac_state)
+	{
+		case SCTP_COMM_UP:        ev->state = HIO_SCTP_ASSOC_COMM_UP;        break;
+		case SCTP_COMM_LOST:      ev->state = HIO_SCTP_ASSOC_COMM_LOST;      break;
+		case SCTP_RESTART:        ev->state = HIO_SCTP_ASSOC_RESTART;        break;
+		case SCTP_SHUTDOWN_COMP:  ev->state = HIO_SCTP_ASSOC_SHUTDOWN_COMP;  break;
+		case SCTP_CANT_STR_ASSOC: ev->state = HIO_SCTP_ASSOC_CANT_STR_ASSOC; break;
+		default:                  ev->state = HIO_SCTP_ASSOC_STATE_UNKNOWN;  break;
+	}
+
+	ev->error = ac.sac_error;
+	ev->assoc_id = (hio_int32_t)ac.sac_assoc_id;
+	/* the negotiated counts - the outcome of both ends' SCTP_INITMSG requests,
+	 * and not reachable any other way */
+	ev->ostreams = ac.sac_outbound_streams;
+	ev->instreams = ac.sac_inbound_streams;
+	return 0;
+}
+
+#else
+
+int hio_dev_sck_parse_assoc_event (const void* data, hio_iolen_t dlen, hio_sctp_assoc_event_t* ev)
+{
+	return -1;
+}
+
+#endif
+
+/* ------------------------------------------------------------------------- */
+/* sctp multi-homing                                                         */
+/* ------------------------------------------------------------------------- */
+
+#if defined(ENABLE_SCTP_MH)
+
+static HIO_INLINE int is_sctp_sck (hio_dev_sck_t* dev)
+{
+	return sck_type_map[dev->type].proto == IPPROTO_SCTP;
+}
+
+/* the sctp calls take addresses as a packed run of sockaddrs of mixed length,
+ * not an array of a fixed-size type. these two convert between that and
+ * hio_skad_t, which is fixed-size and carries an extra area the kernel neither
+ * writes nor reads. */
+
+static int pack_skads (hio_t* hio, const hio_skad_t* addrs, hio_oow_t naddrs, hio_uint8_t** buf)
+{
+	hio_uint8_t* b, * p;
+	hio_oow_t total = 0, i;
+
+	for (i = 0; i < naddrs; i++)
+	{
+		int len = hio_skad_get_size(&addrs[i]);
+		if (len <= 0)
+		{
+			hio_seterrbfmt(hio, HIO_EINVAL, "address #%zu is not of a supported family", i);
+			return -1;
+		}
+		total += len;
+	}
+
+	b = (hio_uint8_t*)hio_allocmem(hio, total);
+	if (HIO_UNLIKELY(!b)) return -1;
+
+	for (i = 0, p = b; i < naddrs; i++)
+	{
+		int len = hio_skad_get_size(&addrs[i]);
+		HIO_MEMCPY(p, &addrs[i], len);
+		p += len;
+	}
+
+	*buf = b;
+	return 0;
+}
+
+static int unpack_skads (hio_t* hio, const struct sockaddr* packed, int count, hio_skad_t* addrs, hio_oow_t* naddrs)
+{
+	hio_oow_t room = *naddrs, i;
+	const hio_uint8_t* p = (const hio_uint8_t*)packed;
+
+	/* every address is reported, so the caller learns how much room it needed
+	 * even when it did not have enough. what does not fit is not written. */
+	for (i = 0; i < (hio_oow_t)count; i++)
+	{
+		hio_oow_t len;
+
+		switch (((const struct sockaddr*)p)->sa_family)
+		{
+			case AF_INET:
+				len = HIO_SIZEOF(struct sockaddr_in);
+				break;
+
+		#if defined(AF_INET6)
+			case AF_INET6:
+				len = HIO_SIZEOF(struct sockaddr_in6);
+				break;
+		#endif
+
+			default:
+				/* the run cannot be walked past an address of unknown length */
+				hio_seterrbfmt(hio, HIO_EINVAL, "unsupported address family %d in the sctp address list", (int)((const struct sockaddr*)p)->sa_family);
+				*naddrs = i;
+				return -1;
+		}
+
+		if (i < room)
+		{
+			/* the extra area past the sockaddr belongs to hio, not the kernel */
+			HIO_MEMSET(&addrs[i], 0, HIO_SIZEOF(addrs[i]));
+			HIO_MEMCPY(&addrs[i], p, len);
+		}
+		p += len;
+	}
+
+	*naddrs = (hio_oow_t)count;
+	if ((hio_oow_t)count > room)
+	{
+		hio_seterrbfmt(hio, HIO_EBUFFULL, "%d addresses do not fit in %zu slot(s)", count, room);
+		return -1;
+	}
+
+	return 0;
+}
+
+int hio_dev_sck_bindx (hio_dev_sck_t* dev, const hio_skad_t* addrs, hio_oow_t naddrs, hio_dev_sck_bindx_flag_t flags)
+{
+	hio_t* hio = dev->hio;
+	hio_uint8_t* buf;
+	int x;
+
+	if (!is_sctp_sck(dev))
+	{
+		hio_seterrbfmt(hio, HIO_ENOIMPL, "not an sctp socket device");
+		return -1;
+	}
+
+	if (naddrs <= 0)
+	{
+		hio_seterrbfmt(hio, HIO_EINVAL, "no address given");
+		return -1;
+	}
+
+	if (pack_skads(hio, addrs, naddrs, &buf) <= -1) return -1;
+
+	x = sctp_bindx(dev->hnd, (struct sockaddr*)buf, (int)naddrs,
+		(flags == HIO_DEV_SCK_BINDX_REM)? SCTP_BINDX_REM_ADDR: SCTP_BINDX_ADD_ADDR);
+	if (x <= -1) hio_seterrwithsyserr(hio, 0, errno);
+
+	hio_freemem (hio, buf);
+	return (x <= -1)? -1: 0;
+}
+
+int hio_dev_sck_getladdrs (hio_dev_sck_t* dev, hio_skad_t* addrs, hio_oow_t* naddrs)
+{
+	hio_t* hio = dev->hio;
+	struct sockaddr* packed = HIO_NULL;
+	int count, x;
+
+	if (!is_sctp_sck(dev))
+	{
+		hio_seterrbfmt(hio, HIO_ENOIMPL, "not an sctp socket device");
+		return -1;
+	}
+
+	/* association 0 asks about the endpoint, which is what local multi-homing
+	 * is about - the addresses belong to the socket, not to one association. */
+	count = sctp_getladdrs(dev->hnd, 0, &packed);
+	if (count <= -1)
+	{
+		hio_seterrwithsyserr(hio, 0, errno);
+		return -1;
+	}
+
+	x = unpack_skads(hio, packed, count, addrs, naddrs);
+	sctp_freeladdrs(packed);
+	return x;
+}
+
+int hio_dev_sck_getpaddrs (hio_dev_sck_t* dev, hio_skad_t* addrs, hio_oow_t* naddrs)
+{
+	hio_t* hio = dev->hio;
+	struct sockaddr* packed = HIO_NULL;
+	int count, x;
+
+	if (!is_sctp_sck(dev))
+	{
+		hio_seterrbfmt(hio, HIO_ENOIMPL, "not an sctp socket device");
+		return -1;
+	}
+
+	/* peer addresses belong to an association, and only a one-to-one socket has
+	 * exactly one. that is what a peeled-off or connected device is; a
+	 * one-to-many socket holds many and cannot answer the question. */
+	if (sck_type_map[dev->type].type == SOCK_SEQPACKET)
+	{
+		hio_seterrbfmt(hio, HIO_EPERM, "peer addresses are per-association - peel the association off first");
+		return -1;
+	}
+
+	count = sctp_getpaddrs(dev->hnd, 0, &packed);
+	if (count <= -1)
+	{
+		hio_seterrwithsyserr(hio, 0, errno);
+		return -1;
+	}
+
+	x = unpack_skads(hio, packed, count, addrs, naddrs);
+	sctp_freepaddrs(packed);
+	return x;
+}
+
+/* SCTP_PRIMARY_ADDR takes the same two members under two names: struct
+ * sctp_prim on linux, struct sctp_setprim on the bsds. */
+#if defined(HAVE_STRUCT_SCTP_PRIM)
+#	define sctp_prim_t struct sctp_prim
+#elif defined(HAVE_STRUCT_SCTP_SETPRIM)
+#	define sctp_prim_t struct sctp_setprim
+#endif
+
+int hio_dev_sck_setprimaryaddr (hio_dev_sck_t* dev, const hio_skad_t* addr)
+{
+	hio_t* hio = dev->hio;
+#if defined(sctp_prim_t)
+	sctp_prim_t prim;
+	int len;
+
+	if (!is_sctp_sck(dev))
+	{
+		hio_seterrbfmt(hio, HIO_ENOIMPL, "not an sctp socket device");
+		return -1;
+	}
+
+	len = hio_skad_get_size(addr);
+	if (len <= 0 || (hio_oow_t)len > HIO_SIZEOF(prim.ssp_addr))
+	{
+		hio_seterrbfmt(hio, HIO_EINVAL, "address is not of a supported family");
+		return -1;
+	}
+
+	HIO_MEMSET(&prim, 0, HIO_SIZEOF(prim));
+	prim.ssp_assoc_id = 0; /* the only association, on a one-to-one socket */
+	HIO_MEMCPY(&prim.ssp_addr, addr, len);
+
+	if (setsockopt(dev->hnd, IPPROTO_SCTP, SCTP_PRIMARY_ADDR, &prim, HIO_SIZEOF(prim)) <= -1)
+	{
+		hio_seterrwithsyserr(hio, 0, errno);
+		return -1;
+	}
+
+	return 0;
+#else
+	hio_seterrbfmt(hio, HIO_ENOIMPL, "no structure for SCTP_PRIMARY_ADDR on this system");
+	return -1;
+#endif
+}
+
+#else /* ENABLE_SCTP_MH */
+
+/* the declarations are unconditional, so the symbols have to exist. they say
+ * so rather than being absent at link time. */
+
+int hio_dev_sck_bindx (hio_dev_sck_t* dev, const hio_skad_t* addrs, hio_oow_t naddrs, hio_dev_sck_bindx_flag_t flags)
+{
+	hio_seterrbfmt(dev->hio, HIO_ENOIMPL, "sctp multi-homing not supported");
+	return -1;
+}
+
+int hio_dev_sck_getladdrs (hio_dev_sck_t* dev, hio_skad_t* addrs, hio_oow_t* naddrs)
+{
+	hio_seterrbfmt(dev->hio, HIO_ENOIMPL, "sctp multi-homing not supported");
+	return -1;
+}
+
+int hio_dev_sck_getpaddrs (hio_dev_sck_t* dev, hio_skad_t* addrs, hio_oow_t* naddrs)
+{
+	hio_seterrbfmt(dev->hio, HIO_ENOIMPL, "sctp multi-homing not supported");
+	return -1;
+}
+
+int hio_dev_sck_setprimaryaddr (hio_dev_sck_t* dev, const hio_skad_t* addr)
+{
+	hio_seterrbfmt(dev->hio, HIO_ENOIMPL, "sctp multi-homing not supported");
+	return -1;
+}
+
+#endif /* ENABLE_SCTP_MH */
+
 int hio_dev_sck_sendfileok (hio_dev_sck_t* dev)
 {
 	/* the transport has to actually implement it. the sctp methods deliberately

lib/sys-mux.c

@@ -252,7 +252,7 @@ void hio_sys_intrmux (hio_t* hio)
 {
 	/* for now, thie only use of the control pipe is to interrupt the multiplexer */
 	hio_sys_mux_t* mux = &hio->sysdep->mux;
-	if (mux->ctrlp[1] != HIO_SYSHND_INVALID) write (mux->ctrlp[1], "Q", 1);
+	if (mux->ctrlp[1] != HIO_SYSHND_INVALID) write(mux->ctrlp[1], "Q", 1);
 }
 
 #if defined(USE_POLL)

lib/tar.c

@@ -215,7 +215,7 @@ static int x_process_content (hio_tar_t* tar)
 	chunksize = tar->x.hi.filesize < tar->x.blk.len? tar->x.hi.filesize: tar->x.blk.len;
 
 /* TODO: error check */
-	fwrite (tar->x.blk.buf, 1, chunksize, tar->x.hi.fp);
+	fwrite(tar->x.blk.buf, 1, chunksize, tar->x.hi.fp);
 
 	tar->x.hi.filesize -= chunksize;
 	if (tar->x.hi.filesize <= 0)

lib/thr.c

@@ -89,14 +89,14 @@ static int ready_to_free_thr_info (hio_t* hio, hio_cfmb_t* cfmb)
 	if (HIO_UNLIKELY(hio->_fini_in_progress))
 	{
 		pthread_join (ti->thr_hnd, HIO_NULL); /* BAD. blocking call in a non-blocking library. not useful to call pthread_tryjoin_np() here. */
-		free_thr_info_resources (hio, ti);
+		free_thr_info_resources(hio, ti);
 		return 1; /* free me */
 	}
 #endif
 
 	if (ti->thr_done)
 	{
-		free_thr_info_resources (hio, ti);
+		free_thr_info_resources(hio, ti);
 #if defined(HAVE_PTHREAD_TRYJOIN_NP)
 		if (pthread_tryjoin_np(ti->thr_hnd) != 0) /* not terminated yet - however, this isn't necessary. z*/
 #endif
@@ -126,7 +126,7 @@ static void* run_thr_func (void* ctx)
 
 	ti->thr_func (ti->hio, &ti->thr_iop, ti->thr_ctx);
 
-	free_thr_info_resources (ti->hio, ti);
+	free_thr_info_resources(ti->hio, ti);
 
 	pthread_cleanup_pop (1);
 	pthread_exit (HIO_NULL);
@@ -301,7 +301,7 @@ static int dev_thr_kill_master (hio_dev_t* dev, int force)
 				 * self-initiated termination or master-driven termination */
 				rdev->slave[i] = HIO_NULL;
 
-				hio_dev_kill ((hio_dev_t*)sdev);
+				hio_dev_kill((hio_dev_t*)sdev);
 			}
 		}
 	}
@@ -310,7 +310,7 @@ static int dev_thr_kill_master (hio_dev_t* dev, int force)
 	if (ti->thr_done)
 	{
 		pthread_detach (ti->thr_hnd); /* pthread_join() may be blocking. detach the thread instead */
-		free_thr_info_resources (hio, ti);
+		free_thr_info_resources(hio, ti);
 		hio_freemem(hio, ti);
 	}
 	else
@@ -319,15 +319,15 @@ static int dev_thr_kill_master (hio_dev_t* dev, int force)
 		/* since pthread_join can be blocking, i'd schedule a resource destroyer with hio_addcfmb().
 		 * see after #else */
 		pthread_join (ti->thr_hnd, HIO_NULL);
-		free_thr_info_resources (hio, ti);
+		free_thr_info_resources(hio, ti);
 		hio_freemem(hio, ti);
 	#else
 		/* schedule a resource destroyer */
-		hio_addcfmb (hio, (hio_cfmb_t*)ti, ready_to_free_thr_info, HIO_NULL);
+		hio_addcfmb(hio, (hio_cfmb_t*)ti, ready_to_free_thr_info, HIO_NULL);
 	#endif
 	}
 
-	if (rdev->on_close) rdev->on_close (rdev, HIO_DEV_THR_MASTER);
+	if (rdev->on_close) rdev->on_close(rdev, HIO_DEV_THR_MASTER);
 	return 0;
 }
 
@@ -344,7 +344,7 @@ static int dev_thr_kill_slave (hio_dev_t* dev, int force)
 		rdev->master = HIO_NULL;
 
 		/* indicate EOF */
-		if (master->on_close) master->on_close (master, rdev->id);
+		if (master->on_close) master->on_close(master, rdev->id);
 
 		HIO_ASSERT(hio, master->slave_count > 0);
 		master->slave_count--;
@@ -355,7 +355,7 @@ static int dev_thr_kill_slave (hio_dev_t* dev, int force)
 			if (master->slave_count <= 0)
 			{
 				/* if this is the last slave, kill the master also */
-				hio_dev_kill ((hio_dev_t*)master);
+				hio_dev_kill((hio_dev_t*)master);
 				/* the master pointer is not valid from this point onwards
 				 * as the actual master device object is freed in hio_dev_kill() */
 			}
@@ -530,7 +530,7 @@ static int dev_thr_ioctl (hio_dev_t* dev, int cmd, void* arg)
 				/* unlike dev_thr_kill_master(), i don't nullify rdev->slave[sid].
 				 * so i treat the closing ioctl as if it's a kill request
 				 * initiated by the slave device itself. */
-				hio_dev_kill ((hio_dev_t*)rdev->slave[sid]);
+				hio_dev_kill((hio_dev_t*)rdev->slave[sid]);
 
 				/* if this is the last slave, the master is destroyed as well.
 				 * therefore, using rdev is unsafe in the assertion below is unsafe.

lib/tmr.c

@@ -33,7 +33,7 @@
 
 void hio_cleartmrjobs (hio_t* hio)
 {
-	while (hio->tmr.size > 0) hio_deltmrjob (hio, 0);
+	while (hio->tmr.size > 0) hio_deltmrjob(hio, 0);
 }
 
 static hio_tmridx_t sift_up (hio_t* hio, hio_tmridx_t index)
@@ -174,10 +174,10 @@ void hio_firetmrjobs (hio_t* hio, const hio_ntime_t* tm, hio_oow_t* firecnt)
 		if (HIO_CMP_NTIME(&hio->tmr.jobs[0].when, &now) > 0) break;
 
 		tmrjob = hio->tmr.jobs[0]; /* copy the scheduled job */
-		hio_deltmrjob (hio, 0); /* deschedule the job */
+		hio_deltmrjob(hio, 0); /* deschedule the job */
 
 		count++;
-		tmrjob.handler (hio, &now, &tmrjob); /* then fire the job */
+		tmrjob.handler(hio, &now, &tmrjob); /* then fire the job */
 	}
 
 	if (firecnt) *firecnt = count;

t/Makefile.in

@@ -588,6 +588,8 @@ PTHREAD_LIBS = @PTHREAD_LIBS@
 QUADMATH_LIBS = @QUADMATH_LIBS@
 RANLIB = @RANLIB@
 SCTP_ENABLED = @SCTP_ENABLED@
+SCTP_MULTIHOMING_ENABLED = @SCTP_MULTIHOMING_ENABLED@
+SCTP_PEELOFF_ENABLED = @SCTP_PEELOFF_ENABLED@
 SED = @SED@
 SENDFILE_LIBS = @SENDFILE_LIBS@
 SET_MAKE = @SET_MAKE@

t/httssvr.c

@@ -654,8 +654,8 @@ int main (int argc, char* argv[])
 		else if (strcmp(argv[i], "-ss") == 0)
 		{
 			i++;
-			g_dev_type4 = HIO_DEV_SCK_SCTP4_SP;
+			g_dev_type4 = HIO_DEV_SCK_SCTP4_SEQPKT;
-			g_dev_type6 = HIO_DEV_SCK_SCTP6_SP;
+			g_dev_type6 = HIO_DEV_SCK_SCTP6_SEQPKT;
 		}
 		else if (strcmp(argv[i], "-t") == 0)
 		{