fixed minor error condition check before setting O_NONBLOCK

introduced ioe for stderr over rxc

Makefile

@@ -13,9 +13,13 @@ SRCS=\
 	bulletin.go \
 	client.go \
 	client-ctl.go \
+	client-cts-rpty.go \
+	client-cts-rpx.go \
+	client-cts-rxc.go \
 	client-metrics.go \
 	client-peer.go \
 	client-pty.go \
+	client-rxc-profile.go \
 	hodu.go \
 	hodu.pb.go \
 	hodu_grpc.pb.go \
@@ -24,11 +28,16 @@ SRCS=\
 	pty.go \
 	server.go \
 	server-ctl.go \
+	server-cts-rpty.go \
+	server-cts-rpx.go \
+	server-cts-rxc.go \
 	server-metrics.go \
 	server-peer.go \
 	server-pty.go \
 	server-pxy.go \
 	server-rpx.go \
+	server-rxc.go \
+	server-rxc-job.go \
 	system.go \
 	transform.go \
 
@@ -63,7 +72,7 @@ clean:
 	rm -f $(NAME) $(NAME).debug
 
 check:
-	go test -x
+	go test -x --count=1 ./...
 
 hodu.pb.go: hodu.proto
 	protoc --go_out=. --go_opt=paths=source_relative \

client-ctl.go

@@ -115,6 +115,7 @@ type json_out_client_stats struct {
 	ClientPtySessions int64 `json:"client-pty-sessions"`
 	ClientRptySessions int64 `json:"client-rpty-sessions"`
 	ClientRpxSessions int64 `json:"client-rpx-sessions"`
+	ClientRxcSessions int64 `json:"client-rxc-sessions"`
 }
 // ------------------------------------
 
@@ -1148,6 +1149,7 @@ func (ctl *client_ctl_stats) ServeHTTP(w http.ResponseWriter, req *http.Request)
 			stats.ClientPtySessions = c.stats.pty_sessions.Load()
 			stats.ClientRptySessions = c.stats.rpty_sessions.Load()
 			stats.ClientRpxSessions = c.stats.rpx_sessions.Load()
+			stats.ClientRxcSessions = c.stats.rxc_sessions.Load()
 			status_code = WriteJsonRespHeader(w, http.StatusOK)
 			if err = je.Encode(stats); err != nil { goto oops }
 

client-cts-rpty.go

@@ -0,0 +1,226 @@
+package hodu
+
+import "errors"
+import "fmt"
+import "io"
+import "strconv"
+import "strings"
+import "sync"
+
+import pts "github.com/creack/pty"
+import "golang.org/x/sys/unix"
+
+// rpty
+func (cts *ClientConn) FindClientRptyById(id uint64) *ClientRpty {
+	var crp *ClientRpty
+	var ok bool
+
+	cts.rpty_mtx.Lock()
+	crp, ok = cts.rpty_map[id]
+	cts.rpty_mtx.Unlock()
+
+	if !ok { crp = nil }
+	return crp
+}
+
+func (cts *ClientConn) RptyLoop(crp *ClientRpty, wg *sync.WaitGroup) {
+
+	var poll_fds []unix.PollFd
+	var buf [2048]byte
+	var n int
+	var out_revents int16
+	var sig_revents int16
+	var err error
+
+	defer wg.Done()
+
+	cts.C.log.Write(cts.Sid, LOG_INFO, "Started rpty(%d) for %s(%s)", crp.id, cts.C.pty_shell, cts.C.pty_user)
+
+	cts.C.stats.rpty_sessions.Add(1)
+
+	poll_fds = []unix.PollFd{
+		unix.PollFd{Fd: int32(crp.tty.Fd()), Events: unix.POLLIN},
+		unix.PollFd{Fd: int32(crp.pfd[0]), Events: unix.POLLIN},
+	}
+
+	for {
+		n, err = unix.Poll(poll_fds, -1) // -1 means wait indefinitely
+		if err != nil {
+			if errors.Is(err, unix.EINTR) { continue }
+			cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to poll rpty(%d) stdout - %s", crp.id, err.Error())
+			break
+		}
+		if n == 0 { // timed out
+			continue
+		}
+
+		out_revents = poll_fds[0].Revents
+		sig_revents = poll_fds[1].Revents
+
+		if (out_revents & unix.POLLIN) != 0 {
+			n, err = crp.tty.Read(buf[:])
+			if n > 0 {
+				var err2 error
+				err2 = cts.psc.Send(MakeRptyDataPacket(crp.id, buf[:n]))
+				if err2 != nil {
+					cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to send %s from rpty(%d) stdout to server - %s", PACKET_KIND_RPTY_DATA.String(), crp.id, err2.Error())
+					break
+				}
+			}
+			if err != nil {
+				if !errors.Is(err, io.EOF) {
+					cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to read rpty(%d) stdout - %s", crp.id, err.Error())
+				}
+				break
+			}
+		}
+		if (sig_revents & unix.POLLIN) != 0 {
+			// don't care to read the pipe as it is closed after the loop
+			//unix.Read(crp.pfd[0], )
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "Stop request noticed on rpty(%d) signal pipe", crp.id)
+			break
+		}
+		if (out_revents & (unix.POLLERR | unix.POLLNVAL)) != 0 {
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "Error detected on rpty(%d) stdout", crp.id)
+			break
+		}
+		if (sig_revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL)) != 0 {
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "EOF detected on rpty(%d) signal pipe", crp.id)
+			break
+		}
+		if (out_revents & unix.POLLHUP) != 0 && (out_revents & unix.POLLIN) == 0 {
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "EOF detected on rpty(%d) stdout", crp.id)
+			break
+		}
+	}
+
+	cts.C.log.Write(cts.Sid, LOG_DEBUG, "Ending rpty(%d) loop", crp.id)
+	err = cts.psc.Send(MakeRptyStopPacket(crp.id, ""))
+	if err != nil {
+		cts.C.log.Write(cts.Sid, LOG_WARN, "Failed to send %s from rpty(%d) to server - %s", PACKET_KIND_RPTY_STOP.String(), crp.id, err.Error())
+	}
+
+	crp.ReqStop()
+	crp.cmd.Wait()
+	crp.tty.Close()
+	unix.Close(crp.pfd[0])
+	unix.Close(crp.pfd[1])
+
+	cts.rpty_mtx.Lock()
+	delete(cts.rpty_map, crp.id)
+	cts.rpty_mtx.Unlock()
+
+	cts.C.stats.rpty_sessions.Add(-1)
+	cts.C.log.Write(cts.Sid, LOG_DEBUG, "Ended rpty(%d) loop", crp.id)
+}
+
+func (cts *ClientConn) StartRpty(id uint64, wg *sync.WaitGroup) error {
+	var crp *ClientRpty
+	var ok bool
+	var i int
+	var err error
+
+	cts.rpty_mtx.Lock()
+	_, ok = cts.rpty_map[id]
+	if ok {
+		cts.rpty_mtx.Unlock()
+		return fmt.Errorf("multiple start on rpty id %d", id)
+	}
+
+	crp = &ClientRpty{ cts: cts, id: id }
+	err = unix.Pipe(crp.pfd[:])
+	if err != nil {
+		cts.rpty_mtx.Unlock()
+		cts.psc.Send(MakeRptyStopPacket(id, err.Error()))
+		return fmt.Errorf("unable to create rpty(%d) event fd for %s(%s) - %s", id, cts.C.pty_shell, cts.C.pty_user, err.Error())
+	}
+	crp.cmd, crp.tty, err = connect_pty(cts.C.pty_shell, cts.C.pty_user)
+	if err != nil {
+		cts.rpty_mtx.Unlock()
+		cts.psc.Send(MakeRptyStopPacket(id, err.Error()))
+		unix.Close(crp.pfd[0])
+		unix.Close(crp.pfd[1])
+		return fmt.Errorf("unable to start rpty(%d) for %s(%s) - %s", id, cts.C.pty_shell, cts.C.pty_user, err.Error())
+	}
+
+	for i = 0; i < 2; i++ {
+		var flags int
+		flags, err = unix.FcntlInt(uintptr(crp.pfd[i]), unix.F_GETFL, 0)
+		if err == nil {
+			unix.FcntlInt(uintptr(crp.pfd[i]), unix.F_SETFL, flags | unix.O_NONBLOCK)
+		}
+	}
+
+	cts.rpty_map[id] = crp
+
+	wg.Add(1)
+	go cts.RptyLoop(crp, wg)
+
+	cts.rpty_mtx.Unlock()
+
+	return nil
+}
+
+func (cts *ClientConn) StopRpty(id uint64) error {
+	var crp *ClientRpty
+
+	crp = cts.FindClientRptyById(id)
+	if crp == nil {
+		return fmt.Errorf("unknown rpty id %d", id)
+	}
+
+	crp.ReqStop()
+	return nil
+}
+
+func (cts *ClientConn) WriteRpty(id uint64, data []byte) error {
+	var crp *ClientRpty
+
+	crp = cts.FindClientRptyById(id)
+	if crp == nil {
+		return fmt.Errorf("unknown rpty id %d", id)
+	}
+
+	crp.tty.Write(data)
+	return nil
+}
+
+func (cts *ClientConn) WriteRptySize(id uint64, data []byte) error {
+	var crp *ClientRpty
+	var flds []string
+
+	crp = cts.FindClientRptyById(id)
+	if crp == nil {
+		return fmt.Errorf("unknown rpty id %d", id)
+	}
+
+	flds = strings.Split(string(data), " ")
+	if len(flds) == 2 {
+		var rows int
+		var cols int
+		rows, _ = strconv.Atoi(flds[0])
+		cols, _ = strconv.Atoi(flds[1])
+		pts.Setsize(crp.tty, &pts.Winsize{Rows: uint16(rows), Cols: uint16(cols)})
+	}
+	return nil
+}
+
+func (cts *ClientConn) HandleRptyEvent(packet_type PACKET_KIND, evt *RptyEvent) error {
+
+	switch packet_type {
+		case PACKET_KIND_RPTY_START:
+			return cts.StartRpty(evt.Id, &cts.C.wg)
+
+		case PACKET_KIND_RPTY_STOP:
+			return cts.StopRpty(evt.Id)
+
+		case PACKET_KIND_RPTY_DATA:
+			return cts.WriteRpty(evt.Id, evt.Data)
+
+		case PACKET_KIND_RPTY_SIZE:
+			return cts.WriteRptySize(evt.Id, evt.Data)
+	}
+
+	// ignore other packet types
+	return nil
+}

client-cts-rpx.go

@@ -0,0 +1,399 @@
+package hodu
+
+import "bufio"
+import "bytes"
+import "context"
+import "crypto/tls"
+import "errors"
+import "fmt"
+import "io"
+import "net"
+import "net/http"  
+import "sync"
+import "strings"
+import "time"
+
+// rpx
+func (cts *ClientConn) FindClientRpxById(id uint64) *ClientRpx {
+	var crpx *ClientRpx
+	var ok bool
+
+	cts.rpx_mtx.Lock()
+	crpx, ok = cts.rpx_map[id]
+	cts.rpx_mtx.Unlock()
+
+	if !ok { crpx = nil }
+	return crpx
+}
+
+func (cts *ClientConn) server_pipe_to_ws_target(crpx* ClientRpx, conn net.Conn, wg *sync.WaitGroup) {
+	var buf [4096]byte
+	var n int
+	var err error
+
+	defer wg.Done()
+
+	for {
+		n, err = crpx.pr.Read(buf[:])
+		if n > 0 {
+			var err2 error
+			_, err2 = conn.Write(buf[:n])
+			if err2 != nil {
+				cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to write websocket for rpx(%d) - %s", crpx.id, err2.Error())
+				break
+			}
+		}
+		if err != nil {
+			if errors.Is(err, io.EOF) { break }
+			cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to read pipe for rpx(%d) - %s", crpx.id, err.Error())
+			break
+		}
+	}
+}
+
+func (cts *ClientConn) proxy_ws(crpx *ClientRpx, raw_req []byte, req *http.Request) (int, error) {
+	var l_wg sync.WaitGroup
+	var conn net.Conn
+	var resp *http.Response
+	var r *bufio.Reader
+	var buf [4096]byte
+	var n int
+	var err error
+
+	if cts.C.rpx_target_tls != nil {
+		var dialer *tls.Dialer
+		dialer = &tls.Dialer{
+			NetDialer: &net.Dialer{},
+			Config: cts.C.rpx_target_tls,
+		}
+		conn, err = dialer.DialContext(crpx.ctx, "tcp", cts.C.rpx_target_addr) // TODO: no hard coding
+	} else {
+		var dialer *net.Dialer
+		dialer = &net.Dialer{}
+		conn, err = dialer.DialContext(crpx.ctx, "tcp", cts.C.rpx_target_addr) // TODO: no hard coding
+	}
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Errorf("failed to dial websocket for rpx(%d) - %s", crpx.id, err.Error())
+	}
+	defer conn.Close()
+
+	// TODO: make this atomic?
+	crpx.ws_conn = conn
+
+	// write the raw request line and headers as sent by the server.
+	// for the upgrade request, i assume no payload.
+	_, err = conn.Write(raw_req)
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Errorf("failed to write websocket request for rpx(%d) - %s", crpx.id, err.Error())
+	}
+
+	r = bufio.NewReader(conn)
+	resp, err = http.ReadResponse(r, req)
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Errorf("failed to write websocket response for rpx(%d) - %s", crpx.id, err.Error())
+	}
+	defer resp.Body.Close()
+
+	err = cts.psc.Send(MakeRpxStartPacket(crpx.id, get_http_resp_line_and_headers(resp)))
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Errorf("failed to send rpx(%d) WebSocket headers to server - %s", crpx.id, err.Error())
+	}
+
+	if resp.StatusCode != http.StatusSwitchingProtocols {
+		// websock upgrade failed. let the code jump to the done
+		// label to skip reading from the pipe. the server side
+		// has the code to ensure no content-length. and the upgrade
+		// fails, the pipe below will be pending forever as the server
+		// side doesn't send data and there's no feeding to the pipe.
+		return resp.StatusCode, fmt.Errorf("protocol switching failed for rpx(%d)", crpx.id)
+	}
+
+	// unlike with the normal request, the actual pipe is not read
+	// until the initial switching protocol response is received.
+
+	l_wg.Add(1)
+	go cts.server_pipe_to_ws_target(crpx, conn, &l_wg)
+
+	for {
+		n, err = conn.Read(buf[:])
+		if n > 0 {
+			var err2 error
+			err2 = cts.psc.Send(MakeRpxDataPacket(crpx.id, buf[:n]))
+			if err2 != nil {
+				crpx.ReqStop() // to break server_pipe_ws_target. don't care about multiple stops
+				return resp.StatusCode, fmt.Errorf("failed to send rpx(%d) data to server - %s", crpx.id, err2.Error())
+			}
+		}
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				cts.psc.Send(MakeRpxEofPacket(crpx.id))
+				cts.C.log.Write(cts.Sid, LOG_DEBUG, "WebSocket rpx(%d) closed by server", crpx.id)
+				break
+			}
+
+			crpx.ReqStop() // to break server_pipe_ws_target. don't care about multiple stops
+			return resp.StatusCode, fmt.Errorf("failed to read WebSocket rpx(%d) - %s", crpx.id, err.Error())
+		}
+	}
+
+	// wait until the pipe reading(from the server side) goroutine is over
+	l_wg.Wait()
+	return resp.StatusCode, nil
+}
+
+func (cts *ClientConn) proxy_http(crpx *ClientRpx, req *http.Request) (int, error) {
+	var tr *http.Transport
+	var resp *http.Response
+	var buf [4096]byte
+	var n int
+	var err error
+
+	tr = &http.Transport {
+		DisableKeepAlives: true, // this implementation can't support keepalive..
+	}
+	if cts.C.rpx_target_tls != nil {
+		tr.TLSClientConfig = cts.C.rpx_target_tls
+	}
+
+	resp, err = tr.RoundTrip(req)
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Errorf("failed to send rpx(%d) request - %s", crpx.id, err.Error())
+	}
+	defer resp.Body.Close()
+
+	err = cts.psc.Send(MakeRpxStartPacket(crpx.id, get_http_resp_line_and_headers(resp)))
+	if err != nil {
+		return resp.StatusCode, fmt.Errorf("failed to send rpx(%d) status and headers to server - %s", crpx.id, err.Error())
+	}
+
+	for {
+		n, err = resp.Body.Read(buf[:])
+		if n > 0 {
+			var err2 error
+			err2 = cts.psc.Send(MakeRpxDataPacket(crpx.id, buf[:n]))
+			if err2 != nil {
+				return resp.StatusCode, fmt.Errorf("failed to send rpx(%d) data to server - %s", crpx.id, err2.Error())
+			}
+		}
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return resp.StatusCode, fmt.Errorf("failed to read response body for rpx(%d) - %s", crpx.id, err.Error())
+		}
+	}
+
+	return resp.StatusCode, nil
+}
+
+func (cts *ClientConn) RpxLoop(crpx *ClientRpx, data []byte, wg *sync.WaitGroup) {
+	var start_time time.Time
+	var time_taken time.Duration
+	var r *bufio.Reader
+	var line string
+	var flds []string
+	var req_meth string
+	var req_path string
+	//var req_proto string
+	var x_forwarded_host string
+	var raw_req bytes.Buffer
+	var status_code int
+	var req *http.Request
+	var err error
+
+	defer wg.Done()
+
+	cts.C.log.Write(cts.Sid, LOG_INFO, "Starting rpx(%d) loop", crpx.id)
+
+	start_time = time.Now()
+
+	const rpx_header_line_max = 65535 // TODO: make this configurable
+
+	r = bufio.NewReader(bytes.NewReader(data))
+	line, err = read_line_limited(r, rpx_header_line_max)
+	if err != nil && !errors.Is(err, io.EOF) {
+		cts.C.log.Write(cts.Sid, LOG_ERROR, "failed to parse request for rpx(%d) - %s", crpx.id, err.Error())
+		goto done
+	}
+	line = strings.TrimRight(line, "\r\n")
+
+	flds = strings.Fields(line)
+	if len(flds) < 3 {
+		cts.C.log.Write(cts.Sid, LOG_ERROR, "Invalid request line for rpx(%d) - %s", crpx.id, line)
+		goto done
+	}
+
+// TODO: handle trailers...
+	req_meth = flds[0]
+	req_path = flds[1]
+	//req_proto = flds[2]
+
+	raw_req.WriteString(line)
+	raw_req.WriteString("\r\n")
+
+	// create a request assuming it's a normal http request
+	req, err = http.NewRequestWithContext(crpx.ctx, req_meth, cts.C.rpx_target_url + req_path, crpx.pr)
+	if err != nil {
+		cts.C.log.Write(cts.Sid, LOG_ERROR, "failed to create request for rpx(%d) - %s", crpx.id, err.Error())
+		goto done
+	}
+
+	for {
+		line, err = read_line_limited(r, rpx_header_line_max)
+		if err != nil && !errors.Is(err, io.EOF) {
+			cts.C.log.Write(cts.Sid, LOG_ERROR, "failed to parse request for rpx(%d) - %s", crpx.id, err.Error())
+			goto done
+		}
+		line = strings.TrimRight(line, "\r\n")
+		if line == "" { break }
+		flds = strings.SplitN(line, ":", 2)
+		if len(flds) == 2 {
+			var k string
+			var v string
+			k = strings.TrimSpace(flds[0])
+			v = strings.TrimSpace(flds[1])
+			req.Header.Add(k, v)
+
+			if strings.EqualFold(k, "Host") {
+				// a normal http client would set HOst to be the target address.
+				// the raw header is coming from the server. so it's different
+				// from the host it's supposed to be. correct it to the right value.
+				fmt.Fprintf(&raw_req, "%s: %s\r\n", k, req.Host)
+			} else {
+				raw_req.WriteString(line)
+				raw_req.WriteString("\r\n")
+
+				if strings.EqualFold(k, "X-Forwarded-Host") {
+					x_forwarded_host = v
+				}
+			}
+		}
+		if errors.Is(err, io.EOF) { break }
+	}
+	raw_req.WriteString("\r\n")
+	if x_forwarded_host == "" {
+		x_forwarded_host = req.Host
+	}
+
+	if strings.EqualFold(req.Header.Get("Upgrade"), "websocket") && strings.Contains(strings.ToLower(req.Header.Get("Connection")), "upgrade") {
+		// websocket
+		status_code, err = cts.proxy_ws(crpx, raw_req.Bytes(), req)
+	} else {
+		// normal http
+		status_code, err = cts.proxy_http(crpx, req)
+	}
+
+	time_taken = time.Since(start_time)
+	if err != nil {
+		cts.C.log.Write(cts.Sid, LOG_ERROR, "rpx(%d) %s - %s %s %d %.9f - failed to proxy - %s", crpx.id, x_forwarded_host, req_meth, req_path, status_code, time_taken.Seconds(), err.Error())
+		goto done
+	} else {
+		cts.C.log.Write(cts.Sid, LOG_INFO, "rpx(%d) %s - %s %s %d %.9f", crpx.id, x_forwarded_host, req_meth, req_path, status_code, time_taken.Seconds())
+	}
+
+done:
+	err = cts.psc.Send(MakeRpxStopPacket(crpx.id))
+	if err != nil {
+		cts.C.log.Write(cts.Sid, LOG_ERROR, "rpx(%d) Failed to send %s to server - %s", crpx.id, PACKET_KIND_RPX_STOP.String(), err.Error())
+	}
+	cts.C.log.Write(cts.Sid, LOG_INFO, "Ending rpx(%d) loop", crpx.id)
+
+	crpx.ReqStop()
+
+	cts.rpx_mtx.Lock()
+	delete(cts.rpx_map, crpx.id)
+	cts.rpx_mtx.Unlock()
+	cts.C.stats.rpx_sessions.Add(-1)
+	cts.C.log.Write(cts.Sid, LOG_INFO, "Ended rpx(%d) loop", crpx.id)
+}
+
+func (cts *ClientConn) StartRpx(id uint64, data []byte, wg *sync.WaitGroup) error {
+	var crpx *ClientRpx
+	var ok bool
+
+	cts.rpx_mtx.Lock()
+	_, ok = cts.rpx_map[id]
+	if ok {
+		cts.rpx_mtx.Unlock()
+		return fmt.Errorf("multiple start on rpx id %d", id)
+	}
+	crpx = &ClientRpx{ id: id }
+	cts.rpx_map[id] = crpx
+
+	// i want the pipe to be created before the goroutine is started
+	// so that the WriteRpx() can write to the pipe. i protect pipe creation
+	// and context creation with a mutex
+	crpx.pr, crpx.pw = io.Pipe()
+	crpx.ctx, crpx.cancel = context.WithCancel(cts.C.Ctx)
+
+	cts.rpx_mtx.Unlock()
+	cts.C.stats.rpx_sessions.Add(1)
+
+	wg.Add(1)
+	go cts.RpxLoop(crpx, data, wg)
+
+	return nil
+}
+
+func (cts *ClientConn) StopRpx(id uint64) error {
+	var crpx *ClientRpx
+
+	crpx = cts.FindClientRpxById(id)
+	if crpx == nil {
+		return fmt.Errorf("unknown rpx id %d", id)
+	}
+
+	crpx.ReqStop()
+	return nil
+}
+
+func (cts *ClientConn) WriteRpx(id uint64, data []byte) error {
+	var crpx *ClientRpx
+	var err error
+
+	crpx = cts.FindClientRpxById(id)
+	if crpx == nil {
+		return fmt.Errorf("unknown rpx id %d", id)
+	}
+
+// TODO: may have to write it in a goroutine to avoid blocking?
+	_, err = crpx.pw.Write(data)
+	if err != nil {
+		cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to write rpx(%d) data - %s", id, err.Error())
+		return err
+	}
+	return nil
+}
+
+func (cts *ClientConn) EofRpx(id uint64, data []byte) error {
+	var crpx *ClientRpx
+
+	crpx = cts.FindClientRpxById(id)
+	if crpx == nil {
+		return fmt.Errorf("unknown rpx id %d", id)
+	}
+
+	// close the writing end only. leave the reading end untouched
+	crpx.pw.Close()
+
+	return nil
+}
+
+func (cts *ClientConn) HandleRpxEvent(packet_type PACKET_KIND, evt *RpxEvent) error {
+	switch packet_type {
+		case PACKET_KIND_RPX_START:
+			return cts.StartRpx(evt.Id, evt.Data, &cts.C.wg)
+
+		case PACKET_KIND_RPX_STOP:
+			return cts.StopRpx(evt.Id)
+
+		case PACKET_KIND_RPX_DATA:
+			return cts.WriteRpx(evt.Id, evt.Data)
+
+		case PACKET_KIND_RPX_EOF:
+			return cts.EofRpx(evt.Id, evt.Data)
+	}
+
+	// ignore other packet types
+	return nil
+}

client-cts-rxc.go

@@ -0,0 +1,688 @@
+package hodu
+
+import "bytes"
+import "encoding/gob"
+import "errors"
+import "fmt"
+import "io"
+import "os"
+import "os/exec"
+import "os/user"
+import "strconv"
+import "sync"
+import "syscall"
+import "unicode"
+import "unicode/utf8"
+
+import "golang.org/x/sys/unix"
+
+func (cts *ClientConn) FindClientRxcById(id uint64) *ClientRxc {
+	var crp *ClientRxc
+	var ok bool
+
+	cts.rxc_mtx.Lock()
+	crp, ok = cts.rxc_map[id]
+	cts.rxc_mtx.Unlock()
+
+	if !ok { crp = nil }
+	return crp
+}
+
+func (cts *ClientConn) RxcLoop(crp *ClientRxc, wg *sync.WaitGroup) {
+
+	var poll_fds []unix.PollFd
+	var buf [2048]byte
+	var n int
+	var out_revents int16
+	var err_revents int16
+	var sig_revents int16
+	var stop_flags uint64
+	var stop_msg string
+	var stdout_open bool
+	var stderr_open bool
+	var err error
+
+	defer wg.Done()
+
+	cts.C.log.Write(cts.Sid, LOG_DEBUG, "Started rxc(%d) loop for %s(%s) - %s", crp.id, crp.req_type, crp.req_script, crp.cmd.String())
+
+	cts.C.stats.rxc_sessions.Add(1)
+
+	stdout_open = true
+	stderr_open = true
+	poll_fds = []unix.PollFd{
+		unix.PollFd{Fd: int32(crp.stdout.Fd()), Events: unix.POLLIN},
+		unix.PollFd{Fd: int32(crp.stderr.Fd()), Events: unix.POLLIN},
+		unix.PollFd{Fd: int32(crp.pfd[0]), Events: unix.POLLIN},
+	}
+
+	for {
+		n, err = unix.Poll(poll_fds, -1) // -1 means wait indefinitely
+		if err != nil {
+			if errors.Is(err, unix.EINTR) { continue }
+			cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to poll rxc(%d) stdout - %s", crp.id, err.Error())
+			break
+		}
+		if n == 0 { continue } // timed out
+
+		out_revents = poll_fds[0].Revents
+		err_revents = poll_fds[1].Revents
+		sig_revents = poll_fds[2].Revents
+
+		if stdout_open && (out_revents & unix.POLLIN) != 0 {
+			n, err = crp.stdout.Read(buf[:])
+			if n > 0 {
+				var err2 error
+				err2 = cts.psc.Send(MakeRxcDataPacket(crp.id, RXC_DATA_FLAG_NONE, buf[:n]))
+				if err2 != nil {
+					cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to send %s from rxc(%d) stdout to server - %s", PACKET_KIND_RXC_DATA.String(), crp.id, err2.Error())
+					break
+				}
+			}
+			if err != nil {
+				if !errors.Is(err, io.EOF) {
+					cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to read rxc(%d) stdout - %s", crp.id, err.Error())
+					break
+				}
+				poll_fds[0].Fd = -1
+				poll_fds[0].Events = 0
+				stdout_open = false
+			}
+		}
+		if stderr_open && (err_revents & unix.POLLIN) != 0 {
+			n, err = crp.stderr.Read(buf[:])
+			if n > 0 {
+				var err2 error
+				err2 = cts.psc.Send(MakeRxcDataPacket(crp.id, RXC_DATA_FLAG_STDERR, buf[:n]))
+				if err2 != nil {
+					cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to send %s from rxc(%d) stderr to server - %s", PACKET_KIND_RXC_DATA.String(), crp.id, err2.Error())
+					break
+				}
+			}
+			if err != nil {
+				if !errors.Is(err, io.EOF) {
+					cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to read rxc(%d) stderr - %s", crp.id, err.Error())
+					break
+				}
+				poll_fds[1].Fd = -1
+				poll_fds[1].Events = 0
+				stderr_open = false
+			}
+		}
+		if (sig_revents & unix.POLLIN) != 0 {
+			// don't care to read the pipe as it is closed after the loop
+			//unix.Read(crp.pfd[0], )
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "Stop request noticed on rxc(%d) signal pipe", crp.id)
+			break
+		}
+		if stdout_open && (out_revents & (unix.POLLERR | unix.POLLNVAL)) != 0 {
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "Error detected on rxc(%d) stdout", crp.id)
+			break
+		}
+		if stderr_open && (err_revents & (unix.POLLERR | unix.POLLNVAL)) != 0 {
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "Error detected on rxc(%d) stderr", crp.id)
+			break
+		}
+		if sig_revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL) != 0 {
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "EOF detected on rxc(%d) signal pipe", crp.id)
+			break
+		}
+		if stdout_open && (out_revents & unix.POLLHUP) != 0 && (out_revents & unix.POLLIN) == 0 {
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "EOF detected on rxc(%d) stdout", crp.id)
+			poll_fds[0].Fd = -1
+			poll_fds[0].Events = 0
+			stdout_open = false
+		}
+		if stderr_open && (err_revents & unix.POLLHUP) != 0 && (err_revents & unix.POLLIN) == 0 {
+			cts.C.log.Write(cts.Sid, LOG_DEBUG, "EOF detected on rxc(%d) stderr", crp.id)
+			poll_fds[1].Fd = -1
+			poll_fds[1].Events = 0
+			stderr_open = false
+		}
+
+		if !stdout_open && !stderr_open { break }
+	}
+
+	cts.C.log.Write(cts.Sid, LOG_DEBUG, "Ending rxc(%d) loop", crp.id)
+	crp.ReqStop()
+	crp.stdin.Close() // close the input before waiting for program termination
+	err = crp.cmd.Wait()
+	if err != nil && crp.cmd.ProcessState == nil {
+		stop_msg = err.Error()
+	}
+	stop_flags = MakeRxcStopFlagsFromProcessState(crp.cmd.ProcessState)
+	err = cts.psc.Send(MakeRxcStopPacket(crp.id, stop_flags, stop_msg))
+	if err != nil {
+		cts.C.log.Write(cts.Sid, LOG_WARN, "Failed to send %s from rxc(%d) to server - %s", PACKET_KIND_RXC_STOP.String(), crp.id, err.Error())
+	}
+
+	crp.stderr.Close()
+	crp.stdout.Close()
+	unix.Close(crp.pfd[0])
+	unix.Close(crp.pfd[1])
+
+	cts.rxc_mtx.Lock()
+	delete(cts.rxc_map, crp.id)
+	cts.rxc_mtx.Unlock()
+
+	cts.C.stats.rxc_sessions.Add(-1)
+	cts.C.log.Write(cts.Sid, LOG_DEBUG, "Ended rxc(%d) loop", crp.id)
+}
+
+func simple_command_escape_digit_value(ch rune, base int) int {
+	switch base {
+		case 8:
+			if ch >= '0' && ch <= '7' {
+				return int(ch - '0')
+			}
+
+		case 16:
+			switch {
+				case ch >= '0' && ch <= '9':
+					return int(ch - '0')
+				case ch >= 'a' && ch <= 'f':
+					return int(ch - 'a') + 10
+				case ch >= 'A' && ch <= 'F':
+					return int(ch - 'A') + 10
+			}
+	}
+
+	return -1
+}
+
+func parse_simple_command_escape_value(runes []rune, start int, base int, digit_count int, tag string) (rune, int, error) {
+	var ch rune
+	var i int
+	var value rune
+	var digit int
+
+	if start + digit_count > len(runes) {
+		return 0, 0, fmt.Errorf("truncated %s escape in simple command", tag)
+	}
+
+	value = 0
+	for i = 0; i < digit_count; i++ {
+		ch = runes[start + i]
+		digit = simple_command_escape_digit_value(ch, base)
+		if digit < 0 {
+			return 0, 0, fmt.Errorf("invalid %s escape in simple command", tag)
+		}
+		value = value * rune(base) + rune(digit)
+	}
+
+	if !utf8.ValidRune(value) {
+		return 0, 0, fmt.Errorf("invalid %s escape value in simple command", tag)
+	}
+
+	return value, digit_count, nil
+}
+
+func parse_simple_command_escape(runes []rune, start int, escape_level int) (rune, int, error) {
+	var ch rune
+
+	if start >= len(runes) {
+		return 0, 0, fmt.Errorf("missing escaped character in simple command")
+	}
+
+	ch = runes[start]
+	if escape_level <= 0 {
+		return ch, 0, nil
+	}
+
+	switch ch {
+		case 'a':
+			return '\a', 0, nil
+		case 'b':
+			return '\b', 0, nil
+		case 'f':
+			return '\f', 0, nil
+		case 'n':
+			return '\n', 0, nil
+		case 'r':
+			return '\r', 0, nil
+		case 't':
+			return '\t', 0, nil
+		case 'v':
+			return '\v', 0, nil
+		case '\\':
+			return '\\', 0, nil
+		case '"':
+			return '"', 0, nil
+		case '\'':
+			return '\'', 0, nil
+	}
+
+	if escape_level < 2 {
+		return ch, 0, nil
+	}
+
+	switch ch {
+		case 'x':
+			return parse_simple_command_escape_value(runes, start + 1, 16, 2, "\\x")
+		case 'o':
+			return parse_simple_command_escape_value(runes, start + 1, 8, 2, "\\o")
+		case 'u':
+			return parse_simple_command_escape_value(runes, start + 1, 16, 4, "\\u")
+		case 'U':
+			return parse_simple_command_escape_value(runes, start + 1, 16, 8, "\\U")
+		default:
+			return ch, 0, nil
+	}
+}
+
+func split_simple_command(script string, escape_level int) ([]string, error) {
+	var args []string
+	var runes []rune
+	var buf bytes.Buffer
+	var ch rune
+	var quote rune
+	var decoded rune
+	var i int
+	var skip_until int
+	var consumed int
+	var token_started bool
+	var escaped bool
+	var err error
+
+	args = make([]string, 0)
+	runes = []rune(script)
+	skip_until = -1
+
+	for i, ch = range runes {
+		if i < skip_until {
+			continue
+		}
+
+		if escaped {
+			if quote == '"' && escape_level > 0 {
+				decoded, consumed, err = parse_simple_command_escape(runes, i, escape_level)
+				if err != nil {
+					return nil, err
+				}
+				buf.WriteRune(decoded)
+				skip_until = i + consumed + 1
+			} else {
+				buf.WriteRune(ch)
+				skip_until = i + 1
+			}
+			token_started = true
+			escaped = false
+			continue
+		}
+
+		if ch == '\\' {
+			escaped = true
+			token_started = true
+			continue
+		}
+
+		if quote != 0 {
+			if ch == quote {
+				quote = 0
+			} else {
+				buf.WriteRune(ch)
+				token_started = true
+			}
+			continue
+		}
+
+		switch ch {
+			case '\'', '"':
+				quote = ch
+				token_started = true
+
+			default:
+				if unicode.IsSpace(ch) {
+					if token_started {
+						args = append(args, buf.String())
+						buf.Reset()
+						token_started = false
+					}
+				} else {
+					buf.WriteRune(ch)
+					token_started = true
+				}
+		}
+	}
+
+	if escaped { return nil, fmt.Errorf("dangling escape in simple command") }
+	if quote != 0 { return nil, fmt.Errorf("unterminated quote in simple command") }
+	if token_started { args = append(args, buf.String()) }
+	if len(args) <= 0 { return nil, fmt.Errorf("blank simple command") }
+
+	return args, nil
+}
+
+func expand_rxc_profile_arg(arg_expr string, input_args []string) (string, error) {
+	var buf bytes.Buffer
+	var r rune
+	var last_pos int
+	var pos int
+	var j int
+	var arg_idx_str string
+	var arg_idx int
+	var arg_expr_len int
+	var input_args_len int
+	var err error
+
+	input_args_len = len(input_args)
+	arg_expr_len = len(arg_expr)
+	last_pos = 0
+	for pos, r = range arg_expr { // use the for .. range expression for rune-based traversal
+		if r != '$' { continue }
+
+		buf.WriteString(arg_expr[last_pos:pos])
+
+		if pos + 1 >= arg_expr_len {
+			// nothing after $
+			buf.WriteByte('$')
+			last_pos = pos + 1
+			break
+		}
+
+		if arg_expr[pos + 1] == '$' {
+			// convert $$ to a literal $
+			buf.WriteByte('$')
+			last_pos = pos + 2
+			continue
+		}
+
+		if arg_expr[pos + 1] != '{' {
+			// $ not followed by {
+			buf.WriteByte('$')
+			last_pos = pos + 1
+			continue
+		}
+
+		j = pos + 2
+		for j < arg_expr_len && arg_expr[j] != '}' { j++ }
+		if j >= arg_expr_len {
+			return "", fmt.Errorf("unterminated rxc profile args expression %s", arg_expr)
+		}
+
+		arg_idx_str = arg_expr[pos + 2:j]
+		if arg_idx_str == "@" {
+			return "", fmt.Errorf("invalid use of ${@} in rxc profile args expression %s", arg_expr)
+		}
+
+		arg_idx, err = strconv.Atoi(arg_idx_str)
+		if err != nil || arg_idx <= 0 {
+			return "", fmt.Errorf("invalid rxc profile argument index ${%s}", arg_idx_str)
+		}
+		if arg_idx > input_args_len {
+			//return "", fmt.Errorf("rxc profile argument index ${%d} out of range", arg_idx)
+			// don't return failure. just carry on without writing anything
+			// write nothing to buf.
+		} else {
+			buf.WriteString(input_args[arg_idx - 1])
+		}
+		last_pos = j + 1
+	}
+
+	if last_pos < arg_expr_len { buf.WriteString(arg_expr[last_pos:]) }
+
+	return buf.String(), nil
+}
+
+func expand_rxc_profile_args(arg_exprs []string, input_args []string) ([]string, error) {
+	var args []string
+	var arg_expr string
+
+	args = make([]string, 0)
+
+	for _, arg_expr = range arg_exprs {
+		if arg_expr == "${@}" {
+			// ${@} must be used alone
+			args = append(args, input_args...)
+		} else {
+			var expanded string
+			var err error
+			// expand_rxc_profile_arg rejects ${@} as it doesn't allow
+			// to combine it with other elements (e.g. xx${@}yy is disallowed).
+			expanded, err = expand_rxc_profile_arg(arg_expr, input_args)
+			if err != nil { return nil, err }
+			args = append(args, expanded)
+		}
+	}
+
+	return args, nil
+}
+
+func apply_rxc_user(cmd *exec.Cmd, rxc_user string) error {
+	var uid int
+	var gid int
+	var u *user.User
+	var err error
+
+	u, err = user.Lookup(rxc_user)
+	if err != nil { return err }
+
+	uid, _ = strconv.Atoi(u.Uid)
+	gid, _ = strconv.Atoi(u.Gid)
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Credential: &syscall.Credential{
+			Uid: uint32(uid),
+			Gid: uint32(gid),
+		},
+		Setsid: true,
+	}
+	cmd.Dir = u.HomeDir
+	cmd.Env = append(cmd.Env,
+		"HOME=" + u.HomeDir,
+		"LOGNAME=" + u.Username,
+		"PATH=" + os.Getenv("PATH"),
+		"USER=" + u.Username,
+	)
+
+	return nil
+}
+
+func connect_cmd(c *Client, type_ string, script string) (*exec.Cmd, *os.File, *os.File, *os.File, error) {
+	var cmd *exec.Cmd
+	var stdin io.WriteCloser
+	var stdout io.ReadCloser
+	var stderr io.ReadCloser
+	var stdin_f *os.File
+	var stdout_f *os.File
+	var stderr_f *os.File
+	var argv []string
+	var cmd_argv []string
+	var profile *ClientRxcProfile
+	var effective_user string
+	var ok bool
+	var err error
+
+	effective_user = c.rxc_user
+
+	if type_ == "simple" {
+		argv, err = split_simple_command(script, 0)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+		if argv[0] == "" {
+			return nil, nil, nil, nil, fmt.Errorf("blank simple command")
+		}
+		profile, err = c.ResolveRxcProfile(argv[0])
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+		if profile == nil {
+			return nil, nil, nil, nil, fmt.Errorf("unknown rxc profile %s", argv[0])
+		}
+		if profile.User != "" { effective_user = profile.User }
+		if profile.Args != nil {
+			cmd_argv, err = expand_rxc_profile_args(profile.Args, argv[1:])
+			if err != nil {
+				return nil, nil, nil, nil, err
+			}
+		} else {
+			cmd_argv = argv[1:]
+		}
+		cmd = exec.Command(profile.Script, cmd_argv...)
+	} else {
+		return nil, nil, nil, nil, fmt.Errorf("unsupported type - %s", type_)
+	}
+
+	if effective_user != "" {
+		err = apply_rxc_user(cmd, effective_user)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+	}
+
+	stdout, err = cmd.StdoutPipe()
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("unable to get stdout pipe - %s", err.Error())
+	}
+	stdout_f, ok = stdout.(*os.File)
+	if !ok {
+		stdout.Close()
+		return nil, nil, nil, nil, fmt.Errorf("unsupported stdout pipe")
+	}
+
+	stderr, err = cmd.StderrPipe()
+	if err != nil {
+		stdout.Close()
+		return nil, nil, nil, nil, fmt.Errorf("unable to get stderr pipe - %s", err.Error())
+	}
+	stderr_f, ok = stderr.(*os.File)
+	if !ok {
+		stderr.Close()
+		stdout.Close()
+		return nil, nil, nil, nil, fmt.Errorf("unsupported stderr pipe")
+	}
+
+	stdin, err = cmd.StdinPipe()
+	if err != nil {
+		stderr.Close()
+		stdout.Close()
+		return nil, nil, nil, nil, fmt.Errorf("unable to get stdin pipe - %s", err.Error())
+	}
+	stdin_f, ok = stdin.(*os.File)
+	if !ok {
+		stdin.Close()
+		stderr.Close()
+		stdout.Close()
+		return nil, nil, nil, nil, fmt.Errorf("unsupported stdin pipe")
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		stdin.Close()
+		stderr.Close()
+		stdout.Close()
+		return nil, nil, nil, nil, err
+	}
+
+	return cmd, stdin_f, stdout_f, stderr_f, nil
+}
+
+func (cts *ClientConn) StartRxc(id uint64, data []byte, wg *sync.WaitGroup) error {
+	var crp *ClientRxc
+	var ok bool
+	var i int
+	var dec *gob.Decoder
+	var args []string
+	var err error
+	var err2 error
+
+	dec = gob.NewDecoder(bytes.NewBuffer(data));
+	err = dec.Decode(&args);
+	if err != nil {
+		return fmt.Errorf("unable to decode data for start on rxc(%d) - %s", id, err.Error())
+	}
+	if len(args) != 2 {
+		return fmt.Errorf("invalid data for start on rxc(%d)", id)
+	}
+
+	cts.rxc_mtx.Lock()
+	_, ok = cts.rxc_map[id]
+	if ok {
+		cts.rxc_mtx.Unlock()
+		return fmt.Errorf("multiple start on rxc(%d)", id)
+	}
+
+	crp = &ClientRxc{ cts: cts, id: id, req_type: args[0], req_script: args[1]  }
+	err = unix.Pipe(crp.pfd[:])
+	if err != nil {
+		cts.rxc_mtx.Unlock()
+
+		err2 = cts.psc.Send(MakeRxcStopPacket(id, 0, err.Error()))
+		if err2 != nil {
+			cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to send %s for rxc(%d) start failure to server - %s", PACKET_KIND_RXC_STOP.String(), id, err2.Error())
+		}
+		return fmt.Errorf("unable to create rxc(%d) event fd for %s(%s) - %s", id, args[0], args[1], err.Error())
+	}
+
+	crp.cmd, crp.stdin, crp.stdout, crp.stderr, err = connect_cmd(cts.C, args[0], args[1])
+	if err != nil {
+		cts.rxc_mtx.Unlock()
+
+		err2 = cts.psc.Send(MakeRxcStopPacket(id, 0, err.Error()))
+		if err2 != nil {
+			cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to send %s for rxc(%d) start failure to server - %s", PACKET_KIND_RXC_STOP.String(), id, err2.Error())
+		}
+		unix.Close(crp.pfd[0])
+		unix.Close(crp.pfd[1])
+		return fmt.Errorf("unable to start rxc(%d) for %s(%s) - %s", id, args[0], args[1], err.Error())
+	}
+
+	for i = 0; i < 2; i++ {
+		var flags int
+		flags, err = unix.FcntlInt(uintptr(crp.pfd[i]), unix.F_GETFL, 0)
+		if err == nil {
+			unix.FcntlInt(uintptr(crp.pfd[i]), unix.F_SETFL, flags | unix.O_NONBLOCK)
+		}
+	}
+
+	cts.rxc_map[id] = crp
+
+	wg.Add(1)
+	go cts.RxcLoop(crp, wg)
+
+	cts.rxc_mtx.Unlock()
+
+	return nil
+}
+
+func (cts *ClientConn) StopRxc(id uint64) error {
+	var crp *ClientRxc
+
+	crp = cts.FindClientRxcById(id)
+	if crp == nil {
+		return fmt.Errorf("unknown rxc id %d", id)
+	}
+
+	crp.ReqStop()
+	return nil
+}
+
+func (cts *ClientConn) WriteRxc(id uint64, data []byte) error {
+	var crp *ClientRxc
+
+	crp = cts.FindClientRxcById(id)
+	if crp == nil {
+		return fmt.Errorf("unknown rxc id %d", id)
+	}
+
+	// TODO:  what if stdin can't be written fast enough and gets blocked?
+	crp.stdin.Write(data)
+	return nil
+}
+
+func (cts *ClientConn) HandleRxcEvent(packet_type PACKET_KIND, evt *RxcEvent) error {
+
+	switch packet_type {
+		case PACKET_KIND_RXC_START:
+			return cts.StartRxc(evt.Id, evt.Data, &cts.C.wg)
+
+		case PACKET_KIND_RXC_STOP:
+			return cts.StopRxc(evt.Id)
+
+		case PACKET_KIND_RXC_DATA:
+			return cts.WriteRxc(evt.Id, evt.Data)
+	}
+
+	// ignore other packet types
+	return nil
+}

client-metrics.go

@@ -13,6 +13,7 @@ type ClientCollector struct {
 	PtySessions      *prometheus.Desc
 	RptySessions     *prometheus.Desc
 	RpxSessions      *prometheus.Desc
+	RxcSessions      *prometheus.Desc
 }
 
 // NewClientCollector returns a new ClientCollector with all prometheus.Desc initialized
@@ -64,6 +65,11 @@ func NewClientCollector(client *Client) ClientCollector {
 			"Number of rpx sessions",
 			nil, nil,
 		),
+		RxcSessions: prometheus.NewDesc(
+			prefix + "rxc_sessions",
+			"Number of rxc sessions",
+			nil, nil,
+		),
 	}
 }
 
@@ -123,4 +129,10 @@ func (c ClientCollector) Collect(ch chan<- prometheus.Metric) {
 		prometheus.GaugeValue,
 		float64(c.client.stats.rpx_sessions.Load()),
 	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.RxcSessions,
+		prometheus.GaugeValue,
+		float64(c.client.stats.rxc_sessions.Load()),
+	)
 }

client-pty.go

@@ -66,6 +66,8 @@ func (pty *client_pty_ws) ServeWebsocket(ws *websocket.Conn) (int, error) {
 			var poll_fds []unix.PollFd
 			var buf [2048]byte
 			var n int
+			var out_revents int16
+			var sig_revents int16
 			var err error
 
 			poll_fds = []unix.PollFd{
@@ -85,16 +87,10 @@ func (pty *client_pty_ws) ServeWebsocket(ws *websocket.Conn) (int, error) {
 					continue
 				}
 
-				if (poll_fds[0].Revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL)) != 0 {
+				out_revents = poll_fds[0].Revents
-					c.log.Write(pty.Id, LOG_DEBUG, "[%s] EOF detected on pty stdout", req.RemoteAddr)
+				sig_revents = poll_fds[1].Revents
-					break
-				}
-				if (poll_fds[1].Revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL)) != 0 {
-					c.log.Write(pty.Id, LOG_DEBUG, "[%s] EOF detected on pty event pipe", req.RemoteAddr)
-					break
-				}
 
-				if (poll_fds[0].Revents & unix.POLLIN) != 0 {
+				if (out_revents & unix.POLLIN) != 0 {
 					n, err = out.Read(buf[:])
 					if n > 0 {
 						var err2 error
@@ -111,8 +107,22 @@ func (pty *client_pty_ws) ServeWebsocket(ws *websocket.Conn) (int, error) {
 						break
 					}
 				}
-				if (poll_fds[1].Revents & unix.POLLIN) != 0 {
+				if (sig_revents & unix.POLLIN) != 0 {
-					c.log.Write(pty.Id, LOG_DEBUG, "[%s] Stop request noticed on pty event pipe", req.RemoteAddr)
+					c.log.Write(pty.Id, LOG_DEBUG, "[%s] Stop request noticed on pty signal pipe", req.RemoteAddr)
+					break
+				}
+
+				if (out_revents & (unix.POLLERR | unix.POLLNVAL)) != 0 {
+					c.log.Write(pty.Id, LOG_DEBUG, "[%s] Error detected on pty stdout", req.RemoteAddr)
+					break
+				}
+				if (sig_revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL)) != 0 {
+					c.log.Write(pty.Id, LOG_DEBUG, "[%s] EOF detected on pty signal pipe", req.RemoteAddr)
+					break
+				}
+
+				if (out_revents & unix.POLLHUP) != 0 && (out_revents & unix.POLLIN) == 0 {
+					c.log.Write(pty.Id, LOG_DEBUG, "[%s] EOF detected on pty stdout", req.RemoteAddr)
 					break
 				}
 			}
@@ -148,7 +158,7 @@ ws_recv_loop:
 
 								err = unix.Pipe(pfd[:])
 								if err != nil {
-									c.log.Write(pty.Id, LOG_ERROR, "[%s] Failed to create event pipe for pty - %s", req.RemoteAddr, err.Error())
+									c.log.Write(pty.Id, LOG_ERROR, "[%s] Failed to create signal pipe for pty - %s", req.RemoteAddr, err.Error())
 									send_ws_data_for_xterm(ws, "error", err.Error())
 									//ws.Close() // dirty way to flag out the error
 									ws.SetReadDeadline(time.Now())  // slightly cleaner way to break the main loop
@@ -240,7 +250,7 @@ done:
 	if cmd != nil { cmd.Wait() }
 	wg.Wait()
 
-	// close the event pipe after all goroutines are over
+	// close the signal pipe after all goroutines are over
 	if pfd[0] >= 0 { unix.Close(pfd[0]) }
 	if pfd[1] >= 0 { unix.Close(pfd[1]) }
 

client-rxc-profile.go

@@ -0,0 +1,225 @@
+package hodu
+
+import "fmt"
+import "os"
+import "path/filepath"
+import "sort"
+import "strings"
+import "time"
+
+import yaml "github.com/goccy/go-yaml"
+
+const CLIENT_RXC_PROFILE_RELOAD_MIN_INTERVAL time.Duration = 5 * time.Second
+
+type ClientRxcProfile struct {
+	Name   string `yaml:"name"`
+	Script string `yaml:"script"`
+	Args   []string `yaml:"args"`
+	User   string `yaml:"user"`
+}
+
+type client_rxc_profile_file_doc struct {
+	Profiles []ClientRxcProfile `yaml:"profiles"`
+}
+
+type client_rxc_profile_file_state struct {
+	mod_time time.Time
+	size     int64
+}
+
+type ClientRxcProfileMap map[string]*ClientRxcProfile
+
+func (c *Client) SetRxcProfileFiles(files []string) {
+	var copied []string
+
+	copied = make([]string, len(files))
+	copy(copied, files)
+
+	c.rxc_profile_mtx.Lock()
+	c.rxc_profile_files = copied
+	c.rxc_profile_map = make(ClientRxcProfileMap)
+	c.rxc_profile_file_states = make(map[string]client_rxc_profile_file_state)
+	c.rxc_profile_last_check = time.Time{}
+	c.rxc_profile_loaded = false
+	c.rxc_profile_mtx.Unlock()
+}
+
+func (c *Client) GetRxcProfileFiles() []string {
+	var copied []string
+
+	c.rxc_profile_mtx.Lock()
+	copied = make([]string, len(c.rxc_profile_files))
+	copy(copied, c.rxc_profile_files)
+	c.rxc_profile_mtx.Unlock()
+
+	return copied
+}
+
+func append_client_rxc_profiles(dst ClientRxcProfileMap, src []ClientRxcProfile, source_file string) error {
+	var profile ClientRxcProfile
+	var copied *ClientRxcProfile
+	var copied_args []string
+	var existing *ClientRxcProfile
+	var ok bool
+
+	for _, profile = range src {
+		profile.Name = strings.TrimSpace(profile.Name)
+		profile.Script = strings.TrimSpace(profile.Script)
+		profile.User = strings.TrimSpace(profile.User)
+
+		if profile.Name == "" {
+			return fmt.Errorf("blank rxc profile name in %s", source_file)
+		}
+		if profile.Script == "" {
+			return fmt.Errorf("blank rxc profile script for %s in %s", profile.Name, source_file)
+		}
+
+		existing, ok = dst[profile.Name]
+		if ok && existing != nil {
+			return fmt.Errorf("duplicate rxc profile %s in %s", profile.Name, source_file)
+		}
+
+		copied = new(ClientRxcProfile)
+		*copied = profile
+		if profile.Args != nil {
+			copied_args = make([]string, len(profile.Args))
+			copy(copied_args, profile.Args)
+			copied.Args = copied_args
+		}
+
+		dst[copied.Name] = copied
+	}
+
+	return nil
+}
+
+func (c *Client) reload_rxc_profiles_if_needed() error {
+	var now time.Time
+	var min_interval time.Duration
+	var patterns []string
+	var matched_file_map map[string]struct{}
+	var matched_files []string
+	var pattern string
+	var file_path string
+	var file *os.File
+	var file_states map[string]client_rxc_profile_file_state
+	var profiles ClientRxcProfileMap
+	var changed bool
+	var current_state client_rxc_profile_file_state
+	var doc client_rxc_profile_file_doc
+	var dec *yaml.Decoder
+	var err error
+
+	now = time.Now()
+
+	c.rxc_profile_mtx.Lock()
+	defer c.rxc_profile_mtx.Unlock()
+
+	min_interval = c.rxc_profile_reload_min_interval
+
+	if min_interval > 0 && !c.rxc_profile_last_check.IsZero() && now.Before(c.rxc_profile_last_check.Add(min_interval)) {
+		return nil
+	}
+	c.rxc_profile_last_check = now
+
+	patterns = make([]string, len(c.rxc_profile_files))
+	copy(patterns, c.rxc_profile_files)
+
+	matched_file_map = make(map[string]struct{})
+	for _, pattern = range patterns {
+		var expanded []string
+
+		if strings.TrimSpace(pattern) == "" { continue }
+
+		expanded, err = filepath.Glob(pattern)
+		if err != nil { return fmt.Errorf("invalid rxc profile file pattern %s - %s", pattern, err.Error()) }
+
+		for _, file_path = range expanded {
+			matched_file_map[file_path] = struct{}{}
+		}
+	}
+
+	matched_files = make([]string, 0, len(matched_file_map))
+	for file_path = range matched_file_map {
+		matched_files = append(matched_files, file_path)
+	}
+	sort.Strings(matched_files)
+
+	file_states = make(map[string]client_rxc_profile_file_state)
+	for _, file_path = range matched_files {
+		var file_info os.FileInfo
+
+		file_info, err = os.Stat(file_path)
+		if err != nil {
+			return fmt.Errorf("failed to stat rxc profile file %s - %s", file_path, err.Error())
+		}
+
+		file_states[file_path] = client_rxc_profile_file_state{
+			mod_time: file_info.ModTime(),
+			size:     file_info.Size(),
+		}
+	}
+
+	if len(file_states) != len(c.rxc_profile_file_states) {
+		changed = true
+	} else {
+		var ok bool
+		for file_path, current_state = range file_states {
+			_, ok = c.rxc_profile_file_states[file_path]
+			if !ok {
+				changed = true
+				break
+			}
+			if !c.rxc_profile_file_states[file_path].mod_time.Equal(current_state.mod_time) || c.rxc_profile_file_states[file_path].size != current_state.size {
+				changed = true
+				break
+			}
+		}
+	}
+
+	if !changed && c.rxc_profile_loaded { return nil }
+
+	profiles = make(ClientRxcProfileMap)
+	for _, file_path = range matched_files {
+		file, err = os.Open(file_path)
+		if err != nil { return fmt.Errorf("failed to open rxc profile file %s - %s", file_path, err.Error()) }
+
+		doc = client_rxc_profile_file_doc{}
+		dec = yaml.NewDecoder(file)
+		err = dec.Decode(&doc)
+		file.Close()
+		if err != nil { return fmt.Errorf("failed to parse rxc profile file %s - %s", file_path, err.Error()) }
+
+		err = append_client_rxc_profiles(profiles, doc.Profiles, file_path)
+		if err != nil { return err }
+	}
+
+	c.rxc_profile_map = profiles
+	c.rxc_profile_file_states = file_states
+	c.rxc_profile_loaded = true
+	c.log.Write("", LOG_DEBUG, "Loaded %d rxc profiles from %d files", len(c.rxc_profile_map), len(matched_files))
+
+	return nil
+}
+
+func (c *Client) ResolveRxcProfile(name string) (*ClientRxcProfile, error) {
+	var profile *ClientRxcProfile
+	var copied ClientRxcProfile
+	var ok bool
+	var err error
+
+	name = strings.TrimSpace(name)
+	if name == "" { return nil, fmt.Errorf("blank rxc profile name") }
+
+	err = c.reload_rxc_profiles_if_needed()
+
+	c.rxc_profile_mtx.Lock()
+	profile, ok = c.rxc_profile_map[name]
+	if ok && profile != nil { copied = *profile }
+	c.rxc_profile_mtx.Unlock()
+
+	if ok && profile != nil { return &copied, nil }
+	if err != nil { return nil, err } // reloading failed above and there is no existing profile found
+
+	return nil, nil // no reloading error but not resolved either
+}

client.go

@@ -1,7 +1,5 @@
 package hodu
 
-import "bufio"
-import "bytes"
 import "container/list"
 import "context"
 import "crypto/tls"
@@ -33,7 +31,6 @@ import "google.golang.org/grpc/keepalive"
 import "google.golang.org/grpc/peer"
 import "google.golang.org/grpc/status"
 
-import pts "github.com/creack/pty"
 import "github.com/prometheus/client_golang/prometheus"
 import "github.com/prometheus/client_golang/prometheus/promhttp"
 
@@ -45,10 +42,11 @@ type ClientPeerConnMap map[PeerId]*ClientPeerConn
 type ClientPeerCancelFuncMap map[PeerId]context.CancelFunc
 type ClientRptyMap map[uint64]*ClientRpty
 type ClientRpxMap map[uint64]*ClientRpx
+type ClientRxcMap map[uint64]*ClientRxc
 
 // --------------------------------------------------------------------
 type ClientRouteConfig struct {
-	Id          RouteId // requested id to be assigned. 0 for automatic assignment
+	Id            RouteId // requested id to be assigned. 0 for automatic assignment
 	PeerAddr      string
 	PeerName      string
 	ServiceAddr   string // server-peer-svc-addr
@@ -181,13 +179,24 @@ type Client struct {
 		pty_sessions atomic.Int64
 		rpty_sessions atomic.Int64
 		rpx_sessions atomic.Int64
+		rxc_sessions atomic.Int64
 	}
 
 	pty_user string
 	pty_shell string
+	rxc_user string
+	rxc_profile_mtx sync.Mutex
+	rxc_profile_files []string
+	rxc_profile_reload_min_interval time.Duration
+	rxc_profile_map ClientRxcProfileMap
+	rxc_profile_file_states map[string]client_rxc_profile_file_state
+	rxc_profile_last_check time.Time
+	rxc_profile_loaded bool
+
 	rpc_ping_intvl time.Duration
 	rpc_ping_tmout time.Duration
 	rpc_seed_tmout time.Duration
+
 	xterm_html string
 }
 
@@ -216,6 +225,18 @@ type ClientRpx struct {
 	ws_conn net.Conn
 }
 
+type ClientRxc struct {
+	cts *ClientConn
+	id uint64
+	req_type string // orignal requested type
+	req_script string // original requested script
+	cmd *exec.Cmd // actual command
+	stdin *os.File
+	stdout *os.File
+	stderr *os.File
+	pfd [2]int
+}
+
 // this struct must contain the key fields contained
 // in the Seed struct generated by protobuf
 type SeedInfo struct {
@@ -259,6 +280,9 @@ type ClientConn struct {
 	rpx_mtx      sync.Mutex
 	rpx_map      ClientRpxMap
 
+	rxc_mtx      sync.Mutex
+	rxc_map      ClientRxcMap
+
 	stop_req      atomic.Bool
 	stop_chan     chan bool
 
@@ -336,6 +360,12 @@ func (g *GuardedPacketStreamClient) Send(data *Packet) error {
 	return g.Hodu_PacketStreamClient.Send(data)
 }
 
+func (g *GuardedPacketStreamClient) CloseSend() error {
+	g.mtx.Lock()
+	defer g.mtx.Unlock()
+	return g.Hodu_PacketStreamClient.CloseSend()
+}
+
 /*func (g *GuardedPacketStreamClient) Recv() (*Packet, error) {
 	return g.psc.Recv()
 }
@@ -347,7 +377,12 @@ func (g *GuardedPacketStreamClient) Context() context.Context {
 // --------------------------------------------------------------------
 
 func (rpty *ClientRpty) ReqStop() {
-	rpty.cmd.Process.Kill()
+	if rpty.cmd != nil {
+		// i don't care about multiple kills
+		rpty.cts.C.log.Write("", LOG_DEBUG, "Terminating process(%d) for rpty(%d) - %s", rpty.cmd.Process.Pid, rpty.id, rpty.cmd.String());
+		rpty.cmd.Process.Kill()
+	}
+
 	// don't check for a write error. the os pipe's buffer must be large enough
 	unix.Write(rpty.pfd[1], []byte{0})
 }
@@ -361,6 +396,17 @@ func (rpx *ClientRpx) ReqStop() {
 	}
 }
 
+func (rxc *ClientRxc) ReqStop() {
+	// i don't care about multiple kills
+	if rxc.cmd != nil {
+		rxc.cts.C.log.Write("", LOG_DEBUG, "Terminating process(%d) for rxc(%d) - %s", rxc.cmd.Process.Pid, rxc.id, rxc.cmd.String());
+		rxc.cmd.Process.Kill()
+	}
+
+	// don't check for a write error. the os pipe's buffer must be large enough
+	unix.Write(rxc.pfd[1], []byte{0})
+}
+
 // --------------------------------------------------------------------
 func NewClientRoute(cts *ClientConn, id RouteId, static bool, client_peer_addr string, client_peer_name string, server_peer_svc_addr string, server_peer_svc_net string, server_peer_option RouteOption, lifetime time.Duration) *ClientRoute {
 	var r ClientRoute
@@ -893,6 +939,7 @@ func NewClientConn(c *Client, cfg *ClientConnConfig) *ClientConn {
 	cts.ptc_list = list.New()
 	cts.rpty_map = make(ClientRptyMap)
 	cts.rpx_map = make(ClientRpxMap)
+	cts.rxc_map = make(ClientRxcMap)
 
 	for i, _ = range cts.cfg.Routes {
 		// override it to static regardless of the value passed in
@@ -1100,10 +1147,11 @@ func (cts *ClientConn) disconnect_from_server(logmsg bool) {
 		var r *ClientRoute
 		var rpty *ClientRpty
 		var rpx *ClientRpx
+		var rxc *ClientRxc
 
 		cts.discon_mtx.Lock()
 
-		if (logmsg) {
+		if logmsg {
 			cts.C.log.Write(cts.Sid, LOG_INFO, "Preparing to disconnect from server[%d] %s", cts.cfg.Index, cts.cfg.ServerAddrs[cts.cfg.Index])
 		}
 
@@ -1127,7 +1175,20 @@ func (cts *ClientConn) disconnect_from_server(logmsg bool) {
 		}
 		cts.rpx_mtx.Unlock()
 
-		// don't care about double closes when this function is called from both RunTask() and ReqStop()
+		cts.rxc_mtx.Lock()
+		for _, rxc = range cts.rxc_map {
+			rxc.ReqStop()
+			// the loop in RxcLoop() is supposed to be broken.
+			// let's not inform the server of this connection.
+			// the server should clean up itself upon connection error
+		}
+		cts.rxc_mtx.Unlock()
+
+		// don't care about double closes when this function is called
+		// from both RunTask() and ReqStop(). Send() may be called after
+		// this CloseSend() in some unwaited goroutines like RxcLoop(),
+		// which is ugly.you may see a message like
+		//   "rpc error: code = Internal desc = SendMsg called after CloseSend" in logs.
 		if cts.psc != nil { cts.psc.CloseSend() }
 		cts.conn.Close()
 
@@ -1142,7 +1203,7 @@ func (cts *ClientConn) disconnect_from_server(logmsg bool) {
 		cts.remote_addr.Set("")
 		// don't reset cts.local_addr_p and cts.remote_addr_p
 
-		if (logmsg) {
+		if logmsg {
 			cts.C.log.Write(cts.Sid, LOG_INFO, "Prepared to disconnect from server[%d] %s", cts.cfg.Index, cts.cfg.ServerAddrs[cts.cfg.Index])
 		}
 		cts.discon_mtx.Unlock()
@@ -1266,7 +1327,6 @@ func (cts *ClientConn) dispatch_packet(pkt *Packet) bool {
 				cts.C.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", pkt.Kind.String(), cts.remote_addr_p)
 			}
 
-
 		case PACKET_KIND_RPTY_START:
 			fallthrough
 		case PACKET_KIND_RPTY_STOP:
@@ -1315,6 +1375,28 @@ func (cts *ClientConn) dispatch_packet(pkt *Packet) bool {
 				cts.C.log.Write(cts.Sid, LOG_ERROR, "Invalid %s event from %s", pkt.Kind.String(), cts.remote_addr_p)
 			}
 
+		case PACKET_KIND_RXC_START:
+			fallthrough
+		case PACKET_KIND_RXC_STOP:
+			fallthrough
+		case PACKET_KIND_RXC_DATA:
+			var x *Packet_RxcEvt
+			x, ok = pkt.U.(*Packet_RxcEvt)
+			if ok {
+				err = cts.HandleRxcEvent(pkt.Kind, x.RxcEvt)
+				if err != nil {
+					cts.C.log.Write(cts.Sid, LOG_ERROR,
+						"Failed to handle %s event for rxc(%d) from %s - %s",
+						pkt.Kind.String(), x.RxcEvt.Id, cts.remote_addr_p, err.Error())
+				} else {
+					cts.C.log.Write(cts.Sid, LOG_DEBUG,
+						"Handled %s event for rxc(%d) from %s",
+						pkt.Kind.String(), x.RxcEvt.Id, cts.remote_addr_p)
+				}
+			} else {
+				cts.C.log.Write(cts.Sid, LOG_ERROR, "Invalid %s event from %s", pkt.Kind.String(), cts.remote_addr_p)
+			}
+
 		default:
 			// do nothing. ignore the rest
 	}
@@ -1556,595 +1638,6 @@ func (cts *ClientConn) ReportPacket(route_id RouteId, pts_id PeerId, packet_type
 	return r.ReportPacket(pts_id, packet_type, event_data)
 }
 
-// rpty
-func (cts *ClientConn) FindClientRptyById(id uint64) *ClientRpty {
-	var crp *ClientRpty
-	var ok bool
-
-	cts.rpty_mtx.Lock()
-	crp, ok = cts.rpty_map[id]
-	cts.rpty_mtx.Unlock()
-
-	if !ok { crp = nil }
-	return crp
-}
-
-func (cts *ClientConn) RptyLoop(crp *ClientRpty, wg *sync.WaitGroup) {
-
-	var poll_fds []unix.PollFd
-	var buf [2048]byte
-	var n int
-	var err error
-
-	defer wg.Done()
-
-	cts.C.log.Write(cts.Sid, LOG_INFO, "Started rpty(%d) for %s(%s)", crp.id, cts.C.pty_shell, cts.C.pty_user)
-
-	cts.C.stats.rpty_sessions.Add(1)
-
-	poll_fds = []unix.PollFd{
-		unix.PollFd{Fd: int32(crp.tty.Fd()), Events: unix.POLLIN},
-		unix.PollFd{Fd: int32(crp.pfd[0]), Events: unix.POLLIN},
-	}
-
-	for {
-		n, err = unix.Poll(poll_fds, -1) // -1 means wait indefinitely
-		if err != nil {
-			if errors.Is(err, unix.EINTR) { continue }
-			cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to poll rpty(%d) stdout - %s", crp.id, err.Error())
-			break
-		}
-		if n == 0 { // timed out
-			continue
-		}
-
-		if (poll_fds[0].Revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL)) != 0 {
-			cts.C.log.Write(cts.Sid, LOG_DEBUG, "EOF detected on rpty(%d) stdout", crp.id)
-			break
-		}
-		if (poll_fds[1].Revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL)) != 0 {
-			cts.C.log.Write(cts.Sid, LOG_DEBUG, "EOF detected on rpty(%d) event pipe", crp.id)
-			break
-		}
-
-		if (poll_fds[0].Revents & unix.POLLIN) != 0 {
-			n, err = crp.tty.Read(buf[:])
-			if n > 0 {
-				var err2 error
-				err2 = cts.psc.Send(MakeRptyDataPacket(crp.id, buf[:n]))
-				if err2 != nil {
-					cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to send %s from rpty(%d) stdout to server - %s", PACKET_KIND_RPTY_DATA.String(), crp.id, err2.Error())
-					break
-				}
-			}
-			if err != nil {
-				if !errors.Is(err, io.EOF) {
-					cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to read rpty(%d) stdout - %s", crp.id, err.Error())
-				}
-				break
-			}
-		}
-		if (poll_fds[1].Revents & unix.POLLIN) != 0 {
-			// don't care to read the pipe as it is closed after the loop
-			//unix.Read(crp.pfd[0], )
-			cts.C.log.Write(cts.Sid, LOG_DEBUG, "Stop request noticed on rpty(%d) event pipe", crp.id)
-			break
-		}
-	}
-
-	cts.C.log.Write(cts.Sid, LOG_DEBUG, "Ending rpty(%d) loop", crp.id)
-	cts.psc.Send(MakeRptyStopPacket(crp.id, ""))
-
-	crp.ReqStop()
-	crp.cmd.Wait()
-	crp.tty.Close()
-	unix.Close(crp.pfd[0])
-	unix.Close(crp.pfd[1])
-
-	cts.rpty_mtx.Lock()
-	delete(cts.rpty_map, crp.id)
-	cts.rpty_mtx.Unlock()
-
-	cts.C.stats.rpty_sessions.Add(-1)
-	cts.C.log.Write(cts.Sid, LOG_DEBUG, "Ended rpty(%d) loop", crp.id)
-}
-
-func (cts *ClientConn) StartRpty(id uint64, wg *sync.WaitGroup) error {
-	var crp *ClientRpty
-	var ok bool
-	var i int
-	var err error
-
-	cts.rpty_mtx.Lock()
-	_, ok = cts.rpty_map[id]
-	if ok {
-		cts.rpty_mtx.Unlock()
-		return fmt.Errorf("multiple start on rpty id %d", id)
-	}
-
-	crp = &ClientRpty{ cts: cts, id: id }
-	err = unix.Pipe(crp.pfd[:])
-	if err != nil {
-		cts.rpty_mtx.Unlock()
-		cts.psc.Send(MakeRptyStopPacket(id, err.Error()))
-		return fmt.Errorf("unable to create rpty(%d) event fd for %s(%s) - %s", id, cts.C.pty_shell, cts.C.pty_user, err.Error())
-	}
-	crp.cmd, crp.tty, err = connect_pty(cts.C.pty_shell, cts.C.pty_user)
-	if err != nil {
-		cts.rpty_mtx.Unlock()
-		cts.psc.Send(MakeRptyStopPacket(id, err.Error()))
-		unix.Close(crp.pfd[0])
-		unix.Close(crp.pfd[1])
-		return fmt.Errorf("unable to start rpty(%d) for %s(%s) - %s", id, cts.C.pty_shell, cts.C.pty_user, err.Error())
-	}
-
-	for i = 0; i < 2; i++ {
-		var flags int
-		flags, err = unix.FcntlInt(uintptr(crp.pfd[i]), unix.F_GETFL, 0)
-		if err != nil {
-			unix.FcntlInt(uintptr(crp.pfd[i]), unix.F_SETFL, flags | unix.O_NONBLOCK)
-		}
-	}
-
-	cts.rpty_map[id] = crp
-
-	wg.Add(1)
-	go cts.RptyLoop(crp, wg)
-
-	cts.rpty_mtx.Unlock()
-
-	return nil
-}
-
-func (cts *ClientConn) StopRpty(id uint64) error {
-	var crp *ClientRpty
-
-	crp = cts.FindClientRptyById(id)
-	if crp == nil {
-		return fmt.Errorf("unknown rpty id %d", id)
-	}
-
-	crp.ReqStop()
-	return nil
-}
-
-func (cts *ClientConn) WriteRpty(id uint64, data []byte) error {
-	var crp *ClientRpty
-
-	crp = cts.FindClientRptyById(id)
-	if crp == nil {
-		return fmt.Errorf("unknown rpty id %d", id)
-	}
-
-	crp.tty.Write(data)
-	return nil
-}
-
-func (cts *ClientConn) WriteRptySize(id uint64, data []byte) error {
-	var crp *ClientRpty
-	var flds []string
-
-	crp = cts.FindClientRptyById(id)
-	if crp == nil {
-		return fmt.Errorf("unknown rpty id %d", id)
-	}
-
-	flds = strings.Split(string(data), " ")
-	if len(flds) == 2 {
-		var rows int
-		var cols int
-		rows, _ = strconv.Atoi(flds[0])
-		cols, _ = strconv.Atoi(flds[1])
-		pts.Setsize(crp.tty, &pts.Winsize{Rows: uint16(rows), Cols: uint16(cols)})
-	}
-	return nil
-}
-
-func (cts *ClientConn) HandleRptyEvent(packet_type PACKET_KIND, evt *RptyEvent) error {
-
-	switch packet_type {
-		case PACKET_KIND_RPTY_START:
-			return cts.StartRpty(evt.Id, &cts.C.wg)
-
-		case PACKET_KIND_RPTY_STOP:
-			return cts.StopRpty(evt.Id)
-
-		case PACKET_KIND_RPTY_DATA:
-			return cts.WriteRpty(evt.Id, evt.Data)
-
-		case PACKET_KIND_RPTY_SIZE:
-			return cts.WriteRptySize(evt.Id, evt.Data)
-	}
-
-	// ignore other packet types
-	return nil
-}
-
-// rpx
-func (cts *ClientConn) FindClientRpxById(id uint64) *ClientRpx {
-	var crpx *ClientRpx
-	var ok bool
-
-	cts.rpx_mtx.Lock()
-	crpx, ok = cts.rpx_map[id]
-	cts.rpx_mtx.Unlock()
-
-	if !ok { crpx = nil }
-	return crpx
-}
-
-func (cts *ClientConn) server_pipe_to_ws_target(crpx* ClientRpx, conn net.Conn, wg *sync.WaitGroup) {
-	var buf [4096]byte
-	var n int
-	var err error
-
-	defer wg.Done()
-
-	for {
-		n, err = crpx.pr.Read(buf[:])
-		if n > 0 {
-			var err2 error
-			_, err2 = conn.Write(buf[:n])
-			if err2 != nil {
-				cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to write websocket for rpx(%d) - %s", crpx.id, err2.Error())
-				break
-			}
-		}
-		if err != nil {
-			if errors.Is(err, io.EOF) { break }
-			cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to read pipe for rpx(%d) - %s", crpx.id, err.Error())
-			break
-		}
-	}
-}
-
-func (cts *ClientConn) proxy_ws(crpx *ClientRpx, raw_req []byte, req *http.Request) (int, error) {
-	var l_wg sync.WaitGroup
-	var conn net.Conn
-	var resp *http.Response
-	var r *bufio.Reader
-	var buf [4096]byte
-	var n int
-	var err error
-
-	if cts.C.rpx_target_tls != nil {
-		var dialer *tls.Dialer
-		dialer = &tls.Dialer{
-			NetDialer: &net.Dialer{},
-			Config: cts.C.rpx_target_tls,
-		}
-		conn, err = dialer.DialContext(crpx.ctx, "tcp", cts.C.rpx_target_addr) // TODO: no hard coding
-	} else {
-		var dialer *net.Dialer
-		dialer = &net.Dialer{}
-		conn, err = dialer.DialContext(crpx.ctx, "tcp", cts.C.rpx_target_addr) // TODO: no hard coding
-	}
-	if err != nil {
-		return http.StatusInternalServerError, fmt.Errorf("failed to dial websocket for rpx(%d) - %s", crpx.id, err.Error())
-	}
-	defer conn.Close()
-
-	// TODO: make this atomic?
-	crpx.ws_conn = conn
-
-	// write the raw request line and headers as sent by the server.
-	// for the upgrade request, i assume no payload.
-	_, err = conn.Write(raw_req)
-	if err != nil {
-		return http.StatusInternalServerError, fmt.Errorf("failed to write websocket request for rpx(%d) - %s", crpx.id, err.Error())
-	}
-
-	r = bufio.NewReader(conn)
-	resp, err = http.ReadResponse(r, req)
-	if err != nil {
-		return http.StatusInternalServerError, fmt.Errorf("failed to write websocket response for rpx(%d) - %s", crpx.id, err.Error())
-	}
-	defer resp.Body.Close()
-
-	err = cts.psc.Send(MakeRpxStartPacket(crpx.id, get_http_resp_line_and_headers(resp)))
-	if err != nil {
-		return http.StatusInternalServerError, fmt.Errorf("failed to send rpx(%d) WebSocket headers to server - %s", crpx.id, err.Error())
-	}
-
-	if resp.StatusCode != http.StatusSwitchingProtocols {
-		// websock upgrade failed. let the code jump to the done
-		// label to skip reading from the pipe. the server side
-		// has the code to ensure no content-length. and the upgrade
-		// fails, the pipe below will be pending forever as the server
-		// side doesn't send data and there's no feeding to the pipe.
-		return resp.StatusCode, fmt.Errorf("protocol switching failed for rpx(%d)", crpx.id)
-	}
-
-	// unlike with the normal request, the actual pipe is not read
-	// until the initial switching protocol response is received.
-
-	l_wg.Add(1)
-	go cts.server_pipe_to_ws_target(crpx, conn, &l_wg)
-
-	for {
-		n, err = conn.Read(buf[:])
-		if n > 0 {
-			var err2 error
-			err2 = cts.psc.Send(MakeRpxDataPacket(crpx.id, buf[:n]))
-			if err2 != nil {
-				crpx.ReqStop() // to break server_pipe_ws_target. don't care about multiple stops
-				return resp.StatusCode, fmt.Errorf("failed to send rpx(%d) data to server - %s", crpx.id, err2.Error())
-			}
-		}
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				cts.psc.Send(MakeRpxEofPacket(crpx.id))
-				cts.C.log.Write(cts.Sid, LOG_DEBUG, "WebSocket rpx(%d) closed by server", crpx.id)
-				break
-			}
-
-			crpx.ReqStop() // to break server_pipe_ws_target. don't care about multiple stops
-			return resp.StatusCode, fmt.Errorf("failed to read WebSocket rpx(%d) - %s", crpx.id, err.Error())
-		}
-	}
-
-	// wait until the pipe reading(from the server side) goroutine is over
-	l_wg.Wait()
-	return resp.StatusCode, nil
-}
-
-func (cts *ClientConn) proxy_http(crpx *ClientRpx, req *http.Request) (int, error) {
-	var tr *http.Transport
-	var resp *http.Response
-	var buf [4096]byte
-	var n int
-	var err error
-
-	tr = &http.Transport {
-		DisableKeepAlives: true, // this implementation can't support keepalive..
-	}
-	if cts.C.rpx_target_tls != nil {
-		tr.TLSClientConfig = cts.C.rpx_target_tls
-	}
-
-	resp, err = tr.RoundTrip(req)
-	if err != nil {
-		return http.StatusInternalServerError, fmt.Errorf("failed to send rpx(%d) request - %s", crpx.id, err.Error())
-	}
-	defer resp.Body.Close()
-
-	err = cts.psc.Send(MakeRpxStartPacket(crpx.id, get_http_resp_line_and_headers(resp)))
-	if err != nil {
-		return resp.StatusCode, fmt.Errorf("failed to send rpx(%d) status and headers to server - %s", crpx.id, err.Error())
-	}
-
-	for {
-		n, err = resp.Body.Read(buf[:])
-		if n > 0 {
-			var err2 error
-			err2 = cts.psc.Send(MakeRpxDataPacket(crpx.id, buf[:n]))
-			if err2 != nil {
-				return resp.StatusCode, fmt.Errorf("failed to send rpx(%d) data to server - %s", crpx.id, err2.Error())
-			}
-		}
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
-			return resp.StatusCode, fmt.Errorf("failed to read response body for rpx(%d) - %s", crpx.id, err.Error())
-		}
-	}
-
-	return resp.StatusCode, nil
-}
-
-func (cts *ClientConn) RpxLoop(crpx *ClientRpx, data []byte, wg *sync.WaitGroup) {
-	var start_time time.Time
-	var time_taken time.Duration
-	var r *bufio.Reader
-	var line string
-	var flds []string
-	var req_meth string
-	var req_path string
-	//var req_proto string
-	var x_forwarded_host string
-	var raw_req bytes.Buffer
-	var status_code int
-	var req *http.Request
-	var err error
-
-	defer wg.Done()
-
-	cts.C.log.Write(cts.Sid, LOG_INFO, "Starting rpx(%d) loop", crpx.id)
-
-	start_time = time.Now()
-
-	const rpx_header_line_max = 65535 // TODO: make this configurable
-
-	r = bufio.NewReader(bytes.NewReader(data))
-	line, err = read_line_limited(r, rpx_header_line_max)
-	if err != nil && !errors.Is(err, io.EOF) {
-		cts.C.log.Write(cts.Sid, LOG_ERROR, "failed to parse request for rpx(%d) - %s", crpx.id, err.Error())
-		goto done
-	}
-	line = strings.TrimRight(line, "\r\n")
-
-	flds = strings.Fields(line)
-	if (len(flds) < 3) {
-		cts.C.log.Write(cts.Sid, LOG_ERROR, "Invalid request line for rpx(%d) - %s", crpx.id, line)
-		goto done
-	}
-
-// TODO: handle trailers...
-	req_meth = flds[0]
-	req_path = flds[1]
-	//req_proto = flds[2]
-
-	raw_req.WriteString(line)
-	raw_req.WriteString("\r\n")
-
-	// create a request assuming it's a normal http request
-	req, err = http.NewRequestWithContext(crpx.ctx, req_meth, cts.C.rpx_target_url + req_path, crpx.pr)
-	if err != nil {
-		cts.C.log.Write(cts.Sid, LOG_ERROR, "failed to create request for rpx(%d) - %s", crpx.id, err.Error())
-		goto done
-	}
-
-	for {
-		line, err = read_line_limited(r, rpx_header_line_max)
-		if err != nil && !errors.Is(err, io.EOF) {
-			cts.C.log.Write(cts.Sid, LOG_ERROR, "failed to parse request for rpx(%d) - %s", crpx.id, err.Error())
-			goto done
-		}
-		line = strings.TrimRight(line, "\r\n")
-		if line == "" { break }
-		flds = strings.SplitN(line, ":", 2)
-		if len(flds) == 2 {
-			var k string
-			var v string
-			k = strings.TrimSpace(flds[0])
-			v = strings.TrimSpace(flds[1])
-			req.Header.Add(k, v)
-
-			if strings.EqualFold(k, "Host") {
-				// a normal http client would set HOst to be the target address.
-				// the raw header is coming from the server. so it's different
-				// from the host it's supposed to be. correct it to the right value.
-				fmt.Fprintf(&raw_req, "%s: %s\r\n", k, req.Host)
-			} else {
-				raw_req.WriteString(line)
-				raw_req.WriteString("\r\n")
-
-				if strings.EqualFold(k, "X-Forwarded-Host") {
-					x_forwarded_host = v
-				}
-			}
-		}
-		if errors.Is(err, io.EOF) { break }
-	}
-	raw_req.WriteString("\r\n")
-	if x_forwarded_host == "" {
-		x_forwarded_host = req.Host
-	}
-
-	if strings.EqualFold(req.Header.Get("Upgrade"), "websocket") && strings.Contains(strings.ToLower(req.Header.Get("Connection")), "upgrade") {
-		// websocket
-		status_code, err = cts.proxy_ws(crpx, raw_req.Bytes(), req)
-	} else {
-		// normal http
-		status_code, err = cts.proxy_http(crpx, req)
-	}
-
-	time_taken = time.Since(start_time)
-	if err != nil {
-		cts.C.log.Write(cts.Sid, LOG_ERROR, "rpx(%d) %s - %s %s %d %.9f - failed to proxy - %s", crpx.id, x_forwarded_host, req_meth, req_path, status_code, time_taken.Seconds(), err.Error())
-		goto done
-	} else {
-		cts.C.log.Write(cts.Sid, LOG_INFO, "rpx(%d) %s - %s %s %d %.9f", crpx.id, x_forwarded_host, req_meth, req_path, status_code, time_taken.Seconds())
-	}
-
-done:
-	err = cts.psc.Send(MakeRpxStopPacket(crpx.id))
-	if err != nil {
-		cts.C.log.Write(cts.Sid, LOG_ERROR, "rpx(%d) Failed to send %s to server - %s", crpx.id, PACKET_KIND_RPX_STOP.String(), err.Error())
-	}
-	cts.C.log.Write(cts.Sid, LOG_INFO, "Ending rpx(%d) loop", crpx.id)
-
-	crpx.ReqStop()
-
-	cts.rpx_mtx.Lock()
-	delete(cts.rpx_map, crpx.id)
-	cts.rpx_mtx.Unlock()
-	cts.C.stats.rpx_sessions.Add(-1)
-	cts.C.log.Write(cts.Sid, LOG_INFO, "Ended rpx(%d) loop", crpx.id)
-}
-
-func (cts *ClientConn) StartRpx(id uint64, data []byte, wg *sync.WaitGroup) error {
-	var crpx *ClientRpx
-	var ok bool
-
-	cts.rpx_mtx.Lock()
-	_, ok = cts.rpx_map[id]
-	if ok {
-		cts.rpx_mtx.Unlock()
-		return fmt.Errorf("multiple start on rpx id %d", id)
-	}
-	crpx = &ClientRpx{ id: id }
-	cts.rpx_map[id] = crpx
-
-	// i want the pipe to be created before the goroutine is started
-	// so that the WriteRpx() can write to the pipe. i protect pipe creation
-	// and context creation with a mutex
-	crpx.pr, crpx.pw = io.Pipe()
-	crpx.ctx, crpx.cancel = context.WithCancel(cts.C.Ctx)
-
-	cts.rpx_mtx.Unlock()
-	cts.C.stats.rpx_sessions.Add(1)
-
-	wg.Add(1)
-	go cts.RpxLoop(crpx, data, wg)
-
-	return nil
-}
-
-func (cts *ClientConn) StopRpx(id uint64) error {
-	var crpx *ClientRpx
-
-	crpx = cts.FindClientRpxById(id)
-	if crpx == nil {
-		return fmt.Errorf("unknown rpx id %d", id)
-	}
-
-	crpx.ReqStop()
-	return nil
-}
-
-func (cts *ClientConn) WriteRpx(id uint64, data []byte) error {
-	var crpx *ClientRpx
-	var err error
-
-	crpx = cts.FindClientRpxById(id)
-	if crpx == nil {
-		return fmt.Errorf("unknown rpx id %d", id)
-	}
-
-// TODO: may have to write it in a goroutine to avoid blocking?
-	_, err = crpx.pw.Write(data)
-	if err != nil {
-		cts.C.log.Write(cts.Sid, LOG_ERROR, "Failed to write rpx(%d) data - %s", id, err.Error())
-		return err
-	}
-	return nil
-}
-
-func (cts *ClientConn) EofRpx(id uint64, data []byte) error {
-	var crpx *ClientRpx
-
-	crpx = cts.FindClientRpxById(id)
-	if crpx == nil {
-		return fmt.Errorf("unknown rpx id %d", id)
-	}
-
-	// close the writing end only. leave the reading end untouched
-	crpx.pw.Close()
-
-	return nil
-}
-
-func (cts *ClientConn) HandleRpxEvent(packet_type PACKET_KIND, evt *RpxEvent) error {
-	switch packet_type {
-		case PACKET_KIND_RPX_START:
-			return cts.StartRpx(evt.Id, evt.Data, &cts.C.wg)
-
-		case PACKET_KIND_RPX_STOP:
-			return cts.StopRpx(evt.Id)
-
-		case PACKET_KIND_RPX_DATA:
-			return cts.WriteRpx(evt.Id, evt.Data)
-
-		case PACKET_KIND_RPX_EOF:
-			return cts.EofRpx(evt.Id, evt.Data)
-	}
-
-	// ignore other packet types
-	return nil
-}
-
 // --------------------------------------------------------------------
 
 func (m ClientPeerConnMap) get_sorted_keys() []PeerId {
@@ -2332,6 +1825,10 @@ func NewClient(ctx context.Context, name string, logger Logger, cfg *ClientConfi
 	c.rpc_ping_intvl = cfg.RpcPingIntvl
 	c.rpc_ping_tmout = cfg.RpcPingTmout
 	c.rpc_seed_tmout = cfg.RpcSeedTmout
+	c.rxc_profile_files = make([]string, 0)
+	c.rxc_profile_reload_min_interval = CLIENT_RXC_PROFILE_RELOAD_MIN_INTERVAL
+	c.rxc_profile_map = make(ClientRxcProfileMap)
+	c.rxc_profile_file_states = make(map[string]client_rxc_profile_file_state)
 
 	c.rpc_tls = cfg.RpcTls
 	c.rpx_target_addr = cfg.RpxTargetAddr
@@ -2452,6 +1949,7 @@ func NewClient(ctx context.Context, name string, logger Logger, cfg *ClientConfi
 	c.stats.pty_sessions.Store(0)
 	c.stats.rpty_sessions.Store(0)
 	c.stats.rpx_sessions.Store(0)
+	c.stats.rxc_sessions.Store(0)
 
 	return &c
 }
@@ -2764,6 +2262,31 @@ func (c *Client) GetPtyShell() string {
 	return c.pty_shell
 }
 
+func (c *Client) SetRxcUser(user string) {
+	c.rxc_user = user
+}
+
+func (c *Client) GetRxcUser() string {
+	return c.rxc_user
+}
+
+func (c *Client) SetRxcProfileReloadMinInterval(interval time.Duration) {
+	c.rxc_profile_mtx.Lock()
+	c.rxc_profile_reload_min_interval = interval
+	c.rxc_profile_last_check = time.Time{}
+	c.rxc_profile_mtx.Unlock()
+}
+
+func (c *Client) GetRxcProfileReloadMinInterval() time.Duration {
+	var interval time.Duration
+
+	c.rxc_profile_mtx.Lock()
+	interval = c.rxc_profile_reload_min_interval
+	c.rxc_profile_mtx.Unlock()
+
+	return interval
+}
+
 func (c *Client) run_single_ctl_server(i int, cs *http.Server, wg *sync.WaitGroup) {
 	var l net.Listener
 	var err error

cmd/config.go

@@ -107,6 +107,8 @@ type ServerAppConfig  struct {
 	MaxPeers        int       `yaml:"max-peer-conns"` // maximum number of connections from peers
 	MaxRpcConns     int       `yaml:"max-rpc-conns"` // maximum number of rpc connections
 	MinRpcPingIntvl time.Duration  `yaml:"min-rpc-ping-interval"`
+	RxcDoneJobRetention *time.Duration  `yaml:"rxc-done-job-retention"`
+	RxcRunOutputMax *int `yaml:"rxc-run-output-max"`
 	HttpReadHeaderTimeout time.Duration `yaml:"http-read-header-timeout"`
 	HttpIdleTimeout       time.Duration `yaml:"http-idle-timeout"`
 	HttpMaxHeaderBytes    int           `yaml:"http-max-header-bytes"`
@@ -126,14 +128,17 @@ type ClientAppConfig  struct {
 	HttpReadHeaderTimeout time.Duration `yaml:"http-read-header-timeout"`
 	HttpIdleTimeout       time.Duration `yaml:"http-idle-timeout"`
 	HttpMaxHeaderBytes    int           `yaml:"http-max-header-bytes"`
-	TokenText     string        `yaml:"token-text"`
+	TokenText       string        `yaml:"token-text"`
-	TokenFile     string        `yaml:"token-file"`
+	TokenFile       string        `yaml:"token-file"`
-	PtyUser       string        `yaml:"pty-user"`
+	PtyUser         string        `yaml:"pty-user"`
-	PtyShell      string        `yaml:"pty-shell"`
+	PtyShell        string        `yaml:"pty-shell"`
-	RpcPingIntvl  time.Duration `yaml:"rpc-ping-interval"`
+	RxcUser         string        `yaml:"rxc-user"`
-	RpcPingTmout  time.Duration `yaml:"rpc-ping-timeout"`
+	RxcProfileFiles []string      `yaml:"rxc-profile-files"`
-	RpcSeedTmout  time.Duration `yaml:"rpc-seed-timeout"`
+	RxcProfileReloadMinInterval *time.Duration `yaml:"rxc-profile-reload-min-interval"`
-	XtermHtmlFile string        `yaml:"xterm-html-file"`
+	RpcPingIntvl    time.Duration `yaml:"rpc-ping-interval"`
+	RpcPingTmout    time.Duration `yaml:"rpc-ping-timeout"`
+	RpcSeedTmout    time.Duration `yaml:"rpc-seed-timeout"`
+	XtermHtmlFile   string        `yaml:"xterm-html-file"`
 }
 
 type ServerConfig struct {

cmd/main.go

@@ -15,6 +15,7 @@ import "strings"
 import "sync"
 import "sync/atomic"
 import "syscall"
+import "time"
 
 // Don't change these items to 'const' as they can be overridden externally with a linker option
 var HODU_NAME string = "hodu"
@@ -151,6 +152,17 @@ func server_main(ctl_addrs []string, rpc_addrs []string, rpx_addrs[] string, pxy
 	config.RpcMaxConns = cfg.APP.MaxRpcConns
 	config.RpcMinPingIntvl = cfg.APP.MinRpcPingIntvl
 	config.MaxPeers = cfg.APP.MaxPeers
+	if cfg.APP.RxcDoneJobRetention == nil {
+		// default to 60 seconds
+		config.RxcDoneJobRetention = hodu.SERVER_RXC_DONE_JOB_RETENTION
+	} else {
+		config.RxcDoneJobRetention = *cfg.APP.RxcDoneJobRetention
+	}
+	if cfg.APP.RxcRunOutputMax == nil {
+		config.RxcRunOutputMax = hodu.SERVER_RXC_RUN_OUTPUT_MAX
+	} else {
+		config.RxcRunOutputMax = *cfg.APP.RxcRunOutputMax
+	}
 	config.HttpReadHeaderTimeout = cfg.APP.HttpReadHeaderTimeout
 	config.HttpIdleTimeout = cfg.APP.HttpIdleTimeout
 	config.HttpMaxHeaderBytes = cfg.APP.HttpMaxHeaderBytes
@@ -274,6 +286,9 @@ func client_main(ctl_addrs []string, rpc_addrs []string, route_configs []string,
 	var logger *AppLogger
 	var pty_user string
 	var pty_shell string
+	var rxc_user string
+	var rxc_profile_files []string
+	var rxc_profile_reload_min_interval time.Duration
 	var xterm_html_file string
 	var xterm_html string
 	var i int
@@ -306,6 +321,13 @@ func client_main(ctl_addrs []string, rpc_addrs []string, route_configs []string,
 	cc.ServerAuthority = cfg.RPC.Endpoint.Authority
 	pty_user = cfg.APP.PtyUser
 	pty_shell = cfg.APP.PtyShell
+	rxc_user = cfg.APP.RxcUser
+	rxc_profile_files = cfg.APP.RxcProfileFiles
+	if cfg.APP.RxcProfileReloadMinInterval == nil {
+		rxc_profile_reload_min_interval = hodu.CLIENT_RXC_PROFILE_RELOAD_MIN_INTERVAL
+	} else {
+		rxc_profile_reload_min_interval = *cfg.APP.RxcProfileReloadMinInterval
+	}
 	xterm_html_file = cfg.APP.XtermHtmlFile
 	config.RpcConnMax = cfg.APP.MaxRpcConns
 	config.PeerConnMax = cfg.APP.MaxPeers
@@ -363,6 +385,9 @@ func client_main(ctl_addrs []string, rpc_addrs []string, route_configs []string,
 
 	if pty_user != "" { c.SetPtyUser(pty_user) }
 	if pty_shell != "" { c.SetPtyShell(pty_shell) }
+	if rxc_user != "" { c.SetRxcUser(rxc_user) }
+	if len(rxc_profile_files) > 0 { c.SetRxcProfileFiles(rxc_profile_files) }
+	c.SetRxcProfileReloadMinInterval(rxc_profile_reload_min_interval)
 	if xterm_html != "" { c.SetXtermHtml(xterm_html) }
 
 	c.StartService(&cc)
@@ -490,11 +515,13 @@ func main() {
 		var logfile string
 		var client_token string
 		var pty_shell string
+		var rxc_profile_files []string
 		var rpx_target_addr string
 		var cfg ClientConfig
 
 		ctl_addrs = make([]string, 0)
 		rpc_addrs = make([]string, 0)
+		rxc_profile_files = make([]string, 0)
 
 		flgs = flag.NewFlagSet("", flag.ContinueOnError)
 		flgs.Func("ctl-on", "specify a listening address for control channel", func(v string) error {
@@ -521,6 +548,10 @@ func main() {
 			pty_shell = v
 			return nil
 		})
+		flgs.Func("rxc-profile-files", "specify a file pattern for rxc profiles", func(v string) error {
+			rxc_profile_files = append(rxc_profile_files, v)
+			return nil
+		})
 		flgs.Func("client-token", "specify a client token", func(v string) error {
 			client_token = v
 			return nil
@@ -572,6 +603,9 @@ func main() {
 		if pty_shell != "" {
 			cfg.APP.PtyShell = pty_shell
 		}
+		if len(rxc_profile_files) > 0 {
+			cfg.APP.RxcProfileFiles = rxc_profile_files
+		}
 		if rpx_target_addr != "" {
 			cfg.RPX.Target.Addr = rpx_target_addr
 		}
@@ -592,7 +626,7 @@ func main() {
 
 wrong_usage:
 	fmt.Fprintf(os.Stderr, "USAGE: %s server --rpc-on=addr:port --ctl-on=addr:port --rpx-on=addr:port --pxy-on=addr:port --wpx-on=addr:port [--config-file=file] [--config-file-pattern=pattern] [--pty-shell=string]\n", os.Args[0])
-	fmt.Fprintf(os.Stderr, "       %s client --rpc-to=addr:port --ctl-on=addr:port [--config-file=file] [--config-file-pattern=pattern] [--pty-shell=string] [--client-token=string] [--rpx-target-addr=addr:port] [peer-addr:peer-port ...]\n", os.Args[0])
+	fmt.Fprintf(os.Stderr, "       %s client --rpc-to=addr:port --ctl-on=addr:port [--config-file=file] [--config-file-pattern=pattern] [--pty-shell=string] [--rxc-profile-files=pattern] [--client-token=string] [--rpx-target-addr=addr:port] [peer-addr:peer-port ...]\n", os.Args[0])
 	fmt.Fprintf(os.Stderr, "       %s version\n", os.Args[0])
 	os.Exit(1)
 

hodu.pb.go

@@ -106,6 +106,9 @@ const (
 	PACKET_KIND_RPX_STOP      PACKET_KIND = 19
 	PACKET_KIND_RPX_DATA      PACKET_KIND = 20
 	PACKET_KIND_RPX_EOF       PACKET_KIND = 21
+	PACKET_KIND_RXC_START     PACKET_KIND = 22
+	PACKET_KIND_RXC_STOP      PACKET_KIND = 23
+	PACKET_KIND_RXC_DATA      PACKET_KIND = 24
 )
 
 // Enum value maps for PACKET_KIND.
@@ -132,6 +135,9 @@ var (
 		19: "RPX_STOP",
 		20: "RPX_DATA",
 		21: "RPX_EOF",
+		22: "RXC_START",
+		23: "RXC_STOP",
+		24: "RXC_DATA",
 	}
 	PACKET_KIND_value = map[string]int32{
 		"RESERVED":      0,
@@ -155,6 +161,9 @@ var (
 		"RPX_STOP":      19,
 		"RPX_DATA":      20,
 		"RPX_EOF":       21,
+		"RXC_START":     22,
+		"RXC_STOP":      23,
+		"RXC_DATA":      24,
 	}
 )
 
@@ -185,6 +194,9 @@ func (PACKET_KIND) EnumDescriptor() ([]byte, []int) {
 	return file_hodu_proto_rawDescGZIP(), []int{1}
 }
 
+// When you add more fields to the Seed message, don't forget to
+// add the fields to the SeedInfo struct in client.go and
+// the copying code from Seed To SeedInfo
 type Seed struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Version       uint32                 `protobuf:"varint,1,opt,name=Version,proto3" json:"Version,omitempty"`
@@ -707,6 +719,66 @@ func (x *RpxEvent) GetData() []byte {
 	return nil
 }
 
+type RxcEvent struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Id            uint64                 `protobuf:"varint,1,opt,name=Id,proto3" json:"Id,omitempty"`
+	Flags         uint64                 `protobuf:"varint,2,opt,name=Flags,proto3" json:"Flags,omitempty"`
+	Data          []byte                 `protobuf:"bytes,3,opt,name=Data,proto3" json:"Data,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *RxcEvent) Reset() {
+	*x = RxcEvent{}
+	mi := &file_hodu_proto_msgTypes[9]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *RxcEvent) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RxcEvent) ProtoMessage() {}
+
+func (x *RxcEvent) ProtoReflect() protoreflect.Message {
+	mi := &file_hodu_proto_msgTypes[9]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RxcEvent.ProtoReflect.Descriptor instead.
+func (*RxcEvent) Descriptor() ([]byte, []int) {
+	return file_hodu_proto_rawDescGZIP(), []int{9}
+}
+
+func (x *RxcEvent) GetId() uint64 {
+	if x != nil {
+		return x.Id
+	}
+	return 0
+}
+
+func (x *RxcEvent) GetFlags() uint64 {
+	if x != nil {
+		return x.Flags
+	}
+	return 0
+}
+
+func (x *RxcEvent) GetData() []byte {
+	if x != nil {
+		return x.Data
+	}
+	return nil
+}
+
 type Packet struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
 	Kind  PACKET_KIND            `protobuf:"varint,1,opt,name=Kind,proto3,enum=PACKET_KIND" json:"Kind,omitempty"`
@@ -720,6 +792,7 @@ type Packet struct {
 	//	*Packet_ConnNoti
 	//	*Packet_RptyEvt
 	//	*Packet_RpxEvt
+	//	*Packet_RxcEvt
 	U             isPacket_U `protobuf_oneof:"U"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
@@ -727,7 +800,7 @@ type Packet struct {
 
 func (x *Packet) Reset() {
 	*x = Packet{}
-	mi := &file_hodu_proto_msgTypes[9]
+	mi := &file_hodu_proto_msgTypes[10]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -739,7 +812,7 @@ func (x *Packet) String() string {
 func (*Packet) ProtoMessage() {}
 
 func (x *Packet) ProtoReflect() protoreflect.Message {
-	mi := &file_hodu_proto_msgTypes[9]
+	mi := &file_hodu_proto_msgTypes[10]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -752,7 +825,7 @@ func (x *Packet) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Packet.ProtoReflect.Descriptor instead.
 func (*Packet) Descriptor() ([]byte, []int) {
-	return file_hodu_proto_rawDescGZIP(), []int{9}
+	return file_hodu_proto_rawDescGZIP(), []int{10}
 }
 
 func (x *Packet) GetKind() PACKET_KIND {
@@ -841,6 +914,15 @@ func (x *Packet) GetRpxEvt() *RpxEvent {
 	return nil
 }
 
+func (x *Packet) GetRxcEvt() *RxcEvent {
+	if x != nil {
+		if x, ok := x.U.(*Packet_RxcEvt); ok {
+			return x.RxcEvt
+		}
+	}
+	return nil
+}
+
 type isPacket_U interface {
 	isPacket_U()
 }
@@ -877,6 +959,10 @@ type Packet_RpxEvt struct {
 	RpxEvt *RpxEvent `protobuf:"bytes,9,opt,name=RpxEvt,proto3,oneof"`
 }
 
+type Packet_RxcEvt struct {
+	RxcEvt *RxcEvent `protobuf:"bytes,10,opt,name=RxcEvt,proto3,oneof"`
+}
+
 func (*Packet_Route) isPacket_U() {}
 
 func (*Packet_Peer) isPacket_U() {}
@@ -893,6 +979,8 @@ func (*Packet_RptyEvt) isPacket_U() {}
 
 func (*Packet_RpxEvt) isPacket_U() {}
 
+func (*Packet_RxcEvt) isPacket_U() {}
+
 var File_hodu_proto protoreflect.FileDescriptor
 
 const file_hodu_proto_rawDesc = "" +
@@ -933,7 +1021,11 @@ const file_hodu_proto_rawDesc = "" +
 	"\x04Data\x18\x02 \x01(\fR\x04Data\".\n" +
 	"\bRpxEvent\x12\x0e\n" +
 	"\x02Id\x18\x01 \x01(\x04R\x02Id\x12\x12\n" +
-	"\x04Data\x18\x02 \x01(\fR\x04Data\"\xd6\x02\n" +
+	"\x04Data\x18\x02 \x01(\fR\x04Data\"D\n" +
+	"\bRxcEvent\x12\x0e\n" +
+	"\x02Id\x18\x01 \x01(\x04R\x02Id\x12\x14\n" +
+	"\x05Flags\x18\x02 \x01(\x04R\x05Flags\x12\x12\n" +
+	"\x04Data\x18\x03 \x01(\fR\x04Data\"\xfb\x02\n" +
 	"\x06Packet\x12 \n" +
 	"\x04Kind\x18\x01 \x01(\x0e2\f.PACKET_KINDR\x04Kind\x12\"\n" +
 	"\x05Route\x18\x02 \x01(\v2\n" +
@@ -946,7 +1038,9 @@ const file_hodu_proto_rawDesc = "" +
 	"\bConnNoti\x18\a \x01(\v2\v.ConnNoticeH\x00R\bConnNoti\x12&\n" +
 	"\aRptyEvt\x18\b \x01(\v2\n" +
 	".RptyEventH\x00R\aRptyEvt\x12#\n" +
-	"\x06RpxEvt\x18\t \x01(\v2\t.RpxEventH\x00R\x06RpxEvtB\x03\n" +
+	"\x06RpxEvt\x18\t \x01(\v2\t.RpxEventH\x00R\x06RpxEvt\x12#\n" +
+	"\x06RxcEvt\x18\n" +
+	" \x01(\v2\t.RxcEventH\x00R\x06RxcEvtB\x03\n" +
 	"\x01U*U\n" +
 	"\fROUTE_OPTION\x12\n" +
 	"\n" +
@@ -956,7 +1050,7 @@ const file_hodu_proto_rawDesc = "" +
 	"\x04TCP6\x10\x04\x12\b\n" +
 	"\x04HTTP\x10\b\x12\t\n" +
 	"\x05HTTPS\x10\x10\x12\a\n" +
-	"\x03SSH\x10 *\xda\x02\n" +
+	"\x03SSH\x10 *\x85\x03\n" +
 	"\vPACKET_KIND\x12\f\n" +
 	"\bRESERVED\x10\x00\x12\x0f\n" +
 	"\vROUTE_START\x10\x01\x12\x0e\n" +
@@ -981,7 +1075,10 @@ const file_hodu_proto_rawDesc = "" +
 	"\tRPX_START\x10\x12\x12\f\n" +
 	"\bRPX_STOP\x10\x13\x12\f\n" +
 	"\bRPX_DATA\x10\x14\x12\v\n" +
-	"\aRPX_EOF\x10\x152I\n" +
+	"\aRPX_EOF\x10\x15\x12\r\n" +
+	"\tRXC_START\x10\x16\x12\f\n" +
+	"\bRXC_STOP\x10\x17\x12\f\n" +
+	"\bRXC_DATA\x10\x182I\n" +
 	"\x04Hodu\x12\x19\n" +
 	"\aGetSeed\x12\x05.Seed\x1a\x05.Seed\"\x00\x12&\n" +
 	"\fPacketStream\x12\a.Packet\x1a\a.Packet\"\x00(\x010\x01B\bZ\x06./hodub\x06proto3"
@@ -999,7 +1096,7 @@ func file_hodu_proto_rawDescGZIP() []byte {
 }
 
 var file_hodu_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
-var file_hodu_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
+var file_hodu_proto_msgTypes = make([]protoimpl.MessageInfo, 11)
 var file_hodu_proto_goTypes = []any{
 	(ROUTE_OPTION)(0),  // 0: ROUTE_OPTION
 	(PACKET_KIND)(0),   // 1: PACKET_KIND
@@ -1012,7 +1109,8 @@ var file_hodu_proto_goTypes = []any{
 	(*ConnNotice)(nil), // 8: ConnNotice
 	(*RptyEvent)(nil),  // 9: RptyEvent
 	(*RpxEvent)(nil),   // 10: RpxEvent
-	(*Packet)(nil),     // 11: Packet
+	(*RxcEvent)(nil),   // 11: RxcEvent
+	(*Packet)(nil),     // 12: Packet
 }
 var file_hodu_proto_depIdxs = []int32{
 	1,  // 0: Packet.Kind:type_name -> PACKET_KIND
@@ -1024,15 +1122,16 @@ var file_hodu_proto_depIdxs = []int32{
 	8,  // 6: Packet.ConnNoti:type_name -> ConnNotice
 	9,  // 7: Packet.RptyEvt:type_name -> RptyEvent
 	10, // 8: Packet.RpxEvt:type_name -> RpxEvent
-	2,  // 9: Hodu.GetSeed:input_type -> Seed
+	11, // 9: Packet.RxcEvt:type_name -> RxcEvent
-	11, // 10: Hodu.PacketStream:input_type -> Packet
+	2,  // 10: Hodu.GetSeed:input_type -> Seed
-	2,  // 11: Hodu.GetSeed:output_type -> Seed
+	12, // 11: Hodu.PacketStream:input_type -> Packet
-	11, // 12: Hodu.PacketStream:output_type -> Packet
+	2,  // 12: Hodu.GetSeed:output_type -> Seed
-	11, // [11:13] is the sub-list for method output_type
+	12, // 13: Hodu.PacketStream:output_type -> Packet
-	9,  // [9:11] is the sub-list for method input_type
+	12, // [12:14] is the sub-list for method output_type
-	9,  // [9:9] is the sub-list for extension type_name
+	10, // [10:12] is the sub-list for method input_type
-	9,  // [9:9] is the sub-list for extension extendee
+	10, // [10:10] is the sub-list for extension type_name
-	0,  // [0:9] is the sub-list for field type_name
+	10, // [10:10] is the sub-list for extension extendee
+	0,  // [0:10] is the sub-list for field type_name
 }
 
 func init() { file_hodu_proto_init() }
@@ -1040,7 +1139,7 @@ func file_hodu_proto_init() {
 	if File_hodu_proto != nil {
 		return
 	}
-	file_hodu_proto_msgTypes[9].OneofWrappers = []any{
+	file_hodu_proto_msgTypes[10].OneofWrappers = []any{
 		(*Packet_Route)(nil),
 		(*Packet_Peer)(nil),
 		(*Packet_Data)(nil),
@@ -1049,6 +1148,7 @@ func file_hodu_proto_init() {
 		(*Packet_ConnNoti)(nil),
 		(*Packet_RptyEvt)(nil),
 		(*Packet_RpxEvt)(nil),
+		(*Packet_RxcEvt)(nil),
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -1056,7 +1156,7 @@ func file_hodu_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_hodu_proto_rawDesc), len(file_hodu_proto_rawDesc)),
 			NumEnums:      2,
-			NumMessages:   10,
+			NumMessages:   11,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

hodu.proto

@@ -93,6 +93,12 @@ message RpxEvent {
 	bytes  Data = 2;
 };
 
+message RxcEvent {
+	uint64 Id = 1;
+	uint64 Flags = 2;
+	bytes  Data = 3;
+};
+
 enum PACKET_KIND {
 	RESERVED      = 0; // not used
 	ROUTE_START   = 1;
@@ -117,6 +123,10 @@ enum PACKET_KIND {
 	RPX_STOP      = 19;
 	RPX_DATA      = 20;
 	RPX_EOF       = 21;
+
+	RXC_START     = 22;
+	RXC_STOP      = 23;
+	RXC_DATA      = 24;
 };
 
 message Packet {
@@ -131,5 +141,6 @@ message Packet {
 		ConnNotice ConnNoti = 7;
 		RptyEvent  RptyEvt  = 8;
 		RpxEvent   RpxEvt   = 9;
+		RxcEvent   RxcEvt = 10;
 	};
 }

packet.go

@@ -1,10 +1,28 @@
 package hodu
 
+import "bytes"
+import "encoding/gob"
+import "os"
+import "syscall"
+
 type ConnId      uint64
 type RouteId     uint32 // keep this in sync with the type of RouteId in hodu.proto
 type PeerId      uint32 // keep this in sync with the type of RouteId in hodu.proto
 type RouteOption uint32
 
+// low 16 bits reserved for various flag bits
+const RXC_FLAG_BITS uint64 = 16
+const RXC_FLAG_MASK uint64 = (uint64(1) << RXC_FLAG_BITS) - 1
+const RXC_DATA_FLAG_NONE uint64 = 0
+const RXC_DATA_FLAG_STDERR uint64 = 1 << 0
+
+// the 32 bits reserved for exit status
+const RXC_STOP_FLAG_STATUS_VALID uint64 = 1 << 0
+const RXC_STOP_STATUS_SHIFT uint64 = RXC_FLAG_BITS
+const RXC_STOP_STATUS_MASK uint64 = uint64(0xFFFFFFFF) << RXC_STOP_STATUS_SHIFT
+
+// we still have high 16 bits unused.
+
 type ConnRouteId struct {
 	conn_id ConnId
 	route_id RouteId
@@ -109,3 +127,44 @@ func MakeRpxDataPacket(id uint64, data_part []byte) *Packet {
 func MakeRpxEofPacket(id uint64) *Packet {
 	return &Packet{Kind: PACKET_KIND_RPX_EOF, U: &Packet_RpxEvt{RpxEvt: &RpxEvent{Id: id}}}
 }
+
+
+func MakeRxcStartPacket(id uint64, kind string, script string) (*Packet, error) {
+	var enc *gob.Encoder
+	var buf bytes.Buffer;
+	var err error
+
+	enc = gob.NewEncoder(&buf)
+	err = enc.Encode([]string{ kind, script });
+	if err != nil { return nil, err }
+	return &Packet{Kind: PACKET_KIND_RXC_START, U: &Packet_RxcEvt{RxcEvt: &RxcEvent{Id: id, Data: buf.Bytes()}}}, nil
+}
+
+func MakeRxcStopFlagsFromProcessState(ps *os.ProcessState) uint64 {
+	var ws syscall.WaitStatus
+	var ok bool
+
+	if ps == nil { return 0 }
+
+	ws, ok = ps.Sys().(syscall.WaitStatus)
+	if !ok { return 0 }
+
+	return RXC_STOP_FLAG_STATUS_VALID | (uint64(uint32(ws)) << RXC_STOP_STATUS_SHIFT)
+}
+
+func GetRxcStopExitCode(flags uint64) int {
+	var ws syscall.WaitStatus
+	if flags & RXC_STOP_FLAG_STATUS_VALID == 0 { return -1 }
+	ws = syscall.WaitStatus(uint32((flags & RXC_STOP_STATUS_MASK) >> RXC_STOP_STATUS_SHIFT))
+	if !ws.Exited() { return -1 }
+	return ws.ExitStatus()
+}
+
+func MakeRxcStopPacket(id uint64, flags uint64, msg string) *Packet {
+	// the rpty stop conveys an error/info message
+	return &Packet{Kind: PACKET_KIND_RXC_STOP, U: &Packet_RxcEvt{RxcEvt: &RxcEvent{Id: id, Flags: flags, Data: []byte(msg)}}}
+}
+
+func MakeRxcDataPacket(id uint64, flags uint64, data []byte) *Packet {
+	return &Packet{Kind: PACKET_KIND_RXC_DATA, U: &Packet_RxcEvt{RxcEvt: &RxcEvent{Id: id, Flags: flags, Data: data}}}
+}

server-ctl.go

@@ -79,6 +79,7 @@ type json_out_server_stats struct {
 	ServerPtySessions int64 `json:"server-pty-sessions"`
 	ServerRptySessions int64 `json:"server-rpty-sessions"`
 	ServerRpxSessions int64 `json:"server-rpx-sessions"`
+	ServerRxcSessions int64 `json:"server-rxc-sessions"`
 }
 
 // this is a more specialized variant of json_in_notice
@@ -240,7 +241,7 @@ func (ctl *server_ctl_server_conns) ServeHTTP(w http.ResponseWriter, req *http.R
 			var ci ConnId
 
 			js = make([]json_out_server_conn, 0)
-			s.cts_mtx.Lock()
+			s.cts_mtx.RLock()
 			for _, ci = range s.cts_map.get_sorted_keys() {
 				var cts *ServerConn
 				var jsp []json_out_server_route
@@ -276,7 +277,7 @@ func (ctl *server_ctl_server_conns) ServeHTTP(w http.ResponseWriter, req *http.R
 					Routes: jsp,
 				})
 			}
-			s.cts_mtx.Unlock()
+			s.cts_mtx.RUnlock()
 
 			status_code = WriteJsonRespHeader(w, http.StatusOK)
 			if err = je.Encode(js); err != nil { goto oops }
@@ -925,6 +926,7 @@ func (ctl *server_ctl_stats) ServeHTTP(w http.ResponseWriter, req *http.Request)
 			stats.ServerPtySessions = s.stats.pty_sessions.Load()
 			stats.ServerRptySessions = s.stats.rpty_sessions.Load()
 			stats.ServerRpxSessions = s.stats.rpx_sessions.Load()
+			stats.ServerRxcSessions = s.stats.rxc_sessions.Load()
 			status_code = WriteJsonRespHeader(w, http.StatusOK)
 			if err = je.Encode(stats); err != nil { goto oops }
 

server-cts-rpty.go

@@ -0,0 +1,200 @@
+package hodu
+
+import "encoding/base64"
+import "fmt"
+
+import "golang.org/x/net/websocket"
+
+// Rpty
+func (cts *ServerConn) StartRpty(ws *websocket.Conn) (*ServerRpty, error) {
+	var ok bool
+	var start_id uint64
+	var assigned_id uint64
+	var rpty *ServerRpty
+	var err error
+
+	cts.rpty_mtx.Lock()
+	start_id = cts.rpty_next_id
+	for {
+		_, ok = cts.rpty_map[cts.rpty_next_id]
+		if !ok {
+			assigned_id = cts.rpty_next_id
+			cts.rpty_next_id++
+			if cts.rpty_next_id == 0 { cts.rpty_next_id++ }
+			break
+		}
+		cts.rpty_next_id++
+		if cts.rpty_next_id == 0 { cts.rpty_next_id++ }
+		if cts.rpty_next_id == start_id {
+			cts.rpty_mtx.Unlock()
+			return nil, fmt.Errorf("unable to assign id")
+		}
+	}
+
+	_, ok = cts.rpty_map_by_ws[ws]
+	if ok {
+		cts.rpty_mtx.Unlock()
+		return nil, fmt.Errorf("connection already associated with rpty. possibly internal error")
+	}
+
+	rpty = &ServerRpty{
+		id: assigned_id,
+		ws: ws,
+	}
+
+	cts.rpty_map[assigned_id] = rpty
+	cts.rpty_map_by_ws[ws] = rpty
+	cts.rpty_mtx.Unlock()
+
+	err = cts.pss.Send(MakeRptyStartPacket(assigned_id))
+	if err != nil {
+		cts.rpty_mtx.Lock()
+		delete(cts.rpty_map, assigned_id)
+		delete(cts.rpty_map_by_ws, ws)
+		cts.rpty_mtx.Unlock()
+		return nil , err
+	}
+
+	cts.S.stats.rpty_sessions.Add(1)
+	return rpty, nil
+}
+
+func (cts *ServerConn) StopRpty(ws *websocket.Conn) error {
+	// called by the websocket handler.
+	var rpty *ServerRpty
+	var id uint64
+	var ok bool
+	var err error
+
+	cts.rpty_mtx.Lock()
+	rpty, ok = cts.rpty_map_by_ws[ws]
+	if !ok {
+		cts.rpty_mtx.Unlock()
+		cts.S.log.Write(cts.Sid, LOG_ERROR, "Unknown websocket connection for rpty - websocket %v", ws.RemoteAddr())
+		return fmt.Errorf("unknown websocket connection for rpty - %v", ws.RemoteAddr())
+	}
+
+	id = rpty.id
+	cts.rpty_mtx.Unlock()
+
+	// send the stop request to the client side
+	err = cts.pss.Send(MakeRptyStopPacket(id, ""))
+	if err !=  nil {
+		cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to send %s(%d) for server %s websocket %v - %s", PACKET_KIND_RPTY_STOP.String(), id, cts.RemoteAddr, ws.RemoteAddr(), err.Error())
+		// carry on
+	}
+
+	// delete the rpty entry from the maps as the websocket
+	// handler is ending
+	cts.rpty_mtx.Lock()
+	delete(cts.rpty_map, id)
+	delete(cts.rpty_map_by_ws, ws)
+	cts.rpty_mtx.Unlock()
+	cts.S.stats.rpty_sessions.Add(-1)
+
+	cts.S.log.Write(cts.Sid, LOG_INFO, "Stopped rpty(%d) for server %s websocket %vs", id, cts.RemoteAddr, ws.RemoteAddr())
+	return nil
+}
+
+func (cts *ServerConn) StopRptyWsById(id uint64, msg string) error {
+	// call this when the stop requested comes from the client.
+	// abort the websocket side.
+
+	var rpty *ServerRpty
+	var ok bool
+
+	cts.rpty_mtx.Lock()
+	rpty, ok = cts.rpty_map[id]
+	if !ok {
+		cts.rpty_mtx.Unlock()
+		return fmt.Errorf("unknown rpty id %d", id)
+	}
+	cts.rpty_mtx.Unlock()
+
+	rpty.ReqStop()
+	cts.S.log.Write(cts.Sid, LOG_INFO, "Stopped rpty(%d) for %s - %s", id, cts.RemoteAddr, msg)
+	return nil
+}
+
+func (cts *ServerConn) WriteRpty(ws *websocket.Conn, data []byte) error {
+	var rpty *ServerRpty
+	var id uint64
+	var ok bool
+	var err error
+
+	cts.rpty_mtx.Lock()
+	rpty, ok = cts.rpty_map_by_ws[ws]
+	if !ok {
+		cts.rpty_mtx.Unlock()
+		return fmt.Errorf("unknown ws connection for rpty - %v", ws.RemoteAddr())
+	}
+
+	id = rpty.id
+	cts.rpty_mtx.Unlock()
+
+	err = cts.pss.Send(MakeRptyDataPacket(id, data))
+	if err != nil {
+		return fmt.Errorf("unable to send rpty data to client - %s", err.Error())
+	}
+
+	return nil
+}
+
+func (cts *ServerConn) WriteRptySize(ws *websocket.Conn, data []byte) error {
+	var rpty *ServerRpty
+	var id uint64
+	var ok bool
+	var err error
+
+	cts.rpty_mtx.Lock()
+	rpty, ok = cts.rpty_map_by_ws[ws]
+	if !ok {
+		cts.rpty_mtx.Unlock()
+		return fmt.Errorf("unknown ws connection for rpty size - %v", ws.RemoteAddr())
+	}
+
+	id = rpty.id
+	cts.rpty_mtx.Unlock()
+
+	err = cts.pss.Send(MakeRptySizePacket(id, data))
+	if err != nil {
+		return fmt.Errorf("unable to send rpty size to client - %s", err.Error())
+	}
+
+	return nil
+}
+
+func (cts *ServerConn) ReadRptyAndWriteWs(id uint64, data []byte) error {
+	var ok bool
+	var rpty *ServerRpty
+	var err error
+
+	cts.rpty_mtx.Lock()
+	rpty, ok = cts.rpty_map[id]
+	if !ok {
+		cts.rpty_mtx.Unlock()
+		return fmt.Errorf("unknown rpty id - %d", id)
+	}
+	cts.rpty_mtx.Unlock()
+
+	err = send_ws_data_for_xterm(rpty.ws, "iov", base64.StdEncoding.EncodeToString(data))
+	if err != nil {
+		return fmt.Errorf("failed to write rpty data(%d) to ws - %s", id, err.Error())
+	}
+
+	return nil
+}
+
+func (cts *ServerConn) HandleRptyEvent(packet_type PACKET_KIND, evt *RptyEvent) error {
+	switch packet_type {
+		case PACKET_KIND_RPTY_STOP:
+			// stop requested from the server
+			return cts.StopRptyWsById(evt.Id, string(evt.Data))
+
+		case PACKET_KIND_RPTY_DATA:
+			return cts.ReadRptyAndWriteWs(evt.Id, evt.Data)
+	}
+
+	// ignore other packet types
+	return nil
+}

server-cts-rpx.go

@@ -0,0 +1,63 @@
+package hodu
+
+import "fmt"
+
+// Rpx
+func (cts *ServerConn) StartRpxWebById(srpx* ServerRpx, id uint64, data []byte) error {
+	// pass the initial response to code in server-rpx.go
+	srpx.start_chan <- data
+	return nil
+}
+
+func (cts *ServerConn) StopRpxWebById(srpx* ServerRpx, id uint64) error {
+	cts.S.log.Write(cts.Sid, LOG_DEBUG, "Requesting to stop rpx(%d)", srpx.id)
+	srpx.ReqStop(true)
+	cts.S.log.Write(cts.Sid, LOG_DEBUG, "Requested to stop rpx(%d)", srpx.id)
+	return nil
+}
+
+func (cts *ServerConn) WriteRpxWebById(srpx* ServerRpx, id uint64, data []byte) error {
+	var err error
+	_, err = srpx.pw.Write(data)
+	if err != nil {
+		cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to write rpx data(%d) to rpx pipe - %s", id, err.Error())
+		srpx.ReqStop(true)
+	}
+	return err
+}
+
+func (cts *ServerConn) EofRpxWebById(srpx* ServerRpx, id uint64) error {
+	srpx.ReqStop(false)
+	return nil
+}
+
+func (cts *ServerConn) HandleRpxEvent(packet_type PACKET_KIND, evt *RpxEvent) error {
+	var ok bool
+	var rpx* ServerRpx
+
+	cts.rpx_mtx.Lock()
+	rpx, ok = cts.rpx_map[evt.Id]
+	if !ok {
+		cts.rpx_mtx.Unlock()
+		return fmt.Errorf("unknown rpx id - %v", evt.Id)
+	}
+	cts.rpx_mtx.Unlock()
+
+	switch packet_type {
+		case PACKET_KIND_RPX_START:
+			return cts.StartRpxWebById(rpx, evt.Id, evt.Data)
+
+		case PACKET_KIND_RPX_STOP:
+			// stop requested from the server
+			return cts.StopRpxWebById(rpx, evt.Id)
+
+		case PACKET_KIND_RPX_EOF:
+			return cts.EofRpxWebById(rpx, evt.Id)
+
+		case PACKET_KIND_RPX_DATA:
+			return cts.WriteRpxWebById(rpx, evt.Id, evt.Data)
+	}
+
+	// ignore other packet types
+	return nil
+}

server-cts-rxc.go

@@ -0,0 +1,225 @@
+package hodu
+
+// Rxc - remote exec (server-side triggered, client-side executed)
+
+import "fmt"
+
+import "golang.org/x/net/websocket"
+
+func (cts *ServerConn) add_new_rxc(sink ServerRxcSink, ws *websocket.Conn, kind string, script string) (*ServerRxc, error) {
+	var ok bool
+	var start_id uint64
+	var assigned_id uint64
+	var rxc *ServerRxc
+	var pkt *Packet
+	var err error
+
+	cts.rxc_mtx.Lock()
+	start_id = cts.rxc_next_id
+	for {
+		_, ok = cts.rxc_map[cts.rxc_next_id]
+		if !ok {
+			assigned_id = cts.rxc_next_id
+			cts.rxc_next_id++
+			if cts.rxc_next_id == 0 { cts.rxc_next_id++ }
+			break
+		}
+		cts.rxc_next_id++
+		if cts.rxc_next_id == 0 { cts.rxc_next_id++ }
+		if cts.rxc_next_id == start_id {
+			cts.rxc_mtx.Unlock()
+			return nil, fmt.Errorf("unable to assign id")
+		}
+	}
+
+	if ws != nil {
+		_, ok = cts.rxc_map_by_ws[ws]
+		if ok {
+			cts.rxc_mtx.Unlock()
+			return nil, fmt.Errorf("connection already associated with rxc. possibly internal error")
+		}
+	}
+
+	rxc = &ServerRxc{
+		id: assigned_id,
+		ws: ws,
+		sink: sink,
+	}
+
+	cts.rxc_map[assigned_id] = rxc
+	if ws != nil { cts.rxc_map_by_ws[ws] = rxc }
+	cts.rxc_mtx.Unlock()
+
+	pkt, err = MakeRxcStartPacket(assigned_id, kind, script)
+	if err != nil {
+		cts.rxc_mtx.Lock()
+		delete(cts.rxc_map, assigned_id)
+		if ws != nil { delete(cts.rxc_map_by_ws, ws) }
+		cts.rxc_mtx.Unlock()
+		return nil, fmt.Errorf("failed to make rxc start packet - %s", err.Error())
+	}
+
+	err = cts.pss.Send(pkt)
+	if err != nil {
+		cts.rxc_mtx.Lock()
+		delete(cts.rxc_map, assigned_id)
+		if ws != nil { delete(cts.rxc_map_by_ws, ws) }
+		cts.rxc_mtx.Unlock()
+		return nil, fmt.Errorf("failed to send rxc start packet - %s", err.Error())
+	}
+
+	cts.S.stats.rxc_sessions.Add(1)
+	return rxc, nil
+}
+
+func (cts *ServerConn) StartRxcForWs(ws *websocket.Conn, kind string, script string) (*ServerRxc, error) {
+	// start a single task over rxc for a websocket
+	return cts.add_new_rxc(&ServerRxcWebsocketSink{ws: ws}, ws, kind, script)
+}
+
+func (cts *ServerConn) RunRxcJob(run *ServerRxcJobRun, kind string, script string) error {
+	var rxc *ServerRxc
+	var err error
+
+	rxc, err = cts.add_new_rxc(run, nil, kind, script)
+	if err != nil { return err }
+	run.mark_started(rxc.id)
+	return nil
+}
+
+func (cts *ServerConn) find_rxc_by_ws(ws *websocket.Conn) (*ServerRxc, bool) {
+	var rxc *ServerRxc
+	var ok bool
+
+	cts.rxc_mtx.Lock()
+	rxc, ok = cts.rxc_map_by_ws[ws]
+	cts.rxc_mtx.Unlock()
+	return rxc, ok
+}
+
+func (cts *ServerConn) find_rxc_by_id(id uint64) (*ServerRxc, bool) {
+	var rxc *ServerRxc
+	var ok bool
+
+	cts.rxc_mtx.Lock()
+	rxc, ok = cts.rxc_map[id]
+	cts.rxc_mtx.Unlock()
+	return rxc, ok
+}
+
+func (cts *ServerConn) remove_rxc_by_id(id uint64) (*ServerRxc, bool) {
+	var rxc *ServerRxc
+	var ok bool
+
+	cts.rxc_mtx.Lock()
+	rxc, ok = cts.rxc_map[id]
+	if ok {
+		delete(cts.rxc_map, id)
+		if rxc.ws != nil {
+			delete(cts.rxc_map_by_ws, rxc.ws)
+		}
+	}
+	cts.rxc_mtx.Unlock()
+	return rxc, ok
+}
+
+func (cts *ServerConn) StopRxcForWs(ws *websocket.Conn) error {
+	// called by the websocket handler.
+	var rxc *ServerRxc
+	var ok bool
+	var err error
+
+	rxc, ok = cts.find_rxc_by_ws(ws)
+	if !ok {
+		cts.S.log.Write(cts.Sid, LOG_ERROR, "Unknown websocket connection for rxc - websocket %v", ws.RemoteAddr())
+		return fmt.Errorf("unknown websocket connection for rxc - %v", ws.RemoteAddr())
+	}
+
+	err = cts.pss.Send(MakeRxcStopPacket(rxc.id, 0, ""))
+	if err != nil {
+		cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to send %s(%d) for server %s websocket %v - %s", PACKET_KIND_RXC_STOP.String(), rxc.id, cts.RemoteAddr, ws.RemoteAddr(), err.Error())
+	}
+
+	_, ok = cts.remove_rxc_by_id(rxc.id)
+	if ok {
+		cts.S.stats.rxc_sessions.Add(-1)
+	}
+
+	rxc.ReqStop()
+	cts.S.log.Write(cts.Sid, LOG_INFO, "Stopped rxc(%d) for server %s websocket %v", rxc.id, cts.RemoteAddr, ws.RemoteAddr())
+	return nil
+}
+
+func (cts *ServerConn) SendStopRxcById(id uint64, flags uint64) error {
+	var ok bool
+	var err error
+
+	_, ok = cts.find_rxc_by_id(id)
+	if !ok {
+		return fmt.Errorf("unknown rxc id %d", id)
+	}
+
+	err = cts.pss.Send(MakeRxcStopPacket(id, flags, ""))
+	if err != nil {
+		return fmt.Errorf("failed to send %s(%d) to client - %s", PACKET_KIND_RXC_STOP.String(), id, err.Error())
+	}
+
+	return nil
+}
+
+func (cts *ServerConn) StopRxcSinkById(id uint64, flags uint64, msg string) error {
+	var rxc *ServerRxc
+	var ok bool
+	var err error
+
+	rxc, ok = cts.remove_rxc_by_id(id)
+	if !ok {
+		return fmt.Errorf("unknown rxc id %d", id)
+	}
+	cts.S.stats.rxc_sessions.Add(-1)
+
+	if rxc.sink != nil {
+		err = rxc.sink.Stop(flags, msg)
+		if err != nil { return err }
+	}
+
+	cts.S.log.Write(cts.Sid, LOG_INFO, "Stopped rxc job(%d) run for client(%s) from %s", id, cts.ClientToken.Get(), cts.RemoteAddr)
+	return nil
+}
+
+func (cts *ServerConn) WriteRxcForWs(ws *websocket.Conn, data []byte) error {
+	var rxc *ServerRxc
+	var ok bool
+	var err error
+
+	rxc, ok = cts.find_rxc_by_ws(ws)
+	if !ok { return fmt.Errorf("unknown ws connection for rxc - %v", ws.RemoteAddr()) }
+
+	err = cts.pss.Send(MakeRxcDataPacket(rxc.id, RXC_DATA_FLAG_NONE, data))
+	if err != nil { return fmt.Errorf("unable to send rxc data to client - %s", err.Error()) }
+
+	return nil
+}
+
+func (cts *ServerConn) ReadRxcAndWriteSinkById(id uint64, flags uint64, data []byte) error {
+	var rxc *ServerRxc
+	var ok bool
+
+	rxc, ok = cts.find_rxc_by_id(id)
+	if !ok { return fmt.Errorf("unknown rxc id - %d", id) }
+	if rxc.sink == nil { return fmt.Errorf("missing rxc sink for id %d", id) }
+
+	return rxc.sink.Write(flags, data)
+}
+
+func (cts *ServerConn) HandleRxcEvent(packet_type PACKET_KIND, evt *RxcEvent) error {
+	switch packet_type {
+		case PACKET_KIND_RXC_STOP:
+			return cts.StopRxcSinkById(evt.Id, evt.Flags, string(evt.Data))
+
+		case PACKET_KIND_RXC_DATA:
+			return cts.ReadRxcAndWriteSinkById(evt.Id, evt.Flags, evt.Data)
+	}
+
+	return nil
+}

server-metrics.go

@@ -14,6 +14,7 @@ type ServerCollector struct {
 	PtySessions      *prometheus.Desc
 	RptySessions     *prometheus.Desc
 	RpxSessions      *prometheus.Desc
+	RxcSessions      *prometheus.Desc
 }
 
 // NewServerCollector returns a new ServerCollector with all prometheus.Desc initialized
@@ -70,6 +71,11 @@ func NewServerCollector(server *Server) ServerCollector {
 			"Number of rpx session",
 			nil, nil,
 		),
+		RxcSessions: prometheus.NewDesc(
+			prefix + "rxc_sessions",
+			"Number of rxc session",
+			nil, nil,
+		),
 	}
 }
 
@@ -82,6 +88,7 @@ func (c ServerCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.PtySessions
 	ch <- c.RptySessions
 	ch <- c.RpxSessions
+	ch <- c.RxcSessions
 }
 
 func (c ServerCollector) Collect(ch chan<- prometheus.Metric) {
@@ -136,4 +143,10 @@ func (c ServerCollector) Collect(ch chan<- prometheus.Metric) {
 		prometheus.GaugeValue,
 		float64(c.server.stats.rpx_sessions.Load()),
 	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.RxcSessions,
+		prometheus.GaugeValue,
+		float64(c.server.stats.rxc_sessions.Load()),
+	)
 }

server-pty.go

@@ -73,6 +73,8 @@ func (pty *server_pty_ws) ServeWebsocket(ws *websocket.Conn) (int, error) {
 			var poll_fds []unix.PollFd
 			var buf [2048]byte
 			var n int
+			var out_revents int16
+			var sig_revents int16
 			var err error
 
 			poll_fds = []unix.PollFd{
@@ -92,16 +94,10 @@ func (pty *server_pty_ws) ServeWebsocket(ws *websocket.Conn) (int, error) {
 					continue
 				}
 
-				if (poll_fds[0].Revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL)) != 0 {
+				out_revents = poll_fds[0].Revents
-					s.log.Write(pty.Id, LOG_DEBUG, "[%s] EOF detected on pty stdout", req.RemoteAddr)
+				sig_revents = poll_fds[1].Revents
-					break
-				}
-				if (poll_fds[1].Revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL)) != 0 {
-					s.log.Write(pty.Id, LOG_DEBUG, "[%s] EOF detected on pty event pipe", req.RemoteAddr)
-					break
-				}
 
-				if (poll_fds[0].Revents & unix.POLLIN) != 0 {
+				if (out_revents & unix.POLLIN) != 0 {
 					n, err = out.Read(buf[:])
 					if n > 0 {
 						var err2 error
@@ -118,8 +114,21 @@ func (pty *server_pty_ws) ServeWebsocket(ws *websocket.Conn) (int, error) {
 						break
 					}
 				}
-				if (poll_fds[1].Revents & unix.POLLIN) != 0 {
+				if (sig_revents & unix.POLLIN) != 0 {
-					s.log.Write(pty.Id, LOG_DEBUG, "[%s] Stop request noticed on pty event pipe", req.RemoteAddr)
+					s.log.Write(pty.Id, LOG_DEBUG, "[%s] Stop request noticed on pty signal pipe", req.RemoteAddr)
+					break
+				}
+
+				if (out_revents & (unix.POLLERR | unix.POLLNVAL)) != 0 {
+					s.log.Write(pty.Id, LOG_DEBUG, "[%s] Error detected on pty stdout", req.RemoteAddr)
+					break
+				}
+				if (sig_revents & (unix.POLLERR | unix.POLLHUP | unix.POLLNVAL)) != 0 {
+					s.log.Write(pty.Id, LOG_DEBUG, "[%s] EOF detected on pty signal pipe", req.RemoteAddr)
+					break
+				}
+				if (out_revents & unix.POLLHUP) != 0 && (out_revents & unix.POLLIN) == 0 {
+					s.log.Write(pty.Id, LOG_DEBUG, "[%s] EOF detected on pty stdout", req.RemoteAddr)
 					break
 				}
 			}
@@ -155,7 +164,7 @@ ws_recv_loop:
 
 								err = unix.Pipe(pfd[:])
 								if err != nil {
-									s.log.Write(pty.Id, LOG_ERROR, "[%s] Failed to create event pipe for pty - %s", req.RemoteAddr, err.Error())
+									s.log.Write(pty.Id, LOG_ERROR, "[%s] Failed to create signal pipe for pty - %s", req.RemoteAddr, err.Error())
 									send_ws_data_for_xterm(ws, "error", err.Error())
 									//ws.Close() // dirty way to flag out the error
 									ws.SetReadDeadline(time.Now()) // slightly cleaner way to break the main loop
@@ -225,6 +234,10 @@ ws_recv_loop:
 							s.log.Write(pty.Id, LOG_DEBUG, "[%s] Resized terminal to %d,%d", req.RemoteAddr, rows, cols)
 							// ignore error
 						}
+
+					default:
+						send_ws_data_for_xterm(ws, "error", fmt.Sprintf("invalid pty event type - %s", ev.Type));
+						s.log.Write(pty.Id, LOG_WARN, "[%s] Invalid pty event type received - %s", req.RemoteAddr, ev.Type)
 				}
 			}
 		}
@@ -247,7 +260,7 @@ done:
 	if cmd != nil { cmd.Wait() }
 	wg.Wait()
 
-	// close the event pipe after all goroutines are over
+	// close the signal pipe after all goroutines are over
 	if pfd[0] >= 0 { unix.Close(pfd[0]) }
 	if pfd[1] >= 0 { unix.Close(pfd[1]) }
 
@@ -348,6 +361,10 @@ ws_recv_loop:
 							s.log.Write(rpty.Id, LOG_DEBUG, "[%s] Requested to resize rpty terminal to %s,%s", req.RemoteAddr, ev.Data[0], ev.Data[1])
 							// ignore error
 						}
+
+					default:
+						send_ws_data_for_xterm(ws, "error", fmt.Sprintf("invalid rpty event type - %s", ev.Type));
+						s.log.Write(rpty.Id, LOG_WARN, "[%s] Invalid rpty event type received - %s", req.RemoteAddr, ev.Type)
 				}
 			}
 		}

server-rpx.go

@@ -60,7 +60,7 @@ func (rpx* server_rpx) handle_header_data(rpx_id uint64, data []byte, w http.Res
 	line = strings.TrimRight(line, "\r\n")
 
 	flds = strings.Fields(line)
-	if (len(flds) < 2) { // i care about the status code..
+	if len(flds) < 2 { // i care about the status code..
 		return http.StatusBadGateway, fmt.Errorf("invalid response status for rpx(%d) - %s", rpx_id, line)
 	}
 	status_code, err = strconv.Atoi(flds[1])

server-rxc-job.go

@@ -0,0 +1,843 @@
+package hodu
+
+import "container/heap"
+import "encoding/base64"
+import "fmt"
+import "sort"
+import "strconv"
+import "strings"
+import "sync"
+import "time"
+import "unsafe"
+
+import "golang.org/x/net/websocket"
+
+const SERVER_RXC_RUN_OUTPUT_MAX int = 1024
+const SERVER_RXC_DONE_JOB_RETENTION time.Duration = 60 * time.Second
+
+const SERVER_RXC_RUN_STATUS_STARTING string = "starting"
+const SERVER_RXC_RUN_STATUS_RUNNING string = "running"
+const SERVER_RXC_RUN_STATUS_STOPPING string = "stopping"
+const SERVER_RXC_RUN_STATUS_STOPPED string = "stopped"
+const SERVER_RXC_RUN_STATUS_FAILED string = "failed"
+
+type ServerRxcJob struct {
+	S *Server
+	Id uint64
+	Type string
+	Script string
+	Created time.Time
+	Done time.Time
+
+	expire_at time.Time
+	heap_index int
+
+	run_mtx sync.Mutex
+	run_map map[ConnId]*ServerRxcJobRun
+	active_run_count int
+	starting_run_count int
+	running_run_count int
+	stopping_run_count int
+	stopped_run_count int
+	failed_run_count int
+}
+
+type ServerRxcJobRun struct {
+	Job *ServerRxcJob
+	Cts *ServerConn
+	ConnId ConnId
+	ClientToken string
+	Created time.Time
+	Started time.Time
+	Stopped time.Time
+	RxcId uint64
+	Status string
+	StopFlags uint64
+	StopMsg string
+
+	Output [2][]byte
+	OutputTruncated [2]bool
+
+	mtx sync.Mutex
+}
+
+type ServerRxcJobMap map[uint64]*ServerRxcJob
+
+type ServerRxcJobExpiryHeap []*ServerRxcJob
+
+type ServerRxcSink interface {
+	ReqStop()
+	Write(flags uint64, data []byte) error
+	Stop(flags uint64, msg string) error
+}
+
+type ServerRxcWebsocketSink struct {
+	ws *websocket.Conn
+}
+
+// ServerRxcWebsocketSink - implements ServerRxcSink
+func (sink *ServerRxcWebsocketSink) ReqStop() {
+	if sink.ws != nil {
+		sink.ws.SetReadDeadline(time.Now())
+	}
+}
+
+func (sink *ServerRxcWebsocketSink) Write(flags uint64, data []byte) error {
+	// TODO: how to distinguish stdout and stderr? when sending iov, encode the info?
+	var data_type string
+	if flags & RXC_DATA_FLAG_STDERR != 0 {
+		data_type = "ioe" // stderr
+	} else {
+		data_type = "iov" // stdout
+	}
+	return send_ws_data_for_xterm(sink.ws, data_type, base64.StdEncoding.EncodeToString(data))
+}
+
+func (sink *ServerRxcWebsocketSink) Stop(flags uint64, msg string) error {
+	sink.ReqStop()
+	return nil
+}
+
+/* implement heap.Interface for ServerRxcJobExpiryMap */
+func (heap ServerRxcJobExpiryHeap) Len() int {
+	return len(heap)
+}
+
+func (heap ServerRxcJobExpiryHeap) Less(i int, j int) bool {
+	if heap[i].expire_at.Before(heap[j].expire_at) { return true }
+	if heap[j].expire_at.Before(heap[i].expire_at) { return false }
+	return heap[i].Id < heap[j].Id
+}
+
+func (heap ServerRxcJobExpiryHeap) Swap(i int, j int) {
+	var x *ServerRxcJob
+	var y *ServerRxcJob
+
+	x = heap[i]
+	y = heap[j]
+	heap[i] = y
+	heap[j] = x
+	if heap[i] != nil { heap[i].heap_index = i }
+	if heap[j] != nil { heap[j].heap_index = j }
+}
+
+func (heap *ServerRxcJobExpiryHeap) Push(x interface{}) {
+	var job *ServerRxcJob
+
+	job = x.(*ServerRxcJob)
+	job.heap_index = len(*heap)
+	*heap = append(*heap, job)
+}
+
+func (heap *ServerRxcJobExpiryHeap) Pop() interface{} {
+	var old ServerRxcJobExpiryHeap
+	var n int
+	var job *ServerRxcJob
+
+	old = *heap
+	n = len(old)
+	job = old[n - 1]
+	old[n - 1] = nil
+	job.heap_index = -1
+	*heap = old[:n - 1]
+	return job
+}
+
+// ServerRxcJobRun
+
+func new_server_rxc_job_run(job *ServerRxcJob, cts *ServerConn) *ServerRxcJobRun {
+	return &ServerRxcJobRun{
+		Job: job,
+		Cts: cts,
+		ConnId: cts.Id,
+		ClientToken: cts.ClientToken.Get(),
+		Created: time.Now(),
+		Status: SERVER_RXC_RUN_STATUS_STARTING,
+	}
+}
+
+func (job *ServerRxcJob) increment_run_status_count_no_lock(status string) {
+	switch status {
+		case SERVER_RXC_RUN_STATUS_STARTING:
+			job.starting_run_count++
+
+		case SERVER_RXC_RUN_STATUS_RUNNING:
+			job.running_run_count++
+
+		case SERVER_RXC_RUN_STATUS_STOPPING:
+			job.stopping_run_count++
+
+		case SERVER_RXC_RUN_STATUS_STOPPED:
+			job.stopped_run_count++
+
+		case SERVER_RXC_RUN_STATUS_FAILED:
+			job.failed_run_count++
+	}
+}
+
+func (job *ServerRxcJob) decrement_run_status_count_no_lock(status string) {
+	switch status {
+		case SERVER_RXC_RUN_STATUS_STARTING:
+			if job.starting_run_count > 0 { job.starting_run_count-- }
+
+		case SERVER_RXC_RUN_STATUS_RUNNING:
+			if job.running_run_count > 0 { job.running_run_count-- }
+
+		case SERVER_RXC_RUN_STATUS_STOPPING:
+			if job.stopping_run_count > 0 { job.stopping_run_count-- }
+
+		case SERVER_RXC_RUN_STATUS_STOPPED:
+			if job.stopped_run_count > 0 { job.stopped_run_count-- }
+
+		case SERVER_RXC_RUN_STATUS_FAILED:
+			if job.failed_run_count > 0 { job.failed_run_count-- }
+	}
+}
+
+func (run *ServerRxcJobRun) mark_started(rxc_id uint64) {
+	run.mtx.Lock()
+	run.RxcId = rxc_id
+	if run.Started.IsZero() {
+		run.Started = time.Now()
+	}
+	if run.Status == SERVER_RXC_RUN_STATUS_STARTING {
+		run.Job.run_mtx.Lock()
+		run.Job.decrement_run_status_count_no_lock(run.Status)
+		run.Job.increment_run_status_count_no_lock(SERVER_RXC_RUN_STATUS_RUNNING)
+		run.Job.run_mtx.Unlock()
+		run.Status = SERVER_RXC_RUN_STATUS_RUNNING
+	}
+	run.mtx.Unlock()
+}
+
+func (run *ServerRxcJobRun) mark_start_failure(msg string) {
+	var transitioned bool
+
+	run.mtx.Lock()
+	if run.Status != SERVER_RXC_RUN_STATUS_STOPPED && run.Status != SERVER_RXC_RUN_STATUS_FAILED {
+		run.Job.run_mtx.Lock()
+		run.Job.decrement_run_status_count_no_lock(run.Status)
+		run.Job.increment_run_status_count_no_lock(SERVER_RXC_RUN_STATUS_FAILED)
+		run.Job.run_mtx.Unlock()
+		run.Stopped = time.Now()
+		run.Status = SERVER_RXC_RUN_STATUS_FAILED
+		run.StopMsg = msg
+		transitioned = true
+	}
+	run.mtx.Unlock()
+
+	if transitioned {
+		run.Cts.S.FireRxcJobRunEvent(SERVER_EVENT_RXC_JOB_RUN_DONE, run)
+		if run.Cts.S.maybe_mark_rxc_job_done(run.Job) {
+			run.Cts.S.notify_rxc_job_purge()
+		}
+	}
+}
+
+func (run *ServerRxcJobRun) request_stop() (*ServerConn, uint64, bool) {
+	var cts *ServerConn
+	var rxc_id uint64
+
+	run.mtx.Lock()
+	if run.Status == SERVER_RXC_RUN_STATUS_RUNNING {
+		run.Job.run_mtx.Lock()
+		run.Job.decrement_run_status_count_no_lock(run.Status)
+		run.Job.increment_run_status_count_no_lock(SERVER_RXC_RUN_STATUS_STOPPING)
+		run.Job.run_mtx.Unlock()
+		run.Status = SERVER_RXC_RUN_STATUS_STOPPING
+		cts = run.Cts
+		rxc_id = run.RxcId
+	}
+	run.mtx.Unlock()
+
+	if cts == nil || rxc_id == 0 {
+		// it wasn't originally at the running state
+		return nil, 0, false
+	}
+
+	return cts, rxc_id, true
+}
+
+func (run *ServerRxcJobRun) is_done() bool {
+	var status string
+
+	run.mtx.Lock()
+	status = run.Status
+	run.mtx.Unlock()
+
+	return status == SERVER_RXC_RUN_STATUS_STOPPED || status == SERVER_RXC_RUN_STATUS_FAILED
+}
+
+func (run *ServerRxcJobRun) append_output(data []byte, outidx int, capture_tail bool) {
+	var fit_len int
+	var keep_len int
+	var old_off int
+	var output_max int
+
+	output_max = run.Job.S.Cfg.RxcRunOutputMax
+	if output_max <= 0 { return } // don't capture
+
+	run.mtx.Lock()
+	if capture_tail {
+		if len(data) >= output_max {
+			if len(data) > output_max || len(run.Output[outidx]) > 0 {
+				run.OutputTruncated[outidx] = true
+			}
+			run.Output[outidx] = append(run.Output[outidx][:0], data[len(data) - output_max:]...)
+		} else {
+			if len(run.Output[outidx]) > output_max {
+				run.Output[outidx] = append([]byte(nil), run.Output[outidx][len(run.Output[outidx]) - output_max:]...)
+				run.OutputTruncated[outidx] = true
+			}
+
+			fit_len = output_max - len(data)
+			if fit_len < 0 { fit_len = 0 }
+			if len(run.Output[outidx]) > fit_len {
+				keep_len = fit_len
+				old_off = len(run.Output[outidx]) - keep_len
+				copy(run.Output[outidx], run.Output[outidx][old_off:])
+				run.Output[outidx] = run.Output[outidx][:keep_len]
+				run.OutputTruncated[outidx]= true
+			}
+
+			run.Output[outidx] = append(run.Output[outidx], data...)
+		}
+	} else {
+		//if len(run.Output[outidx]) > output_max {
+		//	run.Output[outidx] = append([]byte(nil), run.Output[outidx][:output_max]...)
+		//	run.OutputTruncate[outidx]d = true
+		//}
+
+		if !run.OutputTruncated[outidx] {
+			fit_len = output_max - len(run.Output[outidx])
+			if fit_len <= 0 {
+				run.OutputTruncated[outidx] = true
+			} else {
+				if len(data) > fit_len {
+					data = data[:fit_len]
+					run.OutputTruncated[outidx] = true
+				}
+				run.Output[outidx] = append(run.Output[outidx], data...)
+			}
+		}
+	}
+	run.mtx.Unlock()
+}
+
+func (run *ServerRxcJobRun) stop(flags uint64, msg string) {
+	var transitioned bool
+
+	run.mtx.Lock()
+	if run.Status != SERVER_RXC_RUN_STATUS_STOPPED && run.Status != SERVER_RXC_RUN_STATUS_FAILED {
+		run.Job.run_mtx.Lock()
+		run.Job.decrement_run_status_count_no_lock(run.Status)
+		run.Job.increment_run_status_count_no_lock(SERVER_RXC_RUN_STATUS_STOPPED)
+		run.Job.run_mtx.Unlock()
+		run.Status = SERVER_RXC_RUN_STATUS_STOPPED
+		run.Stopped = time.Now()
+		run.StopFlags = flags
+		run.StopMsg = msg
+		transitioned = true
+	}
+	run.mtx.Unlock()
+
+	if transitioned {
+		run.Cts.S.FireRxcJobRunEvent(SERVER_EVENT_RXC_JOB_RUN_DONE, run)
+		if run.Cts.S.maybe_mark_rxc_job_done(run.Job) {
+			run.Cts.S.notify_rxc_job_purge()
+		}
+	}
+}
+
+func (run *ServerRxcJobRun) ReqStop() {
+	// this is called from run.Cts.receive_from_stream() or run.Cts.ReqStop()
+	// I can assume that the cause for this stop request is "connection closed".
+	// If you want to specify your own reason, you must call Stop() instead.
+	run.stop(0, "connection closed")
+}
+
+func (run *ServerRxcJobRun) Write(flags uint64, data []byte) error {
+	var outidx int
+
+	outidx = 0
+	if flags & RXC_DATA_FLAG_STDERR != 0 { outidx = 1 }
+
+	run.append_output(data, outidx, false)
+	return nil
+}
+
+func (run *ServerRxcJobRun) Stop(flags uint64, msg string) error {
+	run.stop(flags, msg)
+	return nil
+}
+
+// ServerRxcJob
+
+func (job *ServerRxcJob) snapshot_runs() []*ServerRxcJobRun {
+	var runs []*ServerRxcJobRun
+	var run *ServerRxcJobRun
+
+	job.run_mtx.Lock()
+	runs = make([]*ServerRxcJobRun, 0, len(job.run_map))
+	for _, run = range job.run_map {
+		runs = append(runs, run)
+	}
+	job.run_mtx.Unlock()
+
+	sort.Slice(runs, func(i int, j int) bool { return runs[i].ConnId < runs[j].ConnId })
+	return runs
+}
+
+func (job *ServerRxcJob) FindRunByConnIdStr(conn_id string) (*ServerRxcJobRun, error) {
+	var conn_nid uint64
+	var err error
+	var run *ServerRxcJobRun
+
+	conn_nid, err = strconv.ParseUint(conn_id, 10, int(unsafe.Sizeof(ConnId(0)) * 8))
+	if err == nil {
+		job.run_mtx.Lock()
+		run = job.run_map[ConnId(conn_nid)]
+		job.run_mtx.Unlock()
+		if run == nil {
+			return nil, fmt.Errorf("non-existent rxc run conn id %d", conn_nid)
+		}
+		return run, nil
+	}
+
+	for _, run = range job.snapshot_runs() {
+		if run.ClientToken == conn_id {
+			return run, nil
+		}
+	}
+
+	return nil, fmt.Errorf("non-existent rxc run client %s", conn_id)
+}
+
+func (job *ServerRxcJob) is_done() bool {
+	var done bool
+
+	job.run_mtx.Lock()
+	done = (job.active_run_count <= 0)
+	job.run_mtx.Unlock()
+	return done
+}
+
+func (job *ServerRxcJob) get_done_time() time.Time {
+	var done time.Time
+
+	job.S.rxc_job_mtx.Lock()
+	done = job.Done
+	job.S.rxc_job_mtx.Unlock()
+	return done
+}
+
+func (s *Server) maybe_mark_rxc_job_done(job *ServerRxcJob) bool {
+	var existing *ServerRxcJob
+	var ok bool
+
+	job.run_mtx.Lock()
+	if job.active_run_count > 0 {
+		job.active_run_count--
+		if job.active_run_count > 0 {
+			job.run_mtx.Unlock()
+			return false
+		}
+	}
+	job.run_mtx.Unlock()
+
+	s.rxc_job_mtx.Lock()
+	existing, ok = s.rxc_job_map[job.Id]
+	if !ok || existing != job {
+		s.rxc_job_mtx.Unlock()
+		return false
+	}
+	if !job.Done.IsZero() {
+		s.rxc_job_mtx.Unlock()
+		return false
+	}
+	job.Done = time.Now()
+	if s.Cfg.RxcDoneJobRetention > 0 {
+		job.expire_at = job.Done.Add(s.Cfg.RxcDoneJobRetention)
+		heap.Push(&s.rxc_job_heap, job)
+	}
+	s.rxc_job_mtx.Unlock()
+	s.FireRxcJobEvent(SERVER_EVENT_RXC_JOB_DONE, job)
+
+	return true
+}
+
+func (job *ServerRxcJob) delete_run(run *ServerRxcJobRun) bool {
+	var existing *ServerRxcJobRun
+	var ok bool
+	var status string
+
+	run.mtx.Lock()
+	status = run.Status
+	run.mtx.Unlock()
+
+	job.run_mtx.Lock()
+	existing, ok = job.run_map[run.ConnId]
+	if ok && existing == run {
+		delete(job.run_map, run.ConnId)
+		job.decrement_run_status_count_no_lock(status)
+	}
+	job.run_mtx.Unlock()
+
+	return ok && existing == run
+}
+
+// Server
+func (s *Server) snapshot_server_rxc_jobs() []*ServerRxcJob {
+	var jobs []*ServerRxcJob
+	var job *ServerRxcJob
+
+	s.rxc_job_mtx.Lock()
+	jobs = make([]*ServerRxcJob, 0, len(s.rxc_job_map))
+	for _, job = range s.rxc_job_map {
+		jobs = append(jobs, job)
+	}
+	s.rxc_job_mtx.Unlock()
+
+	sort.Slice(jobs, func(i int, j int) bool { return jobs[i].Id < jobs[j].Id })
+	return jobs
+}
+
+func (s *Server) FindServerRxcJobById(id uint64) *ServerRxcJob {
+	var job *ServerRxcJob
+	var ok bool
+
+	s.rxc_job_mtx.Lock()
+	job, ok = s.rxc_job_map[id]
+	s.rxc_job_mtx.Unlock()
+	if !ok { return nil }
+	return job
+}
+
+func (s *Server) FindServerRxcJobByIdStr(job_id string) (*ServerRxcJob, error) {
+	var job_nid uint64
+	var err error
+	var job *ServerRxcJob
+
+	job_nid, err = strconv.ParseUint(job_id, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("invalid exec job id %s - %s", job_id, err.Error())
+	}
+
+	job = s.FindServerRxcJobById(job_nid)
+	if job == nil {
+		return nil, fmt.Errorf("non-existent exec job id %d", job_nid)
+	}
+
+	return job, nil
+}
+
+func (s *Server) delete_server_rxc_job_from_heap_no_lock(job *ServerRxcJob) {
+	if job.heap_index >= 0 {
+		heap.Remove(&s.rxc_job_heap, job.heap_index)
+		job.expire_at = time.Time{}
+		//job.heap_index = -1 // skip this as it's set in s.rxc_job_heap.Pop() called by heap.Remove()
+	}
+}
+
+func (s *Server) delete_server_rxc_job_no_lock(job *ServerRxcJob) bool {
+	var existing *ServerRxcJob
+	var ok bool
+
+	existing, ok = s.rxc_job_map[job.Id]
+	if ok && existing == job {
+		delete(s.rxc_job_map, job.Id)
+		s.delete_server_rxc_job_from_heap_no_lock(job)
+	}
+
+	return ok && existing == job
+}
+
+func (s *Server) delete_server_rxc_job(job *ServerRxcJob) bool {
+	var deleted bool
+
+	s.rxc_job_mtx.Lock()
+	deleted = s.delete_server_rxc_job_no_lock(job)
+	s.rxc_job_mtx.Unlock()
+
+	return deleted
+}
+
+func (s *Server) resolve_server_rxc_targets(clients []string) ([]*ServerConn, error) {
+	var conns []*ServerConn
+	var client string
+	var cts *ServerConn
+	var err error
+	var dedupe map[ConnId]*ServerConn
+
+	if len(clients) <= 0 {
+		conns = s.snapshot_server_conns()
+		if len(conns) <= 0 {
+			return nil, fmt.Errorf("no connected clients")
+		}
+		return conns, nil
+	}
+
+	dedupe = make(map[ConnId]*ServerConn)
+	for _, client = range clients {
+		client = strings.TrimSpace(client)
+		if client == "" { continue }
+		cts, err = s.FindServerConnByIdStr(client)
+		if err != nil { return nil, err }
+		dedupe[cts.Id] = cts
+	}
+
+	if len(dedupe) <= 0 {
+		return nil, fmt.Errorf("no connected clients")
+	}
+
+	conns = make([]*ServerConn, 0, len(dedupe))
+	for _, cts = range dedupe {
+		conns = append(conns, cts)
+	}
+	sort.Slice(conns, func(i int, j int) bool { return conns[i].Id < conns[j].Id })
+	return conns, nil
+}
+
+func (s *Server) StartRxcJob(clients []string, type_ string, script string) (*ServerRxcJob, error) {
+	var conns []*ServerConn
+	var cts *ServerConn
+	var job *ServerRxcJob
+	var run *ServerRxcJobRun
+	var err error
+	var ok bool
+
+	if strings.TrimSpace(type_) == "" {
+		return nil, fmt.Errorf("blank rxc type")
+	}
+	if strings.TrimSpace(script) == "" {
+		return nil, fmt.Errorf("blank rxc script")
+	}
+
+	conns, err = s.resolve_server_rxc_targets(clients)
+	if err != nil { return nil, err }
+
+	job = &ServerRxcJob{
+		S: s,
+		Type: type_,
+		Script: script,
+		Created: time.Now(),
+		heap_index: -1,
+		run_map: make(map[ConnId]*ServerRxcJobRun),
+		active_run_count: len(conns),
+		starting_run_count: len(conns),
+	}
+
+	s.rxc_job_mtx.Lock()
+	job.Id = s.rxc_job_next_id
+	s.rxc_job_next_id++
+	if s.rxc_job_next_id == 0 { s.rxc_job_next_id++ }
+	s.rxc_job_map[job.Id] = job
+	s.rxc_job_mtx.Unlock()
+
+	job.run_mtx.Lock()
+	for _, cts = range conns {
+		job.run_map[cts.Id] = new_server_rxc_job_run(job, cts)
+	}
+	job.run_mtx.Unlock()
+
+	for _, cts = range conns {
+		job.run_mtx.Lock()
+		run, ok = job.run_map[cts.Id]
+		job.run_mtx.Unlock()
+		if !ok { continue }
+
+		err = cts.RunRxcJob(run, type_, script)
+		if err != nil {
+			run.mark_start_failure(err.Error())
+			s.log.Write(cts.Sid, LOG_DEBUG, "Failed to run rxc job(%d) on client(%s) from %s(%s)", job.Id, cts.ClientToken.Get(), cts.RemoteAddr)
+		} else {
+			s.log.Write(cts.Sid, LOG_DEBUG, "Ran rxc job(%d) on client(%s) from %s(%s)", job.Id, cts.ClientToken.Get(), cts.RemoteAddr)
+		}
+	}
+
+	s.log.Write("", LOG_DEBUG, "Started rxc job(%d) %s(%s)", job.Id, type_, script)
+	return job, nil
+}
+
+func (s *Server) StopRxcJobRun(run *ServerRxcJobRun) (bool, error) {
+	var cts *ServerConn
+	var rxc_id uint64
+	var ok bool
+	var err error
+
+	cts, rxc_id, ok = run.request_stop()
+	if !ok {
+		return false, nil
+	}
+
+	// sever sends a stop request to abort the run on the client side
+	err = cts.SendStopRxcById(rxc_id, 0)
+	if err != nil {
+		run.stop(0, err.Error())
+		return false, err
+	}
+
+	return true, nil
+}
+
+func (s *Server) StopRxcJob(job *ServerRxcJob) int {
+	var run *ServerRxcJobRun
+	var runs []*ServerRxcJobRun
+	var stop_count int
+
+	job.run_mtx.Lock()
+	runs = make([]*ServerRxcJobRun, 0, len(job.run_map))
+	for _, run = range job.run_map {
+		runs = append(runs, run)
+	}
+	job.run_mtx.Unlock()
+
+	for _, run = range runs {
+		var err error
+		var stopped bool
+
+		stopped, err = s.StopRxcJobRun(run)
+		if stopped { stop_count++ }
+		if err != nil { continue }
+	}
+
+	s.log.Write("", LOG_DEBUG, "Stopped rxc job(%d) after stopping %d runs", job.Id, stop_count)
+	return stop_count
+}
+
+func (s *Server) DeleteRxcJob(job *ServerRxcJob) error {
+	if !job.is_done() {
+		return fmt.Errorf("active rxc job id %d", job.Id)
+	}
+	if !s.delete_server_rxc_job(job) {
+		return fmt.Errorf("non-existent rxc job id %d", job.Id)
+	}
+	if s.Cfg.RxcDoneJobRetention > 0 {
+		// a job is already over and is actually deleted by request.
+		// the purge goroutine needs to re-schedule the next automatic purge.
+		s.notify_rxc_job_purge()
+	}
+
+	s.log.Write("", LOG_DEBUG, "Deleted rxc job(%d)", job.Id)
+	return nil
+}
+
+func (s *Server) DeleteRxcJobRun(run *ServerRxcJobRun) error {
+	if !run.is_done() {
+		return fmt.Errorf("active rxc run conn id %d", run.ConnId)
+	}
+	if !run.Job.delete_run(run) {
+		return fmt.Errorf("non-existent rxc run conn id %d", run.ConnId)
+	}
+
+	return nil
+}
+
+func (s *Server) purge_stale_rxc_jobs(now time.Time, expiry time.Duration) int {
+	var job *ServerRxcJob
+	var purge_count int
+
+	if expiry <= 0 { return 0 }
+
+	s.rxc_job_mtx.Lock()
+	for {
+		if len(s.rxc_job_heap) <= 0 { break }
+		job = s.rxc_job_heap[0]
+		if now.Before(job.expire_at) { break }
+		if s.delete_server_rxc_job_no_lock(job) {
+			s.log.Write("", LOG_INFO, "Purged stale rxc job(%d)", job.Id)
+			purge_count++
+		} else {
+			// this must not happen. but if it happens, it is an internal error and
+			// the job expiry heap and the job map are already out of sync
+			s.log.Write("", LOG_WARN, "Failed to purge stale rxc job(%d): heap/map mismatch", job.Id)
+			// but still attempt to delete it from the heap to prevent future purge blockage
+			s.delete_server_rxc_job_from_heap_no_lock(job)
+		}
+	}
+	s.rxc_job_mtx.Unlock()
+
+	return purge_count
+}
+
+func (s *Server) notify_rxc_job_purge() {
+	select {
+		case s.rxc_job_purge_chan <- struct{}{}:
+			// attempt to write to the channel
+		default:
+			// if not writable, do nothing and return immediately
+	}
+}
+
+func (s *Server) get_next_rxc_job_purge_time(expiry time.Duration) (time.Time, bool) {
+	var next time.Time
+
+	if expiry <= 0 { return time.Time{}, false }
+
+	s.rxc_job_mtx.Lock()
+	if len(s.rxc_job_heap) <= 0 {
+		s.rxc_job_mtx.Unlock()
+		return time.Time{}, false
+	}
+	next = s.rxc_job_heap[0].expire_at
+	s.rxc_job_mtx.Unlock()
+
+	return next, true
+}
+
+func stop_and_drain_timer(timer *time.Timer) {
+	if timer == nil { return }
+	if !timer.Stop() {
+		select {
+			case <-timer.C:
+			default:
+		}
+	}
+}
+
+func (s *Server) run_rxc_job_purger(wg *sync.WaitGroup, expiry time.Duration) {
+	var timer *time.Timer
+	var timer_chan <-chan time.Time
+	var next time.Time
+	var now time.Time
+	var delay time.Duration
+	var ok bool
+
+	defer wg.Done()
+
+	// it's best if the caller ensures expiry > 0
+	if expiry <= 0 { return }
+
+	for {
+		next, ok = s.get_next_rxc_job_purge_time(expiry)
+		if ok {
+			delay = time.Until(next)
+			if delay < 0 { delay = 0 }
+			if timer == nil {
+				timer = time.NewTimer(delay)
+			} else {
+				stop_and_drain_timer(timer)
+				timer.Reset(delay)
+			}
+			timer_chan = timer.C
+		} else {
+			stop_and_drain_timer(timer)
+			timer_chan = nil
+		}
+
+		select {
+			case <-s.rxc_job_purge_chan:
+				// a job was done and/or deleted.
+				// go to the top of the loop to re-calculate when to
+				// run the automatic purge next time.
+				continue
+
+			case now = <-timer_chan:
+				s.purge_stale_rxc_jobs(now, expiry)
+
+			case <-s.Ctx.Done():
+				stop_and_drain_timer(timer)
+				return
+		}
+	}
+}

server-rxc.go

@@ -0,0 +1,592 @@
+package hodu
+
+
+import "encoding/base64"
+import "encoding/json"
+import "fmt"
+import "net/http"
+import "strconv"
+import "sync"
+import "time"
+
+import "golang.org/x/net/websocket"
+
+// rxc - remote exec
+type server_rxc_ws struct {
+	S *Server
+	Id string
+	ws *websocket.Conn
+}
+
+func (rxc *server_rxc_ws) Identity() string {
+	return rxc.Id
+}
+
+func (rxc *server_rxc_ws) ServeWebsocket(ws *websocket.Conn) (int, error) {
+	var s *Server
+	var req *http.Request
+	var token string
+	var cts *ServerConn
+	var rp *ServerRxc
+	var wg sync.WaitGroup
+	var err error
+
+	s = rxc.S
+	req = ws.Request()
+	token = req.FormValue("client-token")
+	if token == "" {
+		ws.Close()
+		return http.StatusBadRequest, fmt.Errorf("no client token specified")
+	}
+
+	cts = s.FindServerConnByClientToken(token)
+	if cts == nil {
+		ws.Close()
+		return http.StatusBadRequest, fmt.Errorf("invalid client token - %s", token)
+	}
+
+ws_recv_loop:
+	for {
+		var msg []byte
+		err = websocket.Message.Receive(ws, &msg)
+		if err != nil {
+			s.log.Write(rxc.Id, LOG_ERROR, "[%s] websocket receive error - %s", req.RemoteAddr, err.Error())
+			goto done
+		}
+
+		if len(msg) > 0 {
+			var ev json_xterm_ws_event
+			err = json.Unmarshal(msg, &ev)
+			if err == nil {
+				switch ev.Type {
+					case "open":
+						if rp == nil && len(ev.Data) == 2 {
+							var type_ string
+							var script string
+
+							type_ = string(ev.Data[0])
+							script = string(ev.Data[1])
+							//options = string(ev.Data[2])
+
+							rp, err = cts.StartRxcForWs(ws, type_, script)
+							if err != nil {
+								s.log.Write(rxc.Id, LOG_ERROR, "[%s] Failed to connect rxc - %s", req.RemoteAddr, err.Error())
+								send_ws_data_for_xterm(ws, "error", err.Error())
+								//ws.Close() // dirty way to flag out the error by making websocket.Message.Receive() fail
+								ws.SetReadDeadline(time.Now()) // slightly cleaner way to break the main loop
+							} else {
+								err = send_ws_data_for_xterm(ws, "status", "opened")
+								if err != nil {
+									s.log.Write(rxc.Id, LOG_ERROR, "[%s] Failed to write 'opened' event to websocket - %s", req.RemoteAddr, err.Error())
+									//ws.Close() // dirty way to flag out the error
+									ws.SetReadDeadline(time.Now()) // slightly cleaner way to break the main loop
+								} else {
+									s.log.Write(rxc.Id, LOG_DEBUG, "[%s] Opened rxc session", req.RemoteAddr)
+								}
+							}
+						}
+
+					case "close":
+						// just break out of the loop and let the remainder to close resources
+						break ws_recv_loop
+
+					case "iov":
+						var i int
+						for i, _ = range ev.Data {
+							var bytes []byte
+							bytes, err = base64.StdEncoding.DecodeString(ev.Data[i])
+							if err != nil {
+								s.log.Write(rxc.Id, LOG_WARN, "[%s] Invalid rxc iov data received - %s", req.RemoteAddr, ev.Data[i])
+							} else {
+								cts.WriteRxcForWs(ws, bytes)
+								// ignore error for now
+							}
+						}
+
+					default:
+						send_ws_data_for_xterm(ws, "error", fmt.Sprintf("invalid rxc event type - %s", ev.Type));
+						s.log.Write(rxc.Id, LOG_WARN, "[%s] Invalid rxc event type received - %s", req.RemoteAddr, ev.Type)
+				}
+			}
+		}
+	}
+
+done:
+	cts.StopRxcForWs(ws)
+	ws.Close() // don't care about multiple closes
+
+	wg.Wait()
+	s.log.Write(rxc.Id, LOG_DEBUG, "[%s] Ended rxc session for %s", req.RemoteAddr, token)
+
+	return http.StatusOK, err
+}
+
+// HTTP control endpoints for RXC jobs:
+//   GET    /_rxc
+//   POST   /_rxc
+//   GET    /_rxc/{job_id}
+//   DELETE /_rxc/{job_id}
+//   POST   /_rxc/{job_id}/stop
+//   GET    /_rxc/{job_id}/runs
+//   GET    /_rxc/{job_id}/runs/{conn_id}
+//   DELETE /_rxc/{job_id}/runs/{conn_id}
+//   POST   /_rxc/{job_id}/runs/{conn_id}/stop
+//
+// The websocket endpoint /_rxc/ws is kept for interactive single-client RXC.
+
+/*
+• Example to run a command on all connected clients:
+
+ curl -X POST http://127.0.0.1:9999/_rxc \
+	-H 'Content-Type: application/json' \
+	-d '{
+		"type": "bash",
+		"script": "uname -a"
+	}'
+
+ Example to run on specific clients only:
+
+curl -X POST http://127.0.0.1:9999/_rxc \
+	-H 'Content-Type: application/json' \
+	-d '{
+		"clients": ["1", "client-token-abc"],
+		"type": "bash",
+		"script": "hostname; id"
+	}'
+
+ Then inspect the created job:
+
+ curl http://127.0.0.1:9999/_rxc/1
+ curl http://127.0.0.1:9999/_rxc/1/runs
+ curl http://127.0.0.1:9999/_rxc/1/runs/1
+
+ Stop a job:
+
+ curl -X POST http://127.0.0.1:9999/_rxc/1/stop
+
+ Stop one run only:
+
+ curl -X POST http://127.0.0.1:9999/_rxc/1/runs/1/stop
+
+ Delete a retained job record:
+
+ curl -X DELETE http://127.0.0.1:9999/_rxc/1
+
+ Delete a retained run record:
+
+ curl -X DELETE http://127.0.0.1:9999/_rxc/1/runs/1
+
+ If your control server uses a prefix, prepend it, for example http://127.0.0.1:9999/api/_rxc.
+*/
+
+type json_in_server_rxc struct {
+	Clients []string `json:"clients"`
+	Type string `json:"type"`
+	Script string `json:"script"`
+}
+
+type json_out_server_rxc_job struct {
+	JobId uint64 `json:"job-id"`
+	Type string `json:"type"`
+	Script string `json:"script"`
+	Status string `json:"status"`
+	CreatedMilli int64 `json:"created-milli"`
+	DoneMilli int64 `json:"done-milli"`
+	TargetCount int `json:"target-count"`
+	RunningCount int `json:"running-count"`
+	StoppingCount int `json:"stopping-count"`
+	StoppedCount int `json:"stopped-count"`
+	FailedCount int `json:"failed-count"`
+}
+
+type json_out_server_rxc_run struct {
+	JobId uint64 `json:"job-id"`
+	ConnId ConnId `json:"conn-id"`
+	ClientToken string `json:"client-token"`
+	RxcId uint64 `json:"rxc-id"`
+	Status string `json:"status"`
+	ExitCode int `json:"exit-code"`
+	StopMsg string `json:"stop-msg"`
+	CreatedMilli int64 `json:"created-milli"`
+	StartedMilli int64 `json:"started-milli"`
+	StoppedMilli int64 `json:"stopped-milli"`
+	Stdout []byte `json:"stdout,omitempty"`
+	StdoutTruncated bool `json:"stdout-truncated"`
+	Stderr []byte `json:"stderr,omitempty"`
+	StderrTruncated bool `json:"stderr-truncated"`
+}
+
+type server_ctl_rxc struct {
+	ServerCtl
+}
+
+type server_ctl_rxc_id struct {
+	ServerCtl
+}
+
+type server_ctl_rxc_id_stop struct {
+	ServerCtl
+}
+
+type server_ctl_rxc_id_runs struct {
+	ServerCtl
+}
+
+type server_ctl_rxc_id_runs_id struct {
+	ServerCtl
+}
+
+type server_ctl_rxc_id_runs_id_stop struct {
+	ServerCtl
+}
+
+func server_rxc_job_to_json(job *ServerRxcJob) json_out_server_rxc_job {
+	var js json_out_server_rxc_job
+	var done time.Time
+
+	js.JobId = job.Id
+	js.Type = job.Type
+	js.Script = job.Script
+	js.CreatedMilli = job.Created.UnixMilli()
+	done = job.get_done_time()
+	if !done.IsZero() { js.DoneMilli = done.UnixMilli() }
+
+	job.run_mtx.Lock()
+	js.TargetCount = len(job.run_map)
+	js.RunningCount = job.running_run_count
+	js.StoppingCount = job.stopping_run_count
+	js.StoppedCount = job.stopped_run_count
+	js.FailedCount = job.failed_run_count
+	job.run_mtx.Unlock()
+
+	switch {
+		case js.TargetCount <= 0:
+			js.Status = "done"
+		case js.RunningCount > 0 || js.StoppingCount > 0:
+			js.Status = "running"
+		case js.StoppedCount > 0 || js.FailedCount > 0:
+			js.Status = "done"
+		default:
+			js.Status = "starting"
+	}
+
+	return js
+}
+
+func server_rxc_run_to_json(run *ServerRxcJobRun, with_output bool) json_out_server_rxc_run {
+	var js json_out_server_rxc_run
+
+	run.mtx.Lock()
+	js.JobId = run.Job.Id
+	js.ConnId = run.ConnId
+	js.ClientToken = run.ClientToken
+	js.RxcId = run.RxcId
+	js.Status = run.Status
+	js.ExitCode = GetRxcStopExitCode(run.StopFlags)
+	js.StopMsg = run.StopMsg
+	js.CreatedMilli = run.Created.UnixMilli()
+	if !run.Started.IsZero() { js.StartedMilli = run.Started.UnixMilli() }
+	if !run.Stopped.IsZero() { js.StoppedMilli = run.Stopped.UnixMilli() }
+	js.Stdout = make([]byte, 0) // i don't want to show nil when empty
+	js.Stderr = make([]byte, 0) // i don't want to show nil when empty
+	js.StdoutTruncated = run.OutputTruncated[0]
+	if with_output { js.Stdout = append(js.Stdout, run.Output[0]...) }
+	js.StderrTruncated = run.OutputTruncated[1]
+	if with_output { js.Stderr = append(js.Stderr, run.Output[1]...) }
+	run.mtx.Unlock()
+
+	return js
+}
+
+func (ctl *server_ctl_rxc) ServeHTTP(w http.ResponseWriter, req *http.Request) (int, error) {
+	var s *Server
+	var status_code int
+	var je *json.Encoder
+	var err error
+
+	s = ctl.S
+	je = json.NewEncoder(w)
+
+	switch req.Method {
+		case http.MethodGet:
+			var jobs []json_out_server_rxc_job
+			var job *ServerRxcJob
+
+			jobs = make([]json_out_server_rxc_job, 0)
+			for _, job = range s.snapshot_server_rxc_jobs() {
+				jobs = append(jobs, server_rxc_job_to_json(job))
+			}
+
+			status_code = WriteJsonRespHeader(w, http.StatusOK)
+			if err = je.Encode(jobs); err != nil { goto oops }
+
+		case http.MethodPost:
+			var x json_in_server_rxc
+			var job *ServerRxcJob
+
+			err = json.NewDecoder(req.Body).Decode(&x)
+			if err != nil {
+				status_code = WriteEmptyRespHeader(w, http.StatusBadRequest)
+				goto oops
+			}
+
+			job, err = s.StartRxcJob(x.Clients, x.Type, x.Script)
+			if err != nil {
+				status_code = WriteJsonRespHeader(w, http.StatusBadRequest)
+				je.Encode(JsonErrmsg{Text: err.Error()})
+				goto oops
+			}
+
+			status_code = WriteJsonRespHeader(w, http.StatusAccepted)
+			if err = je.Encode(server_rxc_job_to_json(job)); err != nil { goto oops }
+
+		default:
+			status_code = WriteEmptyRespHeader(w, http.StatusMethodNotAllowed)
+	}
+
+	return status_code, nil
+
+oops:
+	return status_code, err
+}
+
+func (ctl *server_ctl_rxc_id) ServeHTTP(w http.ResponseWriter, req *http.Request) (int, error) {
+	var s *Server
+	var status_code int
+	var je *json.Encoder
+	var job_id string
+	var job *ServerRxcJob
+	var err error
+
+	s = ctl.S
+	je = json.NewEncoder(w)
+	job_id = req.PathValue("job_id")
+
+	job, err = s.FindServerRxcJobByIdStr(job_id)
+	if err != nil {
+		status_code = WriteJsonRespHeader(w, http.StatusNotFound)
+		je.Encode(JsonErrmsg{Text: err.Error()})
+		goto oops
+	}
+
+	switch req.Method {
+		case http.MethodGet:
+			status_code = WriteJsonRespHeader(w, http.StatusOK)
+			if err = je.Encode(server_rxc_job_to_json(job)); err != nil { goto oops }
+
+		case http.MethodDelete:
+			err = s.DeleteRxcJob(job)
+			if err != nil {
+				status_code = WriteJsonRespHeader(w, http.StatusConflict)
+				je.Encode(JsonErrmsg{Text: err.Error()})
+				goto oops
+			}
+			status_code = WriteEmptyRespHeader(w, http.StatusNoContent)
+
+		default:
+			status_code = WriteEmptyRespHeader(w, http.StatusMethodNotAllowed)
+	}
+
+	return status_code, nil
+
+oops:
+	return status_code, err
+}
+
+func (ctl *server_ctl_rxc_id_stop) ServeHTTP(w http.ResponseWriter, req *http.Request) (int, error) {
+	var s *Server
+	var status_code int
+	var je *json.Encoder
+	var job_id string
+	var job *ServerRxcJob
+	var stop_count int
+	var err error
+
+	s = ctl.S
+	je = json.NewEncoder(w)
+	job_id = req.PathValue("job_id")
+
+	job, err = s.FindServerRxcJobByIdStr(job_id)
+	if err != nil {
+		status_code = WriteJsonRespHeader(w, http.StatusNotFound)
+		je.Encode(JsonErrmsg{Text: err.Error()})
+		goto oops
+	}
+
+	switch req.Method {
+		case http.MethodPost:
+			stop_count = s.StopRxcJob(job)
+			if stop_count > 0 {
+				status_code = WriteEmptyRespHeader(w, http.StatusAccepted)
+			} else {
+				status_code = WriteEmptyRespHeader(w, http.StatusNoContent)
+			}
+
+		default:
+			status_code = WriteEmptyRespHeader(w, http.StatusMethodNotAllowed)
+	}
+
+	return status_code, nil
+
+oops:
+	return status_code, err
+}
+
+func (ctl *server_ctl_rxc_id_runs) ServeHTTP(w http.ResponseWriter, req *http.Request) (int, error) {
+	var s *Server
+	var status_code int
+	var je *json.Encoder
+	var job_id string
+	var job *ServerRxcJob
+	var run *ServerRxcJobRun
+	var runs []json_out_server_rxc_run
+	var err error
+
+	s = ctl.S
+	je = json.NewEncoder(w)
+	job_id = req.PathValue("job_id")
+
+	job, err = s.FindServerRxcJobByIdStr(job_id)
+	if err != nil {
+		status_code = WriteJsonRespHeader(w, http.StatusNotFound)
+		je.Encode(JsonErrmsg{Text: err.Error()})
+		goto oops
+	}
+
+	switch req.Method {
+		case http.MethodGet:
+			var output string
+			var with_output bool
+
+			output = req.URL.Query().Get("output")
+			if output != "" {
+				with_output, err = strconv.ParseBool(output)
+				if err != nil {
+					status_code = WriteJsonRespHeader(w, http.StatusBadRequest)
+					je.Encode(JsonErrmsg{Text: fmt.Sprintf("invalid output parameter - %s", output)})
+					goto oops
+				}
+			}
+			runs = make([]json_out_server_rxc_run, 0)
+			for _, run = range job.snapshot_runs() {
+				runs = append(runs, server_rxc_run_to_json(run, with_output))
+			}
+			status_code = WriteJsonRespHeader(w, http.StatusOK)
+			if err = je.Encode(runs); err != nil { goto oops }
+
+		default:
+			status_code = WriteEmptyRespHeader(w, http.StatusMethodNotAllowed)
+	}
+
+	return status_code, nil
+
+oops:
+	return status_code, err
+}
+
+func (ctl *server_ctl_rxc_id_runs_id) ServeHTTP(w http.ResponseWriter, req *http.Request) (int, error) {
+	var s *Server
+	var status_code int
+	var je *json.Encoder
+	var job_id string
+	var conn_id string
+	var job *ServerRxcJob
+	var run *ServerRxcJobRun
+	var err error
+
+	s = ctl.S
+	je = json.NewEncoder(w)
+	job_id = req.PathValue("job_id")
+	conn_id = req.PathValue("conn_id")
+
+	job, err = s.FindServerRxcJobByIdStr(job_id)
+	if err != nil {
+		status_code = WriteJsonRespHeader(w, http.StatusNotFound)
+		je.Encode(JsonErrmsg{Text: err.Error()})
+		goto oops
+	}
+
+	run, err = job.FindRunByConnIdStr(conn_id)
+	if err != nil {
+		status_code = WriteJsonRespHeader(w, http.StatusNotFound)
+		je.Encode(JsonErrmsg{Text: err.Error()})
+		goto oops
+	}
+
+	switch req.Method {
+		case http.MethodGet:
+			status_code = WriteJsonRespHeader(w, http.StatusOK)
+			if err = je.Encode(server_rxc_run_to_json(run, true)); err != nil { goto oops }
+
+		case http.MethodDelete:
+			err = s.DeleteRxcJobRun(run)
+			if err != nil {
+				status_code = WriteJsonRespHeader(w, http.StatusConflict)
+				je.Encode(JsonErrmsg{Text: err.Error()})
+				goto oops
+			}
+			status_code = WriteEmptyRespHeader(w, http.StatusNoContent)
+
+		default:
+			status_code = WriteEmptyRespHeader(w, http.StatusMethodNotAllowed)
+	}
+
+	return status_code, nil
+
+oops:
+	return status_code, err
+}
+
+func (ctl *server_ctl_rxc_id_runs_id_stop) ServeHTTP(w http.ResponseWriter, req *http.Request) (int, error) {
+	var s *Server
+	var status_code int
+	var je *json.Encoder
+	var job_id string
+	var conn_id string
+	var job *ServerRxcJob
+	var run *ServerRxcJobRun
+	var stopped bool
+	var err error
+
+	s = ctl.S
+	je = json.NewEncoder(w)
+	job_id = req.PathValue("job_id")
+	conn_id = req.PathValue("conn_id")
+
+	job, err = s.FindServerRxcJobByIdStr(job_id)
+	if err != nil {
+		status_code = WriteJsonRespHeader(w, http.StatusNotFound)
+		je.Encode(JsonErrmsg{Text: err.Error()})
+		goto oops
+	}
+
+	run, err = job.FindRunByConnIdStr(conn_id)
+	if err != nil {
+		status_code = WriteJsonRespHeader(w, http.StatusNotFound)
+		je.Encode(JsonErrmsg{Text: err.Error()})
+		goto oops
+	}
+
+	switch req.Method {
+		case http.MethodPost:
+			stopped, err = s.StopRxcJobRun(run)
+			if err != nil {
+				status_code = WriteJsonRespHeader(w, http.StatusInternalServerError)
+				je.Encode(JsonErrmsg{Text: err.Error()})
+				goto oops
+			}
+			if stopped {
+				status_code = WriteEmptyRespHeader(w, http.StatusAccepted)
+			} else {
+				status_code = WriteEmptyRespHeader(w, http.StatusNoContent)
+			}
+
+		default:
+			status_code = WriteEmptyRespHeader(w, http.StatusMethodNotAllowed)
+	}
+
+	return status_code, nil
+
+oops:
+	return status_code, err
+}

server.go

@@ -3,7 +3,6 @@ package hodu
 import "container/list"
 import "context"
 import "crypto/tls"
-import "encoding/base64"
 import "errors"
 import "fmt"
 import "io"
@@ -49,8 +48,12 @@ type ServerSvcPortMap map[PortId]ConnRouteId
 
 type ServerRptyMap map[uint64]*ServerRpty
 type ServerRptyMapByWs map[*websocket.Conn]*ServerRpty
+
 type ServerRpxMap map[uint64]*ServerRpx
 
+type ServerRxcMap map[uint64]*ServerRxc
+type ServerRxcMapByWs map[*websocket.Conn]*ServerRxc
+
 type ServerWpxResponseTransformer func(r *ServerRouteProxyInfo, resp *http.Response) io.Reader
 type ServerWpxForeignPortProxyMaker func(wpx_type string, port_id string) (*ServerRouteProxyInfo, error)
 
@@ -64,6 +67,8 @@ type ServerConfig struct {
 	RpcMaxConns int
 	RpcMinPingIntvl time.Duration
 	MaxPeers int
+	RxcDoneJobRetention time.Duration
+	RxcRunOutputMax int
 	HttpReadHeaderTimeout time.Duration
 	HttpIdleTimeout time.Duration
 	HttpMaxHeaderBytes int
@@ -99,6 +104,8 @@ const (
 	SERVER_EVENT_PEER_STARTED
 	SERVER_EVENT_PEER_UPDATED
 	SERVER_EVENT_PEER_STOPPED
+	SERVER_EVENT_RXC_JOB_RUN_DONE
+	SERVER_EVENT_RXC_JOB_DONE
 )
 
 type ServerEvent struct {
@@ -151,10 +158,15 @@ type Server struct {
 
 	cts_limit        int
 	cts_next_id      ConnId
-	cts_mtx          sync.Mutex
+	cts_mtx          sync.RWMutex
 	cts_map          ServerConnMap
 	cts_map_by_addr  ServerConnMapByAddr
 	cts_map_by_token ServerConnMapByClientToken
+	rxc_job_next_id  uint64
+	rxc_job_mtx      sync.Mutex
+	rxc_job_map      ServerRxcJobMap
+	rxc_job_heap     ServerRxcJobExpiryHeap
+	rxc_job_purge_chan chan struct{}
 	cts_wg           sync.WaitGroup
 
 	pts_limit        int // global pts limit
@@ -181,6 +193,7 @@ type Server struct {
 		pty_sessions atomic.Int64
 		rpty_sessions atomic.Int64
 		rpx_sessions atomic.Int64
+		rxc_sessions atomic.Int64
 	}
 
 	wpx_resp_tf     ServerWpxResponseTransformer
@@ -221,6 +234,11 @@ type ServerConn struct {
 	rpx_mtx        sync.Mutex
 	rpx_map        ServerRpxMap
 
+	rxc_next_id   uint64
+	rxc_mtx       sync.Mutex
+	rxc_map       ServerRxcMap
+	rxc_map_by_ws ServerRxcMapByWs
+
 	wg             sync.WaitGroup
 	stop_req       atomic.Bool
 	stop_chan      chan bool
@@ -269,6 +287,12 @@ type ServerRpx struct {
 	resp_done_chan chan bool
 }
 
+type ServerRxc struct {
+	id uint64
+	ws *websocket.Conn
+	sink ServerRxcSink
+}
+
 type GuardedPacketStreamServer struct {
 	mtx sync.Mutex
 	//pss Hodu_PacketStreamServer
@@ -308,6 +332,17 @@ func (rpx *ServerRpx) ReqStop(close_web bool) {
 	rpx.pw.Close() // close pipe writer
 	if close_web { rpx.br.Close() } // websocket side
 }
+
+func (rxc *ServerRxc) ReqStop() {
+	if rxc.sink != nil {
+		rxc.sink.ReqStop()
+		return
+	}
+	if rxc.ws != nil {
+		//rxc.ws.Close() // dirty way
+		rxc.ws.SetReadDeadline(time.Now()) // slightly cleaner way
+	}
+}
 // ------------------------------------
 
 func NewServerRoute(cts *ServerConn, id RouteId, option RouteOption, ptc_addr string, ptc_name string, svc_requested_addr string, svc_permitted_net string) (*ServerRoute, error) {
@@ -536,7 +571,7 @@ func (cts *ServerConn) make_route_listener(id RouteId, option RouteOption, svc_r
 			return nil, nil, fmt.Errorf("invalid service address %s - %s", svc_requested_addr, err.Error())
 		}
 
-		if (ap.Addr().Is4()) { is4 = true }
+		if ap.Addr().Is4() { is4 = true }
 		svcaddr = &net.TCPAddr{IP: ap.Addr().AsSlice(), Port: int(ap.Port())}
 	}
 
@@ -545,7 +580,7 @@ func (cts *ServerConn) make_route_listener(id RouteId, option RouteOption, svc_r
 		// i don't want the behavior.. I force tcp4 if the ip address given
 		// is ipv4 address
 		nw = "tcp"
-		if (is4) { nw = "tcp4" }
+		if is4 { nw = "tcp4" }
 		if svcaddr == nil {
 			// port 0 for automatic assignment.
 			svcaddr = &net.TCPAddr{Port: 0}
@@ -741,261 +776,6 @@ func (cts *ServerConn) ReqStopAllServerRoutes() {
 	cts.route_mtx.Unlock()
 }
 
-// Rpty
-func (cts *ServerConn) StartRpty(ws *websocket.Conn) (*ServerRpty, error) {
-	var ok bool
-	var start_id uint64
-	var assigned_id uint64
-	var rpty *ServerRpty
-	var err error
-
-	cts.rpty_mtx.Lock()
-	start_id = cts.rpty_next_id
-	for {
-		_, ok = cts.rpty_map[cts.rpty_next_id]
-		if !ok {
-			assigned_id = cts.rpty_next_id
-			cts.rpty_next_id++
-			if cts.rpty_next_id == 0 { cts.rpty_next_id++ }
-			break
-		}
-		cts.rpty_next_id++
-		if cts.rpty_next_id == 0 { cts.rpty_next_id++ }
-		if cts.rpty_next_id == start_id {
-			cts.rpty_mtx.Unlock()
-			return nil, fmt.Errorf("unable to assign id")
-		}
-	}
-
-	_, ok = cts.rpty_map_by_ws[ws]
-	if ok {
-		cts.rpty_mtx.Unlock()
-		return nil, fmt.Errorf("connection already associated with rpty. possibly internal error")
-	}
-
-	rpty = &ServerRpty{
-		id: assigned_id,
-		ws: ws,
-	}
-
-	cts.rpty_map[assigned_id] = rpty
-	cts.rpty_map_by_ws[ws] = rpty
-	cts.rpty_mtx.Unlock()
-
-	err = cts.pss.Send(MakeRptyStartPacket(assigned_id))
-	if err != nil {
-		cts.rpty_mtx.Lock()
-		delete(cts.rpty_map, assigned_id)
-		delete(cts.rpty_map_by_ws, ws)
-		cts.rpty_mtx.Unlock()
-		return nil , err
-	}
-
-	cts.S.stats.rpty_sessions.Add(1)
-	return rpty, nil
-}
-
-func (cts *ServerConn) StopRpty(ws *websocket.Conn) error {
-	// called by the websocket handler.
-	var rpty *ServerRpty
-	var id uint64
-	var ok bool
-	var err error
-
-	cts.rpty_mtx.Lock()
-	rpty, ok = cts.rpty_map_by_ws[ws]
-	if !ok {
-		cts.rpty_mtx.Unlock()
-		cts.S.log.Write(cts.Sid, LOG_ERROR, "Unknown websocket connection for rpty - websocket %v", ws.RemoteAddr())
-		return fmt.Errorf("unknown websocket connection for rpty - %v", ws.RemoteAddr())
-	}
-
-	id = rpty.id
-	cts.rpty_mtx.Unlock()
-
-	// send the stop request to the client side
-	err = cts.pss.Send(MakeRptyStopPacket(id, ""))
-	if err !=  nil {
-		cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to send %s(%d) for server %s websocket %v - %s", PACKET_KIND_RPTY_STOP.String(), id, cts.RemoteAddr, ws.RemoteAddr(), err.Error())
-		// carry on
-	}
-
-	// delete the rpty entry from the maps as the websocket
-	// handler is ending
-	cts.rpty_mtx.Lock()
-	delete(cts.rpty_map, id)
-	delete(cts.rpty_map_by_ws, ws)
-	cts.rpty_mtx.Unlock()
-	cts.S.stats.rpty_sessions.Add(-1)
-
-	cts.S.log.Write(cts.Sid, LOG_INFO, "Stopped rpty(%d) for server %s websocket %vs", id, cts.RemoteAddr, ws.RemoteAddr())
-	return nil
-}
-
-func (cts *ServerConn) StopRptyWsById(id uint64, msg string) error {
-	// call this when the stop requested comes from the client.
-	// abort the websocket side.
-
-	var rpty *ServerRpty
-	var ok bool
-
-	cts.rpty_mtx.Lock()
-	rpty, ok = cts.rpty_map[id]
-	if !ok {
-		cts.rpty_mtx.Unlock()
-		return fmt.Errorf("unknown rpty id %d", id)
-	}
-	cts.rpty_mtx.Unlock()
-
-	rpty.ReqStop()
-	cts.S.log.Write(cts.Sid, LOG_INFO, "Stopped rpty(%d) for %s - %s", id, cts.RemoteAddr, msg)
-	return nil
-}
-
-func (cts *ServerConn) WriteRpty(ws *websocket.Conn, data []byte) error {
-	var rpty *ServerRpty
-	var id uint64
-	var ok bool
-	var err error
-
-	cts.rpty_mtx.Lock()
-	rpty, ok = cts.rpty_map_by_ws[ws]
-	if !ok {
-		cts.rpty_mtx.Unlock()
-		return fmt.Errorf("unknown ws connection for rpty - %v", ws.RemoteAddr())
-	}
-
-	id = rpty.id
-	cts.rpty_mtx.Unlock()
-
-	err = cts.pss.Send(MakeRptyDataPacket(id, data))
-	if err != nil {
-		return fmt.Errorf("unable to send rpty data to client - %s", err.Error())
-	}
-
-	return nil
-}
-
-func (cts *ServerConn) WriteRptySize(ws *websocket.Conn, data []byte) error {
-	var rpty *ServerRpty
-	var id uint64
-	var ok bool
-	var err error
-
-	cts.rpty_mtx.Lock()
-	rpty, ok = cts.rpty_map_by_ws[ws]
-	if !ok {
-		cts.rpty_mtx.Unlock()
-		return fmt.Errorf("unknown ws connection for rpty size - %v", ws.RemoteAddr())
-	}
-
-	id = rpty.id
-	cts.rpty_mtx.Unlock()
-
-	err = cts.pss.Send(MakeRptySizePacket(id, data))
-	if err != nil {
-		return fmt.Errorf("unable to send rpty size to client - %s", err.Error())
-	}
-
-	return nil
-}
-
-func (cts *ServerConn) ReadRptyAndWriteWs(id uint64, data []byte) error {
-	var ok bool
-	var rpty *ServerRpty
-	var err error
-
-	cts.rpty_mtx.Lock()
-	rpty, ok = cts.rpty_map[id]
-	if !ok {
-		cts.rpty_mtx.Unlock()
-		return fmt.Errorf("unknown rpty id - %d", id)
-	}
-	cts.rpty_mtx.Unlock()
-
-	err = send_ws_data_for_xterm(rpty.ws, "iov", base64.StdEncoding.EncodeToString(data))
-	if err != nil {
-		return fmt.Errorf("failed to write rpty data(%d) to ws - %s", id, err.Error())
-	}
-
-	return nil
-}
-
-func (cts *ServerConn) HandleRptyEvent(packet_type PACKET_KIND, evt *RptyEvent) error {
-	switch packet_type {
-		case PACKET_KIND_RPTY_STOP:
-			// stop requested from the server
-			return cts.StopRptyWsById(evt.Id, string(evt.Data))
-
-		case PACKET_KIND_RPTY_DATA:
-			return cts.ReadRptyAndWriteWs(evt.Id, evt.Data)
-	}
-
-	// ignore other packet types
-	return nil
-}
-
-// Rpx
-func (cts *ServerConn) StartRpxWebById(srpx* ServerRpx, id uint64, data []byte) error {
-	// pass the initial response to code in server-rpx.go
-	srpx.start_chan <- data
-	return nil
-}
-
-func (cts *ServerConn) StopRpxWebById(srpx* ServerRpx, id uint64) error {
-	cts.S.log.Write(cts.Sid, LOG_DEBUG, "Requesting to stop rpx(%d)", srpx.id)
-	srpx.ReqStop(true)
-	cts.S.log.Write(cts.Sid, LOG_DEBUG, "Requested to stop rpx(%d)", srpx.id)
-	return nil
-}
-
-func (cts *ServerConn) WroteRpxWebById(srpx* ServerRpx, id uint64, data []byte) error {
-	var err error
-	_, err = srpx.pw.Write(data)
-	if err != nil {
-		cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to write rpx data(%d) to rpx pipe - %s", id, err.Error())
-		srpx.ReqStop(true)
-	}
-	return err
-}
-
-func (cts *ServerConn) EofRpxWebById(srpx* ServerRpx, id uint64) error {
-	srpx.ReqStop(false)
-	return nil
-}
-
-func (cts *ServerConn) HandleRpxEvent(packet_type PACKET_KIND, evt *RpxEvent) error {
-	var ok bool
-	var rpx* ServerRpx
-
-	cts.rpx_mtx.Lock()
-	rpx, ok = cts.rpx_map[evt.Id]
-	if !ok {
-		cts.rpx_mtx.Unlock()
-		return fmt.Errorf("unknown rpx id - %v", evt.Id)
-	}
-	cts.rpx_mtx.Unlock()
-
-	switch packet_type {
-		case PACKET_KIND_RPX_START:
-			return cts.StartRpxWebById(rpx, evt.Id, evt.Data)
-
-		case PACKET_KIND_RPX_STOP:
-			// stop requested from the server
-			return cts.StopRpxWebById(rpx, evt.Id)
-
-		case PACKET_KIND_RPX_EOF:
-			return cts.EofRpxWebById(rpx, evt.Id)
-
-		case PACKET_KIND_RPX_DATA:
-			return cts.WroteRpxWebById(rpx, evt.Id, evt.Data)
-	}
-
-	// ignore other packet types
-	return nil
-}
-
-// Rpx
 func (cts *ServerConn) ReportPacket(route_id RouteId, pts_id PeerId, packet_type PACKET_KIND, event_data interface{}) error {
 	var r *ServerRoute
 	var ok bool
@@ -1013,6 +793,7 @@ func (cts *ServerConn) ReportPacket(route_id RouteId, pts_id PeerId, packet_type
 
 func (cts *ServerConn) receive_from_stream(wg *sync.WaitGroup) {
 	var pkt *Packet
+	var packet_kind PACKET_KIND
 	var err error
 
 	defer wg.Done()
@@ -1028,30 +809,31 @@ func (cts *ServerConn) receive_from_stream(wg *sync.WaitGroup) {
 			goto done
 		}
 
-		switch pkt.Kind {
+		packet_kind = pkt.Kind
+
+		switch packet_kind {
 			case PACKET_KIND_ROUTE_START:
-				var x *Packet_Route
+				var x *RouteDesc
-				var ok bool
+				x = pkt.GetRoute()
-				x, ok = pkt.U.(*Packet_Route)
+				if x != nil {
-				if ok {
 					var r *ServerRoute
 
-					r, err = cts.AddNewServerRoute(RouteId(x.Route.RouteId), RouteOption(x.Route.ServiceOption), x.Route.TargetAddrStr, x.Route.TargetName, x.Route.ServiceAddrStr, x.Route.ServiceNetStr)
+					r, err = cts.AddNewServerRoute(RouteId(x.RouteId), RouteOption(x.ServiceOption), x.TargetAddrStr, x.TargetName, x.ServiceAddrStr, x.ServiceNetStr)
 					if err != nil {
 						cts.S.log.Write(cts.Sid, LOG_ERROR,
 							"Failed to add route(%d,%s) for %s - %s",
-							x.Route.RouteId, x.Route.TargetAddrStr, cts.RemoteAddr, err.Error())
+							x.RouteId, x.TargetAddrStr, cts.RemoteAddr, err.Error())
 
-						err = cts.pss.Send(MakeRouteStoppedPacket(RouteId(x.Route.RouteId), RouteOption(x.Route.ServiceOption), x.Route.TargetAddrStr, x.Route.TargetName, x.Route.ServiceAddrStr, x.Route.ServiceNetStr))
+						err = cts.pss.Send(MakeRouteStoppedPacket(RouteId(x.RouteId), RouteOption(x.ServiceOption), x.TargetAddrStr, x.TargetName, x.ServiceAddrStr, x.ServiceNetStr))
 						if err != nil {
 							cts.S.log.Write(cts.Sid, LOG_ERROR,
 								"Failed to send %s event(%d,%s,%v,%s) to client %s - %s",
-								PACKET_KIND_ROUTE_STOPPED.String(), x.Route.RouteId, x.Route.TargetAddrStr,  x.Route.ServiceOption, x.Route.ServiceNetStr, cts.RemoteAddr, err.Error())
+								PACKET_KIND_ROUTE_STOPPED.String(), x.RouteId, x.TargetAddrStr,  x.ServiceOption, x.ServiceNetStr, cts.RemoteAddr, err.Error())
 							goto done
 						} else {
 							cts.S.log.Write(cts.Sid, LOG_DEBUG,
 								"Sent %s event(%d,%s,%v,%s) to client %s",
-								PACKET_KIND_ROUTE_STOPPED.String(), x.Route.RouteId,x.Route.TargetAddrStr,  x.Route.ServiceOption, x.Route.ServiceNetStr, cts.RemoteAddr)
+								PACKET_KIND_ROUTE_STOPPED.String(), x.RouteId,x.TargetAddrStr,  x.ServiceOption, x.ServiceNetStr, cts.RemoteAddr)
 						}
 
 					} else {
@@ -1077,17 +859,16 @@ func (cts *ServerConn) receive_from_stream(wg *sync.WaitGroup) {
 				}
 
 			case PACKET_KIND_ROUTE_STOP:
-				var x *Packet_Route
+				var x *RouteDesc
-				var ok bool
+				x = pkt.GetRoute()
-				x, ok = pkt.U.(*Packet_Route)
+				if x != nil {
-				if ok {
 					var r *ServerRoute
 
-					r, err = cts.RemoveServerRouteById(RouteId(x.Route.RouteId))
+					r, err = cts.RemoveServerRouteById(RouteId(x.RouteId))
 					if err != nil {
 						cts.S.log.Write(cts.Sid, LOG_ERROR,
 							"Failed to delete route(%d,%s) for client %s - %s",
-							x.Route.RouteId, x.Route.TargetAddrStr, cts.RemoteAddr, err.Error())
+							x.RouteId, x.TargetAddrStr, cts.RemoteAddr, err.Error())
 					} else {
 						cts.S.log.Write(cts.Sid, LOG_ERROR,
 							"Deleted route(%d,%s,%s,%v,%v) for client %s",
@@ -1115,117 +896,113 @@ func (cts *ServerConn) receive_from_stream(wg *sync.WaitGroup) {
 				fallthrough
 			case PACKET_KIND_PEER_STOPPED:
 				// the connection from the client to a peer has been established
-				var x *Packet_Peer
+				var x *PeerDesc
-				var ok bool
+				x = pkt.GetPeer()
-				x, ok = pkt.U.(*Packet_Peer)
+				if x != nil {
-				if ok {
+					err = cts.ReportPacket(RouteId(x.RouteId), PeerId(x.PeerId), packet_kind, x)
-					err = cts.ReportPacket(RouteId(x.Peer.RouteId), PeerId(x.Peer.PeerId), pkt.Kind, x.Peer)
 					if err != nil {
 						cts.S.log.Write(cts.Sid, LOG_ERROR,
 							"Failed to handle %s event from %s for peer(%d,%d,%s,%s) - %s",
-							pkt.Kind.String(), cts.RemoteAddr, x.Peer.RouteId, x.Peer.PeerId, x.Peer.LocalAddrStr, x.Peer.RemoteAddrStr, err.Error())
+							packet_kind.String(), cts.RemoteAddr, x.RouteId, x.PeerId, x.LocalAddrStr, x.RemoteAddrStr, err.Error())
 					} else {
 						cts.S.log.Write(cts.Sid, LOG_DEBUG,
 							"Handled %s event from %s for peer(%d,%d,%s,%s)",
-							pkt.Kind.String(), cts.RemoteAddr, x.Peer.RouteId, x.Peer.PeerId, x.Peer.LocalAddrStr, x.Peer.RemoteAddrStr)
+							packet_kind.String(), cts.RemoteAddr, x.RouteId, x.PeerId, x.LocalAddrStr, x.RemoteAddrStr)
 					}
 				} else {
 					// invalid event data
-					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s event from %s", pkt.Kind.String(), cts.RemoteAddr)
+					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s event from %s", packet_kind.String(), cts.RemoteAddr)
 				}
 
 			case PACKET_KIND_PEER_DATA:
 				// the connection from the client to a peer has been established
-				var x *Packet_Data
+				var x *PeerData
-				var ok bool
+				x = pkt.GetData()
-				x, ok = pkt.U.(*Packet_Data)
+				if x != nil {
-				if ok {
+					err = cts.ReportPacket(RouteId(x.RouteId), PeerId(x.PeerId), packet_kind, x.Data)
-					err = cts.ReportPacket(RouteId(x.Data.RouteId), PeerId(x.Data.PeerId), pkt.Kind, x.Data.Data)
 					if err != nil {
 						cts.S.log.Write(cts.Sid, LOG_ERROR,
 							"Failed to handle %s event from %s for peer(%d,%d) - %s",
-							pkt.Kind.String(), cts.RemoteAddr, x.Data.RouteId, x.Data.PeerId, err.Error())
+							packet_kind.String(), cts.RemoteAddr, x.RouteId, x.PeerId, err.Error())
 					} else {
 						cts.S.log.Write(cts.Sid, LOG_DEBUG,
 							"Handled %s event from %s for peer(%d,%d)",
-							pkt.Kind.String(), cts.RemoteAddr, x.Data.RouteId, x.Data.PeerId)
+							packet_kind.String(), cts.RemoteAddr, x.RouteId, x.PeerId)
 					}
 				} else {
 					// invalid event data
-					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s event from %s", pkt.Kind.String(), cts.RemoteAddr)
+					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s event from %s", packet_kind.String(), cts.RemoteAddr)
 				}
 
 			case PACKET_KIND_CONN_DESC:
-				var x *Packet_Conn
+				var x *ConnDesc
-				var ok bool
+				x = pkt.GetConn();
-				x, ok = pkt.U.(*Packet_Conn)
+				if x != nil {
-				if ok {
+					if x.Token == "" {
-					if x.Conn.Token == "" {
+						cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s - blank token", packet_kind.String(), cts.RemoteAddr)
-						cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s - blank token", pkt.Kind.String(), cts.RemoteAddr)
 						cts.pss.Send(MakeConnErrorPacket(1, "blank token refused"))
 						cts.ReqStop() // TODO: is this desirable to disconnect?
-					} else if x.Conn.Token != cts.ClientToken.Get() {
+					} else if x.Token != cts.ClientToken.Get() {
-						_, err = strconv.ParseUint(x.Conn.Token, 10, int(unsafe.Sizeof(ConnId(0)) * 8))
+						_, err = strconv.ParseUint(x.Token, 10, int(unsafe.Sizeof(ConnId(0)) * 8))
 						if err == nil { // this is not != nil. this is to check if the token is numeric
-							cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s - numeric token '%s'", pkt.Kind.String(), cts.RemoteAddr, x.Conn.Token)
+							cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s - numeric token '%s'", packet_kind.String(), cts.RemoteAddr, x.Token)
 							cts.pss.Send(MakeConnErrorPacket(1, "numeric token refused"))
 							cts.ReqStop() // TODO: is this desirable to disconnect?
 						} else {
+							var ok bool
 							cts.S.cts_mtx.Lock()
-							_, ok = cts.S.cts_map_by_token[x.Conn.Token]
+							_, ok = cts.S.cts_map_by_token[x.Token]
 							if ok {
 								// error
 								cts.S.cts_mtx.Unlock()
-								cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s - duplicate token '%s'", pkt.Kind.String(), cts.RemoteAddr, x.Conn.Token)
+								cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s - duplicate token '%s'", packet_kind.String(), cts.RemoteAddr, x.Token)
-								cts.pss.Send(MakeConnErrorPacket(1, fmt.Sprintf("duplicate token refused - %s", x.Conn.Token)))
+								cts.pss.Send(MakeConnErrorPacket(1, fmt.Sprintf("duplicate token refused - %s", x.Token)))
 								cts.ReqStop() // TODO: is this desirable to disconnect?
 							} else {
 								if cts.ClientToken.Get() != "" { delete(cts.S.cts_map_by_token, cts.ClientToken.Get()) }
-								cts.ClientToken.Set(x.Conn.Token)
+								cts.ClientToken.Set(x.Token)
-								cts.S.cts_map_by_token[x.Conn.Token] = cts
+								cts.S.cts_map_by_token[x.Token] = cts
 								cts.S.cts_mtx.Unlock()
-								cts.S.log.Write(cts.Sid, LOG_INFO, "client(%d) %s - token set to '%s'", cts.Id, cts.RemoteAddr, x.Conn.Token)
+								cts.S.log.Write(cts.Sid, LOG_INFO, "client(%d) %s - token set to '%s'", cts.Id, cts.RemoteAddr, x.Token)
 
 								cts.S.FireConnEvent(SERVER_EVENT_CONN_UPDATED, cts)
 							}
 						}
 					}
 				} else {
-					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", pkt.Kind.String(), cts.RemoteAddr)
+					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", packet_kind.String(), cts.RemoteAddr)
 				}
 
 			case PACKET_KIND_CONN_NOTICE:
 				// the connection from the client to a peer has been established
-				var x *Packet_ConnNoti
+				var x *ConnNotice
-				var ok bool
+				x = pkt.GetConnNoti()
-				x, ok = pkt.U.(*Packet_ConnNoti)
+				if x != nil {
-				if ok {
 					if cts.S.conn_notice_handlers != nil {
 						var handler ServerConnNoticeHandler
 						for _, handler = range cts.S.conn_notice_handlers {
-							handler.Handle(cts, x.ConnNoti.Text)
+							handler.Handle(cts, x.Text)
 						}
 					}
 				} else {
-					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", pkt.Kind.String(), cts.RemoteAddr)
+					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", packet_kind.String(), cts.RemoteAddr)
 				}
 
 			//case PACKET_KIND_RPTY_START: stop is never sent by the client to the server
 			case PACKET_KIND_RPTY_STOP:
 				fallthrough
 			case PACKET_KIND_RPTY_DATA:
-				var x *Packet_RptyEvt
+				var evt *RptyEvent
-				var ok bool
+				evt = pkt.GetRptyEvt()
-				x, ok = pkt.U.(*Packet_RptyEvt)
+				if evt != nil {
-				if ok {
+					err = cts.HandleRptyEvent(packet_kind, evt)
-					err = cts.HandleRptyEvent(pkt.Kind, x.RptyEvt)
 					if err != nil {
-						cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to handle %s event for rpty(%d) from %s - %s", pkt.Kind.String(), x.RptyEvt.Id, cts.RemoteAddr, err.Error())
+						cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to handle %s event for rpty(%d) from %s - %s", packet_kind.String(), evt.Id, cts.RemoteAddr, err.Error())
 					} else {
-						cts.S.log.Write(cts.Sid, LOG_DEBUG, "Handled %s event for rpty(%d) from %s", pkt.Kind.String(), x.RptyEvt.Id, cts.RemoteAddr)
+						cts.S.log.Write(cts.Sid, LOG_DEBUG, "Handled %s event for rpty(%d) from %s", packet_kind.String(), evt.Id, cts.RemoteAddr)
 					}
 				} else {
-					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", pkt.Kind.String(), cts.RemoteAddr)
+					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", packet_kind.String(), cts.RemoteAddr)
 				}
 
 			case PACKET_KIND_RPX_START: // the client sends the response header using START
@@ -1235,18 +1012,35 @@ func (cts *ServerConn) receive_from_stream(wg *sync.WaitGroup) {
 			case PACKET_KIND_RPX_EOF:
 				fallthrough
 			case PACKET_KIND_RPX_DATA:
-				var x *Packet_RpxEvt
+				var evt *RpxEvent
-				var ok bool
+				evt = pkt.GetRpxEvt();
-				x, ok = pkt.U.(*Packet_RpxEvt)
+				if evt != nil {
-				if ok {
+					err = cts.HandleRpxEvent(packet_kind, evt)
-					err = cts.HandleRpxEvent(pkt.Kind, x.RpxEvt)
 					if err != nil {
-						cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to handle %s event for rpx(%d) from %s - %s", pkt.Kind.String(), x.RpxEvt.Id, cts.RemoteAddr, err.Error())
+						cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to handle %s event for rpx(%d) from %s - %s", packet_kind.String(), evt.Id, cts.RemoteAddr, err.Error())
 					} else {
-						cts.S.log.Write(cts.Sid, LOG_DEBUG, "Handled %s event for rpx(%d) from %s", pkt.Kind.String(), x.RpxEvt.Id, cts.RemoteAddr)
+						cts.S.log.Write(cts.Sid, LOG_DEBUG, "Handled %s event for rpx(%d) from %s", packet_kind.String(), evt.Id, cts.RemoteAddr)
 					}
 				} else {
-					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", pkt.Kind.String(), cts.RemoteAddr)
+					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", packet_kind.String(), cts.RemoteAddr)
+				}
+
+			case PACKET_KIND_RXC_START: // the client sends the response header using START
+				fallthrough
+			case PACKET_KIND_RXC_STOP:
+				fallthrough
+			case PACKET_KIND_RXC_DATA:
+				var evt *RxcEvent
+				evt = pkt.GetRxcEvt()
+				if evt != nil {
+					err = cts.HandleRxcEvent(packet_kind, evt)
+					if err != nil {
+						cts.S.log.Write(cts.Sid, LOG_ERROR, "Failed to handle %s event for rxc(%d) from %s - %s", packet_kind.String(), evt.Id, cts.RemoteAddr, err.Error())
+					} else {
+						cts.S.log.Write(cts.Sid, LOG_DEBUG, "Handled %s event for rxc(%d) from %s", packet_kind.String(), evt.Id, cts.RemoteAddr)
+					}
+				} else {
+					cts.S.log.Write(cts.Sid, LOG_ERROR, "Invalid %s packet from %s", packet_kind.String(), cts.RemoteAddr)
 				}
 		}
 	}
@@ -1272,6 +1066,16 @@ done:
 	}
 	cts.rpx_mtx.Unlock()
 
+	// arrange to break all rxc resources
+	cts.rxc_mtx.Lock()
+	if len(cts.rxc_map) > 0 {
+		var rxc *ServerRxc
+		for _, rxc = range cts.rxc_map {
+			rxc.ReqStop()
+		}
+	}
+	cts.rxc_mtx.Unlock()
+
 	cts.S.log.Write(cts.Sid, LOG_INFO, "RPC stream receiver ended")
 }
 
@@ -1342,6 +1146,7 @@ func (cts *ServerConn) ReqStop() {
 		var r *ServerRoute
 		var rpty *ServerRpty
 		var srpx *ServerRpx
+		var rxc *ServerRxc
 
 		cts.route_mtx.Lock()
 		for _, r = range cts.route_map { r.ReqStop() }
@@ -1355,6 +1160,10 @@ func (cts *ServerConn) ReqStop() {
 		for _, srpx = range cts.rpx_map { srpx.ReqStop(true) }
 		cts.rpx_mtx.Unlock()
 
+		cts.rxc_mtx.Lock()
+		for _, rxc = range cts.rxc_map { rxc.ReqStop() }
+		cts.rxc_mtx.Unlock()
+
 		// there is no good way to break a specific connection client to
 		// the grpc server. while the global grpc server is closed in
 		// ReqStop() for Server, the individuation connection is closed
@@ -1445,7 +1254,7 @@ func (cc *ConnCatcher) HandleConn(ctx context.Context, cs stats.ConnStats) {
 	}
 
 	/*
-	md,ok:=metadata.FromIncomingContext(ctx)
+	md, ok = metadata.FromIncomingContext(ctx)
 	if ok {
 	}*/
 
@@ -1728,6 +1537,10 @@ func NewServer(ctx context.Context, name string, logger Logger, cfg *ServerConfi
 	s.cts_map = make(ServerConnMap)
 	s.cts_map_by_addr = make(ServerConnMapByAddr)
 	s.cts_map_by_token = make(ServerConnMapByClientToken)
+	s.rxc_job_next_id = 1
+	s.rxc_job_map = make(ServerRxcJobMap)
+	s.rxc_job_heap = make(ServerRxcJobExpiryHeap, 0)
+	s.rxc_job_purge_chan = make(chan struct{}, 1)
 	s.svc_port_map = make(ServerSvcPortMap)
 	s.stop_chan = make(chan bool, 8)
 	s.stop_req.Store(false)
@@ -1859,6 +1672,21 @@ func NewServer(ctx context.Context, name string, logger Logger, cfg *ServerConfi
 	s.ctl_mux.Handle("/_rpty/",
 		s.WrapHttpHandler(&server_pty_xterm_file{ServerCtl: ServerCtl{S: &s, Id: HS_ID_CTL}, file: "_redir:xterm.html"}))
 
+	s.ctl_mux.Handle("/_rxc",
+		s.WrapHttpHandler(&server_ctl_rxc{ServerCtl{S: &s, Id: HS_ID_CTL}}))
+	s.ctl_mux.Handle("/_rxc/{job_id}",
+		s.WrapHttpHandler(&server_ctl_rxc_id{ServerCtl{S: &s, Id: HS_ID_CTL}}))
+	s.ctl_mux.Handle("/_rxc/{job_id}/stop",
+		s.WrapHttpHandler(&server_ctl_rxc_id_stop{ServerCtl{S: &s, Id: HS_ID_CTL}}))
+	s.ctl_mux.Handle("/_rxc/{job_id}/runs",
+		s.WrapHttpHandler(&server_ctl_rxc_id_runs{ServerCtl{S: &s, Id: HS_ID_CTL}}))
+	s.ctl_mux.Handle("/_rxc/{job_id}/runs/{conn_id}",
+		s.WrapHttpHandler(&server_ctl_rxc_id_runs_id{ServerCtl{S: &s, Id: HS_ID_CTL}}))
+	s.ctl_mux.Handle("/_rxc/{job_id}/runs/{conn_id}/stop",
+		s.WrapHttpHandler(&server_ctl_rxc_id_runs_id_stop{ServerCtl{S: &s, Id: HS_ID_CTL}}))
+	s.ctl_mux.Handle("/_rxc/ws",
+		s.SafeWrapWebsocketHandler(s.WrapWebsocketHandler(&server_rxc_ws{S: &s, Id: HS_ID_CTL})))
+
 	s.ctl = make([]*http.Server, len(cfg.CtlAddrs))
 	for i = 0; i < len(cfg.CtlAddrs); i++ {
 		s.ctl[i] = &http.Server{
@@ -2021,6 +1849,7 @@ func NewServer(ctx context.Context, name string, logger Logger, cfg *ServerConfi
 	s.stats.pty_sessions.Store(0)
 	s.stats.rpty_sessions.Store(0)
 	s.stats.rpx_sessions.Store(0)
+	s.stats.rxc_sessions.Store(0)
 
 	return &s, nil
 
@@ -2385,9 +2214,7 @@ func (s *Server) ReqStop() {
 		}
 
 		// request to stop connections from/to peer held in the cts structure
-		s.cts_mtx.Lock()
+		for _, cts = range s.snapshot_server_conns() { cts.ReqStop() }
-		for _, cts = range s.cts_map { cts.ReqStop() }
-		s.cts_mtx.Unlock()
 
 		s.stop_chan <- true
 	}
@@ -2413,8 +2240,12 @@ func (s *Server) AddNewServerConn(remote_addr *net.Addr, local_addr *net.Addr, p
 
 	cts.rpty_map = make(ServerRptyMap)
 	cts.rpty_map_by_ws = make(ServerRptyMapByWs)
+
 	cts.rpx_map = make(ServerRpxMap)
 
+	cts.rxc_map = make(ServerRxcMap)
+	cts.rxc_map_by_ws = make(ServerRxcMapByWs)
+
 	s.cts_mtx.Lock()
 
 	if s.cts_limit > 0 && len(s.cts_map) >= s.cts_limit {
@@ -2443,6 +2274,7 @@ func (s *Server) AddNewServerConn(remote_addr *net.Addr, local_addr *net.Addr, p
 	cts.Id = assigned_id
 	cts.Sid = fmt.Sprintf("%d", cts.Id) // id in string used for logging
 	cts.rpty_next_id = 1
+	cts.rxc_next_id = 1
 
 	_, ok = s.cts_map_by_addr[cts.RemoteAddr]
 	if ok {
@@ -2470,9 +2302,7 @@ func (s *Server) AddNewServerConn(remote_addr *net.Addr, local_addr *net.Addr, p
 
 func (s *Server) ReqStopAllServerConns() {
 	var cts *ServerConn
-	s.cts_mtx.Lock()
+	for _, cts = range s.snapshot_server_conns() { cts.ReqStop() }
-	for _, cts = range s.cts_map { cts.ReqStop() }
-	s.cts_mtx.Unlock()
 }
 
 func (s *Server) RemoveServerConn(cts *ServerConn) error {
@@ -2547,8 +2377,8 @@ func (s *Server) FindServerConnById(id ConnId) *ServerConn {
 	var cts *ServerConn
 	var ok bool
 
-	s.cts_mtx.Lock()
+	s.cts_mtx.RLock()
-	defer s.cts_mtx.Unlock()
+	defer s.cts_mtx.RUnlock()
 
 	cts, ok = s.cts_map[id]
 	if !ok { return nil }
@@ -2560,8 +2390,8 @@ func (s *Server) FindServerConnByAddr(addr net.Addr) *ServerConn {
 	var cts *ServerConn
 	var ok bool
 
-	s.cts_mtx.Lock()
+	s.cts_mtx.RLock()
-	defer s.cts_mtx.Unlock()
+	defer s.cts_mtx.RUnlock()
 
 	cts, ok = s.cts_map_by_addr[addr]
 	if !ok { return nil }
@@ -2573,8 +2403,8 @@ func (s *Server) FindServerConnByClientToken(token string) *ServerConn {
 	var cts *ServerConn
 	var ok bool
 
-	s.cts_mtx.Lock()
+	s.cts_mtx.RLock()
-	defer s.cts_mtx.Unlock()
+	defer s.cts_mtx.RUnlock()
 
 	cts, ok = s.cts_map_by_token[token]
 	if !ok { return nil }
@@ -2586,8 +2416,8 @@ func (s *Server) FindServerRouteById(id ConnId, route_id RouteId) *ServerRoute {
 	var cts *ServerConn
 	var ok bool
 
-	s.cts_mtx.Lock()
+	s.cts_mtx.RLock()
-	defer s.cts_mtx.Unlock()
+	defer s.cts_mtx.RUnlock()
 
 	cts, ok = s.cts_map[id]
 	if !ok { return nil }
@@ -2599,8 +2429,8 @@ func (s *Server) FindServerRouteByIdAndPtcName(id ConnId, ptc_name string) *Serv
 	var cts *ServerConn
 	var ok bool
 
-	s.cts_mtx.Lock()
+	s.cts_mtx.RLock()
-	defer s.cts_mtx.Unlock()
+	defer s.cts_mtx.RUnlock()
 
 	cts, ok = s.cts_map[id]
 	if !ok { return nil }
@@ -2612,8 +2442,8 @@ func (s *Server) FindServerRouteByClientTokenAndRouteId(token string, route_id R
 	var cts *ServerConn
 	var ok bool
 
-	s.cts_mtx.Lock()
+	s.cts_mtx.RLock()
-	defer s.cts_mtx.Unlock()
+	defer s.cts_mtx.RUnlock()
 
 	cts, ok = s.cts_map_by_token[token]
 	if !ok { return nil }
@@ -2625,8 +2455,8 @@ func (s *Server) FindServerRouteByClientTokenAndPtcName(token string, ptc_name s
 	var cts *ServerConn
 	var ok bool
 
-	s.cts_mtx.Lock()
+	s.cts_mtx.RLock()
-	defer s.cts_mtx.Unlock()
+	defer s.cts_mtx.RUnlock()
 
 	cts, ok = s.cts_map_by_token[token]
 	if !ok { return nil }
@@ -2638,8 +2468,8 @@ func (s *Server) FindServerPeerConnById(id ConnId, route_id RouteId, peer_id Pee
 	var cts *ServerConn
 	var ok bool
 
-	s.cts_mtx.Lock()
+	s.cts_mtx.RLock()
-	defer s.cts_mtx.Unlock()
+	defer s.cts_mtx.RUnlock()
 
 	cts, ok = s.cts_map[id]
 	if !ok { return nil }
@@ -2775,6 +2605,10 @@ func (s *Server) StartService(data interface{}) {
 	go s.bulletin.RunTask(&s.wg)
 	s.wg.Add(1)
 	go s.RunTask(&s.wg)
+	if s.Cfg.RxcDoneJobRetention > 0 {
+		s.wg.Add(1)
+		go s.run_rxc_job_purger(&s.wg, s.Cfg.RxcDoneJobRetention)
+	}
 }
 
 func (s *Server) StartExtService(svc Service, data interface{}) {
@@ -2885,6 +2719,19 @@ func (s *Server) RemoveCtlMetricsCollector(col prometheus.Collector) bool {
 	return s.promreg.Unregister(col)
 }
 
+func (s *Server) snapshot_server_conns() []*ServerConn {
+	var conns []*ServerConn
+	var cts *ServerConn
+
+	s.cts_mtx.RLock()
+	conns = make([]*ServerConn, 0, len(s.cts_map))
+	for _, cts = range s.cts_map {
+		conns = append(conns, cts)
+	}
+	s.cts_mtx.RUnlock()
+	return conns
+}
+
 func (s *Server) SendNotice(id_str string, text string) error {
 	var cts *ServerConn
 	var err error
@@ -2897,14 +2744,10 @@ func (s *Server) SendNotice(id_str string, text string) error {
 			return fmt.Errorf("failed to send conn_notice text '%s' to %s - %s", text, cts.RemoteAddr, err.Error())
 		}
 	} else {
-		s.cts_mtx.Lock()
+		for _, cts = range s.snapshot_server_conns() {
-		// TODO: what if this loop takes too long? in that case,
-		//       lock is held for long. think about how to handle this.
-		for _, cts = range s.cts_map {
 			cts.pss.Send(MakeConnNoticePacket(text))
 			// let's not care about an error when broacasting a notice to all connections
 		}
-		s.cts_mtx.Unlock()
 	}
 
 	return nil
@@ -2996,3 +2839,27 @@ func (s *Server) FirePeerEvent(event_kind ServerEventKind, pts *ServerPeerConn)
 		)
 	}
 }
+
+func (s *Server) FireRxcJobEvent(event_kind ServerEventKind, job *ServerRxcJob) {
+	var js json_out_server_rxc_job
+
+	js = server_rxc_job_to_json(job)
+	s.bulletin.Enqueue(
+		&ServerEvent{
+			Kind: event_kind,
+			Data: &js,
+		},
+	)
+}
+
+func (s *Server) FireRxcJobRunEvent(event_kind ServerEventKind, run *ServerRxcJobRun) {
+	var js json_out_server_rxc_run
+
+	js = server_rxc_run_to_json(run, false)
+	s.bulletin.Enqueue(
+		&ServerEvent{
+			Kind: event_kind,
+			Data: &js,
+		},
+	)
+}