diff --git a/Makefile b/Makefile index c7de9df..7ebf45a 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,16 @@ VERSION=1.0.0 SRCS=\ client.go \ client-ctl.go \ + client-metrics.go \ client-peer.go \ hodu.go \ hodu.pb.go \ hodu_grpc.pb.go \ + jwt.go \ packet.go \ server.go \ server-ctl.go \ + server-metrics.go \ server-peer.go \ server-proxy.go \ system.go \ @@ -49,6 +52,9 @@ clean: go clean -x -i rm -f $(NAME) +test: + go test -x + hodu.pb.go: hodu.proto protoc --go_out=. --go_opt=paths=source_relative \ --go-grpc_out=. --go-grpc_opt=paths=source_relative \ @@ -75,4 +81,4 @@ cmd/tls.crt: cmd/tls.key: openssl req -x509 -newkey rsa:4096 -keyout cmd/tls.key -out cmd/tls.crt -sha256 -days 36500 -nodes -subj "/CN=$(NAME)" --addext "subjectAltName=DNS:$(NAME),IP:10.0.0.1,IP:::1" -.PHONY: clean +.PHONY: clean test diff --git a/client-ctl.go b/client-ctl.go index b1da1dc..95e070b 100644 --- a/client-ctl.go +++ b/client-ctl.go @@ -3,8 +3,6 @@ package hodu import "encoding/json" import "fmt" import "net/http" -//import "net/url" -import "runtime" import "strconv" import "strings" import "time" @@ -85,14 +83,7 @@ type json_out_client_peer struct { } type json_out_client_stats struct { - CPUs int `json:"cpus"` - Goroutines int `json:"goroutines"` - - NumGCs uint32 `json:"num-gcs"` - HeapAllocBytes uint64 `json:"memory-alloc-bytes"` - MemAllocs uint64 `json:"memory-num-allocs"` - MemFrees uint64 `json:"memory-num-frees"` - + json_out_go_stats ClientConns int64 `json:"client-conns"` ClientRoutes int64 `json:"client-routes"` ClientPeers int64 `json:"client-peers"` @@ -140,7 +131,7 @@ type client_ctl_stats struct { // ------------------------------------ -func (ctl *client_ctl) GetId() string { +func (ctl *client_ctl) Id() string { return ctl.id } @@ -831,14 +822,7 @@ func (ctl *client_ctl_stats) ServeHTTP(w http.ResponseWriter, req *http.Request) switch req.Method { case http.MethodGet: var stats json_out_client_stats - var mstat runtime.MemStats - runtime.ReadMemStats(&mstat) - stats.CPUs = runtime.NumCPU() - stats.Goroutines = runtime.NumGoroutine() - stats.NumGCs = mstat.NumGC - stats.HeapAllocBytes = mstat.HeapAlloc - stats.MemAllocs = mstat.Mallocs - stats.MemFrees = mstat.Frees + stats.from_runtime_stats() stats.ClientConns = c.stats.conns.Load() stats.ClientRoutes = c.stats.routes.Load() stats.ClientPeers = c.stats.peers.Load() diff --git a/client-metrics.go b/client-metrics.go new file mode 100644 index 0000000..3d21f86 --- /dev/null +++ b/client-metrics.go @@ -0,0 +1,85 @@ +package hodu + +import "runtime" +import "github.com/prometheus/client_golang/prometheus" + +type ClientCollector struct { + client *Client + BuildInfo *prometheus.Desc + ClientConns *prometheus.Desc + ClientRoutes *prometheus.Desc + ClientPeers *prometheus.Desc +} + +// NewClientCollector returns a new ClientCollector with all prometheus.Desc initialized +func NewClientCollector(client *Client) ClientCollector { + var prefix string + + prefix = client.Name() + "_" + return ClientCollector{ + client: client, + + BuildInfo: prometheus.NewDesc( + prefix + "build_info", + "Build information", + []string{ + "goarch", + "goos", + "goversion", + }, nil, + ), + + ClientConns: prometheus.NewDesc( + prefix + "client_conns", + "Number of client connections from clients", + nil, nil, + ), + ClientRoutes: prometheus.NewDesc( + prefix + "client_routes", + "Number of client-side routes", + nil, nil, + ), + ClientPeers: prometheus.NewDesc( + prefix + "client_peers", + "Number of client-side peers", + nil, nil, + ), + } +} + +func (c ClientCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.BuildInfo + ch <- c.ClientConns + ch <- c.ClientRoutes + ch <- c.ClientPeers +} + +func (c ClientCollector) Collect(ch chan<- prometheus.Metric) { + + ch <- prometheus.MustNewConstMetric( + c.BuildInfo, + prometheus.GaugeValue, + 1, + runtime.GOARCH, + runtime.GOOS, + runtime.Version(), + ) + + ch <- prometheus.MustNewConstMetric( + c.ClientConns, + prometheus.GaugeValue, + float64(c.client.stats.conns.Load()), + ) + + ch <- prometheus.MustNewConstMetric( + c.ClientRoutes, + prometheus.GaugeValue, + float64(c.client.stats.routes.Load()), + ) + + ch <- prometheus.MustNewConstMetric( + c.ClientPeers, + prometheus.GaugeValue, + float64(c.client.stats.peers.Load()), + ) +} diff --git a/client.go b/client.go index ba35a09..8969f05 100644 --- a/client.go +++ b/client.go @@ -18,6 +18,8 @@ import "google.golang.org/grpc/credentials" import "google.golang.org/grpc/credentials/insecure" import "google.golang.org/grpc/peer" import "google.golang.org/grpc/status" +import "github.com/prometheus/client_golang/prometheus" +import "github.com/prometheus/client_golang/prometheus/promhttp" type PacketStreamClient grpc.BidiStreamingClient[Packet, Packet] @@ -51,6 +53,8 @@ type ClientConfigActive struct { } type Client struct { + Named + ctx context.Context ctx_cancel context.CancelFunc ctltlscfg *tls.Config @@ -78,6 +82,7 @@ type Client struct { log Logger route_persister ClientRoutePersister + promreg *prometheus.Registry stats struct { conns atomic.Int64 routes atomic.Int64 @@ -859,7 +864,7 @@ func (cts *ClientConn) add_client_routes(routes []ClientRouteConfig) error { for _, v = range routes { _, err = cts.AddNewClientRoute(&v) if err != nil { - return fmt.Errorf("unable to add client route for %s - %s", v, err.Error()) + return fmt.Errorf("unable to add client route for %v - %s", v, err.Error()) } } @@ -1219,7 +1224,7 @@ func (hlw *client_ctl_log_writer) Write(p []byte) (n int, err error) { } type ClientHttpHandler interface { - GetId() string + Id() string ServeHTTP (w http.ResponseWriter, req *http.Request) (int, error) } @@ -1253,19 +1258,20 @@ func (c *Client) wrap_http_handler(handler ClientHttpHandler) http.Handler { if status_code > 0 { if err == nil { - c.log.Write(handler.GetId(), LOG_INFO, "[%s] %s %s %d %.9f", req.RemoteAddr, req.Method, req.URL.String(), status_code, time_taken.Seconds()) + c.log.Write(handler.Id(), LOG_INFO, "[%s] %s %s %d %.9f", req.RemoteAddr, req.Method, req.URL.String(), status_code, time_taken.Seconds()) } else { - c.log.Write(handler.GetId(), LOG_INFO, "[%s] %s %s %d %.9f - %s", req.RemoteAddr, req.Method, req.URL.String(), status_code, time_taken.Seconds(), err.Error()) + c.log.Write(handler.Id(), LOG_INFO, "[%s] %s %s %d %.9f - %s", req.RemoteAddr, req.Method, req.URL.String(), status_code, time_taken.Seconds(), err.Error()) } } }) } -func NewClient(ctx context.Context, logger Logger, ctl_addrs []string, ctl_prefix string, ctltlscfg *tls.Config, rpctlscfg *tls.Config, rpc_max int, peer_max int, peer_conn_tmout time.Duration) *Client { +func NewClient(ctx context.Context, name string, logger Logger, ctl_addrs []string, ctl_prefix string, ctltlscfg *tls.Config, rpctlscfg *tls.Config, rpc_max int, peer_max int, peer_conn_tmout time.Duration) *Client { var c Client var i int var hs_log *log.Logger + c.name = name c.ctx, c.ctx_cancel = context.WithCancel(ctx) c.ctltlscfg = ctltlscfg c.rpctlscfg = rpctlscfg @@ -1298,6 +1304,14 @@ func NewClient(ctx context.Context, logger Logger, ctl_addrs []string, ctl_prefi c.ctl_mux.Handle(c.ctl_prefix + "/_ctl/stats", c.wrap_http_handler(&client_ctl_stats{client_ctl{c: &c, id: HS_ID_CTL}})) +// TODO: make this optional. add this endpoint only if it's enabled... + c.promreg = prometheus.NewRegistry() + c.promreg.MustRegister(prometheus.NewGoCollector()) + c.promreg.MustRegister(NewClientCollector(&c)) + c.ctl_mux.Handle(c.ctl_prefix + "/_ctl/metrics", + promhttp.HandlerFor(c.promreg, promhttp.HandlerOpts{ EnableOpenMetrics: true })) + + c.ctl_addr = make([]string, len(ctl_addrs)) c.ctl = make([]*http.Server, len(ctl_addrs)) copy(c.ctl_addr, ctl_addrs) @@ -1666,3 +1680,11 @@ func (c *Client) WriteLog(id string, level LogLevel, fmtstr string, args ...inte func (c *Client) SetRoutePersister(persister ClientRoutePersister) { c.route_persister = persister } + +func (c *Client) AddCtlMetricsCollector(col prometheus.Collector) error { + return c.promreg.Register(col) +} + +func (c *Client) RemoveCtlMetricsCollector(col prometheus.Collector) bool { + return c.promreg.Unregister(col) +} diff --git a/cmd/config.go b/cmd/config.go index 9075601..d70890d 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -42,6 +42,12 @@ type ClientTLSConfig struct { ServerName string `yaml:"server-name"` } +type BasicAuthConfig struct { + Enabled bool `yaml:"enabled"` + Users []string `yaml:"users"` + UserFile string `yaml:"user-file"` +} + type CTLServiceConfig struct { Prefix string `yaml:"prefix"` // url prefix for control channel endpoints Addrs []string `yaml:"addresses"` diff --git a/cmd/main.go b/cmd/main.go index d9cda0b..b4f8b24 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -158,6 +158,7 @@ func server_main(ctl_addrs []string, rpc_addrs []string, pxy_addrs []string, wpx s, err = hodu.NewServer( context.Background(), + HODU_NAME, logger, ctl_addrs, rpc_addrs, @@ -315,6 +316,7 @@ func client_main(ctl_addrs []string, rpc_addrs []string, route_configs []string, } c = hodu.NewClient( context.Background(), + HODU_NAME, logger, ctl_addrs, ctl_prefix, diff --git a/go.mod b/go.mod index c94f297..4b8c2fd 100644 --- a/go.mod +++ b/go.mod @@ -12,4 +12,14 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) -require google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect +) diff --git a/go.sum b/go.sum index 190427e..1013442 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,21 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= diff --git a/hodu.go b/hodu.go index 306ac93..0c9fa43 100644 --- a/hodu.go +++ b/hodu.go @@ -10,7 +10,6 @@ import "strings" import "sync" import "time" - const HODU_RPC_VERSION uint32 = 0x010000 type LogLevel int @@ -29,6 +28,10 @@ const LOG_NONE LogMask = LogMask(0) var IPV4_PREFIX_ZERO = netip.MustParsePrefix("0.0.0.0/0") var IPV6_PREFIX_ZERO = netip.MustParsePrefix("::/0") +type Named struct { + name string +} + type Logger interface { Write(id string, level LogLevel, fmtstr string, args ...interface{}) WriteWithCallDepth(id string, level LogLevel, call_depth int, fmtstr string, args ...interface{}) @@ -45,6 +48,43 @@ type Service interface { WriteLog(id string, level LogLevel, fmtstr string, args ...interface{}) } +type json_out_go_stats struct { + CPUs int `json:"cpus"` + Goroutines int `json:"goroutines"` + NumCgoCalls int64 `json:"num-cgo-calls"` + NumGCs uint32 `json:"num-gcs"` + AllocBytes uint64 `json:"memory-alloc-bytes"` + TotalAllocBytes uint64 `json:"memory-total-alloc-bytes"` + SysBytes uint64 `json:"memory-sys-bytes"` + Lookups uint64 `json:"memory-lookups"` + MemAllocs uint64 `json:"memory-num-allocs"` + MemFrees uint64 `json:"memory-num-frees"` + + HeapAllocBytes uint64 `json:"memory-heap-alloc-bytes"` + HeapSysBytes uint64 `json:"memory-heap-sys-bytes"` + HeapIdleBytes uint64 `json:"memory-heap-idle-bytes"` + HeapInuseBytes uint64 `json:"memory-heap-inuse-bytes"` + HeapReleasedBytes uint64 `json:"memory-heap-released-bytes"` + HeapObjects uint64 `json:"memory-heap-objects"` + StackInuseBytes uint64 `json:"memory-stack-inuse-bytes"` + StackSysBytes uint64 `json:"memory-stack-sys-bytes"` + MSpanInuseBytes uint64 `json:"memory-mspan-inuse-bytes"` + MSpanSysBytes uint64 `json:"memory-mspan-sys-bytes"` + MCacheInuseBytes uint64 `json:"memory-mcache-inuse-bytes"` + MCacheSysBytes uint64 `json:"memory-mcache-sys-bytes"` + BuckHashSysBytes uint64 `json:"memory-buck-hash-sys-bytes"` + GCSysBytes uint64 `json:"memory-gc-sys-bytes"` + OtherSysBytes uint64 `json:"memory-other-sys-bytes"` +} + +func (n *Named) SetName(name string) { + n.name = name +} + +func (n *Named) Name() string { + return n.name +} + func TcpAddrStrClass(addr string) string { // the string is supposed to be addr:port @@ -221,3 +261,37 @@ func proxy_info_to_server_route(pi *ServerRouteProxyInfo) *ServerRoute { SvcPermNet: pi.SvcPermNet, } } + +func (stats *json_out_go_stats) from_runtime_stats() { + var mstat runtime.MemStats + + runtime.ReadMemStats(&mstat) + + stats.CPUs = runtime.NumCPU() + stats.Goroutines = runtime.NumGoroutine() + stats.NumCgoCalls = runtime.NumCgoCall() + stats.NumGCs = mstat.NumGC + + stats.AllocBytes = mstat.Alloc + stats.TotalAllocBytes = mstat.TotalAlloc + stats.SysBytes = mstat.Sys + stats.Lookups = mstat.Lookups + stats.MemAllocs = mstat.Mallocs + stats.MemFrees = mstat.Frees + + stats.HeapAllocBytes = mstat.HeapAlloc + stats.HeapSysBytes = mstat.HeapSys + stats.HeapIdleBytes = mstat.HeapIdle + stats.HeapInuseBytes = mstat.HeapInuse + stats.HeapReleasedBytes = mstat.HeapReleased + stats.HeapObjects = mstat.HeapObjects + stats.StackInuseBytes = mstat.StackInuse + stats.StackSysBytes = mstat.StackSys + stats.MSpanInuseBytes = mstat.MSpanInuse + stats.MSpanSysBytes = mstat.MSpanSys + stats.MCacheInuseBytes = mstat.MCacheInuse + stats.MCacheSysBytes = mstat.MCacheSys + stats.BuckHashSysBytes = mstat.BuckHashSys + stats.GCSysBytes = mstat.GCSys + stats.OtherSysBytes = mstat.OtherSys +} diff --git a/jwt.go b/jwt.go new file mode 100644 index 0000000..46197c0 --- /dev/null +++ b/jwt.go @@ -0,0 +1,123 @@ +package hodu + +import "crypto" +import "crypto/hmac" +import "crypto/rand" +import "crypto/rsa" +import "encoding/base64" +import "encoding/json" +import "fmt" +import "hash" +import "strings" + +func Sign(data []byte, privkey *rsa.PrivateKey) ([]byte, error) { + var h hash.Hash + + h = crypto.SHA512.New() + h.Write(data) + + fmt.Printf("%+v\n", h.Sum(nil)) + return rsa.SignPKCS1v15(rand.Reader, privkey, crypto.SHA512, h.Sum(nil)) +} + +func Verify(data []byte, pubkey *rsa.PublicKey, sig []byte) error { + var h hash.Hash + + h = crypto.SHA512.New() + h.Write(data) + + return rsa.VerifyPKCS1v15(pubkey, crypto.SHA512, h.Sum(nil), sig) +} + +func SignHS512(data []byte, key string) ([]byte, error) { + var h hash.Hash + + h = hmac.New(crypto.SHA512.New, []byte(key)) + h.Write(data) + + return h.Sum(nil), nil +} + +func VerifyHS512(data []byte, key string, sig []byte) error { + var h hash.Hash + + h = crypto.SHA512.New() + h.Write(data) + + if !hmac.Equal(h.Sum(nil), sig) { return fmt.Errorf("invalid signature") } + return nil +} + +type JWT struct { +} + +type JWTHeader struct { + Algo string `json:"alg"` + Type string `json:"typ"` +} + +type JWTClaimMap map[string]interface{} + +func (j *JWT) Sign(claims interface{}) (string, error) { + var h JWTHeader + var hb []byte + var cb []byte + var ss string + var sb []byte + var err error + + h.Algo = "HS512" + h.Type = "JWT" + + hb, err = json.Marshal(h) + if err != nil { return "", err } + + cb, err = json.Marshal(claims) + if err != nil { return "", err } + + ss = base64.RawURLEncoding.EncodeToString(hb) + "." + base64.RawURLEncoding.EncodeToString(cb) + sb, err = SignHS512([]byte(ss), "hello") + if err != nil { return "", err } + +fmt.Printf ("%+v %+v %s\n", string(hb), string(cb), (ss + "." + base64.RawURLEncoding.EncodeToString(sb))) + return ss + "." + base64.RawURLEncoding.EncodeToString(sb), nil +} + +func (j *JWT) Verify(tok string) error { + var segs []string + var hb []byte + var cb []byte + var sb []byte + var jh JWTHeader + var jcm JWTClaimMap + var x string + var err error + + segs = strings.Split(tok, ".") + if len(segs) != 3 { return fmt.Errorf("invalid token") } + + hb, err = base64.RawURLEncoding.DecodeString(segs[0]) + if err != nil { return fmt.Errorf("invalid header - %s", err.Error()) } + err = json.Unmarshal(hb, &jh) + if err != nil { return fmt.Errorf("invalid header - %s", err.Error()) } +fmt.Printf ("DECODED HEADER [%+v]\n", jh) + + cb, err = base64.RawURLEncoding.DecodeString(segs[1]) + if err != nil { return fmt.Errorf("invalid claims - %s", err.Error()) } + err = json.Unmarshal(cb, &jcm) + if err != nil { return fmt.Errorf("invalid header - %s", err.Error()) } +fmt.Printf ("DECODED CLAIMS [%+v]\n", jcm) + + x, err = j.Sign(jcm) + if err != nil { return err } +fmt.Printf ("VERIFICATION OK...\n") + + if x != tok { return fmt.Errorf("signature mismatch") } + +// sb, err = base64.RawURLEncoding.DecodeString(segs[2]) +// if err != nil { return fmt.Errorf("invalid signature - %s", err.Error()) } + + _ = sb + + return nil +} diff --git a/jwt_test.go b/jwt_test.go new file mode 100644 index 0000000..cf9e004 --- /dev/null +++ b/jwt_test.go @@ -0,0 +1,26 @@ +package hodu_test + +import "hodu" +import "testing" + +func TestJwt(t *testing.T) { + var j hodu.JWT + var tok string + var err error + + type JWTClaim struct { + Abc string `json:"abc"` + Donkey string `json:"donkey"` + IssuedAt int `json:"iat"` + } + + var jc JWTClaim + jc.Abc = "def" + jc.Donkey = "kong" + jc.IssuedAt = 111 + tok, err = j.Sign(&jc) + if err != nil { t.Fatalf("signing failure - %s", err.Error()) } + + err = j.Verify(tok) + if err != nil { t.Fatalf("verification failure - %s", err.Error()) } +} diff --git a/server-ctl.go b/server-ctl.go index 6841812..e0278fa 100644 --- a/server-ctl.go +++ b/server-ctl.go @@ -2,7 +2,6 @@ package hodu import "encoding/json" import "net/http" -import "runtime" type json_out_server_conn struct { Id ConnId `json:"id"` @@ -21,19 +20,13 @@ type json_out_server_route struct { } type json_out_server_stats struct { - CPUs int `json:"cpus"` - Goroutines int `json:"goroutines"` - - NumGCs uint32 `json:"num-gcs"` - HeapAllocBytes uint64 `json:"memory-alloc-bytes"` - MemAllocs uint64 `json:"memory-num-allocs"` - MemFrees uint64 `json:"memory-num-frees"` + json_out_go_stats ServerConns int64 `json:"server-conns"` ServerRoutes int64 `json:"server-routes"` ServerPeers int64 `json:"server-peers"` - SshProxySessions int64 `json:"ssh-proxy-session"` + SshProxySessions int64 `json:"ssh-pxy-sessions"` } // ------------------------------------ @@ -65,7 +58,7 @@ type server_ctl_stats struct { // ------------------------------------ -func (ctl *server_ctl) GetId() string { +func (ctl *server_ctl) Id() string { return ctl.id } @@ -356,15 +349,7 @@ func (ctl *server_ctl_stats) ServeHTTP(w http.ResponseWriter, req *http.Request) switch req.Method { case http.MethodGet: var stats json_out_server_stats - var mstat runtime.MemStats - - runtime.ReadMemStats(&mstat) - stats.CPUs = runtime.NumCPU() - stats.Goroutines = runtime.NumGoroutine() - stats.NumGCs = mstat.NumGC - stats.HeapAllocBytes = mstat.HeapAlloc - stats.MemAllocs = mstat.Mallocs - stats.MemFrees = mstat.Frees + stats.from_runtime_stats() stats.ServerConns = s.stats.conns.Load() stats.ServerRoutes = s.stats.routes.Load() stats.ServerPeers = s.stats.peers.Load() diff --git a/server-metrics.go b/server-metrics.go new file mode 100644 index 0000000..8b5da83 --- /dev/null +++ b/server-metrics.go @@ -0,0 +1,98 @@ +package hodu + +import "runtime" +import "github.com/prometheus/client_golang/prometheus" + +type ServerCollector struct { + server *Server + BuildInfo *prometheus.Desc + ServerConns *prometheus.Desc + ServerRoutes *prometheus.Desc + ServerPeers *prometheus.Desc + SshProxySessions *prometheus.Desc +} + +// NewServerCollector returns a new ServerCollector with all prometheus.Desc initialized +func NewServerCollector(server *Server) ServerCollector { + var prefix string + + prefix = server.Name() + "_" + return ServerCollector{ + server: server, + + BuildInfo: prometheus.NewDesc( + prefix + "build_info", + "Build information", + []string{ + "goarch", + "goos", + "goversion", + }, nil, + ), + + ServerConns: prometheus.NewDesc( + prefix + "server_conns", + "Number of server connections from clients", + nil, nil, + ), + ServerRoutes: prometheus.NewDesc( + prefix + "server_routes", + "Number of server-side routes", + nil, nil, + ), + ServerPeers: prometheus.NewDesc( + prefix + "server_peers", + "Number of server-side peers", + nil, nil, + ), + SshProxySessions: prometheus.NewDesc( + prefix + "ssh_pxy_sessions", + "Number of SSH proxy sessions", + nil, nil, + ), + } +} + +func (c ServerCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.BuildInfo + ch <- c.ServerConns + ch <- c.ServerRoutes + ch <- c.ServerPeers + ch <- c.SshProxySessions +} + +func (c ServerCollector) Collect(ch chan<- prometheus.Metric) { + + ch <- prometheus.MustNewConstMetric( + c.BuildInfo, + prometheus.GaugeValue, + 1, + runtime.GOARCH, + runtime.GOOS, + runtime.Version(), + ) + + ch <- prometheus.MustNewConstMetric( + c.ServerConns, + prometheus.GaugeValue, + float64(c.server.stats.conns.Load()), + ) + + ch <- prometheus.MustNewConstMetric( + c.ServerRoutes, + prometheus.GaugeValue, + float64(c.server.stats.routes.Load()), + ) + + ch <- prometheus.MustNewConstMetric( + c.ServerPeers, + prometheus.GaugeValue, + float64(c.server.stats.peers.Load()), + ) + + ch <- prometheus.MustNewConstMetric( + c.SshProxySessions, + prometheus.GaugeValue, + float64(c.server.stats.ssh_proxy_sessions.Load()), + ) +} diff --git a/server-proxy.go b/server-proxy.go index 6dd9f58..a82eef8 100644 --- a/server-proxy.go +++ b/server-proxy.go @@ -184,7 +184,7 @@ func mutate_proxy_req_headers(req *http.Request, newreq *http.Request, path_pref return upgrade_required } -func (pxy *server_proxy) GetId() string { +func (pxy *server_proxy) Id() string { return pxy.id } diff --git a/server.go b/server.go index 99ad80e..ac87912 100644 --- a/server.go +++ b/server.go @@ -18,9 +18,10 @@ import "unsafe" import "golang.org/x/net/websocket" import "google.golang.org/grpc" import "google.golang.org/grpc/credentials" -//import "google.golang.org/grpc/metadata" import "google.golang.org/grpc/peer" import "google.golang.org/grpc/stats" +import "github.com/prometheus/client_golang/prometheus" +import "github.com/prometheus/client_golang/prometheus/promhttp" const PTS_LIMIT int = 16384 const CTS_LIMIT int = 16384 @@ -42,6 +43,9 @@ type ServerWpxResponseTransformer func(r *ServerRouteProxyInfo, resp *http.Respo type ServerWpxForeignPortProxyMaker func(wpx_type string, port_id string) (*ServerRouteProxyInfo, error) type Server struct { + UnimplementedHoduServer + Named + ctx context.Context ctx_cancel context.CancelFunc pxytlscfg *tls.Config @@ -88,6 +92,7 @@ type Server struct { svc_port_mtx sync.Mutex svc_port_map ServerSvcPortMap + promreg *prometheus.Registry stats struct { conns atomic.Int64 routes atomic.Int64 @@ -98,8 +103,6 @@ type Server struct { wpx_resp_tf ServerWpxResponseTransformer wpx_foreign_port_proxy_maker ServerWpxForeignPortProxyMaker xterm_html string - - UnimplementedHoduServer } // connection from client. @@ -927,7 +930,7 @@ func (hlw *server_http_log_writer) Write(p []byte) (n int, err error) { } type ServerHttpHandler interface { - GetId() string + Id() string ServeHTTP (w http.ResponseWriter, req *http.Request) (int, error) } @@ -954,15 +957,15 @@ func (s *Server) wrap_http_handler(handler ServerHttpHandler) http.Handler { if status_code > 0 { if err == nil { - s.log.Write(handler.GetId(), LOG_INFO, "[%s] %s %s %d %.9f", req.RemoteAddr, req.Method, req.URL.String(), status_code, time_taken.Seconds()) + s.log.Write(handler.Id(), LOG_INFO, "[%s] %s %s %d %.9f", req.RemoteAddr, req.Method, req.URL.String(), status_code, time_taken.Seconds()) } else { - s.log.Write(handler.GetId(), LOG_INFO, "[%s] %s %s %d %.9f - %s", req.RemoteAddr, req.Method, req.URL.String(), status_code, time_taken.Seconds(), err.Error()) + s.log.Write(handler.Id(), LOG_INFO, "[%s] %s %s %d %.9f - %s", req.RemoteAddr, req.Method, req.URL.String(), status_code, time_taken.Seconds(), err.Error()) } } }) } -func NewServer(ctx context.Context, logger Logger, ctl_addrs []string, rpc_addrs []string, pxy_addrs []string, wpx_addrs []string, ctl_prefix string, ctltlscfg *tls.Config, rpctlscfg *tls.Config, pxytlscfg *tls.Config, wpxtlscfg *tls.Config, rpc_max int, peer_max int) (*Server, error) { +func NewServer(ctx context.Context, name string, logger Logger, ctl_addrs []string, rpc_addrs []string, pxy_addrs []string, wpx_addrs []string, ctl_prefix string, ctltlscfg *tls.Config, rpctlscfg *tls.Config, pxytlscfg *tls.Config, wpxtlscfg *tls.Config, rpc_max int, peer_max int) (*Server, error) { var s Server var l *net.TCPListener var rpcaddr *net.TCPAddr @@ -978,6 +981,7 @@ func NewServer(ctx context.Context, logger Logger, ctl_addrs []string, rpc_addrs } s.ctx, s.ctx_cancel = context.WithCancel(ctx) + s.name = name s.log = logger /* create the specified number of listeners */ s.rpc = make([]*net.TCPListener, 0) @@ -1043,6 +1047,13 @@ func NewServer(ctx context.Context, logger Logger, ctl_addrs []string, rpc_addrs s.ctl_mux.Handle(s.ctl_prefix + "/_ctl/stats", s.wrap_http_handler(&server_ctl_stats{server_ctl{s: &s, id: HS_ID_CTL}})) +// TODO: make this optional. add this endpoint only if it's enabled... + s.promreg = prometheus.NewRegistry() + s.promreg.MustRegister(prometheus.NewGoCollector()) + s.promreg.MustRegister(NewServerCollector(&s)) + s.ctl_mux.Handle(s.ctl_prefix + "/_ctl/metrics", + promhttp.HandlerFor(s.promreg, promhttp.HandlerOpts{ EnableOpenMetrics: true })) + s.ctl_addr = make([]string, len(ctl_addrs)) s.ctl = make([]*http.Server, len(ctl_addrs)) copy(s.ctl_addr, ctl_addrs) @@ -1660,3 +1671,11 @@ func (s *Server) WriteLog(id string, level LogLevel, fmtstr string, args ...inte func (s *Server) AddCtlHandler(path string, handler ServerHttpHandler) { s.ctl_mux.Handle(s.ctl_prefix + "/_ctl" + path, s.wrap_http_handler(handler)) } + +func (s *Server) AddCtlMetricsCollector(col prometheus.Collector) error { + return s.promreg.Register(col) +} + +func (s *Server) RemoveCtlMetricsCollector(col prometheus.Collector) bool { + return s.promreg.Unregister(col) +}