enhanced the logger with file logging and rotation

Makefile

@@ -27,6 +27,7 @@ CMD_DATA=\
 
 CMD_SRCS=\
 	cmd/config.go \
+	cmd/logger.go \
 	cmd/main.go \
 
 all: $(NAME)

client.go

@@ -1477,7 +1477,8 @@ func (c *Client) StopServices() {
 	}
 }
 
-func (s *Client) FixServices() {
+func (c *Client) FixServices() {
+	c.log.Rotate()
 }
 
 func (c *Client) WaitForTermination() {

cmd/config.go

@@ -64,6 +64,8 @@ type RPCEndpointConfig struct { // rpc client-side configuration
 type ServerAppConfig  struct {
 	LogMask     []string  `yaml:"log-mask"`
 	LogFile     string    `yaml:"log-file"`
+	LogMaxSize  int64     `yaml:"log-max-size"`
+	LogRotate   int       `yaml:"log-rotate"`
 	MaxPeers    int       `yaml:"max-peer-conns"` // maximum number of connections from peers
 	MaxRpcConns int       `yaml:"max-rpc-conns"` // maximum number of rpc connections
 }
@@ -71,6 +73,8 @@ type ServerAppConfig  struct {
 type ClientAppConfig  struct {
 	LogMask       []string      `yaml:"log-mask"`
 	LogFile       string        `yaml:"log-file"`
+	LogMaxSize    int64         `yaml:"log-max-size"`
+	LogRotate     int           `yaml:"log-rotate"`
 	MaxPeers      int           `yaml:"max-peer-conns"` // maximum number of connections from peers
 	MaxRpcConns   int           `yaml:"max-rpc-conns"` // maximum number of rpc connections
 	PeerConnTmout time.Duration `yaml:"peer-conn-timeout"`

cmd/logger.go

@@ -0,0 +1,188 @@
+package main
+
+import "fmt"
+import "hodu"
+import "io"
+import "os"
+import "path/filepath"
+import "runtime"
+import "strings"
+import "sync"
+import "time"
+
+type AppLogger struct {
+	Id              string
+	Out             io.Writer
+	Mask            hodu.LogMask
+
+	file            *os.File
+	file_name       string // you can get the file name from file but this is to preserve the original.
+	file_rotate     int
+	file_max_size   int64
+	msg_chan        chan string
+	stop_req_chan   chan struct{}
+	rotate_req_chan chan struct{}
+	wg              sync.WaitGroup
+}
+
+func NewAppLogger (id string, w io.Writer, mask hodu.LogMask) *AppLogger {
+	var l *AppLogger
+	l = &AppLogger{Id: id, Out: w, Mask: mask, msg_chan: make(chan string, 256), stop_req_chan: make(chan struct{}, 1), rotate_req_chan: make(chan struct{}, 16) }
+	l.wg.Add(1)
+	go l.logger_task()
+	return l
+}
+
+func NewAppLoggerToFile (id string, file string, max_size int64, rotate int, mask hodu.LogMask) (*AppLogger, error) {
+	var l *AppLogger
+	var f *os.File
+	var err error
+
+	f, err = os.OpenFile(file, os.O_CREATE | os.O_APPEND | os.O_WRONLY, 0666)
+	if err != nil { return nil, err }
+
+	l = &AppLogger{
+		Id: id,
+		Out: f,
+		Mask: mask,
+		file: f,
+		file_name: file,
+		file_max_size: max_size,
+		file_rotate: rotate,
+		msg_chan: make(chan string, 256),
+		stop_req_chan: make(chan struct{}, 1),
+		rotate_req_chan: make(chan struct{}, 16),
+	}
+	l.wg.Add(1)
+	go l.logger_task()
+	return l, nil
+}
+
+func (l *AppLogger) Close() {
+	l.stop_req_chan <- struct{}{}
+	l.wg.Wait()
+	if l.file != nil { l.file.Close() }
+}
+
+func (l *AppLogger) logger_task() {
+	var msg string
+	defer l.wg.Done()
+
+main_loop:
+	for {
+		select {
+			case msg = <-l.msg_chan:
+				//l.Out.Write([]byte(msg))
+				io.WriteString(l.Out, msg)
+				if l.file_max_size > 0 && l.file != nil {
+					var fi os.FileInfo
+					var err error
+					fi, err = l.file.Stat()
+					if err == nil && fi.Size() >= l.file_max_size {
+						l.rotate()
+					}
+				}
+
+			case <-l.stop_req_chan:
+				break main_loop
+
+			case <- l.rotate_req_chan:
+				l.rotate()
+		}
+	}
+}
+
+func (l *AppLogger) Rotate() {
+	l.rotate_req_chan <- struct{}{}
+}
+
+func (l *AppLogger) Write(id string, level hodu.LogLevel, fmtstr string, args ...interface{}) {
+	if l.Mask & hodu.LogMask(level) == 0 { return }
+	l.write(id, level, 1, fmtstr, args...)
+}
+
+func (l *AppLogger) WriteWithCallDepth(id string, level hodu.LogLevel, call_depth int, fmtstr string, args ...interface{}) {
+	if l.Mask & hodu.LogMask(level) == 0 { return }
+	l.write(id, level, call_depth + 1, fmtstr, args...)
+}
+
+func (l *AppLogger) write(id string, level hodu.LogLevel, call_depth int, fmtstr string, args ...interface{}) {
+	var now time.Time
+	var off_m int
+	var off_h int
+	var off_s int
+	var msg string
+	var callerfile string
+	var caller_line int
+	var caller_ok bool
+	var sb strings.Builder
+
+	//if l.Mask & hodu.LogMask(level) == 0 { return }
+
+	now = time.Now()
+
+	_, off_s = now.Zone()
+	off_m = off_s / 60;
+	off_h = off_m / 60;
+	off_m = off_m % 60;
+	if (off_m < 0) { off_m = -off_m; }
+
+	sb.WriteString(
+		fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d %+03d%02d ",
+		now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), off_h, off_m))
+
+	_, callerfile, caller_line, caller_ok = runtime.Caller(1 + call_depth)
+
+	if caller_ok {
+		sb.WriteString(fmt.Sprintf("[%s:%d] ", filepath.Base(callerfile), caller_line))
+	}
+	sb.WriteString(l.Id)
+	if id != "" {
+		sb.WriteString("(")
+		sb.WriteString(id)
+		sb.WriteString(")")
+	}
+	sb.WriteString(": ")
+	msg = fmt.Sprintf(fmtstr, args...)
+	sb.WriteString(msg)
+	if msg[len(msg) - 1] != '\n' { sb.WriteRune('\n') }
+
+	// use queue to avoid blocking operation as much as possible
+	l.msg_chan <- sb.String()
+}
+
+func (l *AppLogger) rotate() {
+	var f *os.File
+	var fi os.FileInfo
+	var i int
+	var last_rot_no int
+	var err error
+
+	if l.file == nil { return }
+	if l.file_rotate <= 0 { return }
+
+	fi, err = l.file.Stat()
+	if err == nil && fi.Size() <= 0 { return }
+
+	for i = l.file_rotate - 1; i > 0; i-- {
+		if os.Rename(fmt.Sprintf("%s.%d", l.file_name, i), fmt.Sprintf("%s.%d", l.file_name, i + 1)) == nil {
+			if last_rot_no == 0 { last_rot_no = i + 1 }
+		}
+	}
+	if os.Rename(l.file_name, fmt.Sprintf("%s.%d", l.file_name, 1)) == nil {
+		if last_rot_no == 0 { last_rot_no = 1 }
+	}
+
+	f, err = os.OpenFile(l.file_name, os.O_CREATE | os.O_TRUNC | os.O_APPEND | os.O_WRONLY, 0666)
+	if err != nil {
+		l.file.Close()
+		l.file = nil
+		l.Out = os.Stderr
+		// don't reset l.file_name. you can derive that there was an error
+		// if l.file_name is not blank, and if l.Out is os.Stderr, 
+	} else {
+		l.file.Close()
+		l.file = f
+		l.Out = l.file
+	}
+}

cmd/main.go

@@ -9,8 +9,6 @@ import "hodu"
 import "io"
 import "os"
 import "os/signal"
-import "path/filepath"
-import "runtime"
 import "strings"
 import "sync"
 import "syscall"
@@ -26,81 +24,6 @@ var hodu_tls_cert_text []byte
 //go:embed tls.key
 var hodul_tls_key_text []byte
 
-// --------------------------------------------------------------------
-
-type AppLogger struct {
-	Id       string
-	Out      io.Writer
-	Mask     hodu.LogMask
-
-	_mtx     sync.Mutex
-	_file    *os.File
-}
-
-func NewAppLogger (id string, w io.Writer, mask hodu.LogMask) *AppLogger {
-	return &AppLogger{Id: id, Out: w, Mask: mask}
-}
-
-func NewAppLoggerToFile (id string, file string, mask hodu.LogMask) (*AppLogger, error) {
-	var err error
-	var f *os.File
-
-	f, err = os.OpenFile(file, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
-	if err != nil { return nil, err }
-
-	return &AppLogger{Id: id, Out: f, Mask: mask, _file: f}, nil
-}
-
-func (l* AppLogger) Close() {
-	if l._file != nil { l._file.Close() }
-}
-
-func (l* AppLogger) Write(id string, level hodu.LogLevel, fmtstr string, args ...interface{}) {
-	var now time.Time
-	var off_m int
-	var off_h int
-	var off_s int
-	var hdr string
-	var msg string
-	var lid string
-	var caller_file string
-	var caller_line int
-	var caller_ok bool
-
-	if l.Mask & hodu.LogMask(level) == 0 { return }
-
-	now = time.Now()
-
-	_, off_s = now.Zone()
-	off_m = off_s / 60;
-	off_h = off_m / 60;
-	off_m = off_m % 60;
-	if (off_m < 0) { off_m = -off_m; }
-
-	hdr = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d %+03d%02d ",
-		now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), off_h, off_m)
-
-	_, caller_file, caller_line, caller_ok = runtime.Caller(1)
-
-// TODO: add pid?
-	msg = fmt.Sprintf(fmtstr, args...)
-	if id == "" {
-		lid = fmt.Sprintf("%s: ", l.Id)
-	} else {
-		lid = fmt.Sprintf("%s(%s): ", l.Id, id)
-	}
-
-	l._mtx.Lock()
-	l.Out.Write([]byte(hdr))
-	if caller_ok {
-		l.Out.Write([]byte(fmt.Sprintf("[%s:%d] ", filepath.Base(caller_file), caller_line)))
-	}
-	if lid != "" { l.Out.Write([]byte(lid)) }
-	l.Out.Write([]byte(msg))
-	if msg[len(msg) - 1] != '\n' { l.Out.Write([]byte("\n")) }
-	l._mtx.Unlock()
-}
-
 // --------------------------------------------------------------------
 type signal_handler struct {
 	svc hodu.Service
@@ -174,7 +97,9 @@ func server_main(ctl_addrs []string, rpc_addrs []string, pxy_addrs []string, cfg
 	var ctl_prefix string
 	var logger *AppLogger
 	var log_mask hodu.LogMask
-	var log_file string
+	var logfile string
+	var logfile_maxsize int64
+	var logfile_rotate int
 	var max_rpc_conns int
 	var max_peers int
 	var err error
@@ -203,7 +128,9 @@ func server_main(ctl_addrs []string, rpc_addrs []string, pxy_addrs []string, cfg
 
 		ctl_prefix = cfg.CTL.Service.Prefix
 		log_mask = log_strings_to_mask(cfg.APP.LogMask)
-		log_file = cfg.APP.LogFile
+		logfile = cfg.APP.LogFile
+		logfile_maxsize = cfg.APP.LogMaxSize
+		logfile_rotate = cfg.APP.LogRotate
 		max_rpc_conns = cfg.APP.MaxRpcConns
 		max_peers = cfg.APP.MaxPeers
 	}
@@ -212,10 +139,10 @@ func server_main(ctl_addrs []string, rpc_addrs []string, pxy_addrs []string, cfg
 		return fmt.Errorf("no rpc service addresses specified")
 	}
 
-	if log_file == "" {
+	if logfile == "" {
 		logger = NewAppLogger("server", os.Stderr, log_mask)
 	} else {
-		logger, err = NewAppLoggerToFile("server", log_file, log_mask)
+		logger, err = NewAppLoggerToFile("server", logfile, logfile_maxsize, logfile_rotate, log_mask)
 		if err != nil {
 			return fmt.Errorf("failed to initialize logger - %s", err.Error())
 		}
@@ -257,7 +184,9 @@ func client_main(ctl_addrs []string, rpc_addrs []string, peer_addrs []string, cf
 	var cc hodu.ClientConfig
 	var logger *AppLogger
 	var log_mask hodu.LogMask
-	var log_file string
+	var logfile string
+	var logfile_maxsize int64
+	var logfile_rotate int
 	var max_rpc_conns int
 	var max_peers int
 	var peer_conn_tmout time.Duration
@@ -281,7 +210,9 @@ func client_main(ctl_addrs []string, rpc_addrs []string, peer_addrs []string, cf
 		cc.ServerSeedTmout = cfg.RPC.Endpoint.SeedTmout
 		cc.ServerAuthority = cfg.RPC.Endpoint.Authority
 		log_mask = log_strings_to_mask(cfg.APP.LogMask)
-		log_file = cfg.APP.LogFile
+		logfile = cfg.APP.LogFile
+		logfile_maxsize = cfg.APP.LogMaxSize
+		logfile_rotate = cfg.APP.LogRotate
 		max_rpc_conns = cfg.APP.MaxRpcConns
 		max_peers = cfg.APP.MaxPeers
 		peer_conn_tmout = cfg.APP.PeerConnTmout
@@ -292,10 +223,10 @@ func client_main(ctl_addrs []string, rpc_addrs []string, peer_addrs []string, cf
 	cc.ServerAddrs = rpc_addrs
 	cc.PeerAddrs = peer_addrs
 
-	if log_file == "" {
+	if logfile == "" {
 		logger = NewAppLogger("server", os.Stderr, log_mask)
 	} else {
-		logger, err = NewAppLoggerToFile("server", log_file, log_mask)
+		logger, err = NewAppLoggerToFile("server", logfile, logfile_maxsize, logfile_rotate, log_mask)
 		if err != nil {
 			return fmt.Errorf("failed to initialize logger - %s", err.Error())
 		}

hodu.go

@@ -28,6 +28,8 @@ var IPV6_PREFIX_ZERO = netip.MustParsePrefix("::/0")
 
 type Logger interface {
 	Write(id string, level LogLevel, fmtstr string, args ...interface{})
+	WriteWithCallDepth(id string, level LogLevel, call_depth int, fmtstr string, args ...interface{})
+	Rotate()
 }
 
 type Service interface {

server.go

@@ -865,7 +865,8 @@ type server_http_log_writer struct {
 func (hlw *server_http_log_writer) Write(p []byte) (n int, err error) {
 	// the standard http.Server always requires *log.Logger
 	// use this iowriter to create a logger to pass it to the http server.
-	hlw.svr.log.Write("", LOG_INFO, string(p))
+	// since this is another log write wrapper, give adjustment value
+	hlw.svr.log.WriteWithCallDepth("", LOG_INFO, +1, string(p))
 	return len(p), nil
 }
 
@@ -1374,6 +1375,7 @@ func (s *Server) StopServices() {
 }
 
 func (s *Server) FixServices() {
+	s.log.Rotate()
 }
 
 func (s *Server) WaitForTermination() {