From 3d64e38f5ac5bfc9bd2051e8da6d836d049748e4 Mon Sep 17 00:00:00 2001 From: hyung-hwan Date: Thu, 11 Dec 2025 21:08:47 +0900 Subject: [PATCH] added --concurrent to hawkgo --- bin/hawkgo.go | 275 +++++++++++++++++++++++++++++++++----------------- hawk.go | 37 +++++-- hawk_test.go | 10 +- 3 files changed, 214 insertions(+), 108 deletions(-) diff --git a/bin/hawkgo.go b/bin/hawkgo.go index ab1d5bbf..39904a6e 100644 --- a/bin/hawkgo.go +++ b/bin/hawkgo.go @@ -10,7 +10,9 @@ import "os" import "path/filepath" import "runtime" import "runtime/debug" +import "strconv" import "strings" +import "sync" //import "time" type Assign struct { @@ -23,10 +25,13 @@ type Config struct { call string fs string show_extra_info bool + concurrent bool + suffix string srcstr string srcfiles []string - datfiles []string + data_in_files []string + data_out_files []string } func exit_with_error(msg string, err error) { @@ -55,6 +60,17 @@ func parse_args_to_config(cfg *Config) bool { cfg.call = v return nil }) + flgs.BoolFunc("concurrent", "run the script over multiple data files concurrently", func(v string) error { + if v[0] == '.' { + // treat it as a suffix + cfg.suffix = v + cfg.concurrent = true + } else { + cfg.suffix = "" + cfg.concurrent, _ = strconv.ParseBool(v) + } + return nil + }) flgs.Func("field-separator", "set the field separator(FS)", func(v string) error { cfg.fs = v return nil @@ -79,26 +95,170 @@ func parse_args_to_config(cfg *Config) bool { } //if flgs.NArg() == 0 { goto wrong_usage} - cfg.datfiles = flgs.Args() + cfg.data_in_files = flgs.Args() if len(cfg.srcfiles) <= 0 { - if len(cfg.datfiles) == 0 { goto wrong_usage } - cfg.srcstr = cfg.datfiles[0] - cfg.datfiles = cfg.datfiles[1:] + if len(cfg.data_in_files) == 0 { goto wrong_usage } + cfg.srcstr = cfg.data_in_files[0] + cfg.data_in_files = cfg.data_in_files[1:] } + if cfg.concurrent && len(cfg.data_in_files) > 0 { + var i int + var n int + var f []string + n = len(cfg.data_in_files) + cfg.data_out_files = make([]string, n) + for i = 0; i < n; i++ { + f = strings.SplitN(cfg.data_in_files[i], ":", 2) + cfg.data_in_files[i] = f[0] + if len(f) >= 2 { + cfg.data_out_files[i] = f[1] + } else if cfg.suffix != "" && cfg.data_in_files[i] != "" && cfg.data_in_files[i] != "-" { + cfg.data_out_files[i] = cfg.data_in_files[i] + cfg.suffix + } + } + } return true wrong_usage: fmt.Fprintf(os.Stderr, "USAGE: %s [options] sourcestring [datafile]*\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s [options] -f sourcefile [datafile]*\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Options are:\n") + fmt.Fprintf(os.Stderr, " -F, --field-separator string specify the field seperator\n") + fmt.Fprintf(os.Stderr, " -v, --assign name=value set global variable value\n") + fmt.Fprintf(os.Stderr, " -c, --call string call a named function instead of running the\n") + fmt.Fprintf(os.Stderr, " normal loop\n") + fmt.Fprintf(os.Stderr, " --concurrent=[suffix] process multiple data files concurrently\n") + fmt.Fprintf(os.Stderr, " if datafile has two parts delimited by a colon,\n") + fmt.Fprintf(os.Stderr, " the second part specifies the output file (e.g.\n") + fmt.Fprintf(os.Stderr, " infile:outfile). The suffix beginning with a\n") + fmt.Fprintf(os.Stderr, " period is appended to datafile to form the out-\n") + fmt.Fprintf(os.Stderr, " put file name if the second part is missing\n") return false } +func run_script(h *hawk.Hawk, fs_idx int, data_idx int, cfg* Config, wg *sync.WaitGroup) error { + var rtx *hawk.Rtx + var err error + + if wg != nil { defer wg.Done() } + + if data_idx <= -1 { + rtx, err = h.NewRtx(filepath.Base(os.Args[0]), cfg.data_in_files, nil) + } else { + var out_idx_end int = data_idx + + if cfg.data_out_files[data_idx] != "" { out_idx_end++ } + rtx, err = h.NewRtx( + fmt.Sprintf("%s.%d", filepath.Base(os.Args[0]), data_idx), + cfg.data_in_files[data_idx: data_idx + 1], + cfg.data_out_files[data_idx: out_idx_end], + ) + } + if err != nil { + err = fmt.Errorf("failed to make rtx - %s", err.Error()) + goto oops + } else { + var k string + var v Assign + var retv *hawk.Val + + for k, v = range cfg.assigns { + if v.idx >= 0 { + var vv *hawk.Val + vv, err = rtx.NewNumOrStrVal(v.value) + if err != nil { + err = fmt.Errorf("failed to convert value '%s' for global variable '%s' - %s", v.value, k, err.Error()) + goto oops + } + + err = rtx.SetGlobal(v.idx, vv) + vv.Close() + if err != nil { + err = fmt.Errorf("failed to set global variable '%s' - %s", k, err.Error()) + goto oops + } + } + } + + if fs_idx >= 0 { + var vv *hawk.Val + vv, err = rtx.NewStrVal(cfg.fs) + if err != nil { + err = fmt.Errorf("failed to convert field separator '%s' - %s", cfg.fs, err.Error()) + goto oops + } + + rtx.SetGlobal(fs_idx, vv) + vv.Close() + if err != nil { + err = fmt.Errorf("failed to set field separator to '%s' - %s", cfg.fs, err.Error()) + goto oops + } + } + + if cfg.call != "" { + var idx int + var count int + var args []*hawk.Val + + count = len(cfg.data_in_files) + args = make([]*hawk.Val, count) + for idx = 0; idx < count; idx++ { + args[idx], err = rtx.NewStrVal(cfg.data_in_files[idx]) + if err != nil { + err = fmt.Errorf("failed to convert argument [%s] to value - %s", cfg.data_in_files[idx], err.Error()) + goto oops + } + } + retv, err = rtx.Call(cfg.call, args...) + for idx = 0; idx < count; idx++ { + // it's ok not to call Close() on args as rtx.Close() closes them automatically. + // if args are re-created repeatedly, Close() on them is always needed not to + // accumulate too many allocated values. + args[idx].Close() + } + } else { + //v, err = rtx.Loop() + retv, err = rtx.Exec(cfg.data_in_files) + } + if err != nil { + err = fmt.Errorf("failed to run - %s", err.Error()) + goto oops + } + + if cfg.show_extra_info { + var named_vars map[string]*hawk.Val + var vn string + var vv *hawk.Val + + fmt.Printf("[RETURN] - [%v]\n", retv.String()) + + fmt.Printf("NAMED VARIABLES]\n") + named_vars = make(map[string]*hawk.Val) + rtx.GetNamedVars(named_vars) + for vn, vv = range named_vars { + fmt.Printf("%s = %s\n", vn, vv.String()) + } + fmt.Printf("END OF NAMED VARIABLES]\n") + } + } + + // let's not care about closing args or return values + // because rtx.Close() will close them automatically + if rtx != nil { rtx.Close() } + return nil + +oops: + if rtx != nil { rtx.Close() } + return err +} + func main() { var h *hawk.Hawk - var rtx *hawk.Rtx var cfg Config var fs_idx int = -1 + var wg sync.WaitGroup var err error // for profiling @@ -152,100 +312,29 @@ func main() { goto oops } - rtx, err = h.NewRtx(filepath.Base(os.Args[0]), cfg.datfiles) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: failed to make rtx - %s\n", err.Error()) - goto oops - } else { - var k string - var v Assign - var retv *hawk.Val - - for k, v = range cfg.assigns { - if v.idx >= 0 { - var vv *hawk.Val - vv, err = rtx.NewNumOrStrVal(v.value) + if cfg.concurrent && len(cfg.data_in_files) >= 1 { + var n int + var i int + n = len(cfg.data_in_files) + for i = 0; i < n; i++ { + wg.Add(1) + go func(idx int) { + var err error + err = run_script(h, fs_idx, idx, &cfg, &wg) if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: failed to convert value '%s' for global variable '%s' - %s\n", v.value, k, err.Error()) - goto oops + fmt.Fprintf(os.Stderr, "ERROR: [%d]%s\n", idx, err.Error()) } - - err = rtx.SetGlobal(v.idx, vv) - vv.Close() - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: failed to set global variable '%s' - %s\n", k, err.Error()) - goto oops - } - } - } - - if fs_idx >= 0 { - var vv *hawk.Val - vv, err = rtx.NewStrVal(cfg.fs) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: failed to convert field separator '%s' - %s\n", cfg.fs, err.Error()) - goto oops - } - - rtx.SetGlobal(fs_idx, vv) - vv.Close() - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: failed to set field separator to '%s' - %s\n", cfg.fs, err.Error()) - goto oops - } - } - - if cfg.call != "" { - var idx int - var count int - var args []*hawk.Val - - count = len(cfg.datfiles) - args = make([]*hawk.Val, count) - for idx = 0; idx < count; idx++ { - args[idx], err = rtx.NewStrVal(cfg.datfiles[idx]) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: failed to convert argument [%s] to value - %s\n", cfg.datfiles[idx], err.Error()) - goto oops - } - } - retv, err = rtx.Call(cfg.call, args...) - for idx = 0; idx < count; idx++ { - // it's ok not to call Close() on args as rtx.Close() closes them automatically. - // if args are re-created repeatedly, Close() on them is always needed not to - // accumulate too many allocated values. - args[idx].Close() - } - } else { - //v, err = rtx.Loop() - retv, err = rtx.Exec(cfg.datfiles) + }(i) } + wg.Wait() + } else { + err = run_script(h, fs_idx, -1, &cfg, nil) if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: failed to run rtx - %s\n", err.Error()) + fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error()) goto oops } - - if cfg.show_extra_info { - var named_vars map[string]*hawk.Val - var vn string - var vv *hawk.Val - - fmt.Printf("[RETURN] - [%v]\n", retv.String()) - - - fmt.Printf("NAMED VARIABLES]\n") - named_vars = make(map[string]*hawk.Val) - rtx.GetNamedVars(named_vars) - for vn, vv = range named_vars { - fmt.Printf("%s = %s\n", vn, vv.String()) - } - fmt.Printf("END OF NAMED VARIABLES]\n") - } } - // let's not care about closing args or return values - // because rtx.Close() will close them automatically - if rtx != nil { rtx.Close() } if h != nil { h.Close() } runtime.GC() @@ -255,7 +344,7 @@ func main() { return oops: - if rtx != nil { rtx.Close() } +// if rtx != nil { rtx.Close() } if h != nil { h.Close() } os.Exit(255) } diff --git a/hawk.go b/hawk.go index 04779b46..9eabea60 100644 --- a/hawk.go +++ b/hawk.go @@ -491,26 +491,43 @@ func deregister_rtx_instance(rtx *Rtx) { } } -func (hawk *Hawk) NewRtx(id string, in []string) (*Rtx, error) { +func (hawk *Hawk) NewRtx(id string, in []string, out []string) (*Rtx, error) { var rtx *C.hawk_rtx_t var g *Rtx var xtn *C.rtx_xtn_t var cid *C.hawk_bch_t var cin []*C.hawk_bch_t + var cout []*C.hawk_bch_t + var cin_ptr **C.hawk_bch_t + var cout_ptr **C.hawk_bch_t var idx int var in_count int + var out_count int in_count = len(in) - cin = make([]*C.hawk_bch_t, in_count + 1) - for idx = 0; idx < in_count; idx++ { - cin[idx] = C.CString(in[idx]) - } - cin[idx] = (*C.hawk_bch_t)(nil) - cid = C.CString(id) if in_count > 0 { - rtx = C.hawk_rtx_openstdwithbcstr(hawk.c, C.hawk_oow_t(unsafe.Sizeof(*xtn)), cid, &cin[0], nil, nil) - } else { - rtx = C.hawk_rtx_openstdwithbcstr(hawk.c, C.hawk_oow_t(unsafe.Sizeof(*xtn)), cid, nil, nil, nil) + cin = make([]*C.hawk_bch_t, in_count + 1) + for idx = 0; idx < in_count; idx++ { + cin[idx] = C.CString(in[idx]) + } + cin[idx] = (*C.hawk_bch_t)(nil) + cin_ptr = &cin[0] + } + + out_count = len(out) + if out_count > 0 { + cout = make([]*C.hawk_bch_t, out_count + 1) + for idx = 0; idx < out_count; idx++ { + cout[idx] = C.CString(out[idx]) + } + cout[idx] = (*C.hawk_bch_t)(nil) + cout_ptr = &cout[0] + } + + cid = C.CString(id) + rtx = C.hawk_rtx_openstdwithbcstr(hawk.c, C.hawk_oow_t(unsafe.Sizeof(*xtn)), cid, cin_ptr, cout_ptr, nil) + for idx = 0; idx < out_count; idx++ { + C.free(unsafe.Pointer(cout[idx])) } for idx = 0; idx < in_count; idx++ { C.free(unsafe.Pointer(cin[idx])) diff --git a/hawk_test.go b/hawk_test.go index fb12c28f..d5966e15 100644 --- a/hawk_test.go +++ b/hawk_test.go @@ -79,7 +79,7 @@ func run_hawk(h *hawk.Hawk, id int, t *testing.T, wg *sync.WaitGroup) { defer wg.Done() - rtx, err = h.NewRtx(fmt.Sprintf("%d", id), nil) + rtx, err = h.NewRtx(fmt.Sprintf("%d", id), nil, nil) if err != nil { t.Errorf("failed to create rtx id[%d] - %s", id, err.Error()) return @@ -103,9 +103,9 @@ func run_hawk(h *hawk.Hawk, id int, t *testing.T, wg *sync.WaitGroup) { } fmt.Printf("RET[%d] => [%v]\n", id, ret) - // check if ValCount() returns the right number of values created explicitly + // check if ValCount() returns the right number of values created. 3 explicitly and 1 return value i = rtx.ValCount() - if i != 3 { t.Errorf("the number of val objects for rtx id[%d] must be 3. but %d was returned", id, i) } + if i != 4 { t.Errorf("the number of val objects for rtx id[%d] must be 4. but %d was returned", id, i) } rtx.Close() @@ -181,7 +181,7 @@ return x; return } - rtx, err = h.NewRtx("test2", nil) + rtx, err = h.NewRtx("test2", nil, nil) if err != nil { t.Errorf("failed to create rtx - %s", err.Error()) } else { @@ -310,7 +310,7 @@ return x return } - rtx, err = h.NewRtx("test3", nil) + rtx, err = h.NewRtx("test3", nil, nil) if err != nil { t.Errorf("failed to create rtx - %s", err.Error()) } else {