added --concurrent to hawkgo

bin/hawkgo.go

@@ -10,7 +10,9 @@ import "os"
 import "path/filepath"
 import "runtime"
 import "runtime/debug"
+import "strconv"
 import "strings"
+import "sync"
 //import "time"
 
 type Assign struct {
@@ -23,10 +25,13 @@ type Config struct {
 	call string
 	fs string
 	show_extra_info bool
+	concurrent bool
+	suffix string
 
 	srcstr string
 	srcfiles []string
-	datfiles []string
+	data_in_files []string
+	data_out_files []string
 }
 
 func exit_with_error(msg string, err error) {
@@ -55,6 +60,17 @@ func parse_args_to_config(cfg *Config) bool {
 		cfg.call = v
 		return nil
 	})
+	flgs.BoolFunc("concurrent", "run the script over multiple data files concurrently", func(v string) error {
+		if v[0] == '.' {
+			// treat it as a suffix
+			cfg.suffix = v
+			cfg.concurrent = true
+		} else {
+			cfg.suffix = ""
+			cfg.concurrent, _ = strconv.ParseBool(v)
+		}
+		return nil
+	})
 	flgs.Func("field-separator", "set the field separator(FS)", func(v string) error {
 		cfg.fs = v
 		return nil
@@ -79,26 +95,170 @@ func parse_args_to_config(cfg *Config) bool {
 	}
 
 	//if flgs.NArg() == 0 { goto wrong_usage}
-	cfg.datfiles = flgs.Args()
+	cfg.data_in_files = flgs.Args()
 	if len(cfg.srcfiles) <= 0 {
-		if len(cfg.datfiles) == 0 { goto wrong_usage }
+		if len(cfg.data_in_files) == 0 { goto wrong_usage }
-		cfg.srcstr = cfg.datfiles[0]
+		cfg.srcstr = cfg.data_in_files[0]
-		cfg.datfiles = cfg.datfiles[1:]
+		cfg.data_in_files = cfg.data_in_files[1:]
 	}
 
+	if cfg.concurrent && len(cfg.data_in_files) > 0 {
+		var i int
+		var n int
+		var f []string
+		n = len(cfg.data_in_files)
+		cfg.data_out_files = make([]string, n)
+		for i = 0; i < n; i++ {
+			f = strings.SplitN(cfg.data_in_files[i], ":", 2)
+			cfg.data_in_files[i] = f[0]
+			if len(f) >= 2 {
+				cfg.data_out_files[i] = f[1]
+			} else if cfg.suffix != "" && cfg.data_in_files[i] != "" && cfg.data_in_files[i] != "-" {
+				cfg.data_out_files[i] = cfg.data_in_files[i] + cfg.suffix
+			}
+		}
+	}
 	return true
 
 wrong_usage:
 	fmt.Fprintf(os.Stderr, "USAGE: %s [options] sourcestring [datafile]*\n", os.Args[0])
 	fmt.Fprintf(os.Stderr, "       %s [options] -f sourcefile [datafile]*\n", os.Args[0])
+	fmt.Fprintf(os.Stderr, "Options are:\n")
+	fmt.Fprintf(os.Stderr, " -F, --field-separator string     specify the field seperator\n")
+	fmt.Fprintf(os.Stderr, " -v, --assign          name=value set global variable value\n")
+	fmt.Fprintf(os.Stderr, " -c, --call            string     call a named function instead of running the\n")
+	fmt.Fprintf(os.Stderr, "                                  normal loop\n")
+	fmt.Fprintf(os.Stderr, "     --concurrent=[suffix]        process multiple data files concurrently\n")
+	fmt.Fprintf(os.Stderr, "                                  if datafile has two parts delimited by a colon,\n")
+	fmt.Fprintf(os.Stderr, "                                  the second part specifies the output file (e.g.\n")
+	fmt.Fprintf(os.Stderr, "                                  infile:outfile). The suffix beginning with a\n")
+	fmt.Fprintf(os.Stderr, "                                  period is appended to datafile to form the out-\n")
+	fmt.Fprintf(os.Stderr, "                                  put file name if the second part is missing\n")
 	return false
 }
 
+func run_script(h *hawk.Hawk, fs_idx int, data_idx int, cfg* Config, wg *sync.WaitGroup) error {
+	var rtx *hawk.Rtx
+	var err error
+
+	if wg != nil { defer wg.Done() }
+
+	if data_idx <= -1 {
+		rtx, err = h.NewRtx(filepath.Base(os.Args[0]), cfg.data_in_files, nil)
+	} else {
+		var out_idx_end int = data_idx
+
+		if cfg.data_out_files[data_idx] != "" { out_idx_end++ }
+		rtx, err = h.NewRtx(
+			fmt.Sprintf("%s.%d", filepath.Base(os.Args[0]), data_idx),
+			cfg.data_in_files[data_idx: data_idx + 1],
+			cfg.data_out_files[data_idx: out_idx_end],
+		)
+	}
+	if err != nil {
+		err = fmt.Errorf("failed to make rtx - %s", err.Error())
+		goto oops
+	} else  {
+		var k string
+		var v Assign
+		var retv *hawk.Val
+
+		for k, v = range cfg.assigns {
+			if v.idx >= 0  {
+				var vv *hawk.Val
+				vv, err = rtx.NewNumOrStrVal(v.value)
+				if err != nil {
+					err = fmt.Errorf("failed to convert value '%s' for global variable '%s' - %s", v.value, k, err.Error())
+					goto oops
+				}
+
+				err = rtx.SetGlobal(v.idx, vv)
+				vv.Close()
+				if err != nil {
+					err = fmt.Errorf("failed to set global variable '%s' - %s", k, err.Error())
+					goto oops
+				}
+			}
+		}
+
+		if fs_idx >= 0 {
+			var vv *hawk.Val
+			vv, err = rtx.NewStrVal(cfg.fs)
+			if err != nil {
+				err = fmt.Errorf("failed to convert field separator '%s' - %s", cfg.fs, err.Error())
+				goto oops
+			}
+
+			rtx.SetGlobal(fs_idx, vv)
+			vv.Close()
+			if err != nil {
+				err = fmt.Errorf("failed to set field separator to '%s' - %s", cfg.fs, err.Error())
+				goto oops
+			}
+		}
+
+		if cfg.call != "" {
+			var idx int
+			var count int
+			var args []*hawk.Val
+
+			count = len(cfg.data_in_files)
+			args = make([]*hawk.Val, count)
+			for idx = 0; idx < count; idx++ {
+				args[idx], err = rtx.NewStrVal(cfg.data_in_files[idx])
+				if err != nil {
+					err = fmt.Errorf("failed to convert argument [%s] to value - %s", cfg.data_in_files[idx], err.Error())
+					goto oops
+				}
+			}
+			retv, err = rtx.Call(cfg.call, args...)
+			for idx = 0; idx < count; idx++ {
+				// it's ok not to call Close() on args as rtx.Close() closes them automatically.
+				// if args are re-created repeatedly, Close() on them is always needed not to
+				// accumulate too many allocated values.
+				args[idx].Close()
+			}
+		} else {
+			//v, err = rtx.Loop()
+			retv, err = rtx.Exec(cfg.data_in_files)
+		}
+		if err != nil {
+			err = fmt.Errorf("failed to run - %s", err.Error())
+			goto oops
+		}
+
+		if cfg.show_extra_info {
+			var named_vars map[string]*hawk.Val
+			var vn string
+			var vv *hawk.Val
+
+			fmt.Printf("[RETURN] - [%v]\n", retv.String())
+
+			fmt.Printf("NAMED VARIABLES]\n")
+			named_vars = make(map[string]*hawk.Val)
+			rtx.GetNamedVars(named_vars)
+			for vn, vv = range named_vars {
+				fmt.Printf("%s = %s\n", vn, vv.String())
+			}
+			fmt.Printf("END OF NAMED VARIABLES]\n")
+		}
+	}
+
+	// let's not care about closing args or return values
+	// because rtx.Close() will close them automatically
+	if rtx != nil { rtx.Close() }
+	return nil
+
+oops:
+	if rtx != nil { rtx.Close() }
+	return err
+}
+
 func main() {
 	var h *hawk.Hawk
-	var rtx *hawk.Rtx
 	var cfg Config
 	var fs_idx int = -1
+	var wg sync.WaitGroup
 	var err error
 
 	// for profiling
@@ -152,100 +312,29 @@ func main() {
 		goto oops
 	}
 
-	rtx, err = h.NewRtx(filepath.Base(os.Args[0]), cfg.datfiles)
+	if cfg.concurrent && len(cfg.data_in_files) >= 1 {
-	if err != nil {
+		var n int
-		fmt.Fprintf(os.Stderr, "ERROR: failed to make rtx - %s\n", err.Error())
+		var i int
-		goto oops
+		n = len(cfg.data_in_files)
-	} else  {
+		for i = 0; i < n; i++ {
-		var k string
+			wg.Add(1)
-		var v Assign
+			go func(idx int) {
-		var retv *hawk.Val
+				var err error
-
+				err = run_script(h, fs_idx, idx, &cfg, &wg)
-		for k, v = range cfg.assigns {
-			if v.idx >= 0  {
-				var vv *hawk.Val
-				vv, err = rtx.NewNumOrStrVal(v.value)
 				if err != nil {
-					fmt.Fprintf(os.Stderr, "ERROR: failed to convert value '%s' for global variable '%s' - %s\n", v.value, k, err.Error())
+					fmt.Fprintf(os.Stderr, "ERROR: [%d]%s\n", idx, err.Error())
-					goto oops
 				}
-
+			}(i)
-				err = rtx.SetGlobal(v.idx, vv)
-				vv.Close()
-				if err != nil {
-					fmt.Fprintf(os.Stderr, "ERROR: failed to set global variable '%s' - %s\n", k, err.Error())
-					goto oops
-				}
-			}
-		}
-
-		if fs_idx >= 0 {
-			var vv *hawk.Val
-			vv, err = rtx.NewStrVal(cfg.fs)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "ERROR: failed to convert field separator '%s' - %s\n", cfg.fs, err.Error())
-				goto oops
-			}
-
-			rtx.SetGlobal(fs_idx, vv)
-			vv.Close()
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "ERROR: failed to set field separator to '%s' - %s\n", cfg.fs, err.Error())
-				goto oops
-			}
-		}
-
-		if cfg.call != "" {
-			var idx int
-			var count int
-			var args []*hawk.Val
-
-			count = len(cfg.datfiles)
-			args = make([]*hawk.Val, count)
-			for idx = 0; idx < count; idx++ {
-				args[idx], err = rtx.NewStrVal(cfg.datfiles[idx])
-				if err != nil {
-					fmt.Fprintf(os.Stderr, "ERROR: failed to convert argument [%s] to value - %s\n", cfg.datfiles[idx], err.Error())
-					goto oops
-				}
-			}
-			retv, err = rtx.Call(cfg.call, args...)
-			for idx = 0; idx < count; idx++ {
-				// it's ok not to call Close() on args as rtx.Close() closes them automatically.
-				// if args are re-created repeatedly, Close() on them is always needed not to
-				// accumulate too many allocated values.
-				args[idx].Close()
-			}
-		} else {
-			//v, err = rtx.Loop()
-			retv, err = rtx.Exec(cfg.datfiles)
 		}
+		wg.Wait()
+	} else {
+		err = run_script(h, fs_idx, -1, &cfg, nil)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "ERROR: failed to run rtx - %s\n", err.Error())
+			fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
 			goto oops
 		}
-
-		if cfg.show_extra_info {
-			var named_vars map[string]*hawk.Val
-			var vn string
-			var vv *hawk.Val
-
-			fmt.Printf("[RETURN] - [%v]\n", retv.String())
-
-
-			fmt.Printf("NAMED VARIABLES]\n")
-			named_vars = make(map[string]*hawk.Val)
-			rtx.GetNamedVars(named_vars)
-			for vn, vv = range named_vars {
-				fmt.Printf("%s = %s\n", vn, vv.String())
-			}
-			fmt.Printf("END OF NAMED VARIABLES]\n")
-		}
 	}
 
-	// let's not care about closing args or return values
-	// because rtx.Close() will close them automatically
-	if rtx != nil { rtx.Close() }
 	if h != nil { h.Close() }
 
 	runtime.GC()
@@ -255,7 +344,7 @@ func main() {
 	return
 
 oops:
-	if rtx != nil { rtx.Close() }
+//	if rtx != nil { rtx.Close() }
 	if h != nil { h.Close() }
 	os.Exit(255)
 }

hawk.go

@@ -491,26 +491,43 @@ func deregister_rtx_instance(rtx *Rtx) {
 	}
 }
 
-func (hawk *Hawk) NewRtx(id string, in []string) (*Rtx, error) {
+func (hawk *Hawk) NewRtx(id string, in []string, out []string) (*Rtx, error) {
 	var rtx *C.hawk_rtx_t
 	var g *Rtx
 	var xtn *C.rtx_xtn_t
 	var cid *C.hawk_bch_t
 	var cin []*C.hawk_bch_t
+	var cout []*C.hawk_bch_t
+	var cin_ptr **C.hawk_bch_t
+	var cout_ptr **C.hawk_bch_t
 	var idx int
 	var in_count int
+	var out_count int
 
 	in_count = len(in)
-	cin = make([]*C.hawk_bch_t, in_count + 1)
-	for idx = 0; idx < in_count; idx++ {
-		cin[idx] = C.CString(in[idx])
-	}
-	cin[idx] = (*C.hawk_bch_t)(nil)
-	cid = C.CString(id)
 	if in_count > 0 {
-		rtx = C.hawk_rtx_openstdwithbcstr(hawk.c, C.hawk_oow_t(unsafe.Sizeof(*xtn)), cid, &cin[0], nil, nil)
+		cin = make([]*C.hawk_bch_t, in_count + 1)
-	} else {
+		for idx = 0; idx < in_count; idx++ {
-		rtx = C.hawk_rtx_openstdwithbcstr(hawk.c, C.hawk_oow_t(unsafe.Sizeof(*xtn)), cid, nil, nil, nil)
+			cin[idx] = C.CString(in[idx])
+		}
+		cin[idx] = (*C.hawk_bch_t)(nil)
+		cin_ptr = &cin[0]
+	}
+
+	out_count = len(out)
+	if out_count > 0 {
+		cout = make([]*C.hawk_bch_t, out_count + 1)
+		for idx = 0; idx < out_count; idx++ {
+			cout[idx] = C.CString(out[idx])
+		}
+		cout[idx] = (*C.hawk_bch_t)(nil)
+		cout_ptr = &cout[0]
+	}
+
+	cid = C.CString(id)
+	rtx = C.hawk_rtx_openstdwithbcstr(hawk.c, C.hawk_oow_t(unsafe.Sizeof(*xtn)), cid, cin_ptr, cout_ptr, nil)
+	for idx = 0; idx < out_count; idx++ {
+		C.free(unsafe.Pointer(cout[idx]))
 	}
 	for idx = 0; idx < in_count; idx++ {
 		C.free(unsafe.Pointer(cin[idx]))

hawk_test.go

@@ -79,7 +79,7 @@ func run_hawk(h *hawk.Hawk, id int, t *testing.T, wg *sync.WaitGroup) {
 
 	defer wg.Done()
 
-	rtx, err = h.NewRtx(fmt.Sprintf("%d", id), nil)
+	rtx, err = h.NewRtx(fmt.Sprintf("%d", id), nil, nil)
 	if err != nil {
 		t.Errorf("failed to create rtx id[%d] - %s", id, err.Error())
 		return
@@ -103,9 +103,9 @@ func run_hawk(h *hawk.Hawk, id int, t *testing.T, wg *sync.WaitGroup) {
 	}
 	fmt.Printf("RET[%d] => [%v]\n", id, ret)
 
-	// check if ValCount() returns the right number of values created explicitly
+	// check if ValCount() returns the right number of values created. 3 explicitly and 1 return value
 	i = rtx.ValCount()
-	if i != 3 { t.Errorf("the number of val objects for rtx id[%d] must be 3. but %d was returned", id, i) }
+	if i != 4 { t.Errorf("the number of val objects for rtx id[%d] must be 4. but %d was returned", id, i) }
 
 	rtx.Close()
 
@@ -181,7 +181,7 @@ return x;
 		return
 	}
 
-	rtx, err = h.NewRtx("test2", nil)
+	rtx, err = h.NewRtx("test2", nil, nil)
 	if err != nil {
 		t.Errorf("failed to create rtx - %s", err.Error())
 	} else  {
@@ -310,7 +310,7 @@ return x
 		return
 	}
 
-	rtx, err = h.NewRtx("test3", nil)
+	rtx, err = h.NewRtx("test3", nil, nil)
 	if err != nil {
 		t.Errorf("failed to create rtx - %s", err.Error())
 	} else  {