* fix on split-disable, report 'file not found' error issue

.gitignore

@@ -14,4 +14,10 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 
-build/
+build/
+
+cmd/out*
+cmd/cmd
+cmd/ignore*
+
+data/system_reports

cmd/build.sh

@@ -20,19 +20,19 @@ OUTPATH=$BUILDDIR/linux64/
 mkdir -p $OUTPATH
 CGO_ENABLED=0 GOOS="linux" GOARCH="$GOARCH" go build -ldflags="$LDFLAGS" -o $OUT
 mv $OUT $OUTPATH
-cp ../config/config.toml $OUTPATH'config.toml'
+cp ../config.toml $OUTPATH'config.toml'
 echo "${G}${OUT}(linux64) built${W}"
 
 OUTPATH=$BUILDDIR/win64/
 mkdir -p $OUTPATH
 CGO_ENABLED=0 GOOS="windows" GOARCH="$GOARCH" go build -ldflags="$LDFLAGS" -o $OUT.exe
 mv $OUT.exe $OUTPATH
-cp ../config/config.toml $OUTPATH'config.toml'
+cp ../config.toml $OUTPATH'config.toml'
 echo "${G}${OUT}(win64) built${W}"
 
 OUTPATH=$BUILDDIR/mac/
 mkdir -p $OUTPATH
 CGO_ENABLED=0 GOOS="darwin" GOARCH="$GOARCH" go build -ldflags="$LDFLAGS" -o $OUT
 mv $OUT $OUTPATH
-cp ../config/config.toml $OUTPATH'config.toml'
+cp ../config.toml $OUTPATH'config.toml'
 echo "${G}${OUT}(mac) built${W}"

cmd/clean.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+rm -rf ./out
+rm -rf ./out_*
+rm -rf ./ignore

cmd/config.toml

@@ -1,4 +1,4 @@
-InFolder = "../data/"
+InFolder = "../data"
 TrimColAfterSplit = true
 WalkSubFolders = true
 

cmd/main.go

@@ -1,21 +1,17 @@
 package main
 
 import (
-	"log"
 	"os"
 
+	lk "github.com/digisan/logkit"
 	splitter "github.com/nsip/dev-nrt-splitter"
 )
 
 func main() {
-
 	configurations := []string{}
 	if len(os.Args) > 1 {
 		configurations = append(configurations, os.Args[1:]...)
 	}
 	configurations = append(configurations, "./config.toml")
-	if err := splitter.NrtSplit(configurations...); err != nil {
-		log.Fatalln(err)
-	}
-
+	lk.WarnOnErr("%v", splitter.NrtSplit(configurations...))
 }

config.go

@@ -0,0 +1,53 @@
+package splitter
+
+import (
+	cfg "github.com/digisan/go-config"
+	"github.com/gosuri/uiprogress"
+)
+
+var (
+	uip      *uiprogress.Progress
+	bar      *uiprogress.Bar
+	procSize uint64
+	progBar  = true
+
+	enableTrim         bool
+	enableSplit        bool
+	inFolder           string
+	inFolderAbs        string
+	goSubFolder        bool
+	bySplit2           bool
+	ignoreFolder4Split string
+	out4Split          string
+	splitSchema        []string
+	trimColAfterSplit  bool
+	trimCols           []string
+	out4Trim           string
+	merges             []map[string]any
+)
+
+func init() {
+	// setConfig("./config.toml", "../config.toml")
+}
+
+func setConfig(fConfigs ...string) {
+
+	cfg.Init("config", false, fConfigs...)
+
+	inFolder = cfg.Path("InFolder")
+	inFolderAbs = cfg.PathAbs("InFolder")
+	goSubFolder = cfg.Bool("WalkSubFolders")
+	trimColAfterSplit = cfg.Bool("TrimColAfterSplit")
+
+	enableTrim = cfg.Bool("Trim.Enabled")
+	trimCols = cfg.Strs("Trim.Columns")
+	out4Trim = cfg.Path("Trim.OutFolder")
+
+	enableSplit = cfg.Bool("Split.Enabled")
+	bySplit2 = cfg.Bool("Split.Split2")
+	ignoreFolder4Split = cfg.Path("Split.IgnoreFolder")
+	out4Split = cfg.Path("Split.OutFolder")
+	splitSchema = cfg.Strs("Split.Schema")
+
+	merges = cfg.Objects("Merge")
+}

config/config.toml → config.toml

@@ -1,4 +1,4 @@
-InFolder = "./data/"
+InFolder = "../data/"
 TrimColAfterSplit = true
 WalkSubFolders = true
 
@@ -12,7 +12,7 @@ Enabled = true
 IgnoreFolder = "./ignore/"
 OutFolder = "./out_split/"
 Schema = ["School", "YrLevel", "Domain"]
-Split2 = false
+Split2 = true
 
 [[Merge]]
 Enabled = true

go.mod

@@ -1,27 +1,29 @@
 module github.com/nsip/dev-nrt-splitter
 
-go 1.18
+go 1.19
 
 require (
-	github.com/BurntSushi/toml v1.2.0
-	github.com/digisan/csv-tool v0.1.3
-	github.com/digisan/go-generics v0.2.1
-	github.com/digisan/gotk v0.1.18
+	github.com/digisan/csv-tool v0.1.9
+	github.com/digisan/go-config v0.1.5
+	github.com/digisan/go-generics v0.2.17
+	github.com/digisan/gotk v0.2.12
+	github.com/digisan/logkit v0.1.4
 	github.com/gosuri/uiprogress v0.0.1
 )
 
 require (
-	github.com/clbanning/mxj v1.8.4 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/digisan/json-tool v0.0.33 // indirect
-	github.com/digisan/logkit v0.0.10 // indirect
-	github.com/gookit/color v1.5.1 // indirect
+	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/digisan/fileflatter v0.0.3 // indirect
+	github.com/gookit/color v1.5.2 // indirect
 	github.com/gosuri/uilive v0.0.4 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/tidwall/gjson v1.12.1 // indirect
+	github.com/h2non/filetype v1.1.3 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/tidwall/sjson v1.2.4 // indirect
-	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tidwall/sjson v1.2.5 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	golang.org/x/net v0.4.0 // indirect
+	golang.org/x/sys v0.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,38 +1,59 @@
-github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
-github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/digisan/csv-tool v0.1.3 h1:ez1ICsP73Zq5ysZU9wIbEl2juBr9OJMf0hUn81fTPUg=
-github.com/digisan/csv-tool v0.1.3/go.mod h1:UO2elii+wu71KKAd1JForN4MQssSVobFaA6s/1mDxFY=
-github.com/digisan/go-generics v0.2.1 h1:rUQsgfFWWeMTpDBmZZAQSnlbybONe20MzedrN+oG5hE=
-github.com/digisan/go-generics v0.2.1/go.mod h1:uYqkhOsBlydT6uPQaapc6Bk3cbnmQ8ftBm/sWbShxAI=
-github.com/digisan/gotk v0.1.18 h1:IoOhbCUmobW55xAPmZapKqqQ9iIVUU1uaIjouSFf7js=
-github.com/digisan/gotk v0.1.18/go.mod h1:qIohmTpVDN95xoycu2BKbqKRiP1lB5uLhztYFXNPV9E=
-github.com/digisan/json-tool v0.0.33/go.mod h1:XbRbwiX6bSYubwAUx7lktuYiqPZDWVj6AQhIrdPPsAU=
-github.com/digisan/logkit v0.0.10 h1:qN2fA9Ju1oIJia1yPnuTOjddvOWtYwQ+1dg33aTG9Ug=
-github.com/digisan/logkit v0.0.10/go.mod h1:Ve92Db2/jOXmGJ2ArnfNJsWtt8qJn8T8UOpFGnDWvP4=
-github.com/gookit/color v1.5.1 h1:Vjg2VEcdHpwq+oY63s/ksHrgJYCTo0bwWvmmYWdE9fQ=
-github.com/gookit/color v1.5.1/go.mod h1:wZFzea4X8qN6vHOSP2apMb4/+w/orMznEzYsIHPaqKM=
+github.com/digisan/csv-tool v0.1.9 h1:H6A41lxOdTVCkFWFRf8bdPcdcZQSzbSfdTGO7pbSZk8=
+github.com/digisan/csv-tool v0.1.9/go.mod h1:fSfwEJMO+NPBUl01pgUyPyKbz7Mgj16Al9qlxZu9KWg=
+github.com/digisan/fileflatter v0.0.3 h1:uP1iTU1Bu06/NLQPhiqMe9ju+vypMqfltK2qyzBSLmQ=
+github.com/digisan/fileflatter v0.0.3/go.mod h1:7xv/2C0aEsUI5K1+61WhmNOoKbyACIX0jF3i1aHFYtw=
+github.com/digisan/go-config v0.1.5 h1:b3KmSXhF+8evk1avYjUniJAfqngm4gW2qxv9MnoV6AQ=
+github.com/digisan/go-config v0.1.5/go.mod h1:S9bvRIl0woeuGS+Xk7CNUA6as+xOTUPyZhgH5JwsmMs=
+github.com/digisan/go-generics v0.2.17 h1:u4YZn8isb6tnJ15KCTRlbWvhYU9ZYB2wJCHyJi17Lhs=
+github.com/digisan/go-generics v0.2.17/go.mod h1:HoHbkpjEgiw3YQ3QaADIfeFt/kpdODb0qerSbs0PSBA=
+github.com/digisan/gotk v0.2.12 h1:kv8eN7K5v6Bk50tlfTAI7lo4fRfY8CSWHxu3n8DxODE=
+github.com/digisan/gotk v0.2.12/go.mod h1:gkkH06zN+BLeWcjbRnSQvJd1+An5zk4KNrP5Jmtr0HU=
+github.com/digisan/logkit v0.1.4 h1:kbs7nl7AqMou5IFm/RG68/HrxRsbu5G2Zfrvmev+CqQ=
+github.com/digisan/logkit v0.1.4/go.mod h1:U1Cdco16JxWZk/XrxQ3LS4AfW/saqF93zPFJdFa6HhE=
+github.com/gookit/color v1.5.2 h1:uLnfXcaFjlrDnQDT+NCBcfhrXqYTx/rcCa6xn01Y8yI=
+github.com/gookit/color v1.5.2/go.mod h1:w8h4bGiHeeBpvQVePTutdbERIUf3oJE5lZ8HM0UgXyg=
 github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
 github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
 github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
 github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
+github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
+github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
-github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
-github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
+golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
+golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=