Merge pull request #79 from pronovic/ken/findargs-regex

Improvements to argument processing

.gitignore

@@ -17,6 +17,9 @@
 # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
 .glide/
 
+# IntelliJ 
+.idea
+
 ### OSX ###
 *.DS_Store
 .AppleDouble

core/matcher.go

@@ -88,12 +88,13 @@ func handleChatServiceRule(outputMsgs chan<- models.Message, message models.Mess
 		}
 
 		if hit {
-			bot.Log.Debugf("Found rule match '%s'", rule.Name)
+			bot.Log.Debugf("Found rule match '%s' for input '%s'", rule.Name, message.Input)
 			// Don't go through more rules if rule is matched
 			match, stopSearch = true, true
 			// Publish metric to prometheus - metricname will be combination of bot name and rule name
 			Prommetric(bot.Name+"-"+rule.Name, bot)
 			// Capture untouched user input
+
 			message.Vars["_raw_user_input"] = message.Input
 			// Do additional checks on the rule before running
 			if !isValidHitChatRule(&message, rule, processedInput, bot) {
@@ -163,7 +164,7 @@ func isValidHitChatRule(message *models.Message, rule models.Rule, processedInpu
 	// If this wasn't a 'hear' rule, handle the args
 	if rule.Hear == "" {
 		// Get all the args that the message sender supplied
-		args := utils.FindArgs(processedInput)
+		args := utils.RuleArgTokenizer(processedInput)
 		// Are we expecting a number of args but don't have as many as the rule defines? Send a helpful message
 		if len(rule.Args) > 0 && len(args) < len(rule.Args) {
 			msg := fmt.Sprintf("You might be missing an argument or two. This is what I'm looking for\n```%s```", rule.HelpText)

handlers/script.go

@@ -30,13 +30,15 @@ func ScriptExec(args models.Action, msg *models.Message, bot *models.Bot) (*mode
 	defer cancel()
 
 	// Deal with variable substitution in command
+	bot.Log.Debugf("Command is: [%s]", args.Cmd)
 	cmdProcessed, err := utils.Substitute(args.Cmd, msg.Vars)
+	bot.Log.Debugf("Substituted: [%s]", cmdProcessed)
 	if err != nil {
 		return result, err
 	}
 
 	// Parse out all the arguments from the supplied command
-	bin := utils.FindArgs(cmdProcessed)
+	bin := utils.ExecArgTokenizer(cmdProcessed)
 	// Execute the command + arguments with the context
 	cmd := exec.CommandContext(ctx, bin[0], bin[1:]...)
 

utils/parse.go

@@ -67,8 +67,8 @@ func Substitute(value string, tokens map[string]string) (string, error) {
 	return value, nil
 }
 
-// FindArgs goes through a string and tokenizes as parameters
-func FindArgs(stripped string) []string {
+// RuleArgTokenizer goes through a string and tokenizes as parameters for use when identifying rules to be triggered (ignoring empty arguments)
+func RuleArgTokenizer(stripped string) []string {
 	re := regexp.MustCompile(`["“]([^"“”]+)["”]|([^"“”\s]+)`)
 	argmatch := re.FindAllString(stripped, -1)
 
@@ -79,6 +79,18 @@ func FindArgs(stripped string) []string {
 	return argmatch
 }
 
+// ExecArgTokenizer goes through a string and tokenizes as parameters for use when executing a script (respecting empty arguments)
+func ExecArgTokenizer(stripped string) []string {
+	re := regexp.MustCompile(`('[^']*')|("[^"]*")|“([^”]*”)|([^'"“”\s]+)`)
+	argmatch := re.FindAllString(stripped, -1)
+
+	for i, arg := range argmatch {
+		argmatch[i] = strings.Trim(arg, `'"“”`)
+	}
+
+	return argmatch
+}
+
 // find variables within strings with pattern ${var}
 func findVars(value string) (match bool, tokens []string) {
 	match = false

utils/parse_test.go

@@ -76,7 +76,7 @@ func TestSubstitute(t *testing.T) {
 	}
 }
 
-func TestFindArgs(t *testing.T) {
+func TestRuleArgTokenizer(t *testing.T) {
 	type args struct {
 		stripped string
 	}
@@ -98,7 +98,57 @@ func TestFindArgs(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := FindArgs(tt.args.stripped); !reflect.DeepEqual(got, tt.want) {
+			if got := RuleArgTokenizer(tt.args.stripped); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("FindArgs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestExecArgTokenizer(t *testing.T) {
+	type args struct {
+		stripped string
+	}
+
+	var empty []string
+
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{"Single", args{stripped: `one`}, []string{"one"}},
+		{"Single, Empty", args{stripped: ``}, empty},
+		{"Single, Including Slash", args{stripped: `this\that`}, []string{`this\that`}},
+		{"Single, Extra Whitespace", args{stripped: ` one `}, []string{"one"}},
+		{"Multiple", args{stripped: `one two three`}, []string{"one", "two", "three"}},
+		{"Multiple, Extra Whitespace", args{stripped: ` one    two     three  `}, []string{"one", "two", "three"}},
+		{"Single, Quoted (Single Quotes)", args{stripped: `one 'quoted arg'`}, []string{"one", "quoted arg"}},
+		{"Single, Quoted (Double Quotes)", args{stripped: `one "quoted arg"`}, []string{"one", "quoted arg"}},
+		{"Single, Quoted (Smart Quotes)", args{stripped: `one “quoted arg”`}, []string{"one", "quoted arg"}},
+		{"Single, Quoted (Empty)", args{stripped: `one “”`}, []string{"one", ""}},
+		{"Multiple, Quoted (Single Quotes)", args{stripped: `one two 'quoted arg' three`}, []string{"one", "two", "quoted arg", "three"}},
+		{"Multiple, Quoted (Double Quotes)", args{stripped: `one two "quoted arg" three`}, []string{"one", "two", "quoted arg", "three"}},
+		{"Multiple, Quoted (Smart Quotes)", args{stripped: `one two “quoted arg” three`}, []string{"one", "two", "quoted arg", "three"}},
+		{"Multiple, Quoted (Mixed Quotes)", args{stripped: `one two 'quoted arg1' “quoted arg2” "quoted arg3" three`}, []string{"one", "two", "quoted arg1", "quoted arg2", "quoted arg3", "three"}},
+		{"Multiple, Quoted (Embedded Quotes)", args{stripped: `one two 'quoted ”“" arg1' “quoted "' arg2” "quoted ”“' arg3" three`}, []string{"one", "two", `quoted ”“" arg1`, `quoted "' arg2`, `quoted ”“' arg3`, "three"}},
+		{"Multiple, Quoted (Some Empty)", args{stripped: `one two '' “” "" three "" four '' five “” six`}, []string{"one", "two", "", "", "", "three", "", "four", "", "five", "", "six"}},
+		{"Multiple, Quoted (Mismatched Quotes, Single/Double)", args{stripped: `one two 'quoted arg" three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Single/Smart Close)", args{stripped: `one two 'quoted arg” three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Single/Smart Open)", args{stripped: `one two 'quoted arg“ three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Double/Smart Close)", args{stripped: `one two "quoted arg” three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Double/Smart Open)", args{stripped: `one two "quoted arg“ three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Double/Single)", args{stripped: `one two "quoted arg' three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Smart Close/Single)", args{stripped: `one two ”quoted arg' three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Smart Open/Single)", args{stripped: `one two “quoted arg' three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Smart Close/Double)", args{stripped: `one two ”quoted arg" three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Smart Open/Double)", args{stripped: `one two “quoted arg" three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Smart Close/Smart Close)", args{stripped: `one two ”quoted arg” three`}, []string{"one", "two", "quoted", "arg", "three"}},
+		{"Multiple, Quoted (Mismatched Quotes, Smart Open/Smart Open)", args{stripped: `one two “quoted arg“ three`}, []string{"one", "two", "quoted", "arg", "three"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ExecArgTokenizer(tt.args.stripped); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("FindArgs() = %v, want %v", got, tt.want)
 			}
 		})