Merge branch 'master' of github.com:nyaruka/courier into line-handlers

.travis.yml

@@ -20,7 +20,7 @@ before_script:
 - go get github.com/mattn/goveralls
 
 script:
-- go test github.com/nyaruka/courier/... -p=1 -bench=.
+- go test github.com/nyaruka/courier/... -p=1
 - $HOME/gopath/bin/goveralls -service=travis-ci -v
 
 after_success:

cmd/courier/main.go

@@ -30,6 +30,7 @@ import (
 	_ "github.com/nyaruka/courier/handlers/mtarget"
 	_ "github.com/nyaruka/courier/handlers/nexmo"
 	_ "github.com/nyaruka/courier/handlers/shaqodoon"
+	_ "github.com/nyaruka/courier/handlers/smscentral"
 	_ "github.com/nyaruka/courier/handlers/start"
 	_ "github.com/nyaruka/courier/handlers/telegram"
 	_ "github.com/nyaruka/courier/handlers/twilio"

handlers/base.go

@@ -94,7 +94,7 @@ func Validate(form interface{}) error {
 }
 
 // DecodeAndValidateForm takes the passed in form and attempts to parse and validate it from the
-// POST parameters of the passed in request
+// URL query parameters as well as any POST parameters of the passed in request
 func DecodeAndValidateForm(form interface{}, r *http.Request) error {
 	err := r.ParseForm()
 	if err != nil {
@@ -115,23 +115,6 @@ func DecodeAndValidateForm(form interface{}, r *http.Request) error {
 	return nil
 }
 
-// DecodeAndValidateQueryParams takes the passed in form and attempts to parse and validate it from the
-// GET parameters of the passed in request
-func DecodeAndValidateQueryParams(form interface{}, r *http.Request) error {
-	err := decoder.Decode(form, r.URL.Query())
-	if err != nil {
-		return err
-	}
-
-	// check our input is valid
-	err = validate.Struct(form)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // DecodeAndValidateJSON takes the passed in envelope and tries to unmarshal it from the body
 // of the passed in request, then validating it
 func DecodeAndValidateJSON(envelope interface{}, r *http.Request) error {

handlers/external/external.go

@@ -18,7 +18,6 @@ import (
 	"github.com/nyaruka/courier/handlers"
 	"github.com/nyaruka/courier/utils"
 	"github.com/nyaruka/gocommon/urns"
-	"github.com/pkg/errors"
 )
 
 const contentURLEncoded = "application/x-www-form-urlencoded"
@@ -61,17 +60,9 @@ func (h *handler) Initialize(s courier.Server) error {
 // ReceiveMessage is our HTTP handler function for incoming messages
 func (h *handler) ReceiveMessage(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) {
 	externalMessage := &externalMessage{}
-	handlers.DecodeAndValidateQueryParams(externalMessage, r)
-
-	// if this is a post, also try to parse the form body
-	if r.Method == http.MethodPost {
-		handlers.DecodeAndValidateForm(externalMessage, r)
-	}
-
-	// validate whether our required fields are present
-	err := handlers.Validate(externalMessage)
+	err := handlers.DecodeAndValidateForm(externalMessage, r)
 	if err != nil {
-		return nil, err
+		return nil, courier.WriteAndLogRequestError(ctx, w, r, channel, err)
 	}
 
 	// must have one of from or sender set, error if neither
@@ -80,7 +71,7 @@ func (h *handler) ReceiveMessage(ctx context.Context, channel courier.Channel, w
 		sender = externalMessage.From
 	}
 	if sender == "" {
-		return nil, errors.New("must have one of 'sender' or 'from' set")
+		return nil, courier.WriteAndLogRequestError(ctx, w, r, channel, fmt.Errorf("must have one of 'sender' or 'from' set"))
 	}
 
 	// if we have a date, parse it
@@ -93,7 +84,7 @@ func (h *handler) ReceiveMessage(ctx context.Context, channel courier.Channel, w
 	if dateString != "" {
 		date, err = time.Parse(time.RFC3339Nano, dateString)
 		if err != nil {
-			return nil, errors.New("invalid date format, must be RFC 3339")
+			return nil, courier.WriteAndLogRequestError(ctx, w, r, channel, fmt.Errorf("invalid date format, must be RFC 3339"))
 		}
 	}
 
@@ -130,23 +121,15 @@ func (h *handler) buildStatusHandler(status string) courier.ChannelHandleFunc {
 // StatusMessage is our HTTP handler function for status updates
 func (h *handler) StatusMessage(ctx context.Context, statusString string, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) {
 	statusForm := &statusForm{}
-	handlers.DecodeAndValidateQueryParams(statusForm, r)
-
-	// if this is a post, also try to parse the form body
-	if r.Method == http.MethodPost {
-		handlers.DecodeAndValidateForm(statusForm, r)
-	}
-
-	// validate whether our required fields are present
-	err := handlers.Validate(statusForm)
+	err := handlers.DecodeAndValidateForm(statusForm, r)
 	if err != nil {
-		return nil, err
+		return nil, courier.WriteAndLogRequestError(ctx, w, r, channel, err)
 	}
 
-	// get our id
+	// get our status
 	msgStatus, found := statusMappings[strings.ToLower(statusString)]
 	if !found {
-		return nil, fmt.Errorf("unknown status '%s', must be one failed, sent or delivered", statusString)
+		return nil, courier.WriteAndLogRequestError(ctx, w, r, channel, fmt.Errorf("unknown status '%s', must be one failed, sent or delivered", statusString))
 	}
 
 	// write our status

handlers/jiochat/jiochat.go

@@ -73,7 +73,7 @@ type verifyRequest struct {
 // VerifyURL is our HTTP handler function for Jiochat config URL verification callbacks
 func (h *handler) VerifyURL(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) {
 	jcVerify := &verifyRequest{}
-	err := handlers.DecodeAndValidateQueryParams(jcVerify, r)
+	err := handlers.DecodeAndValidateForm(jcVerify, r)
 	if err != nil {
 		return nil, courier.WriteAndLogRequestError(ctx, w, r, channel, err)
 	}

handlers/jiochat/jiochat_test.go

@@ -316,7 +316,7 @@ func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel,
 var defaultSendTestCases = []ChannelSendTestCase{
 	{Label: "Plain Send",
 		Text:           "Simple Message ☺",
-		URN:            "tel:+250788383383",
+		URN:            "jiochat:12345",
 		Status:         "W",
 		ExternalID:     "",
 		ResponseStatus: 200,
@@ -325,11 +325,11 @@ var defaultSendTestCases = []ChannelSendTestCase{
 			"Accept":        "application/json",
 			"Authorization": "Bearer ACCESS_TOKEN",
 		},
-		RequestBody: `{"msgtype":"text","touser":"+250788383383","text":{"content":"Simple Message ☺"}}`,
+		RequestBody: `{"msgtype":"text","touser":"12345","text":{"content":"Simple Message ☺"}}`,
 		SendPrep:    setSendURL},
 	{Label: "Long Send",
 		Text:           "This is a longer message than 160 characters and will cause us to split it into two separate parts, isn't that right but it is even longer than before I say, I need to keep adding more things to make it work",
-		URN:            "tel:+250788383383",
+		URN:            "jiochat:12345",
 		Status:         "W",
 		ExternalID:     "",
 		ResponseStatus: 200,
@@ -338,11 +338,11 @@ var defaultSendTestCases = []ChannelSendTestCase{
 			"Accept":        "application/json",
 			"Authorization": "Bearer ACCESS_TOKEN",
 		},
-		RequestBody: `{"msgtype":"text","touser":"+250788383383","text":{"content":"I need to keep adding more things to make it work"}}`,
+		RequestBody: `{"msgtype":"text","touser":"12345","text":{"content":"I need to keep adding more things to make it work"}}`,
 		SendPrep:    setSendURL},
 	{Label: "Send Attachment",
 		Text:           "My pic!",
-		URN:            "tel:+250788383383",
+		URN:            "jiochat:12345",
 		Attachments:    []string{"image/jpeg:https://foo.bar/image.jpg"},
 		Status:         "W",
 		ExternalID:     "",
@@ -352,11 +352,11 @@ var defaultSendTestCases = []ChannelSendTestCase{
 			"Accept":        "application/json",
 			"Authorization": "Bearer ACCESS_TOKEN",
 		},
-		RequestBody: `{"msgtype":"text","touser":"+250788383383","text":{"content":"My pic!\nhttps://foo.bar/image.jpg"}}`,
+		RequestBody: `{"msgtype":"text","touser":"12345","text":{"content":"My pic!\nhttps://foo.bar/image.jpg"}}`,
 		SendPrep:    setSendURL},
 	{Label: "Error Sending",
 		Text:           "Error Message",
-		URN:            "tel:+250788383383",
+		URN:            "jiochat:12345",
 		Status:         "E",
 		ResponseStatus: 401,
 		Error:          "received non 200 status: 401",

handlers/kannel/kannel.go

@@ -51,7 +51,7 @@ func (h *handler) Initialize(s courier.Server) error {
 func (h *handler) ReceiveMessage(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) {
 	// get our params
 	kannelMsg := &kannelMessage{}
-	err := handlers.DecodeAndValidateQueryParams(kannelMsg, r)
+	err := handlers.DecodeAndValidateForm(kannelMsg, r)
 	if err != nil {
 		return nil, courier.WriteAndLogRequestError(ctx, w, r, channel, err)
 	}
@@ -93,14 +93,14 @@ var kannelStatusMapping = map[int]courier.MsgStatusValue{
 func (h *handler) StatusMessage(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) {
 	// get our params
 	kannelStatus := &kannelStatus{}
-	err := handlers.DecodeAndValidateQueryParams(kannelStatus, r)
+	err := handlers.DecodeAndValidateForm(kannelStatus, r)
 	if err != nil {
-		return nil, err
+		return nil, courier.WriteAndLogRequestError(ctx, w, r, channel, err)
 	}
 
 	msgStatus, found := kannelStatusMapping[kannelStatus.Status]
 	if !found {
-		return nil, fmt.Errorf("unknown status '%d', must be one of 1,2,4,8,16", kannelStatus.Status)
+		return nil, courier.WriteAndLogRequestError(ctx, w, r, channel, fmt.Errorf("unknown status '%d', must be one of 1,2,4,8,16", kannelStatus.Status))
 	}
 
 	// write our status

handlers/mtarget/mtarget.go

@@ -5,9 +5,12 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/buger/jsonparser"
+	"github.com/garyburd/redigo/redis"
 	"github.com/nyaruka/courier"
 	"github.com/nyaruka/courier/handlers"
 	"github.com/nyaruka/courier/utils"
@@ -65,6 +68,65 @@ func (h *handler) receiveMsg(ctx context.Context, c courier.Channel, w http.Resp
 		return nil, courier.WriteAndLogRequestError(ctx, w, r, c, fmt.Errorf("missing required field 'Msisdn'"))
 	}
 
+	// if we have a long message id, then this is part of a multipart message, we don't write the message until
+	// we have received all parts, which we buffer in Redis
+	longID := r.Form.Get("msglong.id")
+	if longID != "" {
+		longCount, _ := strconv.Atoi(r.Form.Get("msglong.msgcount"))
+		longRef, _ := strconv.Atoi(r.Form.Get("msglong.msgref"))
+
+		if longCount == 0 || longRef == 0 {
+			return nil, courier.WriteAndLogRequestError(ctx, w, r, c, fmt.Errorf("invalid or missing 'msglong.msgcount' or 'msglong.msgref' parameters"))
+		}
+
+		if longRef < 1 || longRef > longCount {
+			return nil, courier.WriteAndLogRequestError(ctx, w, r, c, fmt.Errorf("'msglong.msgref' needs to be between 1 and 'msglong.msgcount' inclusive"))
+		}
+
+		rc := h.Backend().RedisPool().Get()
+		defer rc.Close()
+
+		// first things first, populate the new part we just received
+		mapKey := fmt.Sprintf("%s:%s", c.UUID().String(), longID)
+		rc.Send("MULTI")
+		rc.Send("HSET", mapKey, longRef, text)
+		rc.Send("EXPIRE", mapKey, 300)
+		_, err := rc.Do("EXEC")
+		if err != nil {
+			return nil, err
+		}
+
+		// see if we have all the parts we need
+		count, err := redis.Int(rc.Do("HLEN", mapKey))
+		if err != nil {
+			return nil, err
+		}
+
+		// we don't have all the parts yet, say we received the message
+		if count != longCount {
+			return nil, courier.WriteAndLogRequestHandled(ctx, w, r, c, "Message part received")
+		}
+
+		// we have all our parts, grab them and put them together
+		// build up the list of keys we are looking up
+		keys := make([]interface{}, longCount+1)
+		keys[0] = mapKey
+		for i := 1; i < longCount+1; i++ {
+			keys[i] = fmt.Sprintf("%d", i)
+		}
+
+		segments, err := redis.Strings(rc.Do("HMGET", keys...))
+		if err != nil {
+			return nil, err
+		}
+
+		// join our segments in our text
+		text = strings.Join(segments, "")
+
+		// finally delete our key, we are done with this message
+		rc.Do("DEL", mapKey)
+	}
+
 	// create our URN
 	urn := urns.NewTelURNForCountry(from, c.Country())
 
@@ -104,11 +166,12 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat
 	for _, part := range handlers.SplitMsg(handlers.GetTextAndAttachments(msg), maxLength) {
 		// build our request
 		params := url.Values{
-			"username":  []string{username},
-			"password":  []string{password},
-			"msisdn":    []string{msg.URN().Path()},
-			"msg":       []string{part},
-			"serviceid": []string{msg.Channel().Address()},
+			"username":     []string{username},
+			"password":     []string{password},
+			"msisdn":       []string{msg.URN().Path()},
+			"msg":          []string{part},
+			"serviceid":    []string{msg.Channel().Address()},
+			"allowunicode": []string{"true"},
 		}
 
 		msgURL, _ := url.Parse(sendURL)

handlers/mtarget/mtarget_test.go

@@ -9,12 +9,17 @@ import (
 )
 
 var (
-	receiveURL          = "/c/mt/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive"
+	receiveURL = "/c/mt/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive"
+
 	receiveValidMessage = "Msisdn=+923161909799&Content=hello+world&Keyword=Default"
 	receiveStop         = "Msisdn=+923161909799&Content=Stop&Keyword=Stop"
 	receiveMissingFrom  = "Content=hello&Keyword=Default"
 
-	statusURL       = "/c/mt/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status"
+	receivePart2 = "Msisdn=+923161909799&Content=world&Keyword=Default&msglong.id=longmsg&msglong.msgcount=2&msglong.msgref=2"
+	receivePart1 = "Msisdn=+923161909799&Content=hello+&Keyword=Default&msglong.id=longmsg&msglong.msgcount=2&msglong.msgref=1"
+
+	statusURL = "/c/mt/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status"
+
 	statusDelivered = "MsgId=12a7ee90-50ce-11e7-80ae-00000a0a643c&Status=3"
 	statusFailed    = "MsgId=12a7ee90-50ce-11e7-80ae-00000a0a643c&Status=4"
 	statusMissingID = "status?Status=4"
@@ -31,6 +36,10 @@ var handleTestCases = []ChannelHandleTestCase{
 		URN: Sp("tel:+923161909799"), ChannelEvent: Sp("stop_contact")},
 	{Label: "Receive Missing From", URL: receiveURL, Data: receiveMissingFrom, Status: 400, Response: "missing required field 'Msisdn'"},
 
+	{Label: "Receive Part 2", URL: receiveURL, Data: receivePart2, Status: 200, Response: "Handled"},
+	{Label: "Receive Part 1", URL: receiveURL, Data: receivePart1, Status: 200, Response: "Accepted",
+		Text: Sp("hello world"), URN: Sp("tel:+923161909799")},
+
 	{Label: "Status Delivered", URL: statusURL, Data: statusDelivered, Status: 200, Response: "Accepted",
 		ExternalID: Sp("12a7ee90-50ce-11e7-80ae-00000a0a643c"), MsgStatus: Sp("D")},
 	{Label: "Status Failed", URL: statusURL, Data: statusFailed, Status: 200, Response: "Accepted",
@@ -57,11 +66,12 @@ var defaultSendTestCases = []ChannelSendTestCase{
 		Status:       "W",
 		ResponseBody: `{"results":[{"code": "0", "ticket": "externalID"}]}`, ResponseStatus: 200,
 		URLParams: map[string]string{
-			"msisdn":    "+250788383383",
-			"msg":       "Simple Message",
-			"username":  "Username",
-			"password":  "Password",
-			"serviceid": "2020",
+			"msisdn":       "+250788383383",
+			"msg":          "Simple Message",
+			"username":     "Username",
+			"password":     "Password",
+			"serviceid":    "2020",
+			"allowunicode": "true",
 		},
 		SendPrep: setSendURL},
 	{Label: "Unicode Send",

handlers/nexmo/nexmo.go

@@ -79,7 +79,7 @@ var statusMappings = map[string]courier.MsgStatusValue{
 // StatusMessage is our HTTP handler function for status updates
 func (h *handler) StatusMessage(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) {
 	nexmoDeliveryReport := &nexmoDeliveryReport{}
-	handlers.DecodeAndValidateQueryParams(nexmoDeliveryReport, r)
+	handlers.DecodeAndValidateForm(nexmoDeliveryReport, r)
 
 	if nexmoDeliveryReport.MessageID == "" {
 		return nil, courier.WriteAndLogRequestIgnored(ctx, w, r, channel, "no messageId parameter, ignored")
@@ -117,12 +117,7 @@ type nexmoIncomingMessage struct {
 // ReceiveMessage is our HTTP handler function for incoming messages
 func (h *handler) ReceiveMessage(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) {
 	nexmoIncomingMessage := &nexmoIncomingMessage{}
-	handlers.DecodeAndValidateQueryParams(nexmoIncomingMessage, r)
-
-	// if this is a post, also try to parse the form body
-	if r.Method == http.MethodPost {
-		handlers.DecodeAndValidateForm(nexmoIncomingMessage, r)
-	}
+	handlers.DecodeAndValidateForm(nexmoIncomingMessage, r)
 
 	if nexmoIncomingMessage.To == "" {
 		return nil, courier.WriteAndLogRequestIgnored(ctx, w, r, channel, "no to parameter, ignored")