Finish batch messaging

Also moves templates into template space. Has been tested on testwiki already.

feedback.go

@@ -70,6 +70,7 @@ func requestFeedbackFor(requester frsRequesting, w *mwclient.Client) {
 				Title: requester.PageTitle(),
 				RFCID: rfcid,
 			})
+			log.Println("Queued a message for", user.Username, "to give feedback on", requester.PageTitle(), "in", user.Header)
 		}
 	} else {
 		log.Println("WARNING: Headers to send to returned as less than one for page", requester.PageTitle(), "so ignoring for now, but this could be a bug")

go.mod

@@ -5,6 +5,7 @@ go 1.14
 require (
 	cgt.name/pkg/go-mwclient v1.2.0
 	github.com/antonholmquist/jason v1.0.1-0.20180605105355-426ade25b261
+	github.com/gertd/go-pluralize v0.1.7
 	github.com/mashedkeyboard/ybtools/v2 v2.2.2
 	github.com/metal3d/go-slugify v0.0.0-20160607203414-7ac2014b2f23
 )

go.sum

@@ -4,14 +4,16 @@ cgt.name/pkg/go-mwclient v1.2.0 h1:/ZMVH+wF62ITK0Uj1KnM1tPtE/AXYQabXe2cTA6JGSQ=
 cgt.name/pkg/go-mwclient v1.2.0/go.mod h1:sxgLqpaVbtOhM1KiAUPkkRdsE6au+E64Bq9a2GyAQdU=
 github.com/antonholmquist/jason v1.0.1-0.20180605105355-426ade25b261 h1:EhjUMUb2k4WYhEjGTMB3XmD7qf6IAmJQWPpE69sI+sI=
 github.com/antonholmquist/jason v1.0.1-0.20180605105355-426ade25b261/go.mod h1:+GxMEKI0Va2U8h3os6oiUAetHAlGMvxjdpAH/9uvUMA=
-github.com/mashedkeyboard/ybtools/v2 v2.1.0 h1:v+yHJGAGnJtsIxtMpeEFxSSp+L+S2kWcBCQurP7J0Nw=
-github.com/mashedkeyboard/ybtools/v2 v2.1.0/go.mod h1:Wa2S3fUOmMHWO3y27EhASy/dRuVFMv5A2PtIjxYwAF8=
+github.com/gertd/go-pluralize v0.1.7 h1:RgvJTJ5W7olOoAks97BOwOlekBFsLEyh00W48Z6ZEZY=
+github.com/gertd/go-pluralize v0.1.7/go.mod h1:O4eNeeIf91MHh1GJ2I47DNtaesm66NYvjYgAahcqSDQ=
 github.com/mashedkeyboard/ybtools/v2 v2.2.2 h1:oy5zKrmVL+ZhI9V3Fgy9rbTpRbm1Fw3+IzjCpkiMZeI=
 github.com/mashedkeyboard/ybtools/v2 v2.2.2/go.mod h1:Wa2S3fUOmMHWO3y27EhASy/dRuVFMv5A2PtIjxYwAF8=
 github.com/metal3d/go-slugify v0.0.0-20160607203414-7ac2014b2f23 h1:UhdgaX0bR9ZSz+jRK6cPQLU94Q3KB14ijuHum8YbvBA=
 github.com/metal3d/go-slugify v0.0.0-20160607203414-7ac2014b2f23/go.mod h1:sCALRmIiknhX1lHQ8flRsWKMazu5BBjMochEnDupxrk=
 github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450 h1:j2kD3MT1z4PXCiUllUJF9mWUESr9TWKS7iEKsQ/IipM=
 github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

main.go

@@ -49,18 +49,13 @@ func main() {
 
 	frslist.Populate()
 	rfc.LoadRfcsDone(w)
-	defer frslist.FinishRun(w)
-	defer rfc.SaveRfcsDone(w)
 	defer ybtools.SaveEditLimit()
 
 	ga.FetchGATopics()
 
 	processCategory(w, "Category:Wikipedia requests for comment", true)
 	processCategory(w, "Category:Good article nominees", false)
-	// this below line is critical to run, because without it nothing will actually be sent;
-	// however, we do NOT want to defer it, because if we do, it would still run on panicks.
-	// if something has gone wrong, we don't want to send messages, so we oughtn't run this.
-	messages.SendMessageQueue(w)
+	finishRun(w)
 }
 
 // processCategory takes a mwclient instance, a category name, and a bool indicating if the category contains RfCs.
@@ -223,3 +218,19 @@ func processCategory(w *mwclient.Client, category string, rfcCat bool) {
 		}
 	}
 }
+
+// finishRun is called at the end of the FRS run, once everything has completed successfully.
+// The invocation of finishRun is what starts the message queue processing. This is only a
+// separate function really so that we can scope the frslist FinishRun and rfc SaveRfcsDone
+// defers into here; this means that if something goes awfully wrong somewhere else in the
+// program, we don't end up saving rubbish data after having sent nothing at all, but
+// it also means if something goes wrong in the actual sending, the lists are kept up to date.
+func finishRun(w *mwclient.Client) {
+	defer frslist.FinishRun(w)
+	defer rfc.SaveRfcsDone(w)
+
+	// this below line is critical to run, because without it nothing will actually be sent;
+	// however, we do NOT want to defer it, because if we do, it would still run on panicks.
+	// if something has gone wrong, we don't want to send messages, so we oughtn't run this.
+	messages.SendMessageQueue(w)
+}

src/frslist/frsuser.go

@@ -44,7 +44,8 @@ func (f FRSUser) ExceedsLimit() bool {
 	return false
 }
 
-// MarkMessageSent takes a header and increases the number of messages sent for that header by one.
+// MarkMessageSent increases the number of messages sent for the user by one. It's
+// intended for use at the point of queueing a message.
 func (f FRSUser) MarkMessageSent() {
 	sentCountMux.Lock()
 	defer sentCountMux.Unlock()
@@ -56,3 +57,17 @@ func (f FRSUser) MarkMessageSent() {
 
 	sentCount[f.Header][f.Username]++
 }
+
+// MarkMessageUnsent decreases the number of messages sent for the user by one. It
+// should only be used if something goes wrong while we're sending a message to the user.
+func (f FRSUser) MarkMessageUnsent() {
+	sentCountMux.Lock()
+	defer sentCountMux.Unlock()
+
+	// prevent nil map errors
+	if sentCount[f.Header] == nil {
+		return
+	}
+
+	sentCount[f.Header][f.Username]--
+}

src/messages/messages.go

@@ -30,6 +30,7 @@ import (
 
 	"cgt.name/pkg/go-mwclient"
 	"cgt.name/pkg/go-mwclient/params"
+	"github.com/gertd/go-pluralize"
 	"github.com/mashedkeyboard/ybtools/v2"
 )
 
@@ -45,6 +46,31 @@ type Message struct {
 	RFCID string
 }
 
+// headerForMessageSending is a struct used to deduplicate the headers we put in our
+// edit summary, and to produce sanely pluralised values there. It stores the number
+// of messages we've sent this run for the header, along with the FRSUser object.
+type headerForMessageSending struct {
+	countThisRun uint16
+	user         *frslist.FRSUser
+	headerType   string
+}
+
+// editSummaryForFeedbackMsgs is used to generate our edit summary. We run Sprintf over it
+// with the appropriately-formatted values we get back from editSummaryMessagesComponent, joined together with
+// a limitInEditSummary formatted as necessary if the user has a limit set for the category.
+const editSummaryForFeedbackMsgs string = `[[WP:FRS|Feedback Request Service]] notification on %s. You can unsubscribe at [[WP:FRS]].`
+
+// editSummaryMessagesComponent contains the core part of our edit summary. We run Sprintf over it with:
+// %s 1: determiner "a" or "some" depending on if we have plural
+// %s 2: header the user was subscribed to
+// %s 3: the type of request (GA nom, RfC, etc), pluralised if necessary
+// %s 4: limitInEditSummary, or empty string for no limit
+const editSummaryMessagesComponent string = `%s "%s" %s%s`
+
+// limitInEditSummary is used where users have a limit set.
+// Sprintf is run over it with the first param as the used amount, and the second as the limit.
+const limitInEditSummary string = ` (%d/%d this month)`
+
 // messagesToSend is our username-indexed list of messages that we have queued.
 // Each username key maps to a list of messages we have stored up to send them this run.
 var messagesToSend = map[string][]*Message{}
@@ -58,62 +84,60 @@ var commentRegex *regexp.Regexp
 // using the commentRegex.
 var cleanedHeaders = map[string]string{}
 
-// editSummaryForFeedbackMsgs is used to generate our edit summary. We run Sprintf over it
-// with the appropriately-formatted values we get back from editSummaryMessagesComponent, joined together with
-// a limitInEditSummary formatted as necessary if the user has a limit set for the category.
-const editSummaryForFeedbackMsgs string = `[[WP:FRS|Feedback Request Service]] notification on %s. You can unsubscribe at [[WP:FRS]].`
-
-// editSummaryMessagesComponent contains the core part of our edit summary. We run Sprintf over it with:
-// %s 1: header the user was subscribed to
-// %s 2: the type of request (GA nom, RfC, etc)
-// %s 3: limitInEditSummary, or empty string for no limit
-const editSummaryMessagesComponent string = `a "%s" %s%s`
-
-// limitInEditSummary is used where users have a limit set.
-// Sprintf is run over it with the first param as the used amount, and the second as the limit.
-const limitInEditSummary string = ` (%d/%d this month)`
+// pluralizer is used to turn singular words into plurals; specifically,
+// we use it here to pluralise the GA/RfC/whatever requester headers
+// in the edit summary we leave.
+var pluralizer *pluralize.Client
 
 func init() {
 	commentRegex = regexp.MustCompile(`\s*?<!--.*?-->\s*?`)
+	pluralizer = pluralize.NewClient()
 }
 
 // QueueMessage takes a pointer to a Message, and adds it into our queue
 // of messages to send to this user once we've finished our run and we're actually
 // sending the messages that we've processed.
 func QueueMessage(m *Message) {
 	messagesToSend[m.User.Username] = append(messagesToSend[m.User.Username], m)
+	m.User.MarkMessageSent()
 }
 
 // SendMessageQueue takes a pointer to an mwclient instance, and sends all the queued
 // messages from the FRS run.
 func SendMessageQueue(w *mwclient.Client) {
 	for user, messages := range messagesToSend {
-		var textBuilder *strings.Builder
-		var summarySentListBuilder strings.Builder
+		var textBuilder strings.Builder
 
-		textBuilder.WriteString("{{subst:User:Yapperbot/FRS notification")
+		// headersInSummary is just used to make sure our edit summary only has each header once.
+		// it maps each header for the summary to a number of times the header has been used.
+		// each header should be stored against its ''cleaned'' key, not its internal name.
+		var headersInSummary = map[string]*headerForMessageSending{}
+
+		textBuilder.WriteString("{{subst:FRS notification")
 
 		for index, message := range messages {
 			strindex := strconv.Itoa(index)
-			numberedParamToBuilder(textBuilder, strindex, "title")
+			cleanedHeader := cleanedHeaders[message.User.Header]
+			numberedParamToBuilder(&textBuilder, strindex, "title")
 			textBuilder.WriteString(message.Title)
-			numberedParamToBuilder(textBuilder, strindex, "type")
+			numberedParamToBuilder(&textBuilder, strindex, "header")
+			textBuilder.WriteString(cleanedHeader)
+			numberedParamToBuilder(&textBuilder, strindex, "type")
 			textBuilder.WriteString(message.Type)
 			if message.RFCID != "" {
-				numberedParamToBuilder(textBuilder, strindex, "rfcid")
+				numberedParamToBuilder(&textBuilder, strindex, "rfcid")
 				textBuilder.WriteString(message.RFCID)
 			}
 
-			var limitsummary string
-			if message.User.Limited {
-				limitsummary = fmt.Sprintf(limitInEditSummary, message.User.GetCount()+1, message.User.Limit)
-			}
-			summarySentListBuilder.WriteString(fmt.Sprintf(editSummaryMessagesComponent, message.User.Header, message.Type, limitsummary))
-
-			if len(messages) > 1 && index != len(messages)-1 {
-				summarySentListBuilder.WriteString(", ")
-				if index == len(messages)-2 {
-					summarySentListBuilder.WriteString("and ")
+			if header, ok := headersInSummary[cleanedHeader]; ok {
+				// we already have the header in the list. use it.
+				header.countThisRun++
+			} else {
+				// the header hasn't yet been used, create it
+				headersInSummary[cleanedHeader] = &headerForMessageSending{
+					countThisRun: 1,
+					user:         message.User,
+					headerType:   message.Type,
 				}
 			}
 		}
@@ -131,6 +155,38 @@ func SendMessageQueue(w *mwclient.Client) {
 
 		// Drop a note on each user's talk page inviting them to participate
 		if ybtools.CanEdit() {
+			var summarySentListBuilder strings.Builder
+			var index int
+			for headerName, header := range headersInSummary {
+				var limitsummary string
+				if header.user.Limited {
+					limitsummary = fmt.Sprintf(limitInEditSummary, header.user.GetCount(), header.user.Limit)
+				}
+
+				determiner := "a"
+				if header.countThisRun > 1 {
+					determiner = "some"
+					header.headerType = pluralizer.Plural(header.headerType)
+				}
+
+				summarySentListBuilder.WriteString(fmt.Sprintf(
+					editSummaryMessagesComponent,
+					determiner,
+					headerName,
+					header.headerType,
+					limitsummary,
+				))
+
+				if len(messages) > 1 && index != len(messages)-1 {
+					summarySentListBuilder.WriteString(", ")
+					if index == len(messages)-2 {
+						// penultimate
+						summarySentListBuilder.WriteString("and ")
+					}
+				}
+				index++
+			}
+
 			// Generate the edit summary, with their limit
 			editsummary := fmt.Sprintf(editSummaryForFeedbackMsgs, summarySentListBuilder.String())
 
@@ -149,9 +205,6 @@ func SendMessageQueue(w *mwclient.Client) {
 			})
 			if err == nil {
 				log.Println("Successfully invited", user, "to give feedback on", len(messages), "requesting items")
-				for _, message := range messages {
-					message.User.MarkMessageSent()
-				}
 				time.Sleep(5 * time.Second)
 			} else {
 				switch err.(type) {
@@ -167,6 +220,9 @@ func SendMessageQueue(w *mwclient.Client) {
 				default:
 					ybtools.PanicErr("Non-API error returned when trying to notify user ", user, " so dying. Error was ", err)
 				}
+				for _, message := range messages {
+					message.User.MarkMessageUnsent()
+				}
 			}
 		}
 	}