Deadlock bug in Twitter API client from #18

- Reduced the amount of logging that the Reddit API client does (19/04/2023 - 12:25:05)
- Synchronisation bug within SetTweetCap where TweetCapMutex was being acquired twice (19/04/2023 - 12:41:23)

discovery.go

@@ -713,6 +713,7 @@ func RedditDiscoveryPhase(wg *sync.WaitGroup, state *ScoutState, gameScrapers *m
 		state.GetCachedField(StateType).SetOrAdd("Result", "DiscoveryStats", "TotalSnapshots", models.SetOrAddAdd.Func(scoutResult.DiscoveryStats.TotalSnapshots))
 	}
 	close(postResults)
+	log.INFO.Printf("Finished RedditDiscoveryPhase")
 	return
 }
 

reddit/client.go

@@ -335,24 +335,21 @@ func (c *Client) Run(ctx context.Context, bindingName string, attrs map[string]a
 		c.incrementRequestPeriod()
 	}
 
-	i := 0
-	c.rateLimits.Range(func(key, value any) bool {
-		c.Log(fmt.Sprintf("%d: Binding %q - %q", i+1, key, value))
-		i++
-		return true
-	})
-
-	i = 0
+	latestRequestPeriodTime := time.Time{}
+	var latestRequestPeriodCount uint32
 	c.requestsPeriodCount.Range(func(key, value any) bool {
-		count := value.(*atomic.Uint32)
-		c.Log(fmt.Sprintf(
-			"%d: Period %q - %d", i+1,
-			time.Unix(key.(int64), 0).Format("15:04:05"), count.Load(),
-		))
-		i++
+		if currentPeriod := time.Unix(key.(int64), 0); currentPeriod.After(latestRequestPeriodTime) {
+			latestRequestPeriodTime = currentPeriod
+			latestRequestPeriodCount = value.(*atomic.Uint32).Load()
+		}
 		return true
 	})
 
+	c.Log(fmt.Sprintf(
+		"Latest Reddit request period is %s with a count of %d",
+		latestRequestPeriodTime.Format("15:04:05"), latestRequestPeriodCount,
+	))
+
 	return
 }
 

twitter/client.go

@@ -361,19 +361,29 @@ func (w *ClientWrapper) SetTweetCap(used int, remaining int, total int, resets t
 		used, remaining, total, resets.String(), lastFetched.String(),
 	)
 
-	w.TweetCapMutex.Lock()
-	defer w.TweetCapMutex.Unlock()
-	if w.TweetCap == nil {
-		w.TweetCap = &TweetCap{}
-	}
-	w.TweetCap.Used = used
-	w.TweetCap.Remaining = remaining
-	w.TweetCap.Total = total
-	w.TweetCap.Resets = resets
-	w.TweetCap.LastFetched = lastFetched
-
-	if !w.TweetCap.CheckFresh() {
-		log.WARNING.Printf("TweetCap (%s) is not fresh, fetching from web...", w.TweetCap.String())
+	// Set the TweetCap stored in memory as well as retrieving some variables for checks and logging
+	var (
+		fresh          bool
+		tweetCapString string
+	)
+
+	func() {
+		w.TweetCapMutex.Lock()
+		defer w.TweetCapMutex.Unlock()
+		if w.TweetCap == nil {
+			w.TweetCap = &TweetCap{}
+		}
+		w.TweetCap.Used = used
+		w.TweetCap.Remaining = remaining
+		w.TweetCap.Total = total
+		w.TweetCap.Resets = resets
+		w.TweetCap.LastFetched = lastFetched
+		fresh = w.TweetCap.CheckFresh()
+		tweetCapString = w.TweetCap.String()
+	}()
+
+	if !fresh {
+		log.WARNING.Printf("TweetCap (%s) is not fresh, fetching from web...", tweetCapString)
 		if err = w.GetTweetCap(); err != nil {
 			return errors.Wrap(
 				err,
@@ -388,7 +398,7 @@ func (w *ClientWrapper) SetTweetCap(used int, remaining int, total int, resets t
 		// Otherwise, we just write the TweetCap to the cache file
 		log.INFO.Printf(
 			"TweetCap (%s) is fresh, writing to cache: %s",
-			w.TweetCap.String(), w.Config.TwitterTweetCapLocation(),
+			tweetCapString, w.Config.TwitterTweetCapLocation(),
 		)
 		if err = w.WriteTweetCap(); err != nil {
 			return errors.Wrap(err, "could not write TweetCap cache file")