Merge pull request #9 from mattpep/incremental_update

Incremental update

feed.go

@@ -19,6 +19,8 @@ type Feed struct {
 
 var wg sync.WaitGroup
 var isAllUpdate bool
+
+const newArticleDirectory = "new"
 const maxFileNameLength = 255
 
 func truncateString(s string, n int) string {
@@ -31,14 +33,20 @@ func truncateString(s string, n int) string {
 	return s[:n]
 }
 
+func InitialiseNewArticleDirectory() {
+	DeleteFeedFiles(newArticleDirectory)
+	os.MkdirAll(Config.FeedDirectory+"/"+newArticleDirectory, 0755)
+}
+
 func DeleteFeedFiles(name string) {
 	os.RemoveAll(Config.FeedDirectory + "/" + name)
-	os.MkdirAll(Config.FeedDirectory+"/"+name, 0777)
 }
 
-func UpdateFeed(name string) {
+func UpdateFeed(name string, deleteFiles bool) {
 	log.Info("Updating feed '" + name + "'")
 	fp := gofeed.NewParser()
+	downloadCount := 0
+	skipCount := 0
 	feed, err := fp.ParseURL(Config.Feeds[slices.IndexFunc(Config.Feeds, func(f Feed) bool { return f.Name == name })].URL)
 	if err != nil {
 		log.Error("Failed to fetch the feed '" + name + "'")
@@ -47,9 +55,19 @@ func UpdateFeed(name string) {
 		}
 		return
 	}
-	DeleteFeedFiles(name)
+	if deleteFiles {
+		DeleteFeedFiles(name)
+	}
+	os.MkdirAll(Config.FeedDirectory+"/"+name, 0777)
 	for _, item := range feed.Items {
-		file, err := os.Create(Config.FeedDirectory + "/" + name + "/" + truncateString(strings.ReplaceAll(item.Title, "/", ""), maxFileNameLength))
+
+		articlePath := Config.FeedDirectory + "/" + name + "/" + truncateString(strings.ReplaceAll(item.Title, "/", ""), maxFileNameLength)
+		if _, err := os.Stat(articlePath); err == nil {
+			log.Debug("Article " + articlePath + " already exists - skipping download")
+			skipCount++
+			continue
+		}
+		file, err := os.Create(articlePath)
 		if err != nil {
 			log.Error("Failed to create a file for article titled '" + item.Title + "'")
 			continue
@@ -60,18 +78,24 @@ func UpdateFeed(name string) {
 			log.Error("Failed to write content to a file for article titled '" + item.Title + "'")
 			continue
 		}
+		downloadCount++
+		NewLinkPath := Config.FeedDirectory + "/" + newArticleDirectory + "/" + truncateString(strings.ReplaceAll(item.Title, "/", ""), maxFileNameLength)
+		err = os.Symlink(articlePath, NewLinkPath)
+		if err != nil {
+			log.Error("Could not create symlink for newly downloaded article " + articlePath)
+		}
 	}
-	log.Info(strconv.Itoa(len(feed.Items)) + " articles fetched from feed '" + name + "'")
+	log.Info(strconv.Itoa(downloadCount) + " articles fetched from feed '" + name + "' (" + strconv.Itoa(skipCount) + " already seen, " + strconv.Itoa(len(feed.Items)) + " total in feed)")
 	if isAllUpdate {
 		wg.Done()
 	}
 }
 
-func UpdateAllFeeds() {
+func UpdateAllFeeds(deleteFiles bool) {
 	isAllUpdate = true
 	for _, feed := range Config.Feeds {
 		wg.Add(1)
-		go UpdateFeed(feed.Name)
+		go UpdateFeed(feed.Name, deleteFiles)
 	}
 	wg.Wait()
 }

main.go

@@ -62,16 +62,32 @@ func main() {
 					return cmd.Run()
 				},
 			},
+			{
+				Name:    "refetch",
+				Aliases: []string{"r"},
+				Usage:   "delete and refetch given feed(s) or all feeds if no argument is given",
+				Action: func(cCtx *cli.Context) error {
+					InitialiseNewArticleDirectory()
+					if cCtx.Args().Len() == 0 {
+						UpdateAllFeeds(true)
+					}
+					for i := 0; i < cCtx.Args().Len(); i++ {
+						UpdateFeed(cCtx.Args().Get(i), true)
+					}
+					return nil
+				},
+			},
 			{
 				Name:    "update",
 				Aliases: []string{"u"},
 				Usage:   "update given feed(s) or all feeds if no argument is given",
 				Action: func(cCtx *cli.Context) error {
+					InitialiseNewArticleDirectory()
 					if cCtx.Args().Len() == 0 {
-						UpdateAllFeeds()
+						UpdateAllFeeds(false)
 					}
 					for i := 0; i < cCtx.Args().Len(); i++ {
-						UpdateFeed(cCtx.Args().Get(i))
+						UpdateFeed(cCtx.Args().Get(i), false)
 					}
 					return nil
 				},