Improve config and publish

.github/workflows/testing.yml

@@ -66,6 +66,7 @@ jobs:
         uses: ibnesayeed/setup-ipfs@3e5452e358dfa43a0fb92496e96c82cf756be94a
         with:
           run_daemon: true
+          ipfs_version: ${{ vars.IPFS_VERSION }}
 
       - name: Set up Tor
         uses: tor-actions/setup-tor@462b4eef22eecc86cc4abdedafc95a38667e2557
@@ -81,9 +82,9 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         uses: moy2010/meilisearch-github-action@fcc5ef714af0596633665032d459bfb279d3c730 # 0.1.4
         with:
-          meilisearch-version: v0.28.0
-          meilisearch-port: 7700
-          meilisearch-api-key: foobar
+          meilisearch-version: ${{ vars.MEILISEARCH_VERSION }}
+          meilisearch-port: ${{ vars.MEILISEARCH_PORT }}
+          meilisearch-api-key: ${{ vars.MEILISEARCH_APIKEY }}
 
       - name: Set up Chocolatey
         if: matrix.os == 'windows-latest'
@@ -141,10 +142,10 @@ jobs:
         shell: bash
         run: |
           # Set env to enable reduxer
-          echo "WAYBACK_MEILI_ENDPOINT=http://localhost:7700" >> $GITHUB_ENV
-          echo "PLAYBACK_MEILI_ENDPOINT=http://localhost:7700" >> $GITHUB_ENV
-          echo "WAYBACK_MEILI_APIKEY=foobar" >> $GITHUB_ENV
-          echo "PLAYBACK_MEILI_APIKEY=foobar" >> $GITHUB_ENV
+          echo "WAYBACK_MEILI_ENDPOINT=${{ vars.WAYBACK_MEILI_ENDPOINT }}" >> $GITHUB_ENV
+          echo "PLAYBACK_MEILI_ENDPOINT=${{ vars.WAYBACK_MEILI_ENDPOINT }}" >> $GITHUB_ENV
+          echo "WAYBACK_MEILI_APIKEY=${{ vars.MEILISEARCH_APIKEY }}" >> $GITHUB_ENV
+          echo "PLAYBACK_MEILI_APIKEY=${{ vars.MEILISEARCH_APIKEY }}" >> $GITHUB_ENV
 
       - name: Check out code base
         if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'

cmd/playback/main.go

@@ -10,26 +10,33 @@ import (
 	"os"
 
 	"github.com/spf13/cobra"
+	"github.com/wabarc/logger"
 	"github.com/wabarc/playback"
 	"github.com/wabarc/wayback"
+	"github.com/wabarc/wayback/config"
 )
 
 func main() {
+	opts, err := config.NewParser().ParseEnvironmentVariables()
+	if err != nil {
+		logger.Fatal("Parse environment variables or flags failed, error: %v", err)
+	}
+
 	var rootCmd = &cobra.Command{
 		Use:     "playback",
 		Short:   "A toolkit to playback archived webpage from time capsules.",
 		Example: `  playback https://example.com https://example.org`,
 		Version: playback.Version,
 		Run: func(cmd *cobra.Command, args []string) {
-			handle(cmd, args)
+			handle(cmd, opts, args)
 		},
 	}
 
 	// nolint:errcheck
 	rootCmd.Execute()
 }
 
-func handle(cmd *cobra.Command, args []string) {
+func handle(cmd *cobra.Command, opts *config.Options, args []string) {
 	if len(args) < 1 {
 		// nolint:errcheck
 		cmd.Usage()
@@ -42,7 +49,7 @@ func handle(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	collects, err := wayback.Playback(context.TODO(), urls...)
+	collects, err := wayback.Playback(context.TODO(), opts, urls...)
 	if err != nil {
 		cmd.Println(err)
 		os.Exit(1)

cmd/wayback/main.go

@@ -163,28 +163,29 @@ func handle(cmd *cobra.Command, args []string) {
 	setToEnv(cmd)
 	parser := config.NewParser()
 
+	var opts *config.Options
 	if len(daemon) > 0 {
 		logger.Info("Run wayback using configuration file")
-		if config.Opts, err = parser.ParseFile(configFile); err != nil {
+		if _, err = parser.ParseFile(configFile); err != nil {
 			logger.Fatal("Parse configuration file failed, error: %v", err)
 		}
 	}
 
-	if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil {
+	if opts, err = parser.ParseEnvironmentVariables(); err != nil {
 		logger.Fatal("Parse environment variables or flags failed, error: %v", err)
 	}
 
-	if !config.Opts.LogTime() {
+	if !opts.LogTime() {
 		logger.DisableTime()
 	}
 
-	logger.SetLogLevel(config.Opts.LogLevel())
-	if debug || config.Opts.HasDebugMode() {
+	logger.SetLogLevel(opts.LogLevel())
+	if debug || opts.HasDebugMode() {
 		profiling()
 		logger.EnableDebug()
 	}
 
-	if config.Opts.EnabledMetrics() {
+	if opts.EnabledMetrics() {
 		metrics.Gather = metrics.NewCollector()
 	}
 
@@ -194,7 +195,7 @@ func handle(cmd *cobra.Command, args []string) {
 	}
 
 	if print {
-		cmd.Println(spew.Sdump(config.Opts))
+		cmd.Println(spew.Sdump(opts))
 		return
 	}
 
@@ -204,9 +205,9 @@ func handle(cmd *cobra.Command, args []string) {
 	hasArgs := len(args) > 0
 	switch {
 	case hasDaemon:
-		serve(cmd, args)
+		serve(cmd, opts, args)
 	case hasArgs:
-		archive(cmd, args)
+		archive(cmd, opts, args)
 	default:
 		// nolint:errcheck
 		cmd.Help()

cmd/wayback/serve.go

@@ -13,6 +13,7 @@ import (
 	"github.com/wabarc/logger"
 	"github.com/wabarc/wayback/config"
 	"github.com/wabarc/wayback/pooling"
+	"github.com/wabarc/wayback/publish"
 	"github.com/wabarc/wayback/service"
 	"github.com/wabarc/wayback/service/discord"
 	"github.com/wabarc/wayback/service/httpd"
@@ -24,6 +25,8 @@ import (
 	"github.com/wabarc/wayback/service/twitter"
 	"github.com/wabarc/wayback/storage"
 	"github.com/wabarc/wayback/systemd"
+
+	_ "github.com/wabarc/wayback/publish/register"
 )
 
 // Create channel to listen for signals.
@@ -38,30 +41,35 @@ type services struct {
 	targets []target
 }
 
-func serve(_ *cobra.Command, _ []string) {
-	store, err := storage.Open("")
+func serve(_ *cobra.Command, opts *config.Options, _ []string) {
+	store, err := storage.Open(opts, "")
 	if err != nil {
 		logger.Fatal("open storage failed: %v", err)
 	}
 	defer store.Close()
 
+	cfg := []pooling.Option{
+		pooling.Capacity(opts.PoolingSize()),
+		pooling.Timeout(opts.WaybackTimeout()),
+		pooling.MaxRetries(opts.WaybackMaxRetries()),
+	}
 	ctx, cancel := context.WithCancel(context.Background())
-	pool := pooling.New(ctx, config.Opts.PoolingSize())
+	pool := pooling.New(ctx, cfg...)
 	go pool.Roll()
 
-	if config.Opts.EnabledMeilisearch() {
-		endpoint := config.Opts.WaybackMeiliEndpoint()
-		indexing := config.Opts.WaybackMeiliIndexing()
-		apikey := config.Opts.WaybackMeiliApikey()
-		meili := service.NewMeili(endpoint, apikey, indexing)
-		if err := meili.Setup(); err != nil {
-			logger.Error("setup meilisearch failed: %v", err)
-		}
-		logger.Debug("setup meilisearch success")
+	pub := publish.New(ctx, opts)
+	go pub.Start()
+
+	opt := []service.Option{
+		service.Config(opts),
+		service.Storage(store),
+		service.Pool(pool),
+		service.Publish(pub),
 	}
+	options := service.ParseOptions(opt...)
 
 	srv := &services{}
-	_ = srv.run(ctx, store, pool)
+	_ = srv.run(ctx, options)
 
 	if systemd.HasNotifySocket() {
 		logger.Info("sending readiness notification to Systemd")
@@ -71,100 +79,101 @@ func serve(_ *cobra.Command, _ []string) {
 		}
 	}
 
-	go srv.stop(pool, cancel)
+	go srv.daemon(pool, pub, cancel)
 	<-ctx.Done()
 
 	logger.Info("wayback service stopped.")
 }
 
 // nolint:gocyclo
-func (srv *services) run(ctx context.Context, store *storage.Storage, pool *pooling.Pool) *services {
+func (srv *services) run(ctx context.Context, opts service.Options) *services {
 	size := len(daemon)
 	srv.targets = make([]target, 0, size)
 	for _, s := range daemon {
+		s := s
 		switch s {
 		case "irc":
-			irc := relaychat.New(ctx, store, pool)
+			irc := relaychat.New(ctx, opts)
 			go func() {
 				if err := irc.Serve(); err != relaychat.ErrServiceClosed {
-					logger.Error("%v", err)
+					logger.Error("start %s service failed: %v", s, err)
 				}
 			}()
 			srv.targets = append(srv.targets, target{
 				call: func() { irc.Shutdown() }, // nolint:errcheck
 				name: s,
 			})
 		case "slack":
-			sl := slack.New(ctx, store, pool)
+			sl := slack.New(ctx, opts)
 			go func() {
 				if err := sl.Serve(); err != slack.ErrServiceClosed {
-					logger.Error("%v", err)
+					logger.Error("start %s service failed: %v", s, err)
 				}
 			}()
 			srv.targets = append(srv.targets, target{
 				call: func() { sl.Shutdown() }, // nolint:errcheck
 				name: s,
 			})
 		case "discord":
-			d := discord.New(ctx, store, pool)
+			d := discord.New(ctx, opts)
 			go func() {
 				if err := d.Serve(); err != discord.ErrServiceClosed {
-					logger.Error("%v", err)
+					logger.Error("start %s service failed: %v", s, err)
 				}
 			}()
 			srv.targets = append(srv.targets, target{
 				call: func() { d.Shutdown() }, // nolint:errcheck
 				name: s,
 			})
 		case "mastodon", "mstdn":
-			m := mastodon.New(ctx, store, pool)
+			m := mastodon.New(ctx, opts)
 			go func() {
 				if err := m.Serve(); err != mastodon.ErrServiceClosed {
-					logger.Error("%v", err)
+					logger.Error("start %s service failed: %v", s, err)
 				}
 			}()
 			srv.targets = append(srv.targets, target{
 				call: func() { m.Shutdown() }, // nolint:errcheck
 				name: s,
 			})
 		case "telegram":
-			t := telegram.New(ctx, store, pool)
+			t := telegram.New(ctx, opts)
 			go func() {
 				if err := t.Serve(); err != telegram.ErrServiceClosed {
-					logger.Error("%v", err)
+					logger.Error("start %s service failed: %v", s, err)
 				}
 			}()
 			srv.targets = append(srv.targets, target{
 				call: func() { t.Shutdown() }, // nolint:errcheck
 				name: s,
 			})
 		case "twitter":
-			t := twitter.New(ctx, store, pool)
+			t := twitter.New(ctx, opts)
 			go func() {
 				if err := t.Serve(); err != twitter.ErrServiceClosed {
-					logger.Error("%v", err)
+					logger.Error("start %s service failed: %v", s, err)
 				}
 			}()
 			srv.targets = append(srv.targets, target{
 				call: func() { t.Shutdown() }, // nolint:errcheck
 				name: s,
 			})
 		case "matrix":
-			m := matrix.New(ctx, store, pool)
+			m := matrix.New(ctx, opts)
 			go func() {
 				if err := m.Serve(); err != matrix.ErrServiceClosed {
-					logger.Error("%v", err)
+					logger.Error("start %s service failed: %v", s, err)
 				}
 			}()
 			srv.targets = append(srv.targets, target{
 				call: func() { m.Shutdown() }, // nolint:errcheck
 				name: s,
 			})
 		case "web", "httpd":
-			h := httpd.New(ctx, store, pool)
+			h := httpd.New(ctx, opts)
 			go func() {
 				if err := h.Serve(); err != httpd.ErrServiceClosed {
-					logger.Error("%v", err)
+					logger.Error("start %s service failed: %v", s, err)
 				}
 			}()
 			srv.targets = append(srv.targets, target{
@@ -179,7 +188,7 @@ func (srv *services) run(ctx context.Context, store *storage.Storage, pool *pool
 	return srv
 }
 
-func (srv *services) stop(pool *pooling.Pool, cancel context.CancelFunc) {
+func (srv *services) daemon(pool *pooling.Pool, pub *publish.Publish, cancel context.CancelFunc) {
 	// SIGINT handles Ctrl+C locally.
 	// SIGTERM handles termination signal from cloud service.
 	signal.Notify(
@@ -199,6 +208,8 @@ func (srv *services) stop(pool *pooling.Pool, cancel context.CancelFunc) {
 	srv.shutdown()
 	// Gracefully closes the worker pool
 	pool.Close()
+	// Stop publish service
+	pub.Stop()
 	cancel()
 }
 

cmd/wayback/wayback.go

@@ -17,6 +17,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/wabarc/helper"
 	"github.com/wabarc/wayback"
+	"github.com/wabarc/wayback/config"
 	"github.com/wabarc/wayback/errors"
 	"github.com/wabarc/wayback/reduxer"
 	"golang.org/x/sync/errgroup"
@@ -34,15 +35,15 @@ func assets(art reduxer.Artifact) []reduxer.Asset {
 	}
 }
 
-func archive(cmd *cobra.Command, args []string) {
+func archive(cmd *cobra.Command, opts *config.Options, args []string) {
 	// TODO: clean the auto-created temporary directory.
 	archiving := func(ctx context.Context, urls []*url.URL) error {
 		g, ctx := errgroup.WithContext(ctx)
-		rdx, err := reduxer.Do(ctx, urls...)
+		rdx, err := reduxer.Do(ctx, opts, urls...)
 		if err != nil {
 			return errors.Wrap(err, "reduxer unexpected")
 		}
-		cols, err := wayback.Wayback(ctx, rdx, urls...)
+		cols, err := wayback.Wayback(ctx, rdx, opts, urls...)
 		if err != nil {
 			return err
 		}

config/config.go

@@ -5,7 +5,7 @@
 package config // import "github.com/wabarc/wayback/config"
 
 // Opts holds parsed configuration options.
-var Opts *Options
+// var Opts *Options
 
 // nolint:stylecheck
 const (

metrics/metrics.go

@@ -35,6 +35,7 @@ const (
 	PublishMatrix  = "matrix"
 	PublishSlack   = "slack"
 	PublishNostr   = "nostr"
+	PublishMeili   = "meili"
 
 	StatusRequest = "request"
 	StatusSuccess = "success"