Embed a [short lived] JWT token into the test console

cmd/scrape-server/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/efixler/envflags"
 	"github.com/efixler/scrape/fetch"
 	"github.com/efixler/scrape/fetch/trafilatura"
+	"github.com/efixler/scrape/internal/auth"
 	"github.com/efixler/scrape/internal/cmd"
 	"github.com/efixler/scrape/internal/headless"
 	"github.com/efixler/scrape/internal/server"
@@ -34,6 +35,7 @@ const (
 var (
 	flags           flag.FlagSet
 	port            *envflags.Value[int]
+	signingKey      *envflags.Value[*auth.HMACBase64Key]
 	ttl             *envflags.Value[time.Duration]
 	userAgent       *envflags.Value[*ua.UserAgent]
 	dbFlags         *cmd.DatabaseFlags
@@ -56,13 +58,21 @@ func main() {
 		headlessFetcher, _ = trafilatura.Factory(headlessClient)()
 	}
 
+	// TODO: Implement options pattern for NewScrapeServer
 	ss, _ := server.NewScrapeServer(
 		ctx,
 		dbFactory,
 		defaultFetcherFactory,
 		headlessFetcher,
 	)
 
+	if sk := *signingKey.Get(); len(sk) > 0 {
+		ss.SigningKey = sk
+		slog.Info("scrape-server authorization via JWT is enabled")
+	} else {
+		slog.Info("scrape-server authorization is disabled, running in open access mode")
+	}
+
 	mux, err := server.InitMux(ss)
 	if err != nil {
 		slog.Error("scrape-server error initializing the server's mux", "error", err)
@@ -106,6 +116,13 @@ func init() {
 	port = envflags.NewInt("PORT", DefaultPort)
 	port.AddTo(&flags, "port", "Port to run the server on")
 
+	signingKey = envflags.NewText("SIGNING_KEY", &auth.HMACBase64Key{})
+	signingKey.AddTo(
+		&flags,
+		"signing-key",
+		"Base64 encoded HS256 key to verify JWT tokens. Required for JWT auth, and enables JWT auth if set.",
+	)
+
 	ttl = envflags.NewDuration("TTL", resource.DefaultTTL)
 	ttl.AddTo(&flags, "ttl", "TTL for fetched resources")
 

internal/auth/jwt.go

@@ -62,6 +62,13 @@ func (c Claims) Sign(key HMACBase64Key) (string, error) {
 	return c.Token().SignedString([]byte(key))
 }
 
+func ExpiresIn(d time.Duration) option {
+	return func(c *Claims) error {
+		c.ExpiresAt = jwt.NewNumericDate(time.Now().Add(d))
+		return nil
+	}
+}
+
 func ExpiresAt(t time.Time) option {
 	return func(c *Claims) error {
 		if t.Before(time.Now()) {

internal/server/pages/index.html

@@ -55,7 +55,10 @@
 			event.preventDefault();
 			const action = form.action;
 			const headers = new Headers();
-			headers.append('Authorization', 'Bearer not_implemented_yet');
+			const token = '{{AuthToken}}'
+			if (token) {
+				headers.append('Authorization', `Bearer ${token}`);
+			}
 			const formData = new FormData(form);
 
 			try { 

internal/server/routes.go

@@ -1,36 +1,34 @@
 package server
 
 import (
+	"bytes"
 	"context"
-	_ "embed"
+	"embed"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"html/template"
 	"log/slog"
 	"net/http"
 	"net/http/pprof"
 	nurl "net/url"
+	"time"
 
 	"github.com/efixler/webutil/jsonarray"
 
 	"github.com/efixler/scrape"
 	"github.com/efixler/scrape/fetch"
 	"github.com/efixler/scrape/fetch/feed"
 	"github.com/efixler/scrape/fetch/trafilatura"
+	"github.com/efixler/scrape/internal/auth"
 	"github.com/efixler/scrape/internal/server/healthchecks"
 	"github.com/efixler/scrape/resource"
 	"github.com/efixler/scrape/store"
 )
 
-func InitMux(
-	scrapeServer *scrapeServer,
-) (*http.ServeMux, error) {
+func InitMux(scrapeServer *scrapeServer) (*http.ServeMux, error) {
 	mux := http.NewServeMux()
-	mux.HandleFunc("GET /{$}", handleHome)
-	// scrapeServer, err := NewScrapeServer(ctx, sf, headlessClient)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	mux.HandleFunc("GET /{$}", scrapeServer.homeHandler())
 	h := scrapeServer.singleHandler()
 	mux.HandleFunc("GET /extract", h)
 	mux.HandleFunc("POST /extract", h)
@@ -51,22 +49,14 @@ func EnableProfiling(mux *http.ServeMux) {
 	initPProf(mux)
 }
 
-//go:embed pages/index.html
-var home []byte
-
-func handleHome(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	w.WriteHeader(http.StatusOK)
-	w.Write(home)
-}
-
 // The server struct is stateless but uses the same fetchers across all requests,
 // to optimize client and database connections. There's a general fetcher, and
 // special ones for headless scrapes and RSS/Atom feeds.
 type scrapeServer struct {
 	urlFetcher      fetch.URLFetcher
 	headlessFetcher fetch.URLFetcher
 	feedFetcher     fetch.FeedFetcher
+	SigningKey      auth.HMACBase64Key
 }
 
 // When the context passed here is cancelled, the associated fetcher will
@@ -76,7 +66,6 @@ func NewScrapeServer(
 	sf store.Factory,
 	clientFactory fetch.Factory,
 	headlessFetcher fetch.URLFetcher,
-	// headlessClient fetch.Client,
 ) (*scrapeServer, error) {
 	urlFetcher, err := scrape.NewStorageBackedFetcher(
 		trafilatura.Factory(nil),
@@ -116,6 +105,51 @@ func (h *scrapeServer) Storage() store.URLDataStore {
 	return sbf.Storage
 }
 
+//go:embed pages/index.html
+var home embed.FS
+
+func (h scrapeServer) mustHomeTemplate() *template.Template {
+	tmpl := template.New("home")
+	var keyF = func() string { return "" }
+	if (h.SigningKey != nil) && (len(h.SigningKey) > 0) {
+		keyF = func() string {
+			c, err := auth.NewClaims(
+				auth.WithSubject("home"),
+				auth.ExpiresIn(60*time.Minute),
+			)
+			if err != nil {
+				slog.Error("Error creating claims for home view", "error", err)
+				return ""
+			}
+			s, err := c.Sign(h.SigningKey)
+			if err != nil {
+				slog.Error("Error signing claims for home view", "error", err)
+				return ""
+			}
+			return s
+		}
+	}
+	tmpl = tmpl.Funcs(template.FuncMap{"AuthToken": keyF})
+	homeSource, _ := home.ReadFile("pages/index.html")
+	tmpl = template.Must(tmpl.Parse(string(homeSource)))
+	tmpl.Option("missingkey=zero")
+	return tmpl
+}
+
+func (h scrapeServer) homeHandler() http.HandlerFunc {
+	tmpl := h.mustHomeTemplate()
+	return func(w http.ResponseWriter, r *http.Request) {
+		var buf bytes.Buffer
+		if err := tmpl.Execute(&buf, nil); err != nil {
+			http.Error(w, "Error rendering home page", http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		w.WriteHeader(http.StatusOK)
+		w.Write(buf.Bytes())
+	}
+}
+
 func (h *scrapeServer) singleHandler() http.HandlerFunc {
 	return Chain(h.extract, MaxBytes(4096), parseSinglePayload())
 }

internal/server/routes_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/efixler/scrape/fetch"
 	"github.com/efixler/scrape/fetch/trafilatura"
+	"github.com/efixler/scrape/internal/auth"
 	"github.com/efixler/scrape/internal/storage/sqlite"
 	"github.com/efixler/scrape/resource"
 )
@@ -305,3 +306,84 @@ func TestDeleteHandler(t *testing.T) {
 		}
 	}
 }
+
+func TestHomeHandler(t *testing.T) {
+	tests := []struct {
+		name           string
+		key            auth.HMACBase64Key
+		expectedResult int
+	}{
+		{
+			name:           "no key",
+			key:            nil,
+			expectedResult: 200,
+		},
+		{
+			name:           "good key",
+			key:            auth.MustNewHS256SigningKey(),
+			expectedResult: 200,
+		},
+	}
+	for _, test := range tests {
+		ss := &scrapeServer{SigningKey: test.key}
+		req := httptest.NewRequest("GET", "http://foo.bar/", nil)
+		w := httptest.NewRecorder()
+		ss.homeHandler()(w, req)
+		resp := w.Result()
+		if resp.StatusCode != test.expectedResult {
+			t.Errorf("[%s] Expected %d, got %d", test.name, test.expectedResult, resp.StatusCode)
+		}
+	}
+}
+
+func TestMustTemplate(t *testing.T) {
+	tests := []struct {
+		name        string
+		key         auth.HMACBase64Key
+		expectToken bool
+	}{
+		{
+			name:        "with key",
+			key:         auth.MustNewHS256SigningKey(),
+			expectToken: true,
+		},
+		{
+			name:        "no key",
+			key:         nil,
+			expectToken: false,
+		},
+		{
+			name:        "empty key",
+			key:         auth.HMACBase64Key([]byte{}),
+			expectToken: false,
+		},
+	}
+	for _, test := range tests {
+		ss := &scrapeServer{SigningKey: test.key}
+		tmpl := ss.mustHomeTemplate()
+		tmpl, err := tmpl.Parse("{{AuthToken}}")
+		if err != nil {
+			t.Fatalf("[%s] Error parsing template: %s", test.name, err)
+		}
+		var buf bytes.Buffer
+		err = tmpl.Execute(&buf, nil)
+		if err != nil {
+			t.Fatalf("[%s] Error executing template: %s", test.name, err)
+		}
+		output := buf.String()
+		if !test.expectToken && output != "" {
+			t.Fatalf("[%s] Expected empty output, got %s", test.name, output)
+		}
+		if test.expectToken {
+			switch output {
+			case "":
+				t.Fatalf("[%s] Expected non-empty token, got empty", test.name)
+			default:
+				_, err := auth.VerifyToken(test.key, output)
+				if err != nil {
+					t.Fatalf("[%s] Error verifying token: %s", test.name, err)
+				}
+			}
+		}
+	}
+}