internal/frontend: allow refetch for 404

If a path is stored in the version_map table as a 404, allow the path
to be refetched once the taskIDChangeInterval has passed since the
path was last updated.

Fixes golang/go#46117

Change-Id: I985f8c095d4c6af1e092cc686322023cc0abd3b6
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/319050
Trust: Julie Qiu <julie@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>

internal/frontend/404.go

@@ -13,6 +13,7 @@ import (
 	"net/url"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/google/safehtml/template"
 	"github.com/google/safehtml/template/uncheckedconversions"
@@ -107,6 +108,13 @@ func (s *Server) servePathNotFoundPage(w http.ResponseWriter, r *http.Request,
 			http.Redirect(w, r, u, http.StatusFound)
 			return
 		}
+
+		// If a module has a status of 404, but s.taskIDChangeInterval has
+		// passed, allow the module to be refetched.
+		if fr.status == http.StatusNotFound && time.Since(fr.updatedAt) > s.taskIDChangeInterval {
+			return pathNotFoundError(fullPath, requestedVersion)
+		}
+
 		// Redirect to the search result page for an empty directory that is above nested modules.
 		// See https://golang.org/issue/43725 for context.
 		nm, err := ds.GetNestedModules(ctx, fullPath)
@@ -170,7 +178,8 @@ func pathNotFoundError(fullPath, requestedVersion string) error {
 
 // previousFetchStatusAndResponse returns the fetch result from a
 // previous fetch of the fullPath and requestedVersion.
-func previousFetchStatusAndResponse(ctx context.Context, db *postgres.DB, fullPath, modulePath, requestedVersion string) (_ *fetchResult, err error) {
+func previousFetchStatusAndResponse(ctx context.Context, db *postgres.DB,
+	fullPath, modulePath, requestedVersion string) (_ *fetchResult, err error) {
 	defer derrors.Wrap(&err, "previousFetchStatusAndResponse(w, r, %q, %q)", fullPath, requestedVersion)
 
 	// Get all candidate module paths for this path.
@@ -254,6 +263,7 @@ func fetchResultFromVersionMap(vm *internal.VersionMap) *fetchResult {
 		modulePath: vm.ModulePath,
 		goModPath:  vm.GoModPath,
 		status:     vm.Status,
+		updatedAt:  vm.UpdatedAt,
 		err:        err,
 	}
 }

internal/frontend/fetch.go

@@ -124,6 +124,7 @@ type fetchResult struct {
 	status       int
 	err          error
 	responseText string
+	updatedAt    time.Time
 }
 
 func (s *Server) fetchAndPoll(ctx context.Context, ds internal.DataSource, modulePath, fullPath, requestedVersion string) (status int, responseText string) {
@@ -362,6 +363,10 @@ func pollForPath(ctx context.Context, db *postgres.DB, pollEvery time.Duration,
 // process that was initiated is not yet complete.  If the row exists version_map
 // but not paths, it means that a module was found at the requestedVersion, but
 // not the fullPath, so errPathDoesNotExistInModule is returned.
+//
+// Note that if an error occurs while writing to the version_map table,
+// checkForPath will not know. Instead, it will keep running until the request
+// times out.
 func checkForPath(ctx context.Context, db *postgres.DB,
 	fullPath, modulePath, requestedVersion string, taskIDChangeInterval time.Duration) (fr *fetchResult) {
 	defer func() {
@@ -402,11 +407,7 @@ func checkForPath(ctx context.Context, db *postgres.DB,
 	// We successfully retrieved a row in version_map for the modulePath and
 	// requestedVersion. Look at the status of that row to determine whether
 	// an error should be returned.
-	fr = &fetchResult{
-		modulePath: modulePath,
-		status:     vm.Status,
-		goModPath:  vm.GoModPath,
-	}
+	fr = fetchResultFromVersionMap(vm)
 
 	switch fr.status {
 	case http.StatusNotFound,