Consolidate and simplify downloading (#105)

* Consolidate and simplify feed downloading

* Automatically find internal prefix for zip files

dmfr/cmd/dmfr_cmd.go

@@ -27,7 +27,6 @@ func (cmd *Command) Run(args []string) error {
 		log.Print("  sync")
 		log.Print("  import")
 		log.Print("  fetch")
-		log.Print("  recalculate")
 		fl.PrintDefaults()
 	}
 	fl.Parse(args)
@@ -50,8 +49,6 @@ func (cmd *Command) Run(args []string) error {
 		r = &importer.Command{}
 	case "fetch":
 		r = &fetch.Command{}
-	case "recalculate":
-		r = &fetch.RecalculateCommand{}
 	default:
 		return fmt.Errorf("Invalid command: %q", subc)
 	}

dmfr/fetch/fetch.go

@@ -5,14 +5,13 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/interline-io/transitland-lib/dmfr"
+	"github.com/interline-io/transitland-lib/internal/download"
 	"github.com/interline-io/transitland-lib/tl"
 	"github.com/interline-io/transitland-lib/tlcsv"
 	"github.com/interline-io/transitland-lib/tldb"
@@ -27,7 +26,7 @@ type Options struct {
 	Directory               string
 	S3                      string
 	FetchedAt               time.Time
-	Secrets                 Secrets
+	Secrets                 download.Secrets
 }
 
 // Result contains results of a fetch operation.
@@ -97,6 +96,10 @@ func DatabaseFetch(atx tldb.Adapter, opts Options) (Result, error) {
 	return fr, nil
 }
 
+type canSetAuth interface {
+	SetAuth(tl.FeedAuthorization, download.Secret)
+}
+
 // fetchAndCreateFeedVersion from a URL.
 // Returns an error if a serious failure occurs, such as database or filesystem access.
 // Sets Result.FetchError if a regular failure occurs, such as a 404.
@@ -107,14 +110,8 @@ func fetchAndCreateFeedVersion(atx tldb.Adapter, feed tl.Feed, opts Options) (Re
 		fr.FetchError = errors.New("no url")
 		return fr, nil
 	}
-	// Handle fragments
-	u, err := url.Parse(opts.FeedURL)
-	if err != nil {
-		fr.FetchError = errors.New("cannot parse url")
-		return fr, nil
-	}
 	// Get secret
-	secret := Secret{}
+	secret := download.Secret{}
 	if a, err := opts.Secrets.MatchFeed(opts.FeedID); err == nil {
 		secret = a
 	} else if a, err := opts.Secrets.MatchFilename(feed.File); err == nil {
@@ -123,20 +120,14 @@ func fetchAndCreateFeedVersion(atx tldb.Adapter, feed tl.Feed, opts Options) (Re
 		fr.FetchError = errors.New("no secret found")
 		return fr, nil
 	}
-	// Check reader type
+	// Get reader
 	reader, err := tlcsv.NewReader(opts.FeedURL)
 	if err != nil {
 		fr.FetchError = err
 		return fr, nil
 	}
-	// Override the default URLAdapter
-	if u.Scheme == "http" || u.Scheme == "https" || u.Scheme == "ftp" || u.Scheme == "s3" {
-		aa := AuthenticatedURLAdapter{}
-		if err := aa.Download(opts.FeedURL, feed.Authorization, secret); err != nil {
-			fr.FetchError = err
-			return fr, nil
-		}
-		reader.Adapter = &aa
+	if v, ok := reader.Adapter.(canSetAuth); ok {
+		v.SetAuth(feed.Authorization, secret)
 	}
 	// Open
 	if err := reader.Open(); err != nil {
@@ -257,17 +248,3 @@ func copyFileContents(src, dst string) (err error) {
 	err = out.Sync()
 	return
 }
-
-// dmfrGetReaderURL helps load a file from an S3 or Directory location
-func dmfrGetReaderURL(s3 string, directory string, url string) string {
-	if s3 != "" {
-		url = s3 + "/" + url
-	} else if directory != "" {
-		url = filepath.Join(directory, url)
-	}
-	urlsplit := strings.SplitN(url, "#", 2)
-	if len(urlsplit) > 1 {
-		url = url + "#" + urlsplit[1]
-	}
-	return url
-}

dmfr/importer/importer.go

@@ -227,17 +227,7 @@ func MainImportFeedVersion(adapter tldb.Adapter, opts Options) (Result, error) {
 func ImportFeedVersion(atx tldb.Adapter, fv tl.FeedVersion, opts Options) (dmfr.FeedVersionImport, error) {
 	fvi := dmfr.FeedVersionImport{FeedVersionID: fv.ID}
 	// Get Reader
-	url := fv.File
-	if opts.S3 != "" {
-		url = opts.S3 + "/" + fv.File
-	} else if opts.Directory != "" {
-		url = filepath.Join(opts.Directory, fv.File)
-	}
-	urlsplit := strings.SplitN(fv.URL, "#", 2)
-	if len(urlsplit) > 1 && !strings.HasSuffix(fv.URL, ".zip") {
-		url = url + "#" + urlsplit[1]
-	}
-	reader, err := tlcsv.NewReader(url)
+	reader, err := tlcsv.NewReader(dmfrGetReaderURL(opts.S3, opts.Directory, fv.File))
 	if err != nil {
 		return fvi, err
 	}
@@ -280,3 +270,18 @@ func ImportFeedVersion(atx tldb.Adapter, fv tl.FeedVersion, opts Options) (dmfr.
 	fvi.SkipEntityMarkedCount = counts.SkipEntityMarkedCount
 	return fvi, nil
 }
+
+// dmfrGetReaderURL helps load a file from an S3 or Directory location
+func dmfrGetReaderURL(s3 string, directory string, url string) string {
+	if s3 != "" {
+		url = s3 + "/" + url
+	} else if directory != "" {
+		url = filepath.Join(directory, url)
+	}
+	urlsplit := strings.SplitN(url, "#", 2)
+	if len(urlsplit) > 1 && !strings.HasSuffix(url, ".zip") {
+		// Add fragment back only if fragment does not end in ".zip"
+		url = url + "#" + urlsplit[1]
+	}
+	return url
+}

dmfr/importer/importer_test.go

@@ -92,7 +92,7 @@ func TestMainImportFeedVersion(t *testing.T) {
 			}
 			explog := "file does not exist"
 			if fvi.ExceptionLog != explog {
-				t.Errorf("got %s expected %s", fvi.ExceptionLog, explog)
+				t.Errorf("got '%s' expected '%s'", fvi.ExceptionLog, explog)
 			}
 			if fvi.InProgress != false {
 				t.Errorf("expected in_progress = false")