Captain v3.0.1

Now you can download files directly to any object that implements the io.Writer interface

package main

import (
	"log"
	"os"
	"sync"
	"time"

	"cloud.google.com/go/storage"
	"github.com/blacheinc/pixels-golang-engine/config"
	"github.com/blacheinc/pixels-golang-engine/gcs"
	"github.com/opensaucerer/grab/v3"
)

type File struct {
	URL      string `json:"url"`
	Filename string `json:"filename"`
}

func main() {
	grabFiles([]File{
		{
			URL:      "http://ipv4.download.thinkbroadband.com/512MB.zip",
			Filename: "a-random-file.zip",
		},
		{
			URL:      "http://ipv4.download.thinkbroadband.com/512MB.zip",
			Filename: "a-very-random-file.zip",
		},
	})
}

func grabFiles(files []File) {

	var wg sync.WaitGroup

	var activeDownloads []*grab.Response

	// create client
	client := grab.NewClient()

	for _, file := range files {
		// create gcs destination
		ctx := context.Background()

		// create the object
		obj := Client.Bucket("bucket").Object("filename")

		// create the writer
		w := obj.NewWriter(ctx)
		if err != nil {
			log.Printf("gcs.OpenForSaving: Download request failed :: %s :: %s :: %v", file.URL, file.Filename, err)
			return
		}

		// create download request
		req, _ := grab.NewRequestToWriter(w, file.URL)

		// start download
		resp := client.Do(req)

		log.Printf("%s :: Downloading %s to %s ...\n", resp.HTTPResponse.Status, req.URL(), resp.Filename)

		activeDownloads = append(activeDownloads, resp)

		wg.Add(1)
	}

	// start progress ticker
	t := time.NewTicker(5000 * time.Millisecond)
	defer t.Stop()

	for _, resp := range activeDownloads {
		go func(r *grab.Response) {
			<-r.Done
			log.Printf("%s download completed", r.Filename)
		}(resp)
	}

	go func() {
		for {
			<-t.C
			for _, resp := range activeDownloads {
				log.Printf("%s :: transferred %v / %v bytes (%.2f%%)", resp.Filename, resp.BytesComplete(), resp.Size(), 100*resp.Progress())
			}
		}
	}()

	for _, resp := range activeDownloads {
		go func(r *grab.Response) {
			// check for errors
			if err := r.Err(); err != nil {
				log.Printf("%s :: Download failed :: %v\n", r.Filename, err)
				os.Exit(1)
			}

			if err := r.Request.Writer.(*storage.Writer).Close(); err != nil {
				log.Printf("%s :: Download failed :: %v\n", r.Filename, err)
				os.Exit(1)
			}

			log.Printf("Download saved to ./%v \n", r.Filename)

			wg.Done()
		}(resp)
	}

	wg.Wait()
}