Merge pull request #214 from maxmind/naji/parallel-downloads

Naji/parallel downloads

CHANGELOG.md

@@ -1,5 +1,17 @@
 # CHANGELOG
 
+## 5.0.0 (2023-04-11)
+
+* Redefined the `Reader` and `Writer` interface apis in
+  `pkg/geoipupdate/database`. This change aims to to make it easier to
+  introduce custom implementations of these interfaces.
+* Changed the signature of `NewConfig` in `pkg/geoipupdate` to accept
+  optional parameters. This change allows the introduction of new
+  flags or config options without making breaking changes to the function's
+  signature.
+* Introduced `Parallelism` as a new flag and config option to enable
+  concurrent database updates.
+
 ## 4.11.1 (2023-03-16)
 
 * Removed extra underscore in script variables preventing the Docker

README.md

@@ -19,12 +19,12 @@ guide](https://dev.maxmind.com/geoip/upgrading-geoip-update?lang=en).
 ### Installing on Linux via the tarball
 
 Download and extract the appropriate tarball for your system. You will end
-up with a directory named something like `geoipupdate_4.0.0_linux_amd64`
+up with a directory named something like `geoipupdate_5.0.0_linux_amd64`
 depending on the version and architecture.
 
 Copy `geoipupdate` to where you want it to live. To install it to
 `/usr/local/bin/geoipupdate`, run the equivalent of `sudo cp
-geoipupdate_4.0.0_linux_amd64/geoipupdate /usr/local/bin`.
+geoipupdate_5.0.0_linux_amd64/geoipupdate /usr/local/bin`.
 
 `geoipupdate` looks for the config file `/usr/local/etc/GeoIP.conf` by
 default.
@@ -51,7 +51,7 @@ You can also use the tarball.
 
 Download the appropriate .deb for your system.
 
-Run `dpkg -i path/to/geoipupdate_4.0.0_linux_amd64.deb` (replacing the
+Run `dpkg -i path/to/geoipupdate_5.0.0_linux_amd64.deb` (replacing the
 version number and architecture as necessary). You will need to be root.
 For Ubuntu you can prefix the command with `sudo`. This will install
 `geoipupdate` to `/usr/bin/geoipupdate`.
@@ -64,7 +64,7 @@ You can also use the tarball.
 
 Download the appropriate .rpm for your system.
 
-Run `rpm -Uvhi path/to/geoipupdate_4.0.0_linux_amd64.rpm` (replacing the
+Run `rpm -Uvhi path/to/geoipupdate_5.0.0_linux_amd64.rpm` (replacing the
 version number and architecture as necessary). You will need to be root.
 This will install `geoipupdate` to `/usr/bin/geoipupdate`.
 
@@ -87,7 +87,7 @@ $ brew install geoipupdate
 ### Installing on Windows
 
 Download and extract the appropriate zip for your system. You will end up
-with a directory named something like `geoipupdate_4.0.0_windows_amd64`
+with a directory named something like `geoipupdate_5.0.0_windows_amd64`
 depending on the version and architecture.
 
 Copy `geoipupdate.exe` to where you want it to live.
@@ -107,7 +107,7 @@ website](https://golang.org).
 
 The easiest way is via `go get`:
 
-    $ env GO111MODULE=on go get -u github.com/maxmind/geoipupdate/v4/cmd/geoipupdate
+    $ env GO111MODULE=on go get -u github.com/maxmind/geoipupdate/v5/cmd/geoipupdate
 
 This installs `geoipupdate` to `$GOPATH/bin/geoipupdate`.
 

cmd/geoipupdate/args.go

@@ -13,6 +13,7 @@ type Args struct {
 	DatabaseDirectory string
 	StackTrace        bool
 	Verbose           bool
+	Parallelism       int
 }
 
 func getArgs() *Args {
@@ -32,6 +33,7 @@ func getArgs() *Args {
 	stackTrace := flag.Bool("stack-trace", false, "Show a stack trace along with any error message.")
 	verbose := flag.BoolP("verbose", "v", false, "Use verbose output")
 	displayVersion := flag.BoolP("version", "V", false, "Display the version and exit")
+	parallelism := flag.Int("parallelism", 0, "Set the number of parallel database downloads.")
 
 	flag.Parse()
 
@@ -49,11 +51,17 @@ func getArgs() *Args {
 		printUsage()
 	}
 
+	if *parallelism < 0 {
+		log.Printf("Parallelism must be a positive number")
+		printUsage()
+	}
+
 	return &Args{
 		ConfigFile:        *configFile,
 		DatabaseDirectory: *databaseDirectory,
 		StackTrace:        *stackTrace,
 		Verbose:           *verbose,
+		Parallelism:       *parallelism,
 	}
 }
 

cmd/geoipupdate/end_to_end_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"compress/gzip"
+	"context"
 	"crypto/md5"
 	"fmt"
 	"io"
@@ -16,7 +17,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/maxmind/geoipupdate/v4/pkg/geoipupdate"
+	"github.com/maxmind/geoipupdate/v5/pkg/geoipupdate"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -60,8 +61,6 @@ func TestMultipleDatabaseDownload(t *testing.T) {
 	)
 	defer server.Close()
 
-	client := server.Client()
-
 	tempDir, err := ioutil.TempDir("", "gutest-")
 	require.NoError(t, err)
 	defer func() {
@@ -76,12 +75,14 @@ func TestMultipleDatabaseDownload(t *testing.T) {
 		LicenseKey:        "testing",
 		LockFile:          filepath.Join(tempDir, ".geoipupdate.lock"),
 		URL:               server.URL,
+		Parallelism:       1,
 	}
 
 	logOutput := &bytes.Buffer{}
 	log.SetOutput(logOutput)
 
-	err = run(client, config)
+	client := geoipupdate.NewClient(config)
+	err = client.Run(context.Background())
 	assert.NoError(t, err, "run successfully")
 
 	assert.Equal(t, "", logOutput.String(), "no logged output")

cmd/geoipupdate/main.go

@@ -2,30 +2,25 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
-	"net/http"
 	"os"
-	"path/filepath"
 
-	"github.com/maxmind/geoipupdate/v4/pkg/geoipupdate"
-	"github.com/maxmind/geoipupdate/v4/pkg/geoipupdate/database"
+	"github.com/maxmind/geoipupdate/v5/pkg/geoipupdate"
+	"github.com/maxmind/geoipupdate/v5/pkg/geoipupdate/vars"
 )
 
 var (
-	version                  = "unknown"
-	defaultConfigFile        string
-	defaultDatabaseDirectory string
+	version           = "unknown"
+	defaultConfigFile string
 )
 
 func main() {
 	log.SetFlags(0)
 
 	if defaultConfigFile == "" {
-		defaultConfigFile = geoipupdate.DefaultConfigFile
-	}
-	if defaultDatabaseDirectory == "" {
-		defaultDatabaseDirectory = geoipupdate.DefaultDatabaseDirectory
+		defaultConfigFile = vars.DefaultConfigFile
 	}
 
 	args := getArgs()
@@ -39,7 +34,11 @@ func main() {
 	}
 
 	config, err := geoipupdate.NewConfig(
-		args.ConfigFile, defaultDatabaseDirectory, args.DatabaseDirectory, args.Verbose)
+		args.ConfigFile,
+		geoipupdate.WithDatabaseDirectory(args.DatabaseDirectory),
+		geoipupdate.WithParallelism(args.Parallelism),
+		geoipupdate.WithVerbose(args.Verbose),
+	)
 	if err != nil {
 		fatalLogger(fmt.Sprintf("error loading configuration file %s", args.ConfigFile), err)
 	}
@@ -51,28 +50,7 @@ func main() {
 	}
 
 	client := geoipupdate.NewClient(config)
-
-	if err = run(client, config); err != nil {
+	if err = client.Run(context.Background()); err != nil {
 		fatalLogger("error retrieving updates", err)
 	}
 }
-
-func run(client *http.Client, config *geoipupdate.Config) error {
-	dbReader := database.NewHTTPDatabaseReader(client, config)
-
-	for _, editionID := range config.EditionIDs {
-		filename, err := geoipupdate.GetFilename(config, editionID, client)
-		if err != nil {
-			return fmt.Errorf("error retrieving filename for %s: %w", editionID, err)
-		}
-		filePath := filepath.Join(config.DatabaseDirectory, filename)
-		dbWriter, err := database.NewLocalFileDatabaseWriter(filePath, config.LockFile, config.Verbose)
-		if err != nil {
-			return fmt.Errorf("error creating database writer for %s: %w", editionID, err)
-		}
-		if err := dbReader.Get(dbWriter, editionID); err != nil {
-			return fmt.Errorf("error while getting database for %s: %w", editionID, err)
-		}
-	}
-	return nil
-}

conf/GeoIP.conf.default

@@ -44,3 +44,7 @@ EditionIDs GeoLite2-Country GeoLite2-City
 # "s", "m", "h".
 # Defaults to "5m" (5 minutes).
 # RetryFor 5m
+
+# The number of parallel database downloads.
+# Defaults to "1".
+# Parallelism 1

doc/GeoIP.conf.md

@@ -72,6 +72,12 @@ sensitive.
     followed by a unit suffix. Valid time units are `ns`, `us` (or `µs`), `ms`,
     `s`, `m`, `h`. The default is `5m` (5 minutes).
 
+`Parallelism`
+
+:	The maximum number of parallel database downloads. The default is
+	1, which means that databases will be downloaded sequentially. This can be
+	overriden at runtime by the `--parallelism` command line argument.
+
 ## Deprecated settings:
 
 The following are deprecated and will be ignored if present:

doc/geoipupdate.md

@@ -28,6 +28,10 @@ open.
 :   The configuration file to use. See `GeoIP.conf` and its documentation for
     more information. This is optional. It defaults to CONFFILE.
 
+`--parallelism`
+
+:	Set the number of parallel database downloads.
+
 `-h`, `--help`
 
 :   Display help and exit.

go.mod

@@ -1,4 +1,4 @@
-module github.com/maxmind/geoipupdate/v4
+module github.com/maxmind/geoipupdate/v5
 
 go 1.13
 
@@ -8,6 +8,7 @@ require (
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.2
+	golang.org/x/sync v0.1.0
 	golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25 // indirect
 	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
 )

go.sum

@@ -21,6 +21,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25 h1:nwzwVf0l2Y/lkov/+IYgMMbFyI+QypZDds9RxlSmsFQ=
 golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=