From 133565ae1af1a06c8a6c5cb2ba219663febb1f60 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Fri, 8 Feb 2019 22:38:04 +0100 Subject: [PATCH] Release 0.1.0 (#2) * Replace dep with go mod * Move project to github --- Gopkg.lock | 165 ------------------ Gopkg.toml | 58 ------ Makefile | 4 +- README.md | 7 +- api/model.colf | 2 +- build/ci/.gitlab-ci.yml | 10 +- cmd/backup.go | 9 +- cmd/import.go | 9 +- cmd/restore.go | 9 +- cmd/root.go | 13 +- cmd/start.go | 16 +- configs/settings.docker.yaml | 4 +- examples/docker-compose.yml | 2 +- examples/settings.yaml | 6 +- go.mod | 32 ++++ go.sum | 67 +++++++ main.go | 2 +- pkg/common/helpers.go | 2 +- test/test.rest | 20 ++- {internal/distill => urlstore}/Colfer.go | 4 +- {internal => urlstore}/config.go | 12 +- {internal/distill => urlstore}/distill.go | 19 +- .../distill => urlstore}/distill_test.go | 57 +++--- {internal/distill => urlstore}/model.go | 13 +- {internal/distill => urlstore}/model_test.go | 2 +- {internal/distill => urlstore}/stats.go | 7 +- {internal/distill => urlstore}/stats_test.go | 4 +- .../testdata/urls.1123.csv | 0 .../testdata/urls.id.1123.csv | 0 {internal/distill => urlstore}/urlstore.go | 21 ++- {internal/distill => web}/endpoints.go | 82 +++++---- {internal/distill => web}/endpoints_test.go | 8 +- 32 files changed, 280 insertions(+), 386 deletions(-) delete mode 100644 Gopkg.lock delete mode 100644 Gopkg.toml create mode 100644 go.mod create mode 100644 go.sum rename {internal/distill => urlstore}/Colfer.go (99%) rename {internal => urlstore}/config.go (92%) rename {internal/distill => urlstore}/distill.go (88%) rename {internal/distill => urlstore}/distill_test.go (93%) rename {internal/distill => urlstore}/model.go (95%) rename {internal/distill => urlstore}/model_test.go (99%) rename {internal/distill => urlstore}/stats.go (95%) rename {internal/distill => urlstore}/stats_test.go (97%) rename {internal/distill => urlstore}/testdata/urls.1123.csv (100%) rename {internal/distill => urlstore}/testdata/urls.id.1123.csv (100%) rename {internal/distill => urlstore}/urlstore.go (95%) rename {internal/distill => web}/endpoints.go (81%) rename {internal/distill => web}/endpoints_test.go (85%) diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 341b9d2..0000000 --- a/Gopkg.lock +++ /dev/null @@ -1,165 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - branch = "master" - name = "github.com/AndreasBriese/bbloom" - packages = ["."] - revision = "28f7e881ca57bc00e028f9ede9f0d9104cfeef5e" - -[[projects]] - branch = "master" - name = "github.com/bluele/gcache" - packages = ["."] - revision = "472614239ac7e5bc6461e237c798a6ebd5aff8c1" - -[[projects]] - name = "github.com/dgraph-io/badger" - packages = [".","options","protos","skl","table","y"] - revision = "690400e629977977ecddfa8ab3a7f9285e1041a8" - version = "v1.5.1" - -[[projects]] - branch = "master" - name = "github.com/dgryski/go-farm" - packages = ["."] - revision = "2de33835d10275975374b37b2dcfd22c9020a1f5" - -[[projects]] - name = "github.com/fsnotify/fsnotify" - packages = ["."] - revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" - version = "v1.4.7" - -[[projects]] - name = "github.com/go-chi/chi" - packages = [".","middleware"] - revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35" - version = "v3.3.2" - -[[projects]] - name = "github.com/go-chi/render" - packages = ["."] - revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519" - version = "v1.0.0" - -[[projects]] - name = "github.com/golang/protobuf" - packages = ["proto"] - revision = "925541529c1fa6821df4e44ce2723319eb2be768" - version = "v1.0.0" - -[[projects]] - branch = "master" - name = "github.com/hashicorp/hcl" - packages = [".","hcl/ast","hcl/parser","hcl/printer","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] - revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" - -[[projects]] - name = "github.com/inconshreveable/mousetrap" - packages = ["."] - revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" - version = "v1.0" - -[[projects]] - branch = "master" - name = "github.com/jbrodriguez/mlog" - packages = ["."] - revision = "006dc6db226a30dddd9e8eab3cce1e44f651ed8a" - -[[projects]] - name = "github.com/magiconair/properties" - packages = ["."] - revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6" - version = "v1.7.6" - -[[projects]] - branch = "master" - name = "github.com/matoous/go-nanoid" - packages = ["."] - revision = "958d370425a1ea42a4dd3c67d15bf02554a16b82" - -[[projects]] - branch = "master" - name = "github.com/mitchellh/mapstructure" - packages = ["."] - revision = "00c29f56e2386353d58c599509e8dc3801b0d716" - -[[projects]] - name = "github.com/pelletier/go-toml" - packages = ["."] - revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" - version = "v1.1.0" - -[[projects]] - name = "github.com/pkg/errors" - packages = ["."] - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" - -[[projects]] - name = "github.com/spf13/afero" - packages = [".","mem"] - revision = "63644898a8da0bc22138abf860edaf5277b6102e" - version = "v1.1.0" - -[[projects]] - name = "github.com/spf13/cast" - packages = ["."] - revision = "8965335b8c7107321228e3e3702cab9832751bac" - version = "v1.2.0" - -[[projects]] - name = "github.com/spf13/cobra" - packages = ["."] - revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4" - version = "v0.0.2" - -[[projects]] - branch = "master" - name = "github.com/spf13/jwalterweatherman" - packages = ["."] - revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" - -[[projects]] - name = "github.com/spf13/pflag" - packages = ["."] - revision = "583c0c0531f06d5278b7d917446061adc344b5cd" - version = "v1.0.1" - -[[projects]] - name = "github.com/spf13/viper" - packages = ["."] - revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" - version = "v1.0.2" - -[[projects]] - branch = "master" - name = "golang.org/x/net" - packages = ["context","internal/timeseries","trace"] - revision = "5f9ae10d9af5b1c89ae6904293b14b064d4ada23" - -[[projects]] - branch = "master" - name = "golang.org/x/sys" - packages = ["unix"] - revision = "cbbc999da32df943dac6cd71eb3ee39e1d7838b9" - -[[projects]] - name = "golang.org/x/text" - packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"] - revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" - version = "v0.3.0" - -[[projects]] - name = "gopkg.in/yaml.v2" - packages = ["."] - revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" - version = "v2.2.1" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "8fb12fe0a79bc81fc4ff3e55976d5c25a0d890556c75c61e7c0a48fca8e37d4d" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index f83f447..0000000 --- a/Gopkg.toml +++ /dev/null @@ -1,58 +0,0 @@ - -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - - -[[constraint]] - name = "github.com/spf13/cobra" - version = "0.0.2" - -[[constraint]] - name = "github.com/spf13/viper" - version = "1.0.2" - -[[constraint]] - branch = "master" - name = "github.com/jbrodriguez/mlog" - -[[constraint]] - branch = "master" - name = "github.com/matoous/go-nanoid" - -[[constraint]] - name = "github.com/go-chi/chi" - version = "3.3.2" - -[[constraint]] - branch = "master" - name = "github.com/unrolled/render" - -[[constraint]] - name = "github.com/go-chi/cors" - version = "1.0.0" - -[[constraint]] - name = "github.com/dgraph-io/badger" - version = "1.5.0" - -[[constraint]] - branch = "master" - name = "github.com/bluele/gcache" diff --git a/Makefile b/Makefile index 704e4a0..1fbbd99 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GIT_DESCR = $(shell git describe --always) # build output folder OUTPUTFOLDER = dist # docker image -DOCKER_IMAGE = registry.gitlab.com/welance/oss/distill +DOCKER_IMAGE = noandrea/distill DOCKER_TAG = $(shell git describe --always) # build paramters OS = linux @@ -67,4 +67,4 @@ docker-run: @docker run -p 1804:1804 $(DOCKER_IMAGE) debug-start: - @go run main.go -c configs/settings.sample.yaml --debug start + @go run main.go -c examples/settings.yaml --debug start diff --git a/README.md b/README.md index 8f0d5be..bf59974 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,8 @@ -# Distill +# Distill Another url shortener -[![pipeline status](https://gitlab.com/welance/oss/distill/badges/develop/pipeline.svg)](https://gitlab.com/welance/oss/distill/commits/develop) - -[![coverage report](https://gitlab.com/welance/oss/distill/badges/develop/coverage.svg)](https://gitlab.com/welance/oss/distill/commits/develop) +[![pipeline status](https://github.com/noandrea/distill/badges/develop/pipeline.svg)](https://github.com/noandrea/distill/commits/develop) [![coverage report](https://github.com/noandrea/distill/badges/develop/coverage.svg)](https://github.com/noandrea/distill/commits/develop) [![GoDoc](https://godoc.org/github.com/noandrea/distill?status.svg)](https://godoc.org/github.com/noandrea/distill) [![Go Report Card](https://goreportcard.com/badge/github.com/noandrea/distill)](https://goreportcard.com/report/github.com/noandrea/distill) ## Motivations @@ -14,6 +12,7 @@ Existing url shorteners are not suitable for a private deploy use or are too com that can be used for specific events. + ## Features - Choose the alphabet set for the generate short id diff --git a/api/model.colf b/api/model.colf index 6e1b56e..ba67b77 100644 --- a/api/model.colf +++ b/api/model.colf @@ -1,5 +1,5 @@ // Package demo offers a demonstration. -package distill +package urlstore // URLInfo contains the data associated to an id type URLInfo struct { diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index e1b6880..37ab2da 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: welance/build-golang:1.10 +image: welance/build-golang:1.11 stages: - test @@ -9,13 +9,13 @@ variables: DOCKER_HOST: docker before_script: - - cd $GOPATH/src - - mkdir -p gitlab.com/$CI_PROJECT_NAMESPACE - - cd gitlab.com/$CI_PROJECT_NAMESPACE + # - cd $GOPATH/src + - mkdir -p /src/gitlab.com/$CI_PROJECT_NAMESPACE + - cd /src/gitlab.com/$CI_PROJECT_NAMESPACE - ln -s $CI_PROJECT_DIR - cd $CI_PROJECT_NAME - echo -e "machine gitlab.com\nlogin gitlab-ci-token\npassword ${CI_JOB_TOKEN}" > ~/.netrc - - dep ensure -vendor-only + - go mod download lint_code: stage: test diff --git a/cmd/backup.go b/cmd/backup.go index dad3633..e778b8a 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -18,8 +18,9 @@ import ( "os" "path/filepath" + "github.com/noandrea/distill/urlstore" + "github.com/jbrodriguez/mlog" - "gitlab.com/welance/oss/distill/internal/distill" "github.com/spf13/cobra" ) @@ -55,8 +56,8 @@ func init() { } func backup(cmd *cobra.Command, args []string) { - distill.NewSession() - defer distill.CloseSession() + urlstore.NewSession() + defer urlstore.CloseSession() abp, err := filepath.Abs(backupFile) if err != nil { mlog.Fatalf("Invalid path %s: %v", backupFile, err) @@ -65,7 +66,7 @@ func backup(cmd *cobra.Command, args []string) { if err != nil { mlog.Fatalf("Error create backup path to %s: %v", backupFile, err) } - if err = distill.Backup(abp); err != nil { + if err = urlstore.Backup(abp); err != nil { mlog.Fatalf("Error create backup at %s: %v", backupFile, err) } } diff --git a/cmd/import.go b/cmd/import.go index cfc264b..559822e 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -18,9 +18,10 @@ import ( "os" "path/filepath" + "github.com/noandrea/distill/urlstore" + "github.com/jbrodriguez/mlog" "github.com/spf13/cobra" - "gitlab.com/welance/oss/distill/internal/distill" ) // importCmd represents the import command @@ -49,8 +50,8 @@ func init() { } func importCsv(cmd *cobra.Command, args []string) { - distill.NewSession() - defer distill.CloseSession() + urlstore.NewSession() + defer urlstore.CloseSession() abp, err := filepath.Abs(csvFile) if err != nil { mlog.Fatalf("Invalid path %s: %v", csvFile, err) @@ -58,7 +59,7 @@ func importCsv(cmd *cobra.Command, args []string) { if _, err = os.Stat(abp); os.IsNotExist(err) { mlog.Fatalf("Invalid path %s: %v", csvFile, err) } - if rows, err := distill.ImportCSV(abp); err != nil { + if rows, err := urlstore.ImportCSV(abp); err != nil { mlog.Fatalf("Error create backup at %s: %v", csvFile, err) } else { mlog.Info("Import complete, %d url record loaded", rows) diff --git a/cmd/restore.go b/cmd/restore.go index 17b8ef2..fc260af 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -18,9 +18,10 @@ import ( "os" "path/filepath" + "github.com/noandrea/distill/urlstore" + "github.com/jbrodriguez/mlog" "github.com/spf13/cobra" - "gitlab.com/welance/oss/distill/internal/distill" ) // restoreCmd represents the restore command @@ -37,8 +38,8 @@ func init() { } func restore(cmd *cobra.Command, args []string) { - distill.NewSession() - defer distill.CloseSession() + urlstore.NewSession() + defer urlstore.CloseSession() abp, err := filepath.Abs(backupFile) if err != nil { mlog.Fatalf("Invalid path %s: %v", backupFile, err) @@ -46,7 +47,7 @@ func restore(cmd *cobra.Command, args []string) { if _, err := os.Stat(abp); os.IsNotExist(err) { mlog.Fatalf("Invalid path %s: %v", csvFile, err) } - if count, err := distill.Restore(abp); err != nil { + if count, err := urlstore.Restore(abp); err != nil { mlog.Fatalf("Error restoring backup from %s: %v", backupFile, err) } else { mlog.Info("Restored %d URLs from %s ", count, backupFile) diff --git a/cmd/root.go b/cmd/root.go index fc1f9f7..f7c1085 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,11 +19,12 @@ import ( "log" "os" + "github.com/noandrea/distill/urlstore" + "github.com/jbrodriguez/mlog" "github.com/spf13/cobra" "github.com/spf13/viper" - "gitlab.com/welance/oss/distill/internal" - "gitlab.com/welance/oss/distill/pkg/common" + "github.com/noandrea/distill/pkg/common" ) var cfgFile, logFile, version string @@ -91,14 +92,14 @@ func initConfig() { // If a config file is found, read it in. if err := viper.ReadInConfig(); err == nil { mlog.Info("Using config file: %v", viper.ConfigFileUsed()) - viper.Unmarshal(&internal.Config) - internal.Config.Defaults() - internal.Config.Validate() + viper.Unmarshal(&urlstore.Config) + urlstore.Config.Defaults() + urlstore.Config.Validate() } else { switch err.(type) { case viper.ConfigFileNotFoundError: if do := common.AskYes("A configuration file was not found, would you like to generate one?", true); do { - internal.GenerateDefaultConfig("settings.yaml", version) + urlstore.GenerateDefaultConfig("settings.yaml", version) fmt.Println("Configuration settings.yaml created") return } diff --git a/cmd/start.go b/cmd/start.go index 51311a8..0d256a8 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package cmd import ( @@ -19,10 +18,11 @@ import ( "net/http" "strings" + "github.com/noandrea/distill/urlstore" + "github.com/noandrea/distill/web" + "github.com/jbrodriguez/mlog" "github.com/spf13/cobra" - "gitlab.com/welance/oss/distill/internal" - "gitlab.com/welance/oss/distill/internal/distill" ) // startCmd represents the start command @@ -57,16 +57,16 @@ func start(cmd *cobra.Command, args []string) { mlog.Info(" | (_| | \\__ \\ |_| | | |") mlog.Info(" \\__,_|_|___/\\__|_|_|_| v.%v", version) mlog.Info("") - mlog.Info("Listening to %v:%v", internal.Config.Server.Host, internal.Config.Server.Port) + mlog.Info("Listening to %v:%v", urlstore.Config.Server.Host, urlstore.Config.Server.Port) - distill.NewSession() + urlstore.NewSession() if len(strings.TrimSpace(restoreFile)) > 0 { - count, err := distill.Restore(restoreFile) + count, err := urlstore.Restore(restoreFile) if err != nil { mlog.Fatalf("Error restoring URLs from %s: %v ", restoreFile, err) } mlog.Info("Restored %d URLs from %s ", count, restoreFile) } - r := distill.RegisterEndpoints() - http.ListenAndServe(fmt.Sprintf("%s:%d", internal.Config.Server.Host, internal.Config.Server.Port), r) + r := web.RegisterEndpoints() + http.ListenAndServe(fmt.Sprintf("%s:%d", urlstore.Config.Server.Host, urlstore.Config.Server.Port), r) } diff --git a/configs/settings.docker.yaml b/configs/settings.docker.yaml index a416f4f..702f044 100644 --- a/configs/settings.docker.yaml +++ b/configs/settings.docker.yaml @@ -3,8 +3,8 @@ server: host: "0.0.0.0" port: 1804 apiKey: 123123_changeme_changeme - rootRedirect: https://gitlab.com/welance/oss/distill/wikis/welcome - expiredRedirect: https://gitlab.com/welance/oss/distill/wikis/Expired-URL + rootRedirect: https://github.com/noandrea/distill/wikis/welcome + expiredRedirect: https://github.com/noandrea/distill/wikis/Expired-URL dbPath: /data/db enableStats: true diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml index bf67021..6923e62 100644 --- a/examples/docker-compose.yml +++ b/examples/docker-compose.yml @@ -6,7 +6,7 @@ version: '3.1' services: distill: - image: registry.gitlab.com/welance/oss/distill:latest + image: registry.github.com/noandrea/distill:latest container_name: distill restart: unless-stopped ports: diff --git a/examples/settings.yaml b/examples/settings.yaml index 1962ede..75dd8d8 100644 --- a/examples/settings.yaml +++ b/examples/settings.yaml @@ -1,6 +1,6 @@ # # Default configuration for Distill v0.0.3-ci-14-gd67560a -# http://gitlab.com/welance/oss/distill +# http://github.com/noandrea/distill # server: @@ -8,8 +8,8 @@ server: host: 0.0.0.0 port: 1804 dbPath: distill.db - rootRedirect: https://gitlab.com/welance/oss/distill/wikis/welcome - expiredRedirect: https://gitlab.com/welance/oss/distill/wikis/Expired-URL + rootRedirect: https://github.com/noandrea/distill/wikis/welcome + expiredRedirect: https://github.com/noandrea/distill/wikis/Expired-URL shortId: alphabet: abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789 length: 6 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6410fc9 --- /dev/null +++ b/go.mod @@ -0,0 +1,32 @@ +module github.com/noandrea/distill + +require ( + github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 // indirect + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/bluele/gcache v0.0.0-20171010155617-472614239ac7 + github.com/dgraph-io/badger v1.5.4 + github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect + github.com/fsnotify/fsnotify v1.4.7 // indirect + github.com/go-chi/chi v4.0.1+incompatible + github.com/go-chi/render v1.0.0 + github.com/golang/protobuf v1.2.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jbrodriguez/mlog v0.0.0-20160501155140-006dc6db226a + github.com/magiconair/properties v1.8.0 // indirect + github.com/matoous/go-nanoid v0.0.0-20180109130436-958d370425a1 + github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/pelletier/go-toml v1.2.0 // indirect + github.com/pkg/errors v0.8.1 // indirect + github.com/spf13/afero v1.2.1 // indirect + github.com/spf13/cast v1.3.0 // indirect + github.com/spf13/cobra v0.0.2 + github.com/spf13/jwalterweatherman v1.0.0 // indirect + github.com/spf13/pflag v1.0.3 // indirect + github.com/spf13/viper v1.0.2 + github.com/stretchr/testify v1.3.0 // indirect + golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 // indirect + golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect + golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 // indirect + gopkg.in/yaml.v2 v2.2.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8d05d81 --- /dev/null +++ b/go.sum @@ -0,0 +1,67 @@ +github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo= +github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/bluele/gcache v0.0.0-20171010155617-472614239ac7 h1:NpQ+gkFOH27AyDypSCJ/LdsIi/b4rdnEb1N5+IpFfYs= +github.com/bluele/gcache v0.0.0-20171010155617-472614239ac7/go.mod h1:8c4/i2VlovMO2gBnHGQPN5EJw+H0lx1u/5p+cgsXtCk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg= +github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-chi/chi v4.0.1+incompatible h1:RSRC5qmFPtO90t7pTL0DBMNpZFsb/sHF3RXVlDgFisA= +github.com/go-chi/chi v4.0.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA= +github.com/go-chi/render v1.0.0/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jbrodriguez/mlog v0.0.0-20160501155140-006dc6db226a h1:+K+kl0pHXkDTl9Yq4aHZ84BdVkqyUi39E83FM0mDtCA= +github.com/jbrodriguez/mlog v0.0.0-20160501155140-006dc6db226a/go.mod h1:H8HrQQO3i02Ktu5ndZShfTSw3pj2vaHnpfOmUAUcqL4= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matoous/go-nanoid v0.0.0-20180109130436-958d370425a1 h1:Ar0OvPWGq9L0CUY7955quaAQlSu70xMB/Qilkt5n6tw= +github.com/matoous/go-nanoid v0.0.0-20180109130436-958d370425a1/go.mod h1:soqXi4beH2aAljcVvgIDqekDtnM2UZkGl47fniwq3J4= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE= +github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.0.2 h1:Ncr3ZIuJn322w2k1qmzXDnkLAdQMlJqBa9kfAH+irso= +github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 h1:bfLnR+k0tq5Lqt6dflRLcZiz6UaXCMt3vhYJ1l4FQ80= +golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 h1:FDfvYgoVsA7TTZSbgiqjAbfPbK47CNHdWl3h/PJtii0= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 4266806..e02eadd 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ package main -import "gitlab.com/welance/oss/distill/cmd" +import "github.com/noandrea/distill/cmd" // Version hold the version of the program var Version = "0.0.0" diff --git a/pkg/common/helpers.go b/pkg/common/helpers.go index 8150912..44ce559 100644 --- a/pkg/common/helpers.go +++ b/pkg/common/helpers.go @@ -33,7 +33,7 @@ func DefaultIfEmptyInt(v *int, defaultV int) { } } -// RandomString generate a random string of required lenght an with requested alphabet +// RandomString generate a random string of required length using alphabet func RandomString(alphabet string, length int) (s string, err error) { if IsEmptyStr(alphabet) { err = fmt.Errorf("alphabet must not be empty") diff --git a/test/test.rest b/test/test.rest index 829ab4a..3b3974e 100644 --- a/test/test.rest +++ b/test/test.rest @@ -2,19 +2,27 @@ POST http://localhost:1804/api/short -X-API-KEY: naiuraoijodmaojorierjaomdaodorpaokda +X-API-KEY: sSftPgEULMt4UWMh8idQgaTiJh9dKlXfdeiB5qcoCxc!ZehVik Content-Type: application/json { - "url": "https://hackernews.com" + "id": "testtest", + "url": "https://hackernews.com", + "max_requests": 10, + "ttl": 3600 } ### -GET http://localhost:1804/api/stats/qvadbK -X-API-KEY: naiuraoijodmaojorierjaomdaodorpaokda +GET http://localhost:1804/api/stats/testtest +X-API-KEY: sSftPgEULMt4UWMh8idQgaTiJh9dKlXfdeiB5qcoCxc!ZehVik ### -GET http://localhost:1804/qvadbK -X-API-KEY: naiuraoijodmaojorierjaomdaodorpaokda \ No newline at end of file +GET http://localhost:1804/testtest +X-API-KEY: sSftPgEULMt4UWMh8idQgaTiJh9dKlXfdeiB5qcoCxc!ZehVik + +### + +GET http://aet.li/api/stats +X-API-KEY: sSftPgEULMt4UWMh8idQgaTiJh9dKlXfdeiB5qcoCxc!ZehVik \ No newline at end of file diff --git a/internal/distill/Colfer.go b/urlstore/Colfer.go similarity index 99% rename from internal/distill/Colfer.go rename to urlstore/Colfer.go index 619fb2f..3354c55 100644 --- a/internal/distill/Colfer.go +++ b/urlstore/Colfer.go @@ -1,5 +1,5 @@ -// Package demo offers a demonstration. -package distill +// Package urlstore containse the core engine of distill +package urlstore // Code generated by colf(1); DO NOT EDIT. // The compiler used schema file model.colf. diff --git a/internal/config.go b/urlstore/config.go similarity index 92% rename from internal/config.go rename to urlstore/config.go index dc6c7f1..cecb56b 100644 --- a/internal/config.go +++ b/urlstore/config.go @@ -1,4 +1,4 @@ -package internal +package urlstore import ( "fmt" @@ -6,7 +6,7 @@ import ( "strings" "time" - "gitlab.com/welance/oss/distill/pkg/common" + "github.com/noandrea/distill/pkg/common" yaml "gopkg.in/yaml.v2" ) @@ -58,8 +58,8 @@ func (c *ConfigSchema) Defaults() { common.DefaultIfEmptyStr(&c.Server.Host, "0.0.0.0") common.DefaultIfEmptyInt(&c.Server.Port, 1804) common.DefaultIfEmptyStr(&c.Server.DbPath, "distill.db") - common.DefaultIfEmptyStr(&c.Server.RootRedirect, "https://gitlab.com/welance/oss/distill/wikis/welcome") - common.DefaultIfEmptyStr(&c.Server.ExpiredRedirect, "https://gitlab.com/welance/oss/distill/wikis/Expired-URL") + common.DefaultIfEmptyStr(&c.Server.RootRedirect, "https://github.com/noandrea/distill/wikis/welcome") + common.DefaultIfEmptyStr(&c.Server.ExpiredRedirect, "https://github.com/noandrea/distill/wikis/Expired-URL") // for short id common.DefaultIfEmptyStr(&c.ShortID.Alphabet, "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789") @@ -95,7 +95,7 @@ func (c *ConfigSchema) Validate() { } } -// Config sytem configuration +// Config system configuration var Config ConfigSchema // GenerateDefaultConfig generate a default configuration file an writes it in the outFile @@ -106,7 +106,7 @@ func GenerateDefaultConfig(outFile, version string) { data := strings.Join([]string{ "#", fmt.Sprintf("# Default configuration for Distill v%s", version), - "# http://gitlab.com/welance/oss/distill", + "# http://github.com/noandrea/distill", "#\n", fmt.Sprintf("%s", b), "#", diff --git a/internal/distill/distill.go b/urlstore/distill.go similarity index 88% rename from internal/distill/distill.go rename to urlstore/distill.go index bb6b7ce..bc761d5 100644 --- a/internal/distill/distill.go +++ b/urlstore/distill.go @@ -1,4 +1,4 @@ -package distill +package urlstore import ( "encoding/csv" @@ -11,14 +11,13 @@ import ( "time" "github.com/jbrodriguez/mlog" - "gitlab.com/welance/oss/distill/internal" - "gitlab.com/welance/oss/distill/pkg/common" + "github.com/noandrea/distill/pkg/common" ) // generateID generates a new id func generateID() (shortID string) { - a := internal.Config.ShortID.Alphabet - l := internal.Config.ShortID.Length + a := Config.ShortID.Alphabet + l := Config.ShortID.Length // a and l are validated before shortID, _ = common.RandomString(a, l) return @@ -49,12 +48,12 @@ func UpsertURL(url *URLReq, forceAlphabet, forceLength bool, boundAt time.Time) u.ExpireOn = calculateExpiration(u, url.TTL, url.ExpireOn) if u.ExpireOn.IsZero() { // global expiration - u.ExpireOn = calculateExpiration(u, internal.Config.ShortID.TTL, internal.Config.ShortID.ExpireOn) + u.ExpireOn = calculateExpiration(u, Config.ShortID.TTL, Config.ShortID.ExpireOn) } // set max requests, the local version always has priority u.MaxRequests = url.MaxRequests if u.MaxRequests == 0 { - u.MaxRequests = internal.Config.ShortID.MaxRequests + u.MaxRequests = Config.ShortID.MaxRequests } // cleanup the string id u.ID = strings.TrimSpace(url.ID) @@ -63,14 +62,14 @@ func UpsertURL(url *URLReq, forceAlphabet, forceLength bool, boundAt time.Time) err = Insert(u) } else { // TODO: check longest allowed key in badger - p := fmt.Sprintf("[^%s]", regexp.QuoteMeta(internal.Config.ShortID.Alphabet)) + p := fmt.Sprintf("[^%s]", regexp.QuoteMeta(Config.ShortID.Alphabet)) m, _ := regexp.MatchString(p, url.ID) if forceAlphabet && m { err = fmt.Errorf("ID %v doesn't match alphabet and forceAlphabet is active", url.ID) return "", err } - if forceLength && len(url.ID) != internal.Config.ShortID.Length { - err = fmt.Errorf("ID %v doesn't match length and forceLength len %v, required %v", url.ID, len(url.ID), internal.Config.ShortID.Length) + if forceLength && len(url.ID) != Config.ShortID.Length { + err = fmt.Errorf("ID %v doesn't match length and forceLength len %v, required %v", url.ID, len(url.ID), Config.ShortID.Length) return "", err } err = Upsert(u) diff --git a/internal/distill/distill_test.go b/urlstore/distill_test.go similarity index 93% rename from internal/distill/distill_test.go rename to urlstore/distill_test.go index b796755..c356842 100644 --- a/internal/distill/distill_test.go +++ b/urlstore/distill_test.go @@ -1,4 +1,4 @@ -package distill +package urlstore import ( "fmt" @@ -10,10 +10,9 @@ import ( "testing" "time" - "gitlab.com/welance/oss/distill/pkg/common" + "github.com/noandrea/distill/pkg/common" "github.com/jbrodriguez/mlog" - "gitlab.com/welance/oss/distill/internal" ) func setupLog() { @@ -27,83 +26,83 @@ func buildConifgTest() { // path path, _ := ioutil.TempDir("/tmp/", "distill") fmt.Println("test db folder is ", path) - internal.Config = internal.ConfigSchema{ - Server: internal.ServerConfig{ + Config = ConfigSchema{ + Server: ServerConfig{ DbPath: path, APIKey: common.GenerateSecret(), }, - ShortID: internal.ShortIDConfig{ + ShortID: ShortIDConfig{ Alphabet: "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789", Length: 6, }, } - internal.Config.Defaults() - internal.Config.Validate() + Config.Defaults() + Config.Validate() } func buildConifgPanicTest() { setupLog() path := " cann not exists / ssa " fmt.Println("test db folder is ", path) - internal.Config = internal.ConfigSchema{ - Server: internal.ServerConfig{ + Config = ConfigSchema{ + Server: ServerConfig{ DbPath: path, APIKey: common.GenerateSecret(), }, - ShortID: internal.ShortIDConfig{ + ShortID: ShortIDConfig{ Alphabet: "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789", Length: 6, }, } - internal.Config.Defaults() - internal.Config.Validate() + Config.Defaults() + Config.Validate() } func buildConifgTestShortIDParams(alphabet string, length int) { setupLog() path, _ := ioutil.TempDir("/tmp/", "distill") fmt.Println("test db folder is ", path) - internal.Config = internal.ConfigSchema{ - Server: internal.ServerConfig{ + Config = ConfigSchema{ + Server: ServerConfig{ DbPath: path, APIKey: common.GenerateSecret(), }, - ShortID: internal.ShortIDConfig{ + ShortID: ShortIDConfig{ Alphabet: alphabet, Length: length, TTL: 0, MaxRequests: 0, }, - Tuning: internal.TuningConfig{ + Tuning: TuningConfig{ StatsEventsWorkerNum: 2, }, } - internal.Config.Defaults() - internal.Config.Validate() + Config.Defaults() + Config.Validate() } func buildConifgTestExpireParams(ttl, maxr int64, expire time.Time) { setupLog() path, _ := ioutil.TempDir("/tmp/", "distill") fmt.Println("test db folder is ", path) - internal.Config = internal.ConfigSchema{ - Server: internal.ServerConfig{ + Config = ConfigSchema{ + Server: ServerConfig{ DbPath: path, APIKey: common.GenerateSecret(), }, - ShortID: internal.ShortIDConfig{ + ShortID: ShortIDConfig{ Alphabet: "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789", Length: 6, TTL: ttl, MaxRequests: maxr, ExpireOn: expire, }, - Tuning: internal.TuningConfig{ + Tuning: TuningConfig{ StatsEventsWorkerNum: 20, }, } - internal.Config.Defaults() - internal.Config.Validate() + Config.Defaults() + Config.Validate() } func TestGenerateID(t *testing.T) { @@ -121,8 +120,8 @@ func TestGenerateID(t *testing.T) { {"abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789", 30, "[iIl1o0O]"}, } for _, tt := range tests { - internal.Config = internal.ConfigSchema{ - ShortID: internal.ShortIDConfig{ + Config = ConfigSchema{ + ShortID: ShortIDConfig{ Alphabet: tt.Alphabet, Length: tt.Length, }, @@ -547,12 +546,12 @@ func TestExpireTTLUrl(t *testing.T) { defer CloseSession() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - id, err := UpsertURL(&tt.param, true, true, time.Now()) + id, _ := UpsertURL(&tt.param, true, true, time.Now()) //mlog.Info("-- upsert %s --", id) // consume all the requests time.Sleep(time.Duration(tt.wait) * time.Second) now := time.Now() - _, err = GetURLRedirect(id) + _, err := GetURLRedirect(id) u, _ := GetURLInfo(id) fmt.Println(tt.name, tt.wantErr, "\nbat", u.BountAt, "\nexp", u.ExpireOn, "\nnow", now.UTC(), "\ndif", now.Sub(u.BountAt)) //mlog.Info("-- << end %s --", id) diff --git a/internal/distill/model.go b/urlstore/model.go similarity index 95% rename from internal/distill/model.go rename to urlstore/model.go index 8cf4b77..3540ee3 100644 --- a/internal/distill/model.go +++ b/urlstore/model.go @@ -1,8 +1,9 @@ -package distill +package urlstore import ( "encoding/binary" "fmt" + "net/http" "strconv" "time" ) @@ -86,6 +87,16 @@ func (u URLInfo) String() string { return fmt.Sprintf("%v c:%d %v [mr:%d, exp:%v] --> %v", u.ID, u.Counter, u.BountAt.Format(time.Stamp), u.MaxRequests, u.ExpireOn.Format(time.RFC3339Nano), u.URL) } +// Bind will run after the unmarshalling is complete +func (u *URLReq) Bind(r *http.Request) error { + return nil +} + +// Bind will run after the unmarshalling is complete +func (u *ShortID) Bind(r *http.Request) error { + return nil +} + // ______ ______ ____ ____ // .' ___ |.' ____ \|_ _| |_ _| // / .' \_|| (___ \_| \ \ / / diff --git a/internal/distill/model_test.go b/urlstore/model_test.go similarity index 99% rename from internal/distill/model_test.go rename to urlstore/model_test.go index f0df0c8..8528eb8 100644 --- a/internal/distill/model_test.go +++ b/urlstore/model_test.go @@ -1,4 +1,4 @@ -package distill +package urlstore import ( "reflect" diff --git a/internal/distill/stats.go b/urlstore/stats.go similarity index 95% rename from internal/distill/stats.go rename to urlstore/stats.go index 8facb84..a1eef85 100644 --- a/internal/distill/stats.go +++ b/urlstore/stats.go @@ -1,4 +1,4 @@ -package distill +package urlstore import ( "sync" @@ -7,7 +7,6 @@ import ( "github.com/jbrodriguez/mlog" "github.com/dgraph-io/badger" - "gitlab.com/welance/oss/distill/internal" ) var ( @@ -132,7 +131,7 @@ func runDbMaintenance() { // caluclate if gc is necessary deletes := globalStatistics.Deletes - gcLimit := internal.Config.Tuning.DbGCDeletesCount + gcLimit := Config.Tuning.DbGCDeletesCount gcCount := 0 // retrieve the gcCount from the db db.View(func(txn *badger.Txn) (err error) { @@ -151,7 +150,7 @@ func runDbMaintenance() { mlog.Info("") - db.RunValueLogGC(internal.Config.Tuning.DbGCDiscardRation) + db.RunValueLogGC(Config.Tuning.DbGCDiscardRation) mlog.Info("End maintenance n %d for deletes %d > %d", gcCount, int(deletes)-latestGC, gcLimit) // update the gcCount db.Update(func(txn *badger.Txn) (err error) { diff --git a/internal/distill/stats_test.go b/urlstore/stats_test.go similarity index 97% rename from internal/distill/stats_test.go rename to urlstore/stats_test.go index e8d6f9d..a6a0735 100644 --- a/internal/distill/stats_test.go +++ b/urlstore/stats_test.go @@ -1,4 +1,4 @@ -package distill +package urlstore import ( "fmt" @@ -83,7 +83,7 @@ func Test_loadGlobalStatistics(t *testing.T) { t.Errorf("loadGlobalStatistics() = %v, want %v", gotS, tt.wantS) } - // alsot test reset + // also test reset gotS.Deletes = 0 err = ResetStats() if err != nil { diff --git a/internal/distill/testdata/urls.1123.csv b/urlstore/testdata/urls.1123.csv similarity index 100% rename from internal/distill/testdata/urls.1123.csv rename to urlstore/testdata/urls.1123.csv diff --git a/internal/distill/testdata/urls.id.1123.csv b/urlstore/testdata/urls.id.1123.csv similarity index 100% rename from internal/distill/testdata/urls.id.1123.csv rename to urlstore/testdata/urls.id.1123.csv diff --git a/internal/distill/urlstore.go b/urlstore/urlstore.go similarity index 95% rename from internal/distill/urlstore.go rename to urlstore/urlstore.go index 6f7c95d..11ef27a 100644 --- a/internal/distill/urlstore.go +++ b/urlstore/urlstore.go @@ -1,4 +1,5 @@ -package distill +// Package urlstore provides the main functionalities for distill +package urlstore import ( "encoding/csv" @@ -11,7 +12,6 @@ import ( "github.com/bluele/gcache" "github.com/dgraph-io/badger" "github.com/jbrodriguez/mlog" - "gitlab.com/welance/oss/distill/internal" ) const ( @@ -39,15 +39,15 @@ func NewSession() { // open the badger database opts := badger.DefaultOptions opts.SyncWrites = true - opts.Dir = internal.Config.Server.DbPath - opts.ValueDir = internal.Config.Server.DbPath + opts.Dir = Config.Server.DbPath + opts.ValueDir = Config.Server.DbPath var err error db, err = badger.Open(opts) if err != nil { mlog.Fatal(err) } // initialzie internal cache - uc = gcache.New(internal.Config.Tuning.URLCaheSize). + uc = gcache.New(Config.Tuning.URLCaheSize). EvictedFunc(whenRemoved). PurgeVisitorFunc(whenRemoved). ARC(). @@ -233,6 +233,9 @@ func Backup(outFile string) (err error) { return err } ts, err := db.Backup(fp, 0) + if err != nil { + return err + } mlog.Info("Backup completed at %v", ts) case backupExtCsv: err = db.View(func(txn *badger.Txn) (err error) { @@ -248,7 +251,7 @@ func Backup(outFile string) (err error) { // open the iterator opts := badger.DefaultIteratorOptions - opts.PrefetchSize = internal.Config.Tuning.BckCSVIterPrefetchSize + opts.PrefetchSize = Config.Tuning.BckCSVIterPrefetchSize opts.PrefetchValues = true it := txn.NewIterator(opts) defer it.Close() @@ -361,7 +364,7 @@ func (i *URLIterator) Close() { i.Transaction.Discard() } -// Helper fucntions +// Helper functions // dbGet helper functin func dbDel(txn *badger.Txn, keys ...[]byte) (err error) { @@ -396,7 +399,7 @@ func dbGet(txn *badger.Txn, k []byte) (val []byte, err error) { return } val, err = item.Value() - mlog.Trace("dbGet read %s v:%d ", k, item.Version) + mlog.Trace("dbGet read %s v:%d ", k, item.Version()) return } @@ -406,7 +409,7 @@ func dbGetInt64(txn *badger.Txn, k []byte) (i int64) { return } val, err := item.Value() - mlog.Trace("dbGetInt64 read %s v:%d ", k, item.Version) + mlog.Trace("dbGetInt64 read %s v:%d ", k, item.Version()) if err != nil { val = numberZero } diff --git a/internal/distill/endpoints.go b/web/endpoints.go similarity index 81% rename from internal/distill/endpoints.go rename to web/endpoints.go index 1bd83dc..9b9df91 100644 --- a/internal/distill/endpoints.go +++ b/web/endpoints.go @@ -1,4 +1,5 @@ -package distill +// Package web provides the api endpoints for the urstore +package web import ( "crypto/subtle" @@ -7,7 +8,8 @@ import ( "time" "github.com/jbrodriguez/mlog" - "gitlab.com/welance/oss/distill/internal" + + "github.com/noandrea/distill/urlstore" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" @@ -34,10 +36,10 @@ func RegisterEndpoints() (router *chi.Mux) { router.Mount("/profile", middleware.Profiler()) // health check route - router.Get("/health-check", healthCheckHanlder) + router.Get("/health-check", healthCheckHandler) // redirect root to the configured url router.Get("/", func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, internal.Config.Server.RootRedirect, 302) + http.Redirect(w, r, urlstore.Config.Server.RootRedirect, 302) }) // shortener redirect router.Get("/{ID}", handleGetURL) @@ -46,41 +48,25 @@ func RegisterEndpoints() (router *chi.Mux) { r.Use(apiContext) // handle global statistics r.Get("/stats", func(w http.ResponseWriter, r *http.Request) { - render.JSON(w, r, *GetStats()) + render.JSON(w, r, *urlstore.GetStats()) }) r.Delete("/stats", func(w http.ResponseWriter, r *http.Request) { - err := ResetStats() + err := urlstore.ResetStats() if err != nil { render.Render(w, r, ErrInternalError(err, err.Error())) return } - render.JSON(w, r, GetStats()) + render.JSON(w, r, urlstore.GetStats()) }) // handle url statistics - r.Get("/stats/{ID}", func(w http.ResponseWriter, r *http.Request) { - shortID := chi.URLParam(r, "ID") - if urlInfo, err := GetURLInfo(shortID); err == nil { - // send redirect - render.JSON(w, r, urlInfo) - return - } - http.Error(w, "URL not found", 404) - }) + r.Get("/stats/{ID}", handleStatsURL) // handle url setup r.Post("/short", handleShort) // implement kutt.it endpoint r.Post("/url/submit", handleShort) - // delete an id - r.Delete("/short/{ID}", func(w http.ResponseWriter, r *http.Request) { - shortID := chi.URLParam(r, "ID") - err := DeleteURL(shortID) - if err != nil { - render.Render(w, r, ErrNotFound(err, "URL id not found")) - return - } - render.JSON(w, r, ShortID{ID: shortID}) - }) + r.Delete("/short/{ID}", handleDeleteURL) + // backup }) return router @@ -95,7 +81,7 @@ func RegisterEndpoints() (router *chi.Mux) { // func handleShort(w http.ResponseWriter, r *http.Request) { - urlReq := &URLReq{} + urlReq := &urlstore.URLReq{} if err := render.Bind(r, urlReq); err != nil { render.Render(w, r, ErrInvalidRequest(err, err.Error())) return @@ -111,19 +97,19 @@ func handleShort(w http.ResponseWriter, r *http.Request) { forceLenght = true } // upsert the data - id, err := UpsertURL(urlReq, forceAlphabet, forceLenght, time.Now()) + id, err := urlstore.UpsertURL(urlReq, forceAlphabet, forceLenght, time.Now()) mlog.Trace("creted %v", id) // TODO: check the actual error if err != nil { render.Render(w, r, ErrInvalidRequest(err, err.Error())) return } - render.JSON(w, r, ShortID{ID: id}) + render.JSON(w, r, urlstore.ShortID{ID: id}) } func handleGetURL(w http.ResponseWriter, r *http.Request) { shortID := chi.URLParam(r, "ID") - targetURL, err := GetURLRedirect(shortID) + targetURL, err := urlstore.GetURLRedirect(shortID) if err != nil { http.Error(w, "URL not found", 404) } @@ -132,10 +118,30 @@ func handleGetURL(w http.ResponseWriter, r *http.Request) { return } -func healthCheckHanlder(w http.ResponseWriter, r *http.Request) { +func healthCheckHandler(w http.ResponseWriter, r *http.Request) { w.Write([]byte("good")) } +func handleStatsURL(w http.ResponseWriter, r *http.Request) { + shortID := chi.URLParam(r, "ID") + if urlInfo, err := urlstore.GetURLInfo(shortID); err == nil { + // send redirect + render.JSON(w, r, urlInfo) + return + } + http.Error(w, "URL not found", 404) +} + +func handleDeleteURL(w http.ResponseWriter, r *http.Request) { + shortID := chi.URLParam(r, "ID") + err := urlstore.DeleteURL(shortID) + if err != nil { + render.Render(w, r, ErrNotFound(err, "URL id not found")) + return + } + render.JSON(w, r, urlstore.ShortID{ID: shortID}) +} + // ____ ____ ______ ______ ______ // |_ \ / _|.' ____ \ .' ___ | .' ____ \ // | \/ | | (___ \_|/ .' \_| | (___ \_| @@ -144,16 +150,6 @@ func healthCheckHanlder(w http.ResponseWriter, r *http.Request) { // |_____||_____| \______.' `._____.' \______.' // -// Bind will run after the unmarshalling is complete -func (u *URLReq) Bind(r *http.Request) error { - return nil -} - -// Bind will run after the unmarshalling is complete -func (u *ShortID) Bind(r *http.Request) error { - return nil -} - // ErrInvalidRequest render an invalid request func ErrInvalidRequest(err error, message string) render.Renderer { return &ErrResponse{ @@ -214,8 +210,8 @@ type ErrResponse struct { // apiContext verify the api key header func apiContext(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - apiKey := r.Header.Get(internal.Config.Tuning.APIKeyHeaderName) - keyMatch := subtle.ConstantTimeCompare([]byte(apiKey), []byte(internal.Config.Server.APIKey)) + apiKey := r.Header.Get(urlstore.Config.Tuning.APIKeyHeaderName) + keyMatch := subtle.ConstantTimeCompare([]byte(apiKey), []byte(urlstore.Config.Server.APIKey)) if keyMatch == 0 { http.Error(w, http.StatusText(403), 403) return diff --git a/internal/distill/endpoints_test.go b/web/endpoints_test.go similarity index 85% rename from internal/distill/endpoints_test.go rename to web/endpoints_test.go index ae72ca9..405a6bc 100644 --- a/internal/distill/endpoints_test.go +++ b/web/endpoints_test.go @@ -1,4 +1,4 @@ -package distill +package web import ( "net/http" @@ -12,13 +12,13 @@ func TestRegisterEndpoints(t *testing.T) { name string route string method string - hanlder func(w http.ResponseWriter, r *http.Request) + handler func(w http.ResponseWriter, r *http.Request) }{ { "hc", "/health-check", "GET", - healthCheckHanlder, + healthCheckHandler, }, } @@ -31,7 +31,7 @@ func TestRegisterEndpoints(t *testing.T) { } rr := httptest.NewRecorder() - handler := http.HandlerFunc(tt.hanlder) + handler := http.HandlerFunc(tt.handler) // Our handlers satisfy http.Handler, so we can call their ServeHTTP method // directly and pass in our Request and ResponseRecorder.