Skip to content

Commit

Permalink
feature: add json & query param providers
Browse files Browse the repository at this point in the history
  • Loading branch information
mdshack committed Aug 7, 2023
1 parent 0a5aa04 commit 456fd65
Show file tree
Hide file tree
Showing 9 changed files with 91 additions and 143 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go:
- 1.14
- 1.15

go_import_path: github.com/mpchadwick/dbanon
go_import_path: github.com/MicahDShackelford/dbanon

before_script:
- curl -sfL https://raw.githubusercontent.com/knqyf263/cob/master/install.sh | sudo sh -s -- -b /usr/local/bin
Expand Down
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ build:
go install github.com/shuLhan/go-bindata/v4/cmd/go-bindata@master
$$GOPATH/bin/go-bindata -pkg bindata -o bindata/bindata.go etc/*
go get ./...
go test -coverprofile=coverage.out -covermode=atomic -race $$GOPATH/src/github.com/mpchadwick/dbanon/src
go test -coverprofile=coverage.out -covermode=atomic -race $$GOPATH/src/github.com/MicahDShackelford/dbanon/src
go build -ldflags "$(LDFLAGS)" -o dbanon main.go
go test $$GOPATH/src/github.com/mpchadwick/dbanon/integration
go test $$GOPATH/src/github.com/MicahDShackelford/dbanon/integration
rm -rf bindata

bench:
$$GOPATH/bin/go-bindata -pkg bindata -o bindata/bindata.go etc/*
go test -run=XXX -bench=. -benchtime=20s $$GOPATH/src/github.com/mpchadwick/dbanon/src
go test -run=XXX -bench=. -benchtime=20s $$GOPATH/src/github.com/MicahDShackelford/dbanon/src
rm -rf bindata
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
# dbanon

[![Build Status](https://github.com/mpchadwick/dbanon/actions/workflows/ci.yml/badge.svg)](https://github.com/mpchadwick/dbanon/actions) [![codecov](https://codecov.io/gh/mpchadwick/dbanon/branch/master/graph/badge.svg)](https://codecov.io/gh/mpchadwick/dbanon)
[![Build Status](https://github.com/MicahDShackelford/dbanon/actions/workflows/ci.yml/badge.svg)](https://github.com/MicahDShackelford/dbanon/actions) [![codecov](https://codecov.io/gh/mpchadwick/dbanon/branch/master/graph/badge.svg)](https://codecov.io/gh/mpchadwick/dbanon)

A run-anywhere, dependency-less database anonymizer.

## Installation

Download [the latest release from GitHub](https://github.com/mpchadwick/dbanon/releases).
Download [the latest release from GitHub](https://github.com/MicahDShackelford/dbanon/releases).

## Usage

Expand Down Expand Up @@ -56,9 +56,9 @@ mysqldump mydb | dbanon -config=myconfig.yml | gzip > mydb.sql.gz

See [the `etc` directory](etc/) for examples.

Columns are specified as key / value pairs. The value string winds up getting passed to [this function](https://github.com/mpchadwick/dbanon/blob/ade634a10bc282c06fecef115afbdd6661a94277/src/provider.go#L36), which gets random values from [`dmgk/faker`](https://github.com/dmgk/faker).
Columns are specified as key / value pairs. The value string winds up getting passed to [this function](https://github.com/MicahDShackelford/dbanon/blob/ade634a10bc282c06fecef115afbdd6661a94277/src/provider.go#L36), which gets random values from [`dmgk/faker`](https://github.com/dmgk/faker).

It is also possible to pass direct Faker function calls for [supported "raw providers"](https://github.com/mpchadwick/dbanon/blob/ade634a10bc282c06fecef115afbdd6661a94277/src/provider.go#L13-L17)
It is also possible to pass direct Faker function calls for [supported "raw providers"](https://github.com/MicahDShackelford/dbanon/blob/ade634a10bc282c06fecef115afbdd6661a94277/src/provider.go#L13-L17)

## Logging

Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
module github.com/mpchadwick/dbanon
module github.com/MicahDShackelford/dbanon

go 1.12

Expand All @@ -11,6 +11,7 @@ require (
github.com/blang/semver v3.5.1+incompatible
github.com/blastrain/vitess-sqlparser v0.0.0-20200914074247-af18b79da035
github.com/golang/glog v1.0.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
github.com/prometheus/client_golang v1.12.2 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,8 @@ github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ
github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE=
github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
github.com/go-chi/chi v1.5.0/go.mod h1:REp24E+25iKvxgeTfHmdUoL5x15kBiDBlnIl5bCwe2k=
github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
Expand Down
2 changes: 1 addition & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"flag"
"fmt"
"github.com/blang/semver"
"github.com/mpchadwick/dbanon/src"
"github.com/MicahDShackelford/dbanon/src"
"github.com/rhysd/go-github-selfupdate/selfupdate"
"github.com/sirupsen/logrus"
"io/ioutil"
Expand Down
2 changes: 1 addition & 1 deletion src/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package dbanon

import (
"errors"
"github.com/mpchadwick/dbanon/bindata"
"github.com/MicahDShackelford/dbanon/bindata"
"gopkg.in/yaml.v2"
"io/ioutil"
"strings"
Expand Down
4 changes: 2 additions & 2 deletions src/processor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@ func TestProcessLine(t *testing.T) {
// todod: Not really expecting to receive input like this, but there are several reports
// that the sqlparser.Parse is choking. Update this when we receive an example on
// one of these GitHub issues, or on any other new issue:
// https://github.com/mpchadwick/dbanon/issues/18
// https://github.com/mpchadwick/dbanon/issues/17
// https://github.com/MicahDShackelford/dbanon/issues/18
// https://github.com/MicahDShackelford/dbanon/issues/17
r3aInput := "INSERT INTO this is not valid sql"
r3aResult := processor.ProcessLine(r3aInput)
if r3aResult != r3aInput {
Expand Down
205 changes: 75 additions & 130 deletions src/provider.go
Original file line number Diff line number Diff line change
@@ -1,153 +1,98 @@
package dbanon

import (
"reflect"
"strconv"
"strings"
"math/rand"
"syreclabs.com/go/faker"
"github.com/google/uuid"
)

var fakeEmail = faker.Internet().Email

var rawProviders = map[string]interface{}{
"Lorem()": faker.Lorem(),
"Internet()": faker.Internet(),
"Commerce()": faker.Commerce(),
"Code()": faker.Code(),
"Number()": faker.Number(),
}

type Provider struct {
providedUniqueEmails map[string]int
}
type Provider struct {}

func NewProvider() *Provider {
made := make(map[string]int)

p := &Provider{providedUniqueEmails: made}
p := &Provider{}

return p
}

type ProviderInterface interface {
Get(s string) string
Get(fakeType string) string
}

func (p Provider) Get(s string) string {
i := strings.Index(s, "faker.")
if i == 0 {
return p.raw(s)
}

switch s {
case "firstname":
return faker.Name().FirstName()
case "lastname":
return faker.Name().LastName()
case "fullname":
return faker.Name().FirstName() + " " + faker.Name().LastName()
case "email":
return faker.Internet().Email()
case "unique_email":
new := fakeEmail()

_, exists := p.providedUniqueEmails[new]
if !exists {
p.providedUniqueEmails[new] = 0
return new
}

p.providedUniqueEmails[new]++
ret := strings.Join([]string{strconv.Itoa(p.providedUniqueEmails[new]), new}, "")

return ret
case "username":
return faker.Internet().UserName()
case "password":
return faker.Internet().Password(8, 14)
case "datetime":
return faker.Date().Birthday(0, 40).Format("2006-01-02 15:04:05")
case "customer_suffix":
return faker.Name().Suffix()
case "ipv4":
return faker.Internet().IpV4Address()
case "state":
return faker.Address().State()
case "city":
return faker.Address().City()
case "postcode":
return faker.Address().Postcode()
case "street":
return faker.Address().StreetAddress()
case "telephone":
return faker.PhoneNumber().PhoneNumber()
case "title":
return faker.Name().Prefix()
case "company":
return faker.Company().Name()
case "md5":
return faker.Lorem().Characters(32)
case "note255":
return faker.Lorem().Characters(50)
case "region_id":
// https://github.com/meanbee/magedbm2/blob/fc8bbf9a97db2c27d0cd8a1153dda8c95b6f5996/src/Anonymizer/Formatter/Address/RegionId.php#L24
return faker.Number().Between(1, 550)
case "gender":
// https://github.com/meanbee/magedbm2/blob/fc8bbf9a97db2c27d0cd8a1153dda8c95b6f5996/src/Anonymizer/Formatter/Person/Gender.php#L20
return faker.Number().Between(1, 3)
case "country_code":
return faker.Address().CountryCode()
case "shipment_tracking_number":
return faker.RandomString(18)
case "vat_number":
// https://github.com/meanbee/magedbm2/blob/fc8bbf9a97db2c27d0cd8a1153dda8c95b6f5996/src/Anonymizer/Formatter/Company/VatNumber.php#L21
return "GB" + faker.Number().Between(100000000, 999999999)
func (p Provider) Get(fakeType string) string {
switch fakeType {
// Name
case "first_name":
return faker.Name().FirstName()
case "last_name":
return faker.Name().LastName()
case "full_name":
return faker.Name().FirstName() + " " + faker.Name().LastName()

// Company
case "company_name":
return faker.Company().Name()

// Internet
case "username":
return faker.Internet().UserName()
case "password":
return faker.Internet().Password(8, 14)
case "ipv4":
return faker.Internet().IpV4Address()
case "url":
return faker.Internet().Url()
case "linkedin_url":
return "https://linkedin.com/in/" + faker.Internet().UserName()
case "md5":
return faker.Lorem().Characters(32)
case "uuid":
return uuid.NewString()
case "json":
return p.fakeJson()
case "query_params":
return p.fakeQueryParams()

// Dates
case "datetime":
return faker.Date().Birthday(0, 40).Format("2006-01-02 15:04:05")

// Geo
case "state":
return faker.Address().State()
case "city":
return faker.Address().City()
case "postcode":
return faker.Address().Postcode()
case "street":
return faker.Address().StreetAddress()
case "country_code":
return faker.Address().CountryCode()

// Currency
case "money":
numberOfDigits := rand.Intn(5 - 1) + 1
return faker.Number().Number(numberOfDigits)
case "money_decimal":
numberOfDigits := rand.Intn(5 - 1) + 1
return faker.Number().Decimal(numberOfDigits, 2)

// Contact
case "email":
return faker.Internet().Email()
case "telephone":
return faker.PhoneNumber().PhoneNumber()
}

logger := GetLogger()
logger.Error(s + " does not match any known type")
logger.Error(fakeType + " does not match any known type")

return ""
}

func (p Provider) raw(s string) string {
logger := GetLogger()
parts := strings.Split(s, ".")

className, ok := rawProviders[parts[1]]
if !ok {
logger.Error(parts[1] + " is not a supported provider")
return ""
}

class := reflect.ValueOf(className)
methodName := strings.Split(parts[2], "(")[0]
method := class.MethodByName(methodName)
if !method.IsValid() {
logger.Error(parts[2] + " is not a valid method")
return ""
}

argsStart := strings.Index(parts[2], "(")
argsEnd := strings.Index(parts[2], ")")
if argsStart == -1 || argsEnd == -1 || argsEnd < argsStart {
logger.Error("Could not identify arguments for " + parts[2])
return ""
}

args := parts[2][argsStart+1 : argsEnd]
if args == "" {
out := method.Call(nil)
return out[0].String()
} else {
argsArray := strings.Split(args, ",")
in := make([]reflect.Value, len(argsArray))
for i, param := range argsArray {
intVal, _ := strconv.Atoi(strings.Replace(param, " ", "", -1))
in[i] = reflect.ValueOf(intVal)
}

out := method.Call(in)
return out[0].String()
}
func (p Provider) fakeJson() string {
return "{\"fake\": true}"
}

func (p Provider) fakeQueryParams() string {
return "?fake=true"
}

0 comments on commit 456fd65

Please sign in to comment.