Skip to content

Commit

Permalink
use copyin inline
Browse files Browse the repository at this point in the history
  • Loading branch information
pjdufour-truss committed Jun 7, 2019
1 parent d141533 commit 8633296
Show file tree
Hide file tree
Showing 14 changed files with 567 additions and 40 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ coverage.out
__snapshots__

# development
/mnt
/tmp

# Ignore
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ __snapshots__
/build

# development
/mnt
/tmp

# All compiled binaries
Expand Down
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,7 @@ endif

.PHONY: db_dev_start
db_dev_start: ## Start Dev DB
mkdir -p mnt/postgresql/data
ifndef CIRCLECI
brew services stop postgresql 2> /dev/null || true
endif
Expand All @@ -465,6 +466,7 @@ endif
-e POSTGRES_PASSWORD=$(PGPASSWORD) \
-p $(DB_PORT_DEV):$(DB_PORT_DOCKER)\
--shm-size=256MB \
-v $(abspath mnt/postgresql/data):/var/lib/postgresql/data \
$(DB_DOCKER_CONTAINER_IMAGE) \
-c "shared_buffers=256MB" \
-c "work_mem=16MB" \
Expand Down
76 changes: 56 additions & 20 deletions pkg/migrate/Builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,12 @@ func (b *Builder) Compile(s3Client *s3.S3) (*pop.Migration, error) {
m.Path = b.path
m.Runner = func(m pop.Migration, tx *pop.Connection) error {

defer func() {
if r := recover(); r != nil {
fmt.Fprintln(os.Stderr, "recoverd:", r)
}
}()

f, err := os.Open(b.path[len("file://"):])
if err != nil {
return err
Expand All @@ -111,18 +117,30 @@ func (b *Builder) Compile(s3Client *s3.S3) (*pop.Migration, error) {
inputReader = strings.NewReader(content)
}

lines := make(chan string, 100000)
go CleanMigraton(inputReader, lines)

statements := make(chan string, 100000)
go SplitStatements(lines, statements)
errExec := Exec(inputReader, tx)
if errExec != nil {
fmt.Fprintln(os.Stderr, errors.Wrapf(errExec, "error executing %s", m.Path).Error())
return errors.Wrapf(errExec, "error executing %s", m.Path)
}

for stmt := range statements {
err = tx.RawQuery(stmt).Exec()
if err != nil {
return errors.Wrapf(err, "error executing %s, sql: %s", m.Path, stmt)
/*
//lines := make(chan string, 100000)
//go CleanMigraton(inputReader, lines)
//statements := make(chan string, 100000)
//go SplitStatements(lines, statements)
for stmt := range statements {
//fmt.Println(fmt.Sprintf("Type %T", tx.Store))
//_, errExec := tx.Store.Tx.Tx.Exec(stmt)
//_, errExec := tx.Store.Exec(stmt)
errExec := tx.RawQuery(stmt).Exec()
//_, errExec := tx.Store.Exec(stmt)
if errExec != nil {
return errors.Wrapf(errExec, "error executing %s, sql: %s", m.Path, stmt)
}
}
}
*/

return nil
}
Expand Down Expand Up @@ -162,18 +180,36 @@ func (b *Builder) Compile(s3Client *s3.S3) (*pop.Migration, error) {
inputReader = strings.NewReader(content)
}

lines := make(chan string, 100000)
go CleanMigraton(inputReader, lines)

statements := make(chan string, 100000)
go SplitStatements(lines, statements)
errExec := Exec(inputReader, tx)
if errExec != nil {
return errors.Wrapf(errExec, "error executing %s", m.Path)
}

for stmt := range statements {
errExec := tx.RawQuery(stmt).Exec()
if errExec != nil {
return errors.Wrapf(errExec, "error executing %s, sql: %s", m.Path, stmt)
/*
lines := make(chan string, 100000)
go CleanMigraton(inputReader, lines)
statements := make(chan string, 100000)
go SplitStatements(lines, statements)
i := 0
for stmt := range statements {
//fmt.Println(fmt.Sprintf("Type %T", tx.Store))
//_, errExec := tx.Store.Tx.Tx.Exec(stmt)
_, errExec := tx.Store.Exec(stmt)
//errExec := tx.RawQuery(stmt).Exec()
if errExec != nil {
return errors.Wrapf(errExec, "error executing %s, sql: %s", m.Path, stmt)
}
i++
if i % 100 == 0 {
fmt.Println("Statements Executed:", i)
}
}
}
*/

return nil
}
return m, nil
Expand Down
Loading

0 comments on commit 8633296

Please sign in to comment.