Skip to content

Commit

Permalink
Add go-sqlite3-stdlib functions (#59)
Browse files Browse the repository at this point in the history
* Bump runner version

* Add mode test

Just embed taxi.csv

7zip

* 7z steps
  • Loading branch information
eatonphil committed May 14, 2022
1 parent 094b8c0 commit 4ca4a31
Show file tree
Hide file tree
Showing 9 changed files with 48 additions and 26 deletions.
8 changes: 4 additions & 4 deletions go.mod
Expand Up @@ -6,8 +6,9 @@ go 1.18
// replace github.com/multiprocessio/datastation/runner => ../datastation/runner

require (
github.com/chzyer/readline v1.5.0
github.com/google/uuid v1.3.0
github.com/multiprocessio/datastation/runner v0.0.0-20220420211013-6135c59970ef
github.com/multiprocessio/datastation/runner v0.0.0-20220514140833-05821ae05f52
github.com/olekukonko/tablewriter v0.0.5
)

Expand Down Expand Up @@ -35,10 +36,8 @@ require (
github.com/aws/smithy-go v1.9.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chzyer/readline v1.5.0 // indirect
github.com/deepmap/oapi-codegen v1.8.2 // indirect
github.com/denisenkom/go-mssqldb v0.12.0 // indirect
github.com/elastic/go-elasticsearch/v6 v6.8.10 // indirect
github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 // indirect
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
github.com/gabriel-vasile/mimetype v1.4.0 // indirect
Expand Down Expand Up @@ -66,13 +65,14 @@ require (
github.com/matoous/go-nanoid/v2 v2.0.0 // indirect
github.com/mattn/go-ieproxy v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/mattn/go-sqlite3 v1.14.12 // indirect
github.com/mattn/go-sqlite3 v1.14.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e // indirect
github.com/multiprocessio/go-openoffice v0.0.0-20220110232726-064f5dda1956 // indirect
github.com/multiprocessio/go-sqlite3-stdlib v0.0.0-20220513170836-f1c26f349467 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/neo4j/neo4j-go-driver/v4 v4.4.2 // indirect
github.com/paulmach/orb v0.4.0 // indirect
Expand Down
14 changes: 8 additions & 6 deletions go.sum
Expand Up @@ -178,11 +178,13 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/logex v1.2.0 h1:+eqR0HfOetur4tgnC8ftU5imRnhi4te+BadWS95c5AM=
github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/readline v1.5.0 h1:lSwwFrbNviGePhkewF1az4oLmcwqCZijQ2/Wi3BGHAI=
github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23 h1:dZ0/VyGgQdVGAss6Ju0dt5P0QltE0SFY5Woh6hbIfiQ=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
Expand All @@ -205,8 +207,6 @@ github.com/denisenkom/go-mssqldb v0.12.0 h1:VtrkII767ttSPNRfFekePK3sctr+joXgO58s
github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/elastic/go-elasticsearch/v6 v6.8.10 h1:2lN0gJ93gMBXvkhwih5xquldszpm8FlUwqG5sPzr6a8=
github.com/elastic/go-elasticsearch/v6 v6.8.10/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
Expand Down Expand Up @@ -442,8 +442,8 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0=
github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I=
github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM=
Expand All @@ -457,12 +457,14 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/multiprocessio/datastation/runner v0.0.0-20220420211013-6135c59970ef h1:0hIkTTdmuENzEwnZFArTusOWoxou6Xkbsx5lgDftH/c=
github.com/multiprocessio/datastation/runner v0.0.0-20220420211013-6135c59970ef/go.mod h1:WcwUD6bH7yCftknbUJRzjRvX/4yTsVCby6XIJOhLsjM=
github.com/multiprocessio/datastation/runner v0.0.0-20220514140833-05821ae05f52 h1:fsoqK6tOGelY8QIGmc1j+vfya4O1X7D4VUnQndNNxHM=
github.com/multiprocessio/datastation/runner v0.0.0-20220514140833-05821ae05f52/go.mod h1:9xziND3qjo/BHhJdxmYGkvRMCjC65sij4u/4/vVu6Uw=
github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e h1:NlPl7amllnQyVAkZgjBvFEkKxJSba/R8ZpaTodc7SIQ=
github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e/go.mod h1:huI4M/MrI5px/SgmXYi0a2byKikSLgDrnMQuXOqKtw4=
github.com/multiprocessio/go-openoffice v0.0.0-20220110232726-064f5dda1956 h1:WVofL03Eq+z3LbDOfH5eKzu2U85LFZZngOMBlNaO/H0=
github.com/multiprocessio/go-openoffice v0.0.0-20220110232726-064f5dda1956/go.mod h1:JMhCobMOTCgIL4EwBPVrRB60Iq9hknRP9mX/MYiWHAc=
github.com/multiprocessio/go-sqlite3-stdlib v0.0.0-20220513170836-f1c26f349467 h1:KCF4uQe2+KU0tDHfXQN3KR82rfJWw7mTcBeMubrMZlg=
github.com/multiprocessio/go-sqlite3-stdlib v0.0.0-20220513170836-f1c26f349467/go.mod h1:UCZsRpQ3yrTCiZq7GFwmgvfuFGnlUz2USCg1gRCFU40=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
Expand Down
8 changes: 7 additions & 1 deletion main.go
@@ -1,6 +1,7 @@
package main

import (
"bufio"
"crypto/sha1"
"encoding/hex"
"encoding/json"
Expand Down Expand Up @@ -50,9 +51,12 @@ func evalFileInto(file, mimetype string, out *os.File) error {
return fmt.Errorf("Unknown mimetype for file: %s.\n", file)
}

w := bufio.NewWriterSize(out, 1e6)
defer w.Flush()

return runner.TransformFile(file, runner.ContentTypeInfo{
Type: mimetype,
}, out)
}, w)
}

func getShape(resultFile, panelId string) (*runner.Shape, error) {
Expand Down Expand Up @@ -260,6 +264,8 @@ func runQuery(queryRaw string, project *runner.ProjectState, ec *runner.EvalCont
}
return fmt.Errorf("Input is not an array of objects%s\n", rest)
}

return err
}

resultFile := ec.GetPanelResultsFile(project.Id, panel.Id)
Expand Down
3 changes: 3 additions & 0 deletions scripts/ci/prepare_linux.sh
Expand Up @@ -7,3 +7,6 @@ sudo rm -rf /usr/local/go
sudo tar -C /usr/local -xzf go1.18.linux-amd64.tar.gz
sudo ln -s /usr/local/go/bin/go /usr/local/bin/go
sudo ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt

sudo apt install -y p7zip
7z e testdata/taxi.csv.7z
5 changes: 5 additions & 0 deletions scripts/ci/prepare_macos.sh
Expand Up @@ -7,3 +7,8 @@ sudo rm -rf /usr/local/go
sudo tar -C /usr/local -xzf go1.18.darwin-amd64.tar.gz
sudo mv /usr/local/go/bin/go /usr/local/bin/go
sudo mv /usr/local/go/bin/gofmt /usr/local/bin/gofmt

# Install 7z
curl -LO https://github.com/jinfeihan57/p7zip/releases/download/v17.04/macos-10.15-p7zip.zip
unzip macos-10.15-p7zip.zip
./7z e testdata/taxi.csv.7z
6 changes: 4 additions & 2 deletions scripts/ci/prepare_windows.ps1
@@ -1,8 +1,10 @@
iwr -useb 'https://raw.githubusercontent.com/scoopinstaller/install/master/install.ps1' -outfile 'install.ps1'
.\install.ps1 -RunAsAdmin
Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH
scoop install jq zip curl
scoop install jq zip curl 7zip

curl -L -O "https://go.dev/dl/go1.18.windows-amd64.zip"
unzip go1.18.windows-amd64.zip
Join-Path $pwd "go\bin" >> $Env:GITHUB_PATH
Join-Path $pwd "go\bin" >> $Env:GITHUB_PATH

7z e testdata/taxi.csv.7z
19 changes: 6 additions & 13 deletions scripts/test.py
Expand Up @@ -281,7 +281,6 @@ def test(name, to_run, want, fail=False, sort=False, winSkip=False, within_secon
{"passenger_count":"9","COUNT(*)":1,"AVG(total_amount)":113.6}]"""
want_stderr = "Cache invalid, re-import required.\n"

cmd("curl https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2021-04.csv -o taxi.csv", doNotReplaceWin=True)
test("Caching from file (first time so import is required)", to_run, want, want_stderr=want_stderr, sort=True)

to_run = """
Expand All @@ -290,25 +289,19 @@ def test(name, to_run, want, fail=False, sort=False, winSkip=False, within_secon

test("Caching from pipe (second time so import not required)", to_run, want, sort=True, winSkip=True, within_seconds=5)

# Truncate to first 10 lines to test cache busting
with open('taxi.csv') as t:
lines = []
for line in t:
lines.append(line)
if len(lines) > 10:
break

with open('taxi.csv', 'w') as f:
f.write(''.join(lines))

to_run = """
cat taxi.csv | ./dsq --cache -s csv 'SELECT passenger_count, COUNT(*), AVG(total_amount) FROM {} GROUP BY passenger_count ORDER BY COUNT(*) DESC'"""
cat testdata/taxi_trunc.csv | ./dsq --cache -s csv 'SELECT passenger_count, COUNT(*), AVG(total_amount) FROM {} GROUP BY passenger_count ORDER BY COUNT(*) DESC'"""
want = """[{"COUNT(*)":9,"AVG(total_amount)":20.571111111111115,"passenger_count":"1"},
{"passenger_count":"0","COUNT(*)":1,"AVG(total_amount)":43.67}]"""
want_stderr = "Cache invalid, re-import required.\n"

test("Re-imports when file changes with cache on", to_run, want, want_stderr=want_stderr, sort=True, winSkip=True)

# Mode support
to_run = """./dsq testdata/userdata.json 'SELECT mode(Activated) mostly_activated FROM {}' """
want = '[{"mostly_activated":1}]'
test("Mode support", to_run, want=want)

# END OF TESTS

# START OF REGRESSION TESTS
Expand Down
Binary file added testdata/taxi.csv.7z
Binary file not shown.
11 changes: 11 additions & 0 deletions testdata/taxi_trunc.csv
@@ -0,0 +1,11 @@
VendorID,tpep_pickup_datetime,tpep_dropoff_datetime,passenger_count,trip_distance,RatecodeID,store_and_fwd_flag,PULocationID,DOLocationID,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount,congestion_surcharge
1,2021-04-01 00:00:18,2021-04-01 00:21:54,1,8.40,1,N,79,116,1,25.5,3,0.5,5.85,0,0.3,35.15,2.5
1,2021-04-01 00:42:37,2021-04-01 00:46:23,1,.90,1,N,75,236,2,5,3,0.5,0,0,0.3,8.8,2.5
1,2021-04-01 00:57:56,2021-04-01 01:08:22,1,3.40,1,N,236,168,2,11.5,3,0.5,0,0,0.3,15.3,2.5
1,2021-04-01 00:01:58,2021-04-01 00:54:27,1,.00,1,N,47,61,1,44.2,0,0.5,0,0,0.3,45,0
2,2021-04-01 00:24:55,2021-04-01 00:34:33,1,1.96,1,N,238,152,1,9,0.5,0.5,3.09,0,0.3,13.39,0
2,2021-04-01 00:19:16,2021-04-01 00:21:46,1,.77,1,N,142,238,1,4.5,0.5,0.5,1.24,0,0.3,9.54,2.5
2,2021-04-01 00:25:11,2021-04-01 00:31:53,1,3.65,1,N,238,244,1,11.5,0.5,0.5,2.56,0,0.3,15.36,0
1,2021-04-01 00:27:53,2021-04-01 00:47:03,0,8.90,1,N,138,239,1,26.5,3,0.5,7.25,6.12,0.3,43.67,2.5
2,2021-04-01 00:24:24,2021-04-01 00:37:50,1,2.98,1,N,151,244,2,12,0.5,0.5,0,0,0.3,13.3,0
1,2021-04-01 00:19:18,2021-04-01 00:41:25,1,8.90,1,N,132,196,2,28,0.5,0.5,0,0,0.3,29.3,0

0 comments on commit 4ca4a31

Please sign in to comment.