From 9c3adc27c7cf2859466413e9075808e4688fc0a3 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Wed, 22 Dec 2021 00:35:31 -0500 Subject: [PATCH] Pack program info rather than reading JSON files dynamically in Go runner (#129) * Pack program info * Add basic integration tests for dsq * Add test script * Debug * Create results directory if not exists * Generate packed info for diff * Drop all tests but dsq * Run tests on win/mac/linux * No integration test version * With powershell * Switch go * Correct image * Uninstall 1.15 * Drop cat * Add ids incase * Add more comments * Dont fail if dir exists * Dont fail if dir exists * Debugging * Run windows test in bash * Rearrange tests * Try with verbose * More debugging * Try another pipe detection * Verbose again * Add back old tests * Add support for pipe fallback * one more whitespace * one more whitespace * one more whitespace * Fix for fmt --- .github/workflows/pull_requests.yml | 39 ++++++++++++++ .gitignore | 3 +- runner/cmd/dsq/main.go | 56 +++++++++++++++----- runner/cmd/dsq/scripts/test.sh | 25 +++++++++ runner/file.go | 2 + runner/program.go | 3 +- runner/program_info.go | 11 ++++ runner/scripts/generate_program_type_info.sh | 16 ++++++ 8 files changed, 140 insertions(+), 15 deletions(-) create mode 100755 runner/cmd/dsq/scripts/test.sh create mode 100644 runner/program_info.go create mode 100755 runner/scripts/generate_program_type_info.sh diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml index d752e6607..cb8a3e9ed 100644 --- a/.github/workflows/pull_requests.yml +++ b/.github/workflows/pull_requests.yml @@ -35,6 +35,7 @@ jobs: - run: yarn format - run: yarn tsc - run: cd runner && gofmt -w . + - run: ./runner/scripts/generate_program_type_info.sh - run: ./scripts/fail_on_diff.sh - run: yarn test-licenses # Needed so we can have ./build/desktop_runner.js and ./build/go_desktop_runner ready for tests @@ -166,3 +167,41 @@ jobs: - run: cd runner && go test -cover - run: yarn release-desktop 0.0.0-e2etest - run: yarn e2e-test + + dsq-tests-ubuntu: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@master + with: + ref: ${{ github.ref }} + + - run: ./scripts/ci/prepare_linux.sh + - run: cd runner && go build -o ../dsq cmd/dsq/main.go + - run: ./runner/cmd/dsq/scripts/test.sh + + dsq-tests-windows: + runs-on: windows-latest + + steps: + - uses: actions/checkout@master + with: + ref: ${{ github.ref }} + + - run: ./scripts/ci/prepare_windows.ps1 + shell: pwsh + - run: cd runner && go build -o ../dsq cmd/dsq/main.go + - run: ./runner/cmd/dsq/scripts/test.sh + shell: bash + + dsq-tests-macos: + runs-on: macos-latest + + steps: + - uses: actions/checkout@master + with: + ref: ${{ github.ref }} + + - run: ./scripts/ci/prepare_macos.sh + - run: cd runner && go build -o ../dsq cmd/dsq/main.go + - run: ./runner/cmd/dsq/scripts/test.sh diff --git a/.gitignore b/.gitignore index da4e5b329..cff8014d3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ TAGS all.out runner/report runner/runner -runner/runner.exe \ No newline at end of file +runner/runner.exe +dsq \ No newline at end of file diff --git a/runner/cmd/dsq/main.go b/runner/cmd/dsq/main.go index 6e300637a..fb645f1b1 100644 --- a/runner/cmd/dsq/main.go +++ b/runner/cmd/dsq/main.go @@ -5,16 +5,24 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "log" "os" "strings" "github.com/multiprocessio/datastation/runner" + + "github.com/google/uuid" ) func isinpipe() bool { fi, _ := os.Stdin.Stat() - return (fi.Mode() & os.ModeCharDevice) == 0 + if fi == nil { + return false + } + + // This comes back incorrect in automated environments like Github Actions. + return !(fi.Mode()&os.ModeNamedPipe == 0) } func resolveContentType(fileExtensionOrContentType string) string { @@ -31,20 +39,33 @@ func getResult(res interface{}) error { out := bytes.NewBuffer(nil) arg := firstNonFlagArg - var internalErr error - if isinpipe() { - mimetype := resolveContentType(arg) - if mimetype == "" { - return fmt.Errorf(`First argument when used in a pipe should be file extension or content type. e.g. 'cat test.csv | dsq csv "SELECT * FROM {}"'`) + mimetype := resolveContentType(arg) + if mimetype == "" { + return fmt.Errorf(`First argument when used in a pipe should be file extension or content type. e.g. 'cat test.csv | dsq csv "SELECT * FROM {}"'`) + } + + cti := runner.ContentTypeInfo{Type: mimetype} + + // isinpipe() is sometimes incorrect. If the first arg + // is a file, fall back to acting like this isn't in a + // pipe. + runAsFile := !isinpipe() + if !runAsFile && cti.Type == arg { + if _, err := os.Stat(arg); err == nil { + runAsFile = true } + } - cti := runner.ContentTypeInfo{Type: mimetype} - internalErr = runner.TransformReader(os.Stdin, "", cti, out) + if !runAsFile { + err := runner.TransformReader(os.Stdin, "", cti, out) + if err != nil { + return err + } } else { - internalErr = runner.TransformFile(arg, runner.ContentTypeInfo{}, out) - } - if internalErr != nil { - return internalErr + err := runner.TransformFile(arg, runner.ContentTypeInfo{}, out) + if err != nil { + return err + } } decoder := json.NewDecoder(out) @@ -97,8 +118,17 @@ func main() { ResultMeta: runner.PanelResult{ Shape: *shape, }, + Id: uuid.New().String(), + Name: uuid.New().String(), + } + + projectTmp, err := ioutil.TempFile("", "dsq-project") + if err != nil { + log.Fatal(err) } + defer os.Remove(projectTmp.Name()) project := &runner.ProjectState{ + Id: projectTmp.Name(), Pages: []runner.ProjectPage{ { Panels: []runner.PanelInfo{p0}, @@ -117,6 +147,8 @@ func main() { panel := &runner.PanelInfo{ Type: runner.DatabasePanel, Content: query, + Id: uuid.New().String(), + Name: uuid.New().String(), DatabasePanelInfo: &runner.DatabasePanelInfo{ Database: runner.DatabasePanelInfoDatabase{ ConnectorId: connector.Id, diff --git a/runner/cmd/dsq/scripts/test.sh b/runner/cmd/dsq/scripts/test.sh new file mode 100755 index 000000000..0372e815c --- /dev/null +++ b/runner/cmd/dsq/scripts/test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -e + +types="csv parquet json" + +for t in $types; do + echo "Testing $t (pipe)." + sqlcount="$(cat ./testdata/userdata.$t | ./dsq $t 'SELECT COUNT(1) AS c FROM {}' | jq '.[0].c')" + if [[ "$sqlcount" != "1000" ]]; then + echo "Bad SQL count for $t (pipe). Expected 1000, got $sqlcount." + exit 1 + else + echo "Pipe $t test successful." + fi + + echo "Testing $t (file)." + sqlcount="$(./dsq ./testdata/userdata.$t 'SELECT COUNT(1) AS c FROM {}' | jq '.[0].c')" + if [[ "$sqlcount" != "1000" ]]; then + echo "Bad SQL count for $t (file). Expected 1000, got $sqlcount." + exit 1 + else + echo "File $t test successful." + fi +done diff --git a/runner/file.go b/runner/file.go index b894e3d2c..634a54efc 100644 --- a/runner/file.go +++ b/runner/file.go @@ -69,6 +69,8 @@ func withJSONArrayOutWriter(w io.Writer, cb func(w *JSONArrayWriter) error) erro } func openTruncate(out string) (*os.File, error) { + base := filepath.Dir(out) + _ = os.Mkdir(base, os.ModePerm) return os.OpenFile(out, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, os.ModePerm) } diff --git a/runner/program.go b/runner/program.go index ea8b468c8..c39a197a5 100644 --- a/runner/program.go +++ b/runner/program.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "os/exec" - "path" "strings" "github.com/google/uuid" @@ -90,7 +89,7 @@ func (ec EvalContext) evalProgramPanel(project *ProjectState, pageIndex int, pan } var p ProgramEvalInfo - err := readJSONFileInto(path.Join("shared", "languages", string(panel.Program.Type)+".json"), &p) + err := json.Unmarshal([]byte(packedProgramTypeInfo[panel.Program.Type]), &p) if err != nil { return err } diff --git a/runner/program_info.go b/runner/program_info.go new file mode 100644 index 000000000..5e3f904f4 --- /dev/null +++ b/runner/program_info.go @@ -0,0 +1,11 @@ +package runner + +// GENERATED BY ./runner/scripts/generate_program_type_info.sh. DO NOT MODIFY. + +var packedProgramTypeInfo = map[SupportedLanguages]string{ + "javascript": `{"id":"javascript","preamble":"function DM_getPanelFile(i){ return '$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i];}\nfunction DM_getPanel(i) {\n const fs = require('fs');\n return JSON.parse(fs.readFileSync('$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i]));\n}\nfunction DM_setPanel(v) {\n const fs = require('fs');\n const fd = fs.openSync('$$PANEL_RESULTS_FILE$$', 'w');\n if (Array.isArray(v)) {\n fs.writeSync(fd, '[');\n for (let i = 0; i < v.length; i++) {\n const row = v[i];\n let rowJSON = JSON.stringify(row);\n if (i < v.length - 1) {\n rowJSON += ',';\n }\n fs.writeSync(fd, rowJSON);\n }\n fs.writeSync(fd, ']');\n } else {\n fs.writeSync(fd, JSON.stringify(v));\n }\n}","defaultPath":"node","name":"JavaScript"}`, + "julia": `{"id":"julia","name":"Julia","defaultPath":"julia","preamble":"\ntry\n import JSON\ncatch e\n import Pkg\n Pkg.add(\"JSON\")\n import JSON\nend\nfunction DM_getPanel(i)\n panelId = JSON.parse(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[string(i)]\n JSON.parsefile(string(\"$$RESULTS_FILE$$\", panelId))\nend\nfunction DM_setPanel(v)\n open(\"$$PANEL_RESULTS_FILE$$\", \"w\") do f\n JSON.print(f, v)\n end\nend\nfunction DM_getPanelFile(i)\n string(\"$$RESULTS_FILE$$\", JSON.parse(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[string(i)])\nend"}`, + "python": `{"id":"python","name":"Python","defaultPath":"python3","preamble":"def DM_getPanelFile(i):\n return r'$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[str(i)]\ndef DM_getPanel(i):\n import json\n with open(r'$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[str(i)]) as f:\n return json.load(f)\ndef DM_setPanel(v):\n import json\n with open(r'$$PANEL_RESULTS_FILE$$', 'w') as f:\n json.dump(v, f)"}`, + "r": `{"id":"r","name":"R","defaultPath":"Rscript","preamble":"\ntryCatch(library(\"rjson\"), error=function(cond) {install.packages(\"rjson\", repos=\"https://cloud.r-project.org\")}, finally=library(\"rjson\"))\nDM_getPanel <- function(i) {\n panelId = fromJSON(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[[toString(i)]]\n fromJSON(file=paste(\"$$RESULTS_FILE$$\", panelId, sep=\"\"))\n}\nDM_setPanel <- function(v) {\n write(toJSON(v), \"$$PANEL_RESULTS_FILE$$\")\n}\nDM_getPanelFile <- function(i) {\n paste(\"$$RESULTS_FILE$$\", fromJSON(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[[toString(i)]], sep=\"\")\n}\n"}`, + "ruby": `{"id":"ruby","name":"Ruby","defaultPath":"ruby","preamble":"\ndef DM_getPanel(i)\n require 'json'\n JSON.parse(File.read('$$RESULTS_FILE$$' + JSON.parse('$$JSON_ID_MAP$$')[i.to_s]))\nend\ndef DM_setPanel(v)\n require 'json'\n File.write('$$PANEL_RESULTS_FILE$$', v.to_json)\nend\ndef DM_getPanelFile(i)\n require 'json'\n '$$RESULTS_FILE$$' + JSON.parse('$$JSON_ID_MAP$$')[i.to_s]\nend\n"}`, +} diff --git a/runner/scripts/generate_program_type_info.sh b/runner/scripts/generate_program_type_info.sh new file mode 100755 index 000000000..b77202e01 --- /dev/null +++ b/runner/scripts/generate_program_type_info.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -e + +echo "package runner + +// GENERATED BY ./runner/scripts/generate_program_type_info.sh. DO NOT MODIFY. + +var packedProgramTypeInfo = map[SupportedLanguages]string{" > runner/program_info.go +for file in $(ls ./shared/languages/*.json); do + echo "$(cat $file | jq '.id'): \`$(cat $file | jq -c '.')\`," >> runner/program_info.go +done + +echo "}" >> runner/program_info.go + +gofmt -w -s .