diff --git a/desktop/panel/http.test.js b/desktop/panel/http.test.js index 7c2bec222..215f7fb99 100644 --- a/desktop/panel/http.test.js +++ b/desktop/panel/http.test.js @@ -32,30 +32,36 @@ const baseline = JSON.parse( const USERDATA_FILES = ['json', 'xlsx', 'csv', 'parquet', 'jsonl', 'cjson']; const PORT = '9799'; +const PORT2 = '9798'; -let server; +let server, server2; // Kill the existing server if it wasn't killed correctly already. beforeAll(async () => { // TODO: port this logic to other platforms... - if (process.platform === 'linux') { - try { - cp.execSync( - `bash -c "ps aux | grep 'http.server ${PORT}' | grep -v grep | awk '{print \\$2}' | xargs -I {} kill {}"` - ); - } catch (e) { - console.error(e); + for (const port of [PORT, PORT2]) { + if (process.platform === 'linux') { + try { + cp.execSync( + `bash -c "ps aux | grep 'http.server ${port}' | grep -v grep | awk '{print \\$2}' | xargs -I {} kill {}"` + ); + } catch (e) { + console.error(e); + } } } // Start a new server for all tests server = spawn('python3', ['-m', 'http.server', PORT]); + server2 = spawn('httpmirror', [PORT2]); - server.stdout.on('data', (data) => { - console.log(data.toString()); - }); + [server, server2].forEach((server) => { + server.stdout.on('data', (data) => { + console.log(data.toString()); + }); - server.stderr.on('data', (data) => { - console.warn(data.toString()); + server.stderr.on('data', (data) => { + console.warn(data.toString()); + }); }); // Keep trying to connect to the server until it's ready @@ -240,7 +246,7 @@ for (const subprocessName of RUNNERS) { }); describe('http with macro', () => { - test('correct result', () => { + test('macro as url', () => { const lp = new LiteralPanelInfo({ contentTypeInfo: { type: 'text/plain' }, content: '/testdata/allformats/unknown', @@ -269,6 +275,40 @@ for (const subprocessName of RUNNERS) { { evalPanels: true, subprocessName } ); }); + + test('macro as body', () => { + const lp = new LiteralPanelInfo({ + contentTypeInfo: { type: 'application/json' }, + content: '[{"name": "jim", "age": 12}]', + name: 'Raw Data', + }); + + const hp = new HTTPPanelInfo( + '', + new HTTPConnectorInfo('', 'http://localhost:9798') + ); + hp.http.http.method = 'POST'; + hp.content = '{{ DM_getPanel("0") | json }}'; + hp.http.http.contentTypeInfo.type = 'text/plain'; + + const panels = [lp, hp]; + + return withSavedPanels( + panels, + (project) => { + // Grab result + const value = JSON.parse( + fs + .readFileSync(getProjectResultsFile(project.projectName) + hp.id) + .toString() + ); + expect([{ name: 'jim', age: 12 }]).toStrictEqual( + JSON.parse(value.split('\r\n\r\n')[1]) + ); + }, + { evalPanels: true, subprocessName } + ); + }); }); if (process.platform === 'linux') { @@ -313,4 +353,5 @@ for (const subprocessName of RUNNERS) { afterAll(() => { process.kill(server.pid); + process.kill(server2.pid); }); diff --git a/runner/database_athena.go b/runner/database_athena.go index e55788307..9b8819313 100644 --- a/runner/database_athena.go +++ b/runner/database_athena.go @@ -8,6 +8,7 @@ import ( "github.com/multiprocessio/go-json" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/athena" ) @@ -35,7 +36,14 @@ func mapAthenaType(value, t string) interface{} { } func evalAthena(panel *PanelInfo, dbInfo DatabaseConnectorInfoDatabase, w io.Writer) error { + secret, err := dbInfo.Password.decrypt() + if err != nil { + return err + } + cfg := aws.NewConfig().WithRegion(dbInfo.Extra["aws_region"]) + cfg.Credentials = credentials.NewStaticCredentials(dbInfo.Username, secret, "") + sess := session.Must(session.NewSession(cfg)) svc := athena.New(sess, cfg) diff --git a/runner/file.go b/runner/file.go index 71f07d15a..f85521ac4 100644 --- a/runner/file.go +++ b/runner/file.go @@ -317,8 +317,6 @@ func transformGeneric(in io.Reader, out io.Writer) error { return err } - var prev byte = ' ' - for { b, err := r.ReadByte() if err == io.EOF { @@ -329,19 +327,29 @@ func transformGeneric(in io.Reader, out io.Writer) error { return err } - if b == '"' && prev != '\\' { - err = o.WriteByte('\\') - if err != nil { - return err - } + // Escape necessary characters + switch b { + case '\b': + _, err = o.WriteString(`\b`) + case '\f': + _, err = o.WriteString(`\f`) + case '\n': + _, err = o.WriteString(`\n`) + case '\r': + _, err = o.WriteString(`\r`) + case '\t': + _, err = o.WriteString(`\t`) + case '"': + _, err = o.WriteString(`\"`) + case '\\': + _, err = o.WriteString(`\\`) + default: + err = o.WriteByte(b) } - err = o.WriteByte(b) if err != nil { return err } - - prev = b } err = o.WriteByte('"') diff --git a/runner/file_test.go b/runner/file_test.go index 109705924..c47cf446e 100644 --- a/runner/file_test.go +++ b/runner/file_test.go @@ -192,18 +192,11 @@ func Test_transformORCFile(t *testing.T) { } func Test_transformGeneric(t *testing.T) { - tests := []struct { - in string - out interface{} - }{ - { - in: `abcdef`, - out: `abcdef`, - }, - { - in: `ab""cdef`, - out: `ab""cdef`, - }, + tests := []string{ + `abcdef`, + `ab""cdef`, + `ab +cdef`, } for _, test := range tests { @@ -211,7 +204,7 @@ func Test_transformGeneric(t *testing.T) { defer os.Remove(inTmp.Name()) assert.Nil(t, err) - inTmp.WriteString(test.in) + inTmp.WriteString(test) outTmp, err := ioutil.TempFile("", "") defer os.Remove(outTmp.Name()) @@ -226,7 +219,7 @@ func Test_transformGeneric(t *testing.T) { err = json.Unmarshal(outTmpBs, &m) assert.Nil(t, err) - assert.Equal(t, test.out, m) + assert.Equal(t, test, m) } } diff --git a/runner/http.go b/runner/http.go index 0ed57e32a..fcdbb04d8 100644 --- a/runner/http.go +++ b/runner/http.go @@ -201,6 +201,7 @@ func (ec EvalContext) evalHTTPPanel(project *ProjectState, pageIndex int, panel sendBody: panel.Content != "" && (h.Method == http.MethodPut || h.Method == http.MethodPatch || h.Method == http.MethodPost), customCaCerts: customCaCerts, + method: h.Method, }) if err != nil { return err diff --git a/scripts/ci/prepare_linux_integration_test_setup_only.sh b/scripts/ci/prepare_linux_integration_test_setup_only.sh index dce2640a7..84cf6e8e3 100755 --- a/scripts/ci/prepare_linux_integration_test_setup_only.sh +++ b/scripts/ci/prepare_linux_integration_test_setup_only.sh @@ -9,6 +9,9 @@ set -ex # Set up Julia, SSH, etc. sudo apt-get install -y jq julia openssh-server php +# Set up http helper +go install github.com/multiprocessio/httpmirror@latest + # Set up coverage tools go install github.com/axw/gocov/gocov@v1.0.0 go install github.com/wadey/gocovmerge@b5bfa59 diff --git a/scripts/ci/prepare_macos.sh b/scripts/ci/prepare_macos.sh index 474549a8c..5a9c10079 100755 --- a/scripts/ci/prepare_macos.sh +++ b/scripts/ci/prepare_macos.sh @@ -7,5 +7,7 @@ brew uninstall go@1.15 brew install cmake jq go r julia node@16 npm brew link --overwrite node@16 go install github.com/google/go-jsonnet/cmd/jsonnet@latest +go install github.com/multiprocessio/httpmirror@latest +cp ~/go/bin/httpmirror /usr/local/bin/httpmirror npm install --global yarn yarn diff --git a/scripts/ci/prepare_windows.ps1 b/scripts/ci/prepare_windows.ps1 index 87240398d..5ce08e470 100755 --- a/scripts/ci/prepare_windows.ps1 +++ b/scripts/ci/prepare_windows.ps1 @@ -4,5 +4,7 @@ Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH scoop install nodejs@16 cmake python yarn zip jq curl go r julia yarn go install github.com/google/go-jsonnet/cmd/jsonnet@latest +go install github.com/multiprocessio/httpmirror@latest +Join-Path (Resolve-Path ~).Path "go\bin" >> $Env:GITHUB_PATH New-Alias zip 7z \ No newline at end of file diff --git a/ui/Panel.tsx b/ui/Panel.tsx index 09a422ca0..a11c20b2d 100644 --- a/ui/Panel.tsx +++ b/ui/Panel.tsx @@ -314,7 +314,11 @@ export function Panel({ className={`panel ${fullScreen === panel.id ? 'panel--fullscreen' : ''} ${ hidden ? 'panel--hidden' : '' } ${ - panelUIDetails.body === null && !results.exception ? 'panel--empty' : '' + (panelUIDetails.body === null || + (panelUIDetails.hideBody && panelUIDetails.hideBody(panel))) && + !results.exception + ? 'panel--empty' + : '' } ${results.loading ? 'panel--loading' : ''}`} tabIndex={1001} ref={panelRef} @@ -431,27 +435,25 @@ export function Panel({ 'Run to apply changes' )} - {!VISUAL_PANELS.includes(panel.type) ? ( - + - - ) : null} + {results.loading ? : } + +