New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.
Already on GitHub? Sign in to your account
DB Connector for Impala #203
Merged
Changes from all commits
Commits
Show all changes
14 commits
Select commit
Hold shift + click to select a range
2d1477f
Added UI updates for Impala
tarzzz d5474f6
Added tooltip over connection-selectors
tarzzz 5a0ccb6
package.json
tarzzz 67ae51d
Added backend for Impala.
tarzzz b1beb26
Updated package.json and yarn.lock
tarzzz 9e85a2b
Updated sample-db ip
tarzzz 7248f8e
Added tests.
tarzzz e8efab3
Typos + added/updated comments
tarzzz 0c7a096
Added test for invalid credentials
tarzzz b1c9262
Made some updates to impala API.
tarzzz b026a1f
Added routes tests for impala
tarzzz 479304d
Added back 'connect' method
tarzzz ae2fff9
Fixed tests
tarzzz 1a42a7f
Use connection details from utils.js
tarzzz File filter
Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| import fetch from 'node-fetch'; | ||
| import * as impala from 'node-impala'; | ||
| import {dissoc, keys, values, init, map, prepend, unnest} from 'ramda'; | ||
|
|
||
| import Logger from '../../logger'; | ||
|
|
||
|
|
||
| export function createClient(connection) { | ||
| const client = impala.createClient(); | ||
| client.connect({ | ||
| host: connection.host, | ||
| port: connection.port, | ||
| resultType: 'json-array' | ||
| }); | ||
| return client; | ||
| } | ||
|
|
||
| export function connect(connection) { | ||
|
|
||
| // Runs a blank query to check connection has been established: | ||
| return createClient(connection).query('SELECT ID FROM (SELECT 1 ID) DUAL WHERE ID=0') | ||
| .catch(err => { | ||
| Logger.log(err); | ||
| throw new Error(err); | ||
| }); | ||
| } | ||
|
|
||
| export function tables(connection) { | ||
| const code = (connection.database) ? | ||
| `show tables in ${connection.database}` : | ||
| 'show tables'; | ||
| return createClient(connection).query(code) | ||
| .then(json => { | ||
| let tableNames = json.map(t => t.name); | ||
| if (connection.database) tableNames = tableNames.map(tn => `${connection.database}.${tn}`); | ||
| tableNames = tableNames.map(tn => tn.toUpperCase()); | ||
|
|
||
| return tableNames; | ||
| }).catch(err => { | ||
| Logger.log(err); | ||
| throw new Error(err); | ||
| }); | ||
| } | ||
|
|
||
| export function schemas(connection) { | ||
| let columnnames = ['tablename', 'column_name', 'data_type']; | ||
| const showTables = (connection.database) ? | ||
| `show tables in ${connection.database}` : | ||
| 'show tables'; | ||
|
|
||
| return createClient(connection).query(showTables) | ||
| .then(json => { | ||
| let tableNames = json.map(t => t.name); | ||
| if (connection.database) tableNames = tableNames.map(tn => `${connection.database}.${tn}`); | ||
|
|
||
| /* | ||
| * The last column in the output of describe statement is 'comment', | ||
| * so we remove it(using Ramda.init) before sending out the result. | ||
| */ | ||
| const promises = map(tableName => { | ||
| return query(`describe ${tableName}`, connection) | ||
| .then(json => map(row => prepend(tableName, init(row)), json.rows)); | ||
| }, tableNames); | ||
|
|
||
| // Wait for all the describe-table promises to resolve before resolving: | ||
| return Promise.all(promises); | ||
| }).then(res => { | ||
|
|
||
| // The results are nested inside a list, so we need to un-nest first: | ||
| const rows = unnest(res); | ||
| return {columnnames, rows}; | ||
| }).catch(err => { | ||
| Logger.log(err); | ||
| throw new Error(err); | ||
| }); | ||
| } | ||
|
|
||
| export function query(query, connection) { | ||
|
|
||
| return createClient(connection).query(query) | ||
| .then(json => { | ||
| let columnnames = []; | ||
| let rows = [[]]; | ||
| if (json.length !== 0) { | ||
| columnnames = keys(json[0]); | ||
| rows = json.map(obj => values(obj)); | ||
| } | ||
| return {columnnames, rows}; | ||
| }).catch(err => { | ||
| Logger.log(err); | ||
| throw new Error(err) | ||
| }); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| import {assert} from 'chai'; | ||
|
|
||
| import {DIALECTS} from '../../app/constants/constants.js'; | ||
| import {apacheImpalaConnection as connection} from './utils.js'; | ||
| import { | ||
| query, connect, tables | ||
| } from '../../backend/persistent/datastores/Datastores.js'; | ||
|
|
||
| describe('Apache Impala:', function () { | ||
|
|
||
| it('connect succeeds', function() { | ||
| this.timeout(180 * 1000); | ||
| return connect(connection); | ||
| }); | ||
|
|
||
| it('tables returns list of tables', function() { | ||
| return tables(connection).then(result => { | ||
| const tableName = (connection.database) ? | ||
| `${connection.database}.ALCOHOL_CONSUMPTION_BY_COUNTRY_2010`.toUpperCase() : | ||
| 'ALCOHOL_CONSUMPTION_BY_COUNTRY_2010'; | ||
|
|
||
| assert.deepEqual(result, [tableName]); | ||
| }); | ||
| }); | ||
|
|
||
| it('query returns rows and column names', function() { | ||
| const tableName = (connection.database) ? | ||
| `${connection.database}.ALCOHOL_CONSUMPTION_BY_COUNTRY_2010`.toUpperCase() : | ||
| 'ALCOHOL_CONSUMPTION_BY_COUNTRY_2010'; | ||
|
|
||
| return query(`SELECT * FROM ${tableName}\nLIMIT 5`, connection).then(results => { | ||
| assert.deepEqual(results.rows, [ | ||
| ['Belarus', "17.5"], | ||
| ['Moldova', "16.8"], | ||
| ['Lithuania', "15.4"], | ||
| ['Russia', "15.1"], | ||
| ['Romania', "14.4"] | ||
| ]); | ||
| assert.deepEqual(results.columnnames, ['loc', 'alcohol']); | ||
| }); | ||
| }); | ||
|
|
||
| it('connect for invalid credentials fails', function() { | ||
| connection.host = 'http://lah-lah.lemons.com'; | ||
|
|
||
| return connect(connection).catch(err => { | ||
| // reset hostname | ||
| connection.host = '35.184.155.127'; | ||
|
|
||
| assert.equal(err, ('Error: Error: getaddrinfo ENOTFOUND ' + | ||
| 'http://lah-lah.lemons.com ' + | ||
| 'http://lah-lah.lemons.com:21000')); | ||
| }); | ||
| }); | ||
| }); | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
could we add another test that bad credentials causes the connection to fail with an appropriate error message?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
and since this is sql-like, it would be good to add this to the set of routes tests here: https://github.com/plotly/falcon-sql-client/blob/4dbbde6bfcbd43c6753053d80a283c5338bbcba4/test/backend/routes.spec.js#L544-L995