Sync

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: cargo
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+  - package-ecosystem: cargo
+    directory: "/sqlparser_bench"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10

.github/workflows/rust.yml

@@ -1,31 +1,78 @@
 name: Rust
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
-  build:
 
+  codestyle:
     runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: hecrj/setup-rust-action@v1
+        with:
+          components: rustfmt
+          # Note that `nightly` is required for `license_template_path`, as
+          # it's an unstable feature.
+          rust-version: nightly
+      - uses: actions/checkout@v2
+      - run: cargo +nightly fmt -- --check --config-path <(echo 'license_template_path = "HEADER"')
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: hecrj/setup-rust-action@v1
+        with:
+          components: clippy
+      - uses: actions/checkout@v2
+      - run: cargo clippy --all-targets --all-features -- -D warnings
 
+  compile:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: hecrj/setup-rust-action@v1
+      - uses: actions/checkout@master
+      - run: cargo check --all-targets --all-features
+
+  test:
+    strategy:
+      matrix:
+        rust: [stable, beta, nightly]
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
     - name: Setup Rust
-      run: |
-        rustup toolchain install nightly && rustup component add --toolchain nightly rustfmt
-        rustup toolchain install stable
-        rustup override set stable
-    # Clippy must be run first, as its lints are only triggered during
-    # compilation. Put another way: after a successful `cargo build`, `cargo
-    # clippy` is guaranteed to produce no results. This bug is known upstream:
-    # https://github.com/rust-lang/rust-clippy/issues/2604.
-#    - name: Clippy
-#      run: cargo clippy -- --all-targets --all-features -- -D warnings
-    - name: Check formatting
-      run: |
-        cargo +nightly fmt -- --check --config-path <(echo 'license_template_path = "HEADER"')
-    - name: Build
-      run: cargo build --verbose
-    - name: Run tests
-      run: cargo test --verbose
-    - name: Run tests for all features
-      run: cargo test --verbose -- all-features
+      uses: hecrj/setup-rust-action@v1
+      with:
+        rust-version: ${{ matrix.rust }}
+    - name: Install Tarpaulin
+      uses: actions-rs/install@v0.1
+      with:
+        crate: cargo-tarpaulin
+        version: 0.14.2
+        use-tool-cache: true
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Test
+      run: cargo test --all-features
+    - name: Coverage
+      if: matrix.rust == 'stable'
+      run: cargo tarpaulin -o Lcov --output-dir ./coverage
+    - name: Coveralls
+      if: matrix.rust == 'stable'
+      uses: coverallsapp/github-action@master
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  publish-crate:
+    if: startsWith(github.ref, 'refs/tags/v0')
+    runs-on: ubuntu-latest
+    needs: [test]
+    steps:
+      - name: Set up Rust
+        uses: hecrj/setup-rust-action@v1
+      - uses: actions/checkout@v2
+      - name: Publish
+        shell: bash
+        run: |
+          cargo publish --token ${{ secrets.CRATES_TOKEN }}

.gitignore

@@ -1,6 +1,7 @@
 # Generated by Cargo
 # will have compiled files and executables
 /target/
+/sqlparser_bench/target/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock

.travis.yml

@@ -23,6 +23,16 @@ rust:
   - stable
 
 before_script:
+  # Travis installs rust with a non-default "minimal" profile, and advises us
+  # to add clippy manually:
+  - rustup component add clippy
+  # Unfortunately, this method often breaks on the nightly channel, where the
+  # most recent build might not have all the optional components.
+  # We explicitly specify `--profile default` to obtain the most recent nightly
+  # that has rustfmt (we don't care if it's a week old, as we need it only for
+  # an experimental flag):
+  - rustup toolchain install nightly --profile default
+
   - pip install 'travis-cargo<0.2' --user && export PATH=$HOME/.local/bin:$PATH
   - export PATH=$HOME/.cargo/bin:$PATH
   # `cargo install` fails if the specified binary is already installed, and
@@ -33,13 +43,9 @@ before_script:
   # rebuilds from scratch, ignoring the cache entirely.
   #
   # [0]: https://github.com/rust-lang/cargo/issues/2082
-  - cargo install cargo-update || echo "cargo-update already installed"
-  - cargo install cargo-travis || echo "cargo-travis already installed"
+  - cargo install cargo-update || echo "cargo-update already installed"  # for `cargo install-update`
+  - cargo install cargo-travis || echo "cargo-travis already installed"  # for `cargo coveralls`
   - cargo install-update -a  # updates cargo-travis, if the cached version is outdated
-  - rustup component add clippy
-  # The license_template_path setting we use to verify copyright headers is
-  # only available on the nightly rustfmt.
-  - rustup toolchain install nightly && rustup component add --toolchain nightly rustfmt
 
 script:
   # Clippy must be run first, as its lints are only triggered during
@@ -50,6 +56,8 @@ script:
   - travis-cargo build
   - travis-cargo test
   - travis-cargo test -- all-features
+  # The license_template_path setting we use to verify copyright headers is
+  # only available on the nightly rustfmt.
   - cargo +nightly fmt -- --check --config-path <(echo 'license_template_path = "HEADER"')
 
 after_success:

CHANGELOG.md

@@ -6,10 +6,96 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 Given that the parser produces a typed AST, any changes to the AST will technically be breaking and thus will result in a `0.(N+1)` version. We document changes that break via addition as "Added".
 
 ## [Unreleased]
-Nothing here yet! Check https://github.com/andygrove/sqlparser-rs/commits/master for undocumented changes.
+Check https://github.com/ballista-compute/sqlparser-rs/commits/main for undocumented changes.
 
 
-## [0.5.0] - 2019-10-10 
+## [0.8.0] 2020-02-20
+
+### Added
+* Introduce Hive QL dialect `HiveDialect` and syntax (#235) - Thanks @hntd187!
+* Add `SUBSTRING(col [FROM <expr>] [FOR <expr>])` syntax (#293)
+* Support parsing floats without leading digits `.01` (#294)
+* Support parsing multiple show variables (#290) - Thanks @francis-du!
+* Support SQLite `INSERT OR [..]` syntax (#281) - Thanks @zhangli-pear!
+
+## [0.7.0] 2020-12-28
+
+### Changed
+- Change the MySQL dialect to support `` `identifiers` `` quoted with backticks instead of the standard `"double-quoted"` identifiers (#247) - thanks @mashuai!
+- Update bigdecimal requirement from 0.1 to 0.2 (#268)
+
+### Added
+- Enable dialect-specific behaviours in the parser (`dialect_of!()`) (#254) - thanks @eyalleshem!
+- Support named arguments in function invocations (`ARG_NAME => val`) (#250) - thanks @eyalleshem!
+- Support `TABLE()` functions in `FROM` (#253) - thanks @eyalleshem!
+- Support Snowflake's single-line comments starting with '#' or '//' (#264) - thanks @eyalleshem!
+- Support PostgreSQL `PREPARE`, `EXECUTE`, and `DEALLOCATE` (#243) - thanks @silathdiir!
+- Support PostgreSQL math operators (#267) - thanks @alex-dukhno!
+- Add SQLite dialect (#248) - thanks @mashuai!
+- Add Snowflake dialect (#259) - thanks @eyalleshem!
+- Support for Recursive CTEs - thanks @rhanqtl!
+- Support `FROM (table_name) alias` syntax - thanks @eyalleshem!
+- Support for `EXPLAIN [ANALYZE] VERBOSE` - thanks @ovr!
+- Support `ANALYZE TABLE`
+- DDL:
+    - Support `OR REPLACE` in `CREATE VIEW`/`TABLE` (#239)  - thanks @Dandandan!
+    - Support specifying `ASC`/`DESC` in index columns (#249) - thanks @mashuai!
+    - Support SQLite `AUTOINCREMENT` and MySQL `AUTO_INCREMENT` column option in `CREATE TABLE` (#234) - thanks @mashuai!
+    - Support PostgreSQL `IF NOT EXISTS` for `CREATE SCHEMA` (#276) - thanks @alex-dukhno!
+
+### Fixed
+- Fix a typo in `JSONFILE` serialization, introduced in 0.3.1 (#237)
+- Change `CREATE INDEX` serialization to not end with a semicolon, introduced in 0.5.1 (#245)
+- Don't fail parsing `ALTER TABLE ADD COLUMN` ending with a semicolon, introduced in 0.5.1 (#246) - thanks @mashuai
+
+## [0.6.1] - 2020-07-20
+
+### Added
+- Support BigQuery `ASSERT` statement (#226)
+
+## [0.6.0] - 2020-07-20
+
+### Added
+- Support SQLite's `CREATE TABLE (...) WITHOUT ROWID` (#208) - thanks @mashuai!
+- Support SQLite's `CREATE VIRTUAL TABLE` (#209) - thanks @mashuai!
+
+## [0.5.1] - 2020-06-26
+This release should have been called `0.6`, as it introduces multiple incompatible changes to the API. If you don't want to upgrade yet, you can revert to the previous version by changing your `Cargo.toml` to:
+
+    sqlparser = "= 0.5.0"
+
+
+### Changed
+- **`Parser::parse_sql` now accepts a `&str` instead of `String` (#182)** - thanks @Dandandan!
+- Change `Ident` (previously a simple `String`) to store the parsed (unquoted) `value` of the identifier and the `quote_style` separately (#143) - thanks @apparebit!
+- Support Snowflake's `FROM (table_name)` (#155) - thanks @eyalleshem!
+- Add line and column number to TokenizerError (#194) - thanks @Dandandan!
+- Use Token::EOF instead of Option<Token> (#195)
+- Make the units keyword following `INTERVAL '...'` optional (#184) - thanks @maxcountryman!
+- Generalize `DATE`/`TIME`/`TIMESTAMP` literals representation in the AST (`TypedString { data_type, value }`) and allow `DATE` and other keywords to be used as identifiers when not followed by a string (#187) - thanks @maxcountryman!
+- Output DataType capitalized (`fmt::Display`) (#202) - thanks @Dandandan!
+
+### Added
+- Support MSSQL `TOP (<N>) [ PERCENT ] [ WITH TIES ]` (#150) - thanks @alexkyllo!
+- Support MySQL `LIMIT row_count OFFSET offset` (not followed by `ROW` or `ROWS`) and remember which variant was parsed (#158) - thanks @mjibson!
+- Support PostgreSQL `CREATE TABLE IF NOT EXISTS table_name` (#163) - thanks @alex-dukhno!
+- Support basic forms of `CREATE INDEX` and `DROP INDEX` (#167) - thanks @mashuai!
+- Support `ON { UPDATE | DELETE } { RESTRICT | CASCADE | SET NULL | NO ACTION | SET DEFAULT }` in `FOREIGN KEY` constraints (#170) - thanks @c7hm4r!
+- Support basic forms of `CREATE SCHEMA` and `DROP SCHEMA` (#173) - thanks @alex-dukhno!
+- Support `NULLS FIRST`/`LAST` in `ORDER BY` expressions (#176) - thanks @houqp!
+- Support `LISTAGG()` (#174) - thanks @maxcountryman!
+- Support the string concatentation operator `||` (#178) - thanks @Dandandan!
+- Support bitwise AND (`&`), OR (`|`), XOR (`^`) (#181) - thanks @Dandandan!
+- Add serde support to AST structs and enums (#196) - thanks @panarch!
+- Support `ALTER TABLE ADD COLUMN`, `RENAME COLUMN`, and `RENAME TO` (#203) - thanks @mashuai!
+- Support `ALTER TABLE DROP COLUMN` (#148) - thanks @ivanceras!
+- Support `CREATE TABLE ... AS ...` (#206) - thanks @Dandandan!
+
+### Fixed
+- Report an error for unterminated string literals (#165)
+- Make file format (`STORED AS`) case insensitive (#200) and don't allow quoting it (#201) - thanks @Dandandan!
+
+## [0.5.0] - 2019-10-10
 
 ### Changed
 - Replace the `Value::Long(u64)` and `Value::Double(f64)` variants with `Value::Number(String)` to avoid losing precision when parsing decimal literals (#130) - thanks @benesch!
@@ -122,3 +208,4 @@ We don't have a changelog for the changes made in 2018, but thanks to @crw5996,
 
 ## [0.1.0] - 2018-09-03
 Initial release
+

Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "sqlparser"
 description = "Extensible SQL Lexer and Parser with support for ANSI SQL:2011"
-version = "0.5.1-alpha-0"
+version = "0.8.1-alpha.0"
 authors = ["Andy Grove <andygrove73@gmail.com>"]
-homepage = "https://github.com/andygrove/sqlparser-rs"
+homepage = "https://github.com/ballista-compute/sqlparser-rs"
 documentation = "https://docs.rs/sqlparser/"
 keywords = [ "ansi", "sql", "lexer", "parser" ]
-repository = "https://github.com/andygrove/sqlparser-rs"
+repository = "https://github.com/ballista-compute/sqlparser-rs"
 license = "Apache-2.0"
 include = [
     "src/**/*.rs",
@@ -18,10 +18,25 @@ edition = "2018"
 name = "sqlparser"
 path = "src/lib.rs"
 
+[features]
+# Enable JSON output in the `cli` example:
+json_example = ["serde_json", "serde"]
+
 [dependencies]
-bigdecimal = { version = "0.1.0", optional = true }
-log = "0.4.5"
+bigdecimal = { version = "0.2", features = ["serde"], optional = true }
+log = "0.4"
+serde = { version = "1.0", features = ["derive"], optional = true }
+# serde_json is only used in examples/cli, but we have to put it outside
+# of dev-dependencies because of
+# https://github.com/rust-lang/cargo/issues/1596
+serde_json = { version = "1.0", optional = true }
 
 [dev-dependencies]
-simple_logger = "1.0.1"
+simple_logger = "1.9"
 matches = "0.1"
+
+[package.metadata.release]
+# Instruct `cargo release` to not run `cargo publish` locally:
+# https://github.com/sunng87/cargo-release/blob/master/docs/reference.md#config-fields
+# See docs/releasing.md for details.
+disable-publish = true

README.md

@@ -2,17 +2,17 @@
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Version](https://img.shields.io/crates/v/sqlparser.svg)](https://crates.io/crates/sqlparser)
-[![Build Status](https://travis-ci.org/andygrove/sqlparser-rs.svg?branch=master)](https://travis-ci.org/andygrove/sqlparser-rs)
-[![Coverage Status](https://coveralls.io/repos/github/andygrove/sqlparser-rs/badge.svg?branch=master)](https://coveralls.io/github/andygrove/sqlparser-rs?branch=master)
+[![Build Status](https://github.com/ballista-compute/sqlparser-rs/workflows/Rust/badge.svg?branch=main)](https://github.com/ballista-compute/sqlparser-rs/actions?query=workflow%3ARust+branch%3Amain)
+[![Coverage Status](https://coveralls.io/repos/github/ballista-compute/sqlparser-rs/badge.svg?branch=main)](https://coveralls.io/github/ballista-compute/sqlparser-rs?branch=main)
 [![Gitter Chat](https://badges.gitter.im/sqlparser-rs/community.svg)](https://gitter.im/sqlparser-rs/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 The goal of this project is to build a SQL lexer and parser capable of parsing
 SQL that conforms with the [ANSI/ISO SQL standard][sql-standard] while also
 making it easy to support custom dialects so that this crate can be used as a
 foundation for vendor-specific parsers.
 
-This parser is currently being used by the [DataFusion] query engine and
-[LocustDB].
+This parser is currently being used by the [DataFusion] query engine,
+[LocustDB], and [Ballista].
 
 ## Example
 
@@ -29,7 +29,7 @@ let sql = "SELECT a, b, 123, myfunc(b) \
 
 let dialect = GenericDialect {}; // or AnsiDialect, or your own dialect ...
 
-let ast = Parser::parse_sql(&dialect, sql.to_string()).unwrap();
+let ast = Parser::parse_sql(&dialect, sql).unwrap();
 
 println!("AST: {:?}", ast);
 ```
@@ -40,17 +40,21 @@ This outputs
 AST: [Query(Query { ctes: [], body: Select(Select { distinct: false, projection: [UnnamedExpr(Identifier("a")), UnnamedExpr(Identifier("b")), UnnamedExpr(Value(Long(123))), UnnamedExpr(Function(Function { name: ObjectName(["myfunc"]), args: [Identifier("b")], over: None, distinct: false }))], from: [TableWithJoins { relation: Table { name: ObjectName(["table_1"]), alias: None, args: [], with_hints: [] }, joins: [] }], selection: Some(BinaryOp { left: BinaryOp { left: Identifier("a"), op: Gt, right: Identifier("b") }, op: And, right: BinaryOp { left: Identifier("b"), op: Lt, right: Value(Long(100)) } }), group_by: [], having: None }), order_by: [OrderByExpr { expr: Identifier("a"), asc: Some(false) }, OrderByExpr { expr: Identifier("b"), asc: None }], limit: None, offset: None, fetch: None })]
 ```
 
+## Command line
+To parse a file and dump the results as JSON:
+```
+$ cargo run --features json_example --example cli FILENAME.sql [--dialectname]
+```
+
 ## SQL compliance
 
 SQL was first standardized in 1987, and revisions of the standard have been
 published regularly since. Most revisions have added significant new features to
 the language, and as a result no database claims to support the full breadth of
 features. This parser currently supports most of the SQL-92 syntax, plus some
-syntax from newer versions that have been explicitly requested, plus some MSSQL-
-and PostgreSQL-specific syntax. Whenever possible, the [online SQL:2011
-grammar][sql-2011-grammar] is used to guide what syntax to accept. (We will
-happily accept changes that conform to the SQL:2016 syntax as well, but that
-edition's grammar is not yet available online.)
+syntax from newer versions that have been explicitly requested, plus some MSSQL,
+PostgreSQL, and other dialect-specific syntax. Whenever possible, the [online
+SQL:2016 grammar][sql-2016-grammar] is used to guide what syntax to accept.
 
 Unfortunately, stating anything more specific about compliance is difficult.
 There is no publicly available test suite that can assess compliance
@@ -105,8 +109,8 @@ If you are unable to submit a patch, feel free to file an issue instead. Please
 try to include:
 
   * some representative examples of the syntax you wish to support or fix;
-  * the relevant bits of the [SQL grammar][sql-2011-grammar], if the syntax is
-    part of SQL:2011; and
+  * the relevant bits of the [SQL grammar][sql-2016-grammar], if the syntax is
+    part of SQL:2016; and
   * links to documentation for the feature for a few of the most popular
     databases that support it.
 
@@ -119,9 +123,10 @@ resources.
 
 [tdop-tutorial]: https://eli.thegreenplace.net/2010/01/02/top-down-operator-precedence-parsing
 [`cargo fmt`]: https://github.com/rust-lang/rustfmt#on-the-stable-toolchain
-[current issues]: https://github.com/andygrove/sqlparser-rs/issues
+[current issues]: https://github.com/ballista-compute/sqlparser-rs/issues
 [DataFusion]: https://github.com/apache/arrow/tree/master/rust/datafusion
 [LocustDB]: https://github.com/cswinter/LocustDB
+[Ballista]: https://github.com/ballista-compute/ballista
 [Pratt Parser]: https://tdop.github.io/
-[sql-2011-grammar]: https://jakewheat.github.io/sql-overview/sql-2011-foundation-grammar.html
-[sql-standard]: https://en.wikipedia.org/wiki/ISO/IEC_9075
+[sql-2016-grammar]: https://jakewheat.github.io/sql-overview/sql-2016-foundation-grammar.html
+[sql-standard]: https://en.wikipedia.org/wiki/ISO/IEC_9075