fix parse 'FOREIGN KEY', block comments

shinichi-takii · Feb 17, 2019 · 998e327 · 998e327
1 parent 349788c
commit 998e327
Show file tree

Hide file tree

Showing 6 changed files with 57 additions and 33 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## 1.2.3
+- Fix parse error for MySQL DDL with 'FOREIGN KEY'.
+- Fix not completely parsed with block comments.
+
 ## 1.2.2
 - Fix FutureWarning of Python 3.7.
 - Add supports PostgreSQL data type.

diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 # DDL Parse
 
-[![PyPI version](https://img.shields.io/pypi/v/ddlparse.svg)](https://pypi.python.org/pypi/ddlparse)
-[![Python version](https://img.shields.io/pypi/pyversions/ddlparse.svg)](https://pypi.python.org/pypi/ddlparse)
+[![PyPI version](https://img.shields.io/pypi/v/ddlparse.svg)](https://pypi.org/project/ddlparse/)
+[![Python version](https://img.shields.io/pypi/pyversions/ddlparse.svg)](https://pypi.org/project/ddlparse/)
 [![Travis CI Build Status](https://travis-ci.org/shinichi-takii/ddlparse.svg?branch=master)](https://travis-ci.org/shinichi-takii/ddlparse)
 [![Coveralls Coverage Status](https://coveralls.io/repos/github/shinichi-takii/ddlparse/badge.svg?branch=master)](https://coveralls.io/github/shinichi-takii/ddlparse?branch=master)
 [![codecov Coverage Status](https://codecov.io/gh/shinichi-takii/ddlparse/branch/master/graph/badge.svg)](https://codecov.io/gh/shinichi-takii/ddlparse)
@@ -22,7 +22,7 @@
 ## Requirement
 
 1. Python >= 3.4
-1. [pyparsing](http://pyparsing.wikispaces.com/)
+1. [pyparsing](https://github.com/pyparsing/pyparsing)
 
 ## Installation
 
@@ -118,11 +118,11 @@ for col in table.columns.values():
     col_info.append("precision(=length) = {}".format(col.precision))
     col_info.append("scale = {}".format(col.scale))
     col_info.append("constraint = {}".format(col.constraint))
-    col_info.append("not_null =  {}".format(col.not_null))
-    col_info.append("PK =  {}".format(col.primary_key))
-    col_info.append("unique =  {}".format(col.unique))
-    col_info.append("bq_legacy_data_type =  {}".format(col.bigquery_legacy_data_type))
-    col_info.append("bq_standard_data_type =  {}".format(col.bigquery_standard_data_type))
+    col_info.append("not_null = {}".format(col.not_null))
+    col_info.append("PK = {}".format(col.primary_key))
+    col_info.append("unique = {}".format(col.unique))
+    col_info.append("bq_legacy_data_type = {}".format(col.bigquery_legacy_data_type))
+    col_info.append("bq_standard_data_type = {}".format(col.bigquery_standard_data_type))
     col_info.append("BQ {}".format(col.to_bigquery_field()))
     print(" : ".join(col_info))
 
@@ -148,8 +148,8 @@ Shinichi Takii <shinichi.takii@gmail.com>
 ## Links
 
 - Repository : https://github.com/shinichi-takii/ddlparse
-- PyPI Package : https://pypi.python.org/pypi/ddlparse
+- PyPI Package : https://pypi.org/project/ddlparse/
 
 ## Special Thanks
 
-- pyparsing : http://pyparsing.wikispaces.com/
+- pyparsing : https://github.com/pyparsing/pyparsing
diff --git a/README.rst b/README.rst
@@ -2,11 +2,11 @@ DDL Parse
 =========
 
 .. image:: https://img.shields.io/pypi/v/ddlparse.svg
-   :target: https://pypi.python.org/pypi/ddlparse
+   :target: https://pypi.org/project/ddlparse/
    :alt: PyPI version
 
 .. image:: https://img.shields.io/pypi/pyversions/ddlparse.svg
-   :target: https://pypi.python.org/pypi/ddlparse
+   :target: https://pypi.org/project/ddlparse/
    :alt: Python version
 
 .. image:: https://travis-ci.org/shinichi-takii/ddlparse.svg?branch=master
@@ -49,7 +49,7 @@ Requirement
 -----------
 
 1. Python >= 3.4
-2. `pyparsing <http://pyparsing.wikispaces.com/>`__
+2. `pyparsing <https://github.com/pyparsing/pyparsing>`__
 
 Installation
 ------------
@@ -154,11 +154,11 @@ Example
        col_info.append("precision(=length) = {}".format(col.precision))
        col_info.append("scale = {}".format(col.scale))
        col_info.append("constraint = {}".format(col.constraint))
-       col_info.append("not_null =  {}".format(col.not_null))
-       col_info.append("PK =  {}".format(col.primary_key))
-       col_info.append("unique =  {}".format(col.unique))
-       col_info.append("bq_legacy_data_type =  {}".format(col.bigquery_legacy_data_type))
-       col_info.append("bq_standard_data_type =  {}".format(col.bigquery_standard_data_type))
+       col_info.append("not_null = {}".format(col.not_null))
+       col_info.append("PK = {}".format(col.primary_key))
+       col_info.append("unique = {}".format(col.unique))
+       col_info.append("bq_legacy_data_type = {}".format(col.bigquery_legacy_data_type))
+       col_info.append("bq_standard_data_type = {}".format(col.bigquery_standard_data_type))
        col_info.append("BQ {}".format(col.to_bigquery_field()))
        print(" : ".join(col_info))
 
@@ -187,9 +187,9 @@ Links
 -----
 
 -  Repository : https://github.com/shinichi-takii/ddlparse
--  PyPI Package : https://pypi.python.org/pypi/ddlparse
+-  PyPI Package : https://pypi.org/project/ddlparse/
 
 Special Thanks
 --------------
 
--  pyparsing : http://pyparsing.wikispaces.com/
+-  pyparsing : https://github.com/pyparsing/pyparsing
diff --git a/ddlparse/ddlparse.py b/ddlparse/ddlparse.py
@@ -454,8 +454,8 @@ class DdlParse(DdlParseBase):
     """DDL parser"""
 
     _LPAR, _RPAR, _COMMA, _SEMICOLON, _DOT, _DOUBLEQUOTE, _BACKQUOTE, _SPACE = map(Suppress, "(),;.\"` ")
-    _CREATE, _TABLE, _TEMP, _CONSTRAINT, _NOT_NULL, _PRIMARY_KEY, _UNIQUE, _UNIQUE_KEY, _KEY, _CHAR_SEMANTICS, _BYTE_SEMANTICS = \
-        map(CaselessKeyword, "CREATE, TABLE, TEMP, CONSTRAINT, NOT NULL, PRIMARY KEY, UNIQUE, UNIQUE KEY, KEY, CHAR, BYTE".replace(", ", ",").split(","))
+    _CREATE, _TABLE, _TEMP, _CONSTRAINT, _NOT_NULL, _PRIMARY_KEY, _UNIQUE, _UNIQUE_KEY, _FOREIGN_KEY, _REFERENCES, _KEY, _CHAR_SEMANTICS, _BYTE_SEMANTICS = \
+        map(CaselessKeyword, "CREATE, TABLE, TEMP, CONSTRAINT, NOT NULL, PRIMARY KEY, UNIQUE, UNIQUE KEY, FOREIGN KEY, REFERENCES, KEY, CHAR, BYTE".replace(", ", ",").split(","))
     _SUPPRESS_QUOTE = _BACKQUOTE | _DOUBLEQUOTE
 
     _COMMENT = Suppress("--" + Regex(r".+"))
@@ -465,19 +465,33 @@ class DdlParse(DdlParseBase):
         + _LPAR \
         + delimitedList(
             OneOrMore(
+                _COMMENT
+                |
                 # Ignore Index
                 Suppress(_KEY + Word(alphanums+"_'`() "))
                 |
                 Group(
                     Optional(Suppress(_CONSTRAINT) + Optional(_SUPPRESS_QUOTE) + Word(alphanums+"_")("name") + Optional(_SUPPRESS_QUOTE))
-                    + (_PRIMARY_KEY ^ _UNIQUE ^ _UNIQUE_KEY ^ _NOT_NULL)("type")
-                    + Optional(_SUPPRESS_QUOTE) + Optional(Word(alphanums+"_"))("name") + Optional(_SUPPRESS_QUOTE)
-                    + _LPAR + Group(delimitedList(Optional(_SUPPRESS_QUOTE) + Word(alphanums+"_") + Optional(_SUPPRESS_QUOTE)))("constraint_columns") + _RPAR
+                    + (
+                        (
+                            (_PRIMARY_KEY ^ _UNIQUE ^ _UNIQUE_KEY ^ _NOT_NULL)("type")
+                            + Optional(_SUPPRESS_QUOTE) + Optional(Word(alphanums+"_"))("name") + Optional(_SUPPRESS_QUOTE)
+                            + _LPAR + Group(delimitedList(Optional(_SUPPRESS_QUOTE) + Word(alphanums+"_") + Optional(_SUPPRESS_QUOTE)))("constraint_columns") + _RPAR
+                        )
+                        |
+                        (
+                            (_FOREIGN_KEY)("type")
+                            + _LPAR + Group(delimitedList(Optional(_SUPPRESS_QUOTE) + Word(alphanums+"_") + Optional(_SUPPRESS_QUOTE)))("constraint_columns") + _RPAR
+                            + Optional(Suppress(_REFERENCES)
+                                + Optional(_SUPPRESS_QUOTE) + Word(alphanums+"_")("references_table") + Optional(_SUPPRESS_QUOTE)
+                                + _LPAR + Group(delimitedList(Optional(_SUPPRESS_QUOTE) + Word(alphanums+"_") + Optional(_SUPPRESS_QUOTE)))("references_columns") + _RPAR
+                            )
+                        )
+                    )
                 )("constraint")
                 |
                 Group(
-                    Optional(_COMMENT)
-                    + Optional(_SUPPRESS_QUOTE) + Word(alphanums+"_")("name") + Optional(_SUPPRESS_QUOTE)
+                    Optional(_SUPPRESS_QUOTE) + Word(alphanums+"_")("name") + Optional(_SUPPRESS_QUOTE)
                     + Group(
                           Word(alphanums+"_")
                         + Optional(CaselessKeyword("WITHOUT TIME ZONE") ^ CaselessKeyword("WITH TIME ZONE") ^ CaselessKeyword("PRECISION") ^ CaselessKeyword("VARYING"))
@@ -486,6 +500,8 @@ class DdlParse(DdlParseBase):
                     + Optional(Word(r"\[\]"))("array_brackets")
                     + Optional(Regex(r"DEFAULT\s+[^,]+", re.IGNORECASE) | Word(alphanums+"_': -"))("constraint")
                 )("column")
+                |
+                _COMMENT
             )
         )("columns")
 

diff --git a/example/example.py b/example/example.py
@@ -73,11 +73,11 @@
     col_info.append("precision(=length) = {}".format(col.precision))
     col_info.append("scale = {}".format(col.scale))
     col_info.append("constraint = {}".format(col.constraint))
-    col_info.append("not_null =  {}".format(col.not_null))
-    col_info.append("PK =  {}".format(col.primary_key))
-    col_info.append("unique =  {}".format(col.unique))
-    col_info.append("bq_legacy_data_type =  {}".format(col.bigquery_legacy_data_type))
-    col_info.append("bq_standard_data_type =  {}".format(col.bigquery_standard_data_type))
+    col_info.append("not_null = {}".format(col.not_null))
+    col_info.append("PK = {}".format(col.primary_key))
+    col_info.append("unique = {}".format(col.unique))
+    col_info.append("bq_legacy_data_type = {}".format(col.bigquery_legacy_data_type))
+    col_info.append("bq_standard_data_type = {}".format(col.bigquery_standard_data_type))
     col_info.append("BQ {}".format(col.to_bigquery_field()))
     print(" : ".join(col_info))
 

diff --git a/test/test_ddlparse.py b/test/test_ddlparse.py
@@ -13,6 +13,8 @@
             """
             CREATE TABLE Sample_Table (
               Col_01 varchar(100) PRIMARY KEY,
+              -- comment
+              -- comment
               Col_02 char(200) NOT NULL UNIQUE,
               Col_03 text UNIQUE,
               Col_04 integer,
@@ -151,7 +153,8 @@
               Col_04 double,
               Col_05 datetime,
               CONSTRAINT const_01 PRIMARY KEY (Col_01, Col_02),
-              CONSTRAINT \"const_02\" UNIQUE (Col_03, Col_04)
+              CONSTRAINT \"const_02\" UNIQUE (Col_03, Col_04),
+              CONSTRAINT \"const_03\" FOREIGN KEY (Col_04, \"Col_05\") REFERENCES ref_table_01 (\"Col_04\", Col_05)
             );
             """,
         "database" : DdlParse.DATABASE.mysql,
@@ -193,6 +196,7 @@
               PRIMARY KEY (Col_01, Col_02),
               UNIQUE (Col_03, Col_04),
               NOT NULL (Col_04, Col_05)
+              FOREIGN KEY (Col_04, Col_05) REFERENCES ref_table_01 (Col_04, Col_05)
             );
             """,
         "database" : None,