Merge d4af230 into 3c973a2

shinichi-takii · May 2, 2018 · 1f89b2d · 1f89b2d
2 parents 3c973a2 + d4af230
commit 1f89b2d
Show file tree

Hide file tree

Showing 7 changed files with 122 additions and 81 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,14 +1,20 @@
 # Changelog
 
+## 1.1.2
+- Add support Oracle data type.
+    - `CLOB`, `NCLOB`
+    - `NUMBER` with no length & scale specification
+- Miner fix.
+
 ## 1.1.1
 - Fix Postgres/Redshift parse of "::" syntax in field attribute.
 
 ## 1.1.0
 - Add `source_database` option.
 - Add `to_bigquery_fields` method to Columns dicttionary(`DdlParseColumnDict` class).
 - Fix BigQuery convert of Oracle data type.
-  - Oracle 'DATE' -> BigQuery 'DATETIME'
-  - Oracle 'NUMBER' -> BigQuery 'INTEGER' or 'FLOAT'
+    - Oracle 'DATE' -> BigQuery 'DATETIME'
+    - Oracle 'NUMBER' -> BigQuery 'INTEGER' or 'FLOAT'
 
 ## 1.0.2
 - Miner enhancement.

diff --git a/README.md b/README.md
@@ -75,6 +75,10 @@ table = DdlParse().parse(ddl=sample_ddl, source_database=DdlParse.DATABASE.oracl
 parser = DdlParse(sample_ddl)
 table = parser.parse()
 
+print("* BigQuery Fields * : normal")
+print(table.to_bigquery_fields())
+
+
 # parse pattern (2-2) : Specify source database
 parser = DdlParse(ddl=sample_ddl, source_database=DdlParse.DATABASE.oracle)
 table = parser.parse()
@@ -91,6 +95,9 @@ parser.source_database = DdlParse.DATABASE.oracle
 parser.ddl = sample_ddl
 table = parser.parse()
 
+print("* BigQuery Fields * : Oracle")
+print(table.to_bigquery_fields())
+
 
 print("* TABLE *")
 print("schema = {} : name = {} : is_temp = {}".format(table.schema, table.name, table.is_temp))

diff --git a/README.rst b/README.rst
@@ -1,9 +1,15 @@
 DDL Parse
 =========
 
-|PyPI version| |Python version| |Travis CI Build Status| |Coveralls
-Coverage Status| |codecov Coverage Status| |Requirements Status|
-|License|
+`PyPI version <https://pypi.python.org/pypi/ddlparse>`__ `Python
+version <https://pypi.python.org/pypi/ddlparse>`__ `Travis CI Build
+Status <https://travis-ci.org/shinichi-takii/ddlparse>`__ `Coveralls
+Coverage
+Status <https://coveralls.io/github/shinichi-takii/ddlparse?branch=master>`__
+`codecov Coverage
+Status <https://codecov.io/gh/shinichi-takii/ddlparse>`__ `Requirements
+Status <https://requires.io/github/shinichi-takii/ddlparse/requirements/?branch=master>`__
+`License <https://github.com/shinichi-takii/ddlparse/blob/master/LICENSE.md>`__
 
 *DDL parase and Convert to BigQuery JSON schema module, available in
 Python.*
@@ -35,13 +41,13 @@ pip install:
 
 .. code:: bash
 
-    $ pip install ddlparse
+   $ pip install ddlparse
 
 command install:
 
 .. code:: bash
 
-    $ python setup.py install
+   $ python setup.py install
 
 Update
 ~~~~~~
@@ -50,7 +56,7 @@ pip update:
 
 .. code:: bash
 
-    $ pip install ddlparse --upgrade
+   $ pip install ddlparse --upgrade
 
 Usage
 -----
@@ -60,75 +66,82 @@ Example
 
 .. code:: python
 
-    from ddlparse import DdlParse
+   from ddlparse import DdlParse
 
-    sample_ddl = """
-    CREATE TABLE My_Schema.Sample_Table (
-      ID integer PRIMARY KEY,
-      NAME varchar(100) NOT NULL,
-      TOTAL bigint NOT NULL,
-      AVG decimal(5,1) NOT NULL,
-      CREATED_AT date, -- Oracle 'DATE' -> BigQuery 'DATETIME'
-      UNIQUE (NAME)
-    );
-    """
+   sample_ddl = """
+   CREATE TABLE My_Schema.Sample_Table (
+     ID integer PRIMARY KEY,
+     NAME varchar(100) NOT NULL,
+     TOTAL bigint NOT NULL,
+     AVG decimal(5,1) NOT NULL,
+     CREATED_AT date, -- Oracle 'DATE' -> BigQuery 'DATETIME'
+     UNIQUE (NAME)
+   );
+   """
 
 
-    # parse pattern (1-1)
-    table = DdlParse().parse(sample_ddl)
+   # parse pattern (1-1)
+   table = DdlParse().parse(sample_ddl)
 
-    # parse pattern (1-2) : Specify source database
-    table = DdlParse().parse(ddl=sample_ddl, source_database=DdlParse.DATABASE.oracle)
+   # parse pattern (1-2) : Specify source database
+   table = DdlParse().parse(ddl=sample_ddl, source_database=DdlParse.DATABASE.oracle)
 
 
-    # parse pattern (2-1)
-    parser = DdlParse(sample_ddl)
-    table = parser.parse()
+   # parse pattern (2-1)
+   parser = DdlParse(sample_ddl)
+   table = parser.parse()
 
-    # parse pattern (2-2) : Specify source database
-    parser = DdlParse(ddl=sample_ddl, source_database=DdlParse.DATABASE.oracle)
-    table = parser.parse()
+   print("* BigQuery Fields * : normal")
+   print(table.to_bigquery_fields())
 
 
-    # parse pattern (3-1)
-    parser = DdlParse()
-    parser.ddl = sample_ddl
-    table = parser.parse()
+   # parse pattern (2-2) : Specify source database
+   parser = DdlParse(ddl=sample_ddl, source_database=DdlParse.DATABASE.oracle)
+   table = parser.parse()
 
-    # parse pattern (3-2) : Specify source database
-    parser = DdlParse()
-    parser.source_database = DdlParse.DATABASE.oracle
-    parser.ddl = sample_ddl
-    table = parser.parse()
 
+   # parse pattern (3-1)
+   parser = DdlParse()
+   parser.ddl = sample_ddl
+   table = parser.parse()
 
-    print("* TABLE *")
-    print("schema = {} : name = {} : is_temp = {}".format(table.schema, table.name, table.is_temp))
+   # parse pattern (3-2) : Specify source database
+   parser = DdlParse()
+   parser.source_database = DdlParse.DATABASE.oracle
+   parser.ddl = sample_ddl
+   table = parser.parse()
 
-    print("* BigQuery Fields *")
-    print(table.to_bigquery_fields())
+   print("* BigQuery Fields * : Oracle")
+   print(table.to_bigquery_fields())
 
-    print("* BigQuery Fields - column name to lower case / upper case *")
-    print(table.to_bigquery_fields(DdlParse.NAME_CASE.lower))
-    print(table.to_bigquery_fields(DdlParse.NAME_CASE.upper))
 
-    print("* COLUMN *")
-    for col in table.columns.values():
-        print("name = {} : data_type = {} : length = {} : precision(=length) = {} : scale = {} : constraint = {} : not_null =  {} : PK =  {} : unique =  {} : BQ {}".format(
-            col.name,
-            col.data_type,
-            col.length,
-            col.precision,
-            col.scale,
-            col.constraint,
-            col.not_null,
-            col.primary_key,
-            col.unique,
-            col.to_bigquery_field()
-            ))
+   print("* TABLE *")
+   print("schema = {} : name = {} : is_temp = {}".format(table.schema, table.name, table.is_temp))
 
-    print("* Get Column object (case insensitive) *")
-    print(table.columns["total"])
+   print("* BigQuery Fields *")
+   print(table.to_bigquery_fields())
+
+   print("* BigQuery Fields - column name to lower case / upper case *")
+   print(table.to_bigquery_fields(DdlParse.NAME_CASE.lower))
+   print(table.to_bigquery_fields(DdlParse.NAME_CASE.upper))
+
+   print("* COLUMN *")
+   for col in table.columns.values():
+       print("name = {} : data_type = {} : length = {} : precision(=length) = {} : scale = {} : constraint = {} : not_null =  {} : PK =  {} : unique =  {} : BQ {}".format(
+           col.name,
+           col.data_type,
+           col.length,
+           col.precision,
+           col.scale,
+           col.constraint,
+           col.not_null,
+           col.primary_key,
+           col.unique,
+           col.to_bigquery_field()
+           ))
+
+   print("* Get Column object (case insensitive) *")
+   print(table.columns["total"])
 
 License
 -------
@@ -151,18 +164,3 @@ Special Thanks
 --------------
 
 -  pyparsing : http://pyparsing.wikispaces.com/
-
-.. |PyPI version| image:: https://img.shields.io/pypi/v/ddlparse.svg
-   :target: https://pypi.python.org/pypi/ddlparse
-.. |Python version| image:: https://img.shields.io/pypi/pyversions/ddlparse.svg
-   :target: https://pypi.python.org/pypi/ddlparse
-.. |Travis CI Build Status| image:: https://travis-ci.org/shinichi-takii/ddlparse.svg?branch=master
-   :target: https://travis-ci.org/shinichi-takii/ddlparse
-.. |Coveralls Coverage Status| image:: https://coveralls.io/repos/github/shinichi-takii/ddlparse/badge.svg?branch=master
-   :target: https://coveralls.io/github/shinichi-takii/ddlparse?branch=master
-.. |codecov Coverage Status| image:: https://codecov.io/gh/shinichi-takii/ddlparse/branch/master/graph/badge.svg
-   :target: https://codecov.io/gh/shinichi-takii/ddlparse
-.. |Requirements Status| image:: https://requires.io/github/shinichi-takii/ddlparse/requirements.svg?branch=master
-   :target: https://requires.io/github/shinichi-takii/ddlparse/requirements/?branch=master
-.. |License| image:: https://img.shields.io/badge/License-BSD%203--Clause-blue.svg
-   :target: https://github.com/shinichi-takii/ddlparse/blob/master/LICENSE.md
diff --git a/ddlparse/__init__.py b/ddlparse/__init__.py
@@ -8,7 +8,7 @@
 from .ddlparse import *
 
 __copyright__    = 'Copyright (C) 2018 Shinichi Takii'
-__version__      = '1.1.1'
+__version__      = '1.1.2'
 __license__      = 'BSD-3-Clause'
 __author__       = 'Shinichi Takii'
 __author_email__ = 'shinichi.takii@gmail.com'

diff --git a/ddlparse/ddlparse.py b/ddlparse/ddlparse.py
@@ -165,7 +165,7 @@ def bigquery_data_type(self):
 
         # BigQuery data type = {source_database: [data type, ...], ...}
         BQ_DATA_TYPE_DIC = OrderedDict()
-        BQ_DATA_TYPE_DIC["STRING"] = {None: [re.compile(r"(CHAR|TEXT)")]}
+        BQ_DATA_TYPE_DIC["STRING"] = {None: [re.compile(r"(CHAR|TEXT|CLOB)")]}
         BQ_DATA_TYPE_DIC["INTEGER"] = {None: [re.compile(r"INT|SERIAL|YEAR")]}
         BQ_DATA_TYPE_DIC["FLOAT"] = {None: [re.compile(r"(FLOAT|DOUBLE)"), "REAL", "MONEY"]}
         BQ_DATA_TYPE_DIC["DATETIME"] = {
@@ -193,7 +193,15 @@ def bigquery_data_type(self):
                         return bq_type
 
         if self._data_type in ["NUMERIC", "NUMBER", "DECIMAL"]:
-            return "INTEGER" if self._scale is None else "FLOAT"
+            if self._scale is not None:
+                return "FLOAT"
+
+            if self._data_type == "NUMBER" \
+                and self._source_database == self.DATABASE.oracle \
+                and self._length is None:
+                return "FLOAT"
+
+            return "INTEGER"
 
         raise ValueError("Unknown data type : '{}'".format(self._data_type))
 
@@ -329,7 +337,7 @@ class DdlParse(DdlParseBase):
                         + Optional(CaselessKeyword("WITHOUT TIME ZONE") ^ CaselessKeyword("WITH TIME ZONE") ^ CaselessKeyword("PRECISION"))
                         + Optional(_LPAR + Regex(r"\d+\s*,*\s*\d*") + _RPAR)
                         )("type")
-                    + Optional(Word(alphanums+"_': "))("constraint")
+                    + Optional(Word(alphanums+"_': -"))("constraint")
                 )("column")
             )
         )("columns")

diff --git a/example/example.py b/example/example.py
@@ -30,6 +30,10 @@
 parser = DdlParse(sample_ddl)
 table = parser.parse()
 
+print("* BigQuery Fields * : normal")
+print(table.to_bigquery_fields())
+
+
 # parse pattern (2-2) : Specify source database
 parser = DdlParse(ddl=sample_ddl, source_database=DdlParse.DATABASE.oracle)
 table = parser.parse()
@@ -46,6 +50,9 @@
 parser.ddl = sample_ddl
 table = parser.parse()
 
+print("* BigQuery Fields * : Oracle")
+print(table.to_bigquery_fields())
+
 
 print("* TABLE *")
 print("schema = {} : name = {} : is_temp = {}".format(table.schema, table.name, table.is_temp))

diff --git a/test/test_ddlparse.py b/test/test_ddlparse.py
@@ -201,16 +201,31 @@
         "ddl" :
             """
             CREATE TABLE Sample_Table (
-              Col_01 date
+              Col_01 date,
+              Col_02 number(1),
+              Col_03 number(1,2),
+              Col_04 number,
+              Col_05 clob,
+              Col_06 nclob
             );
             """,
         "database" : DdlParse.DATABASE.oracle,
         "table" : {"schema" : None, "name" : "Sample_Table", "temp" : False},
         "columns" : [
             {"name" : "Col_01", "type" : "DATE", "length" : None, "scale" : None, "not_null" : False, "pk" : False, "unique" : False, "constraint" : ""},
+            {"name" : "Col_02", "type" : "NUMBER", "length" : 1, "scale" : None, "not_null" : False, "pk" : False, "unique" : False, "constraint" : ""},
+            {"name" : "Col_03", "type" : "NUMBER", "length" : 1, "scale" : 2, "not_null" : False, "pk" : False, "unique" : False, "constraint" : ""},
+            {"name" : "Col_04", "type" : "NUMBER", "length" : None, "scale" : None, "not_null" : False, "pk" : False, "unique" : False, "constraint" : ""},
+            {"name" : "Col_05", "type" : "CLOB", "length" : None, "scale" : None, "not_null" : False, "pk" : False, "unique" : False, "constraint" : ""},
+            {"name" : "Col_06", "type" : "NCLOB", "length" : None, "scale" : None, "not_null" : False, "pk" : False, "unique" : False, "constraint" : ""},
         ],
         "bq_field" : [
             '{"name": "Col_01", "type": "DATETIME", "mode": "NULLABLE"}',
+            '{"name": "Col_02", "type": "INTEGER", "mode": "NULLABLE"}',
+            '{"name": "Col_03", "type": "FLOAT", "mode": "NULLABLE"}',
+            '{"name": "Col_04", "type": "FLOAT", "mode": "NULLABLE"}',
+            '{"name": "Col_05", "type": "STRING", "mode": "NULLABLE"}',
+            '{"name": "Col_06", "type": "STRING", "mode": "NULLABLE"}',
         ],
     },