Skip to content
Permalink
Browse files

add fonction to convert dbase into sqllite

  • Loading branch information...
sdpython committed Sep 30, 2014
1 parent 14cb12d commit c44a33e63391432886404abe3b948c064d2bbf84
Showing with 82 additions and 5 deletions.
  1. +18 −2 _unittests/ut_datasource/test_convert.py
  2. +1 −1 src/pyensae/__init__.py
  3. +63 −2 src/pyensae/datasource/convert.py
@@ -18,7 +18,7 @@
import pyquickhelper

from pyquickhelper import fLOG
from src.pyensae.datasource.convert import dBase2df
from src.pyensae.datasource.convert import dBase2df, dBase2sqllite


class TestConvert(unittest.TestCase):
@@ -28,8 +28,24 @@ def test_dbf2df(self):
fold = os.path.abspath(os.path.split(__file__)[0])
file = os.path.join(fold, "data", "varmod_naissances.dbf")
df = dBase2df(file)
fLOG(df)
assert df is not None
#fLOG(df)

def test_dbf2sqlite(self):
fLOG (__file__, self._testMethodName, OutputPrint = __name__ == "__main__", LogFile = "temp_hal_log2.txt")
fold = os.path.abspath(os.path.split(__file__)[0])
file = os.path.join(fold, "data", "varmod_naissances.dbf")

outf = os.path.join(fold, "out_dbfsqlite")
if not os.path.exists(outf) : os.mkdir(outf)

dbfile = os.path.join(outf, "naissance.db3")
if os.path.exists(dbfile) : os.remove(dbfile)

df = dBase2df(file)
dBase2sqllite(dbfile, file)
assert df is not None
assert os.path.exists(dbfile)

if __name__ == "__main__" :
unittest.main ()
@@ -27,4 +27,4 @@ def check( log = False):
from .finance.astock import StockPrices
from .sql.database_main import Database
from .datasource.data_velib import DataVelibCollect
from .datasource.convert import dBase2df
from .datasource.convert import dBase2df, dBase2sqllite
@@ -5,6 +5,8 @@

import pandas

from ..sql.database_main import Database

def dBase2df(file, encoding="cp437"):
"""
converts a dBase file into a list of dataframe (one per table)
@@ -15,10 +17,69 @@ def dBase2df(file, encoding="cp437"):
The module relies on `dbfread <https://pypi.python.org/pypi/dbfread/>`_.
"""

import dbfread
table = dbfread.open(file, load=True, encoding=encoding)
res = [ _ for _ in table ]
return pandas.DataFrame(table.records)


def dBase2sqllite(db, table, encoding="cp437"):
"""
Put all rows from a dBase database into sqlite
Add a dbase table to an open sqlite database.
@param db cursor on SQLite or file name
@param table DBF object
@param encoding encoding if table is a filename
The table will be removed if it exists.
"""

typemap = {
'F': 'FLOAT',
'L': 'BOOLEAN',
'I': 'INTEGER',
'C': 'TEXT',
'N': 'REAL', # because it can be integer or float
'M': 'TEXT',
'D': 'DATE',
'T': 'DATETIME',
'0': 'INTEGER',
}

if isinstance(db,str):
cursor = Database(db)
cursor.connect()
else :
cursor = db

if isinstance(table, str):
import dbfread
table = dbfread.open(table, load=False, encoding=encoding)

cursor.execute('drop table if exists %s' % table.name)

field_types = {}
for f in table.fields:
field_types[f.name] = typemap.get(f.type, 'TEXT')

#
# Create the table
#
defs = ', '.join(['%s %s' % (f, field_types[f])
for f in table.field_names])
sql = 'create table %s (%s)' % (table.name, defs)
cursor.execute(sql)

# Create data rows
refs = ', '.join([':' + f for f in table.field_names])
sql = 'insert into %s values (%s)' % (table.name, refs)

for rec in table:
cursor._connection.execute(sql, list(rec.values()))

if isinstance(db, str):
cursor.commit()
cursor.close()


0 comments on commit c44a33e

Please sign in to comment.
You can’t perform that action at this time.