Skip to content
This repository has been archived by the owner on Jan 13, 2024. It is now read-only.

Commit

Permalink
Browse files Browse the repository at this point in the history
fix the method exporting the result of a SQL query in class Database
  • Loading branch information
sdpython committed Sep 15, 2013
1 parent 1d23ee6 commit 89973cf
Show file tree
Hide file tree
Showing 11 changed files with 76 additions and 43 deletions.
2 changes: 1 addition & 1 deletion README.rst
Expand Up @@ -10,7 +10,7 @@ README
Introduction
------------

This project contain helpers used at the `ENSAE <http://www.ensae.fr/>`_ for teaching puroposes.
This project contain helpers used at the `ENSAE <http://www.ensae.fr/>`_ for teaching purposes.
The project is hosted `here <http://www.xavierdupre.fr/site2013/index_code.html>`_
and on gitbug: `pyensae <https://github.com/sdpython/pyensae/>`_.

Expand Down
2 changes: 1 addition & 1 deletion _doc/sphinxdoc/source/index.rst
Expand Up @@ -34,7 +34,7 @@ From the page `Pieces of codes, libraries <http://www.xavierdupre.fr/site2013/in
sudo su
python3.3 setup.py install
* Using pip::
* Using pip (see `pipy/pyensae <https://pypi.python.org/pypi/pyensae/>`_)::

pip install pyensae

Expand Down
15 changes: 15 additions & 0 deletions _doc/sphinxdoc/source/usecase.rst
Expand Up @@ -35,3 +35,18 @@ Import a flat file into a SQLite database
file = "textfile.txt"
import_flatfile_into_database(dbf, file)

Export the results of a SQL query into a flat file
++++++++++++++++++++++++++++++++++++++++++++++++++

::

from pyensae.sql.database_main import Database
dbfile = "filename.db3"
filetxt = "fileview.txt"
sql = "..."
db = Database(dbfile)
db.connect()
db.export_view_into_flat_file (sql, fileview, header = True)
db.close()
2 changes: 1 addition & 1 deletion setup.py
Expand Up @@ -59,7 +59,7 @@
project_var_name + ', first name, last name'

DESCRIPTION = \
"""This a project template including a setup and the generation of sphinx generation."""
"""Helpers for teaching purposes."""

CLASSIFIERS = \
[
Expand Down
3 changes: 2 additions & 1 deletion src/pyensae/__init__.py
Expand Up @@ -14,4 +14,5 @@ def check( log = False):
return True

from .resources.http_retrieve import download_data
from .sql.database_helper import import_flatfile_into_database
from .sql.database_helper import import_flatfile_into_database
from .file.file_helper import replace_comma_by_point
2 changes: 1 addition & 1 deletion src/pyensae/resources/__init__.py
Expand Up @@ -21,6 +21,6 @@ def check_dependency( name ):
sys.path.append(ab)
import pyhome3
else :
raise Exception("unexpected library name" + name)
raise ImportError("unexpected library name" + name)


6 changes: 3 additions & 3 deletions src/pyensae/resources/http_retrieve.py
Expand Up @@ -87,7 +87,7 @@ def download_data ( name,
website = "http://www.xavierdupre.fr/enseignement/complements/"

if not os.path.exists (whereTo) :
raise Exception("this folder should exists " + whereTo)
raise FileExistsError("this folder should exists " + whereTo)

origname = name
if name in sys.modules :
Expand Down Expand Up @@ -121,7 +121,7 @@ def download_data ( name,
u = urllib2.urlopen (url, "r")
all = u.read ()
if "404 Not Found" in all and 'if "404 Not Found" in all :' not in all :
raise Exception ("fichier introuvable: " + name )
raise FileNotFoundError ("fichier introuvable: " + name )
u.close ()
u = open (outfile, "wb" if ".exe" in file or ".zip" in file else "r")
u.write ( all )
Expand Down Expand Up @@ -208,7 +208,7 @@ def download_data ( name,
raise e

if name not in temp.__name__ :
raise Exception ("name should be present in __name__ " + name + " ? " + temp.__name__ )
raise NameError ("name should be present in __name__ " + name + " ? " + temp.__name__ )
glo[moduleName] = temp
sys.modules[moduleName] = temp
sys.modules[origname] = temp
Expand Down
11 changes: 7 additions & 4 deletions src/pyensae/sql/database_core.py
Expand Up @@ -105,8 +105,8 @@ def blind(s) :
LOG = blind
self.LOG = LOG

if isinstance (LOG, dict) : raise Exception ("fLOG should be a function, not a dictionary")
if isinstance (self.LOG, dict) : raise Exception ("LOG should be a function, not a dictionary")
if isinstance (LOG, dict) : raise TypeError ("fLOG should be a function, not a dictionary")
if isinstance (self.LOG, dict) : raise TypeError ("LOG should be a function, not a dictionary")

if engine == "SQLite" :
self._sql_file = sql_file
Expand Down Expand Up @@ -320,7 +320,8 @@ def get_file (self, attached_db = False) :
def has_table (self, table) :
"""say if the table belongs to the database
@param table table name
@return boolean"""
@return boolean
"""
return table in self.get_table_list ("." in table)

def has_index (self, index) :
Expand Down Expand Up @@ -416,8 +417,10 @@ def get_attached_database_list (self, file = False) :

def get_table_list (self, add_attached = False) :
"""
return the list of table
@param add_attached if True, add the list of tables included in the attached databases
return the table list
@return the table list
"""
self._check_connection ()
if self.isMSSQL () :
Expand Down
4 changes: 2 additions & 2 deletions src/pyensae/sql/database_core2.py
Expand Up @@ -25,7 +25,7 @@ def _check_connection (self) :
"""check the SQL connection"""
if "_connection" not in self.__dict__ :
message = "use connect method before doing operation on this database"
raise HalException (message)
raise Exception (message)

def _check_values (self, values) :
"""when values are inserted or updated, this method doubles "'"
Expand Down Expand Up @@ -264,7 +264,7 @@ def _process_text_line (self, line, columns, format, lower_case, num_line,
elif v[1] is datetime.datetime :
if isinstance (val, datetime.datetime) : val = val
elif isinstance (val, str) : val = datetime.datetime.parse(val)
else : raise Exception("unable to convert %s into datetime" % str(type(val)))
else : raise TypeError("unable to convert %s into datetime" % str(type(val)))
else : val = v [1] (val)
except ValueError as e :
self.LOG("(b)line number ", num_line, "**unable to process a line columns ",c, "#", v [0], " type ", v [1], " value ", repr (line [c]))
Expand Down
8 changes: 4 additions & 4 deletions src/pyensae/sql/database_exception.py
Expand Up @@ -19,14 +19,14 @@ def __init__ (self, description, ex, sql, log = True) :
@param sql SQL instruction
@param log log the exception
"""
HalException.__init__ (self, description, log = False)
Exception.__init__ (self, description + "\n" + sql)
self.ex = ex
self.sql = sql
if log :
self._log ()
if False and log :
print (description + "\n" + sql)

def __str__ (self) :
mes = HalException.__str__ (self)
mes = Exception.__str__ (self)
mes += "\n" + str (self.ex)
mes += "\n" + "\n".join ( repr (self.sql).split ("\\n") )
if len (mes) > 10000 :
Expand Down
64 changes: 39 additions & 25 deletions src/pyensae/sql/database_import_export.py
Expand Up @@ -58,15 +58,15 @@ def export_view_into_flat_file (self, view_sql,
filename,
header = False,
post_process = None,
encoding = None) :
encoding = "utf8") :
"""export a table into a flat file
@param view_sql SQL request
@param filename filename
@param header if != None, add a header on the first line (header is a list of string)
@param post_process if != None, use this function to post-process a text line extracted from the file
@param encoding if != None, use this as a parameter to convert any value into str
"""
sepline = GetSepLine ()
sepline = "\n"

self._check_connection ()

Expand All @@ -85,18 +85,15 @@ def export_view_into_flat_file (self, view_sql,
cur = self.execute (sql)
nbline = 0

f = open (filename, "w", encoding="utf-8")
f = open (filename, "w", encoding=encoding)
f.write (header_line)

for line_ in cur :

if post_process != None : line = post_process (line_, memo)
else : line = line_

if encoding != None :
pr = "\t".join ([self._clean_string (str (x, encoding)) for x in line])
else :
pr = "\t".join ([self._clean_string (str (x)) for x in line])
pr = "\t".join ([self._clean_string (str (x)) for x in line])

f.write (pr + sepline)
nbline += 1
Expand Down Expand Up @@ -145,15 +142,7 @@ def _append_table (self, file,
@param file file name or a matrix (this matrix can be an iterator)
@param table table name
@param columns columns definition, dictionary { key:(column_name,python_type) }
or { key:(column_name,python_type,preprocessing_function) }
where preprocessing_function is a function whose prototype is for example:
@code
def preprocessing_score (s) :
return s.replace (",",".")
@endcode
- if PRIMARYKEY is added, the key is considered as the primary key
- if AUTOINCREMENT is added, the key will automatically filled (like an id)
@param columns columns definition (see below)
@param format tsv, the only one accepted for the time being, it can be a function (line, **params)
@param header the file has a header of not, if True, skip the first line
@param stop if -1, insert every line, otherwise stop when the number of inserted lines is stop
Expand All @@ -168,6 +157,22 @@ def preprocessing_score (s) :
@param skip_exception skip exception while inserting an element
@return number of inserted elements
The columns definition must follow the schema:
- dictionary ``{ key:(column_name,python_type) }``
- or ``{ key:(column_name,python_type,preprocessing_function) }``
``preprocessing_function`` is a function whose prototype is for example:
@code
def preprocessing_score (s) :
return s.replace (",",".")
@endcode
And:
- if ``PRIMARYKEY`` is added, the key is considered as the primary key
- if ``AUTOINCREMENT`` is added, the key will automatically filled (like an id)
"""

if stop != -1 : self.LOG ("SQL append table stop is ", stop)
Expand Down Expand Up @@ -284,15 +289,7 @@ def import_table_from_flat_file ( self,
@param file file name or matrix
@param table table name
@param columns columns definition, dictionary { key:(column_name,python_type) }
or { key:(column_name,python_type,preprocessing_function) }
where preprocessing_function is a function whose prototype is for example:
@code
def preprocessing_score (s) :
return s.replace (",",".")
@endcode
- if PRIMARYKEY is added, the key is considered as the primary key
- if AUTOINCREMENT is added, the key will automatically filled (like an id)
@param columns columns definition (see below)
if None: columns are guessed
@param format tsv, the only one accepted for the time being, it can be a function whose parameter are a line and **params
@param header the file has a header of not, if True, skip the first line
Expand All @@ -306,6 +303,23 @@ def preprocessing_score (s) :
@param change_to_text changes the format from any to TEXT
@param display if True, print more information on stdout
@param strict_separator strict number of columns, it assumes there is no separator in the content of every column
The columns definition must follow the schema:
- dictionary ``{ key:(column_name,python_type) }``
- or ``{ key:(column_name,python_type,preprocessing_function) }``
``preprocessing_function`` is a function whose prototype is for example:
@code
def preprocessing_score (s) :
return s.replace (",",".")
@endcode
And:
- if ``PRIMARYKEY`` is added, the key is considered as the primary key
- if ``AUTOINCREMENT`` is added, the key will automatically filled (like an id)
"""
if isinstance (file, list) : self.LOG ("processing file ", file [:min (len (file),10)])
else : self.LOG ("processing file ", file)
Expand Down

0 comments on commit 89973cf

Please sign in to comment.