Permalink
Browse files

Debug import enquete

  • Loading branch information...
1 parent bbbca81 commit 5eaaae638a8eb595905075d8f71655ab0a73ec7f @danieleguido danieleguido committed Feb 21, 2013
Showing with 26 additions and 15 deletions.
  1. +2 −0 outside/api.py
  2. +8 −4 reanalyseapp/imexport.py
  3. +6 −3 reanalyseapp/update.py
  4. +6 −5 reanalyseapp/utils.py
  5. +1 −1 reanalyseapp/views.py
  6. +3 −2 settings.py
View
2 outside/api.py
@@ -452,7 +452,9 @@ def test( request ):
full_url = "%s://%s%s"% ( 'https' if request.is_secure() else 'http', request.get_host(), reverse('outside.views.confirm', args=( "dsdsdsds", 12 ) ) )
response.add( 'full_url', full_url )
+ request.session.set_test_cookie()
+ response.add( 'test cookie worked', request.session.test_cookie_worked() )
return response.json()
View
12 reanalyseapp/imexport.py
@@ -200,7 +200,7 @@ def importEnqueteUsingMeta(upPath,folderPath):
newEnquete.metadata = simplejson.dumps(allmeta,indent=4,ensure_ascii=False)
newEnquete.save()
- eidstr = "["+str(newEnquete.id)+"] " # for logger prefix
+ eidstr = "[enquete_id:"+str(newEnquete.id)+"] " # for logger prefix
### create permission for this enquete
content_type,isnew = ContentType.objects.get_or_create(app_label='reanalyseapp', model='Enquete')
@@ -214,6 +214,7 @@ def importEnqueteUsingMeta(upPath,folderPath):
doc = csv.DictReader(open(docPath),delimiter='\t')
for row in doc:
#try:
+ logger.info( "%s storing document " % ( eidstr, row['*id'] ) )
if row['*id']!='*descr':
#try:
file_location = folderPath+row['*file'] # if LINK > url , else REF > nothing
@@ -238,9 +239,9 @@ def importEnqueteUsingMeta(upPath,folderPath):
### fetch document date
try:
- doc_date = datetime.datetime.strptime(row['*date'], "%d/%m/%y") #"31-12-12"
+ doc_date = datetime.datetime.strptime(row['*date'], "%Y_%m_%d") #"31-12-12"
except:
- logger.info(eidstr+"EXCEPT malformed or empty date @ "+row['*id']+" | "+row['*file'])
+ logger.exception(eidstr+"EXCEPT malformed or empty date @ "+row['*id']+" | "+row['*file'])
doc_date = datetime.datetime.today()
### very special for ese, don't create any texte() model, just parse ese.xml and fill enquete.ese with a json
@@ -256,7 +257,8 @@ def importEnqueteUsingMeta(upPath,folderPath):
elif doc_category1 in DOC_CAT_1.keys() and doc_category2 in DOC_CAT_2.keys():
if doc_mimetype in DOCUMENT_MIMETYPES:
newDocument = Texte(enquete=newEnquete, name=doc_name, doccat1=doc_category1, doccat2=doc_category2, description=doc_description, locationpath=file_location, date=doc_date, location=doc_location, status='1', public=doc_public)
-
+ logger.info(eidstr+"creating document: "+doc_mimetype+" | "+doc_category1+" | "+doc_category2+" | "+file_location)
+
newDocument.doctype = doc_mimetype.upper()
if doc_mimetype in ['link','ref']:
@@ -305,6 +307,8 @@ def importEnqueteUsingMeta(upPath,folderPath):
### unknown cat
else:
logger.info(eidstr+"EXCEPT unconsidered or empty *category: ("+doc_category1+") | ("+doc_category2+")")
+
+
#except:
#logger.info(eidstr+" EXCEPT on meta_document.csv line: "+row['*id'])
else:
View
9 reanalyseapp/update.py
@@ -4,8 +4,7 @@
#
import sys, os, csv, re
from optparse import OptionParser
-
-
+from datetime import datetime
# get path of the django project
path = ("/").join( sys.path[0].split("/")[:-1] )
@@ -48,6 +47,10 @@ def update( textes, enquete, csvdict ):
locationgeo = re.sub( r'[^0-9\.,]', '', row['*locationgeo'])
researcher = row['*researcher']
article = row['*article']
+ date = datetime.strptime(row['*date'], "%Y_%m_%d")
+
+ print " date: %s" % date
+
except KeyError, e:
print " Field format is not valid: %s " % ( e )
break
@@ -149,4 +152,4 @@ def error( message="generic error", parser=None):
# execute srcipt
if __name__ == '__main__':
- main(sys.argv[1:])
+ main(sys.argv[1:])
View
11 reanalyseapp/utils.py
@@ -460,19 +460,20 @@ def extract(self, file, dir):
divisions = 100 / percent
perc = int(num_files / divisions)
- try :
+
# extract files to directory structure
- for i, name in enumerate(zf.namelist()) :
+ for i, name in enumerate(zf.namelist()) :
+ try :
name=name.decode("utf-8")
logger.info(name)
if not name.endswith('/'):
outfile = open(os.path.join(dir, name), 'wb')
outfile.write(zf.read(name.encode("latin-1")))
outfile.flush()
outfile.close()
- except Exception,e:
- logger.info("ERROR : A FILE NAME CONTAINS A COMPLEX CHARACTER, please use standard ascii charcater a-Z")
- exit()
+ except Exception,e:
+ logger.exception("ERROR : %s CONTAINS A COMPLEX CHARACTER, please use standard ascii charcater a-Z" % name )
+ exit()
def _createstructure(self, file, dir):
self._makedirs(self._listdirs(file), dir)
View
2 reanalyseapp/views.py
@@ -272,7 +272,7 @@ def deleteThis(path):
else:
logger.info("weird file path ! not removing: "+path)
except:
- logger.info("["+str(eid)+"] EXCEPT trying to remove: "+path)
+ logger.exception("["+str(eid)+"] EXCEPT trying to remove: "+path)
###########################################################################
@login_required
def eDelete(request,eid):
View
5 settings.py
@@ -157,7 +157,7 @@
# Examples: "http://foo.com/static/admin/", "/static/admin/".
ADMIN_MEDIA_PREFIX = BASE_URL+'media/admin/'
-SESSION_EXPIRE_AT_BROWSER_CLOSE = True
+# SESSION_EXPIRE_AT_BROWSER_CLOSE = True
# Additional locations of static files
STATICFILES_DIRS = (
@@ -247,8 +247,9 @@
'disable_existing_loggers': False,
'formatters': {
'verbose': {
+ 'format': '%(levelname)s %(asctime)s | %(module)s:%(lineno)d | %(thread)d %(message)s'
#'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
- 'format': '%(levelname)s %(asctime)s | %(module)s | %(message)s'
+ #'format': '%(levelname)s %(asctime)s | %(module)s | %(message)s'
},
'simple': {
'format': '%(levelname)s %(message)s'

0 comments on commit 5eaaae6

Please sign in to comment.