Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Setting up a loading routing for buoy data; also set up logging

  • Loading branch information...
commit 6d1ac67a279b793c7dc1c543703326581b7c4f5c 1 parent f112a2f
@arthur-e authored
View
117 campaigns/loaders.py
@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+import os, sys
+import datetime, numpy, re, shutil, random
+
+sys.path.append('/usr/local/project/GreatLakesObs/')
+try:
+ os.environ["DJANGO_SETTINGS_MODULE"] = "settings_production"
+ import settings_production as settings # Assumed to be in the same directory.
+except ImportError:
+ sys.stderr.write("Couldn't find the django settings file\n")
+ sys.exit(1)
+
+try:
+ from django.db.utils import IntegrityError
+except ImportError:
+ from django.db import IntegrityError
+except:
+ raise
+from django.core.management import setup_environ
+from django.contrib.gis.geos import *
+from settings import SUPPORTED_BUOYS
+from campaigns.models import *
+from xml.dom import minidom
+
+#output_folder = '/netfs/cluster8/data/gis_lab/project/GLOS/data/buoy/'
+#failed_folder = '/netfs/cluster8/data/gis_lab/project/GLOS/data/buoy/failed_import'
+#dir_path = '/netfs/cluster8/data/public/ftp/incoming'
+
+def load_buoys(path, id_list=None):
+ '''
+ Loads all available buoy data in a specified location for, optionally, only
+ the buoys specified in a list of buoy IDs.
+ '''
+ finder = re.compile('\d{5}_\d*\.xml')
+
+ def load_from_xml(xml_name, model):
+ '''
+ Loads buoy data from an XML file in the NDBC schema.
+ '''
+ expected_fields = []
+ for field in model._meta.fields:
+ expected_fields.append(field.name)
+
+ try:
+ xmldoc = minidom.parse(xml_name)
+ objects = []
+ attrs = {}
+ attrs['date_time'] = datetime.datetime.strptime(str(xmldoc.childNodes[0].childNodes[3].childNodes[0].data),"%m/%d/%Y %H:%M:%S")
+ attrs['station'] = str(xmldoc.childNodes[0].childNodes[1].childNodes[0].data)
+
+ for node in xmldoc.childNodes[0].childNodes[5].childNodes:
+
+ if node.__class__.__name__ == 'Element':
+
+ # Don't assume every tag in the feed is a model field
+ if str(node.tagName) in expected_fields:
+
+ if node.childNodes[0].data not in ['NAN', '']:
+ attrs[str(node.tagName)] = eval(node.childNodes[0].data)
+
+ else:
+ attrs[str(node.tagName)] = None
+
+ except:
+ sys.stderr.write("Unhandled error while parsing %s\n" % xml_name)
+
+ for errorline in sys.exc_info():
+ sys.stderr.write(" " + str(errorline) + "\n")
+ return -1
+
+ inst = model(**attrs)
+
+ try:
+ inst.save()
+ except IntegrityError:
+ sys.stderr.write("IntegrityError encountered while saving record.\n")
+ return -1
+ except:
+ sys.stderr.write("Unhandled error while running load_BuoyRecord()\n")
+ for errorline in sys.exc_info():
+ sys.stderr.write(" " + str(errorline) + "\n")
+ return -1
+
+def process_incoming(dir_path, output_folder, failed_folder):
+ '''
+ Adds the contents of a directory to the database
+ Leaves behind a processed.txt file that lists the files
+ that have already been processed. Will not process a file
+ that has already been processed according to the text file.
+ '''
+
+ sys.stdout.write('Starting data import at ' + str(datetime.datetime.now()) + '\n')
+ files = os.listdir(dir_path)
+
+ for item in files:
+ if northbuoy_finder.match(item) != None:
+ sys.stdout.write( "Processing: " + os.path.join(dir_path, item)+ '\n')
+ if load_buoy(os.path.join(dir_path, item), NorthBuoyRecord) == -1:
+ shutil.move(os.path.join(dir_path, item),os.path.join(failed_folder, item))
+ else:
+ shutil.move(os.path.join(dir_path, item),os.path.join(output_folder, item))
+
+ if southbuoy_finder.match(item) != None:
+ sys.stdout.write( "Processing: " + os.path.join(dir_path, item)+ '\n')
+ if load_buoy(os.path.join(dir_path, item), SouthBuoyRecord) == -1:
+ shutil.move(os.path.join(dir_path, item),os.path.join(failed_folder, item))
+ else:
+ shutil.move(os.path.join(dir_path, item),os.path.join(output_folder, item))
+
+if __name__=="__main__":
+ setup_environ(settings)
+
+ process_incoming(sys.argv[1], sys.argv[2], sys.argv[3])
+
+ print 'Done!'
+ print datetime.datetime.now()
+
View
0  campaigns/management/__init__.py
No changes.
View
0  campaigns/management/commands/__init__.py
No changes.
View
43 campaigns/management/commands/loadbuoys.py
@@ -0,0 +1,43 @@
+import os, sys
+import datetime, numpy, re, shutil, random
+import logging
+logger = logging.getLogger('buoy.logging')
+
+sys.path.append('/usr/local/project/GreatLakesObs/')
+
+try:
+ os.environ["DJANGO_SETTINGS_MODULE"] = "settings"
+ import settings as settings # Assumed to be in the same directory.
+except ImportError:
+ sys.stderr.write("Couldn't find the django settings file\n")
+ sys.exit(1)
+
+try:
+ from django.db.utils import IntegrityError
+except ImportError:
+ from django.db import IntegrityError
+except:
+ sys.stderr.write("Couldn't import the IntegrityError class\n")
+ sys.exit(1)
+
+from xml.dom import minidom
+from django.core.management import setup_environ
+from django.core.management.base import BaseCommand, CommandError
+from django.contrib.gis.geos import *
+from settings import SUPPORTED_BUOYS
+from campaigns.models import *
+
+class Command(BaseCommand):
+ args = '<path>'
+ help = 'Loads all available buoy data for SUPPORTED_BUOYS from the specified path'
+
+ def handle(self, *args, **options):
+ path = args[0]
+ finder = re.compile('\d{5}_\d*\.xml')
+
+ logger.info('\nStarting data import at ' + str(datetime.datetime.now()) + '\n')
+ files = os.listdir(path)
+
+ for item in files:
+ if finder.match(item) != None:
+ logger.debug(item)
View
5 settings.ini.TEMPLATE
@@ -17,7 +17,10 @@ DATABASE_PORT:
DATABASE_NAME: greatlakesobs
[logging]
-LOG_FILENAME: /var/log/greatlakesobs/django.log
+LOG_FILENAME: /var/log/GreatLakesObs/django.log
+ERROR_LOG_FILENAME: /var/log/GreatLakesObs/error.log
+GENERAL_LOG_FILENAME: /var/log/GreatLakesObs/general.log
+BUOY_LOG_FILENAME: /var/log/GreatLakesObs/buoy.log
[media]
MEDIA_ROOT: /path/to/static/media/directory/used/for/development
View
39 settings.py
@@ -14,6 +14,8 @@
MANAGERS = tuple(config.items('404 mail'))
+SUPPORTED_BUOYS = ('45023', '45025')
+
DATABASES = {
'default': {
'ENGINE': 'django.contrib.gis.db.backends.postgis',
@@ -129,6 +131,7 @@
'django.contrib.admindocs',
'public',
'assets',
+ 'campaigns',
)
# A sample logging configuration. The only tangible logging
@@ -139,11 +142,35 @@
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
+ 'formatters': {
+ 'default': {
+ 'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+ },
+ },
'handlers': {
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler'
- }
+ },
+ 'error': {
+ 'level': 'WARNING',
+ 'class': 'logging.FileHandler',
+ #'filename':os.path.join(PARENT_DIR, 'django.log'),
+ 'filename': config.get('logging', 'ERROR_LOG_FILENAME'),
+ 'formatter': 'default'
+ },
+ 'general_handler': {
+ 'level': 'DEBUG',
+ 'class': 'logging.FileHandler',
+ 'filename': config.get('logging', 'GENERAL_LOG_FILENAME'),
+ 'formatter': 'default'
+ },
+ 'buoy_handler': {
+ 'level': 'INFO',
+ 'class': 'logging.FileHandler',
+ 'filename': config.get('logging', 'BUOY_LOG_FILENAME'),
+ 'formatter': 'default'
+ },
},
'loggers': {
'django.request': {
@@ -151,5 +178,15 @@
'level': 'ERROR',
'propagate': True,
},
+ 'buoy': {
+ 'handlers': ['general_handler'],
+ 'level': 'DEBUG',
+ 'propagate': False
+ },
+ 'buoy.loading': {
+ 'handler': ['buoy_handler'],
+ 'level': 'INFO',
+ 'propagate': True,
+ }
}
}
Please sign in to comment.
Something went wrong with that request. Please try again.