Updated adsputils to v1.2.4 and logging fixes

adsdata/metrics.py

@@ -10,37 +10,45 @@
 from sqlalchemy import create_engine
 from collections import defaultdict
 from datetime import datetime
+import os
 import time
 import sys
 import json
 import argparse
 
-from adsputils import load_config, setup_logging
+from adsputils import setup_logging, load_config
 import nonbib
 import models
 
+# ============================= INITIALIZATION ==================================== #
 
 Base = declarative_base()
 meta = MetaData()
-metrics_logger = None
 
+# ================================ CLASSES ======================================== #
 
 class Metrics():
     """computes and provides interface for metrics data"""
 
     def __init__(self, schema_='metrics'):
-        self.logger = setup_logging('AdsDataSqlSync', 'INFO')
+        # - Use app logger:
+        #import logging
+        #logger = logging.getLogger('ads-data')
+        # - Or individual logger for this file:
+        proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))
+        self.config = load_config(proj_home=proj_home)
+        self.logger = setup_logging(__name__, proj_home=proj_home,
+                                level=self.config.get('LOGGING_LEVEL', 'INFO'),
+                                attach_stdout=self.config.get('LOG_STDOUT', False))
 
         self.schema =  schema_
         self.table = models.MetricsTable()
         self.table.schema = self.schema
 
-        # used to buffer writes                                                                                         
+        # used to buffer writes
         self.upserts = []
         self.tmp_update_buffer = []
         self.tmp_count = 0
-        self.config = {}
-        self.config.update(load_config())
 
 
     def create_metrics_table(self, db_engine):
@@ -109,7 +117,7 @@ def get_by_bibcodes(self, session, bibcodes):
         return results
 
 
-    def update_metrics_changed(self, db_conn, nonbib_conn, row_view_schema='ingest'):  
+    def update_metrics_changed(self, db_conn, nonbib_conn, row_view_schema='ingest'):
         """changed bibcodes are in sql table, for each we update metrics record"""
         Nonbib_Session = sessionmaker(bind=nonbib_conn)
         nonbib_sess = Nonbib_Session()
@@ -118,7 +126,7 @@ def update_metrics_changed(self, db_conn, nonbib_conn, row_view_schema='ingest')
         Metrics_Session = sessionmaker()
         metrics_sess = Metrics_Session(bind=db_conn)
         metrics_sess.execute('set search_path to {}'.format('metrics'))
-        
+
         sql_sync = nonbib.NonBib(row_view_schema)
         query = nonbib_sess.query(models.NonBibTable)
         count = 0
@@ -149,7 +157,7 @@ def update_metrics_bibcode(self, bibcode, db_conn, nonbib_conn, row_view_schema=
         row = sql_sync.get_by_bibcode(nonbib_conn, bibcode)
         metrics_old = metrics_sess.query(models.MetricsTable).filter(models.MetricsTable.bibcode == bibcode).first()
         metrics_new  = self.row_view_to_metrics(row, nonbib_conn, row_view_schema, metrics_old)
-	if metrics_old:	
+	if metrics_old:
             metrics_sess.merge(metrics_new)
         else:
             metrics_sess.add(metrics_new)
@@ -221,7 +229,7 @@ def update_metrics_all(self, db_conn, nonbib_conn, row_view_schema='ingest', sta
     def row_view_to_metrics(self, passed_row_view, nonbib_db_conn, row_view_schema='nonbib', m=None):
         """convert the passed row view into a complete metrics dictionary"""
         if m is None:
-            m = models.MetricsTable()            
+            m = models.MetricsTable()
         # first do easy fields
         bibcode = passed_row_view.bibcode
         m.bibcode = bibcode

adsdata/models.py

@@ -12,7 +12,6 @@
 from collections import defaultdict
 from datetime import datetime
 
-from adsputils import load_config, setup_logging
 
 Base = declarative_base()
 
@@ -54,7 +53,7 @@ class CanonicalTable(Base):
 class AuthorTable(Base):
     __tablename__ = 'author'
     bibcode = Column(String, primary_key=True)
-    authors = Column(ARRAY(String)) 
+    authors = Column(ARRAY(String))
 
 
 class RefereedTable(Base):
@@ -87,28 +86,28 @@ class SimbadTable(Base):
     __tablename__ = 'simbad'
     bibcode = Column(String, primary_key=True)
     simbad_objects = Column(ARRAY(String))
-                     
+
 
 class NedTable(Base):
     __tablename__ = 'ned'
     bibcode = Column(String, primary_key=True)
     ned_objects = Column(ARRAY(String))
-                     
+
 
 class GrantsTable(Base):
     __tablename__ = 'grants'
     bibcode = Column(String, primary_key=True)
     grants = Column(ARRAY(String))
-                     
+
 
 class CitationTable(Base):
     __tablename__ = 'citation'
     bibcode = Column(String, primary_key=True)
     citations = Column(ARRAY(String))
-                     
+
 
 class RelevanceTable(Base):
-    __tablename__ = 'relevance' 
+    __tablename__ = 'relevance'
     bibcode = Column(String, primary_key=True)
     boost = Column(Float)
     citation_count = Column(Integer)
@@ -120,25 +119,25 @@ class ReaderTable(Base):
     __tablename__ = 'reader'
     bibcode = Column(String, primary_key=True)
     readers = Column(ARRAY(String))
-                     
+
 
 class ReadsTable(Base):
     __tablename__ = 'reads'
     bibcode = Column(String, primary_key=True)
     reads = Column(ARRAY(Integer))
-                     
-                     
+
+
 class DownloadTable(Base):
     __tablename__ = 'download'
     bibcode = Column(String, primary_key=True)
     downloads = Column(ARRAY(Integer))
-                     
+
 
 class ReferenceTable(Base):
     __tablename__ = 'reference'
     bibcode = Column(String, primary_key=True)
     reference = Column(ARRAY(String))
-    
+
 class ChangedTable(Base):
     __tablename__ = 'changedrows'
     bibcode = Column(String, primary_key=True)

adsdata/nonbib.py

@@ -8,14 +8,19 @@
 from sqlalchemy.sql import select
 from sqlalchemy.dialects.postgresql import ARRAY
 from sqlalchemy.schema import CreateSchema, DropSchema
+import os
 import sys
 import argparse
 
 from adsputils import load_config, setup_logging
 import models
 
+# ============================= INITIALIZATION ==================================== #
+
 Base = declarative_base()
 
+# ================================ CLASSES ======================================== #
+
 
 class NonBib:
     """manages 12 fields of nonbibliographic data
@@ -37,7 +42,15 @@ def __init__(self, schema_='nonbib'):
         self.meta = MetaData()
         self.table = models.NonBibTable()
         self.table.schema = self.schema
-        self.logger = setup_logging('AdsDataSqlSync', 'INFO')
+        # - Use app logger:
+        #import logging
+        #logger = logging.getLogger('ads-data')
+        # - Or individual logger for this file:
+        proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))
+        config = load_config(proj_home=proj_home)
+        self.logger = setup_logging(__name__, proj_home=proj_home,
+                                level=config.get('LOGGING_LEVEL', 'INFO'),
+                                attach_stdout=config.get('LOG_STDOUT', False))
 
 
     def create_column_tables(self, db_engine):