Merge branch 'release/2.0.2'

thespacedoctor · May 5, 2020 · d1bbda0 · d1bbda0
2 parents 8b6d12d + a57522f
commit d1bbda0
Show file tree

Hide file tree

Showing 7 changed files with 45 additions and 31 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,14 +1,19 @@
+# v2.0.2
+
+* **enhanced**: annotation writing & peak magnitude calculations now faster
+* **fixed**: issue with verbosity = 0 not being quiet enough
+
 # v2.0.1
 
-* fixed: AGN with secure distance info now given higher weight in rank algotithm
+* **fixed**: AGN with secure distance info now given higher weight in rank algotithm
 
 # v2.0.0
 
-* enhancement: annotations provided for transient webpages are now more readable readable and the user is given a link to an external resource where possible. e.g.
+* **enhancement**: annotations provided for transient webpages are now more readable readable and the user is given a link to an external resource where possible. e.g.
 
     > The transient is possibly associated with [SDSS J175627.38+033241.9](http://skyserver.sdss.org/dr12/en/tools/explore/Summary.aspx?id=1237668571478362594); a B=15.67 mag galaxy found in the SDSS/GLADE/2MASS/PS1 catalogues. It's located 5.62" S, 0.27" W from the galaxy centre. 
 
-* fixed: sometimes in the catalogues saturated stars spawn multiple nearby catalogues entries, all marked as galaxies. In the SDSS catalogue these spurious 'galaxy' are sometimes detected well below the saturation limit of SDSS (12-14th) and so bypass our BS checker. This has been resolved in the MySQL database views.
-* fixed: entries with null values in some magnitude columns in the new PS1 catalogue generating fake NT
-* fixed: SDSS galaxies in the PhotoObjAll table with photoz getting prioritised over assoicated galaxies which have specz, but are at a great angukar distance than SDSS sources. 
-* fixed: AGN given NT classifications instead of 'AGN'
+* **fixed**: sometimes in the catalogues saturated stars spawn multiple nearby catalogues entries, all marked as galaxies. In the SDSS catalogue these spurious 'galaxy' are sometimes detected well below the saturation limit of SDSS (12-14th) and so bypass our BS checker. This has been resolved in the MySQL database views.
+* **fixed**: entries with null values in some magnitude columns in the new PS1 catalogue generating fake NT
+* **fixed**: SDSS galaxies in the PhotoObjAll table with photoz getting prioritised over assoicated galaxies which have specz, but are at a great angukar distance than SDSS sources. 
+* **fixed**: AGN given NT classifications instead of 'AGN'
diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@ sherlock
 
 [![Documentation Status](https://readthedocs.org/projects/qub-sherlock/badge/)](http://qub-sherlock.readthedocs.io/en/latest/?badge)
 
-[![Coverage Status](https://cdn.rawgit.com/thespacedoctor/sherlock/master/coverage.svg)](https://cdn.rawgit.com/thespacedoctor/sherlock/master/htmlcov/index.html)
+[![Coverage Status](https://cdn.jsdelivr.net/gh/thespacedoctor/sherlock@master/coverage.svg)](https://cdn.jsdelivr.net/gh/thespacedoctor/sherlock@master/htmlcov/index.html)
 
 *A python package with command-line tools for contextually classifying
 variable/transient astronomical sources. Sherlock mines a library of

diff --git a/README.rst b/README.rst
@@ -5,8 +5,8 @@ sherlock
     :target: http://qub-sherlock.readthedocs.io/en/latest/?badge
     :alt: Documentation Status
 
-.. image:: https://cdn.rawgit.com/thespacedoctor/sherlock/master/coverage.svg
-    :target: https://cdn.rawgit.com/thespacedoctor/sherlock/master/htmlcov/index.html
+.. image:: https://cdn.jsdelivr.net/gh/thespacedoctor/sherlock@master/coverage.svg
+    :target: https://cdn.jsdelivr.net/gh/thespacedoctor/sherlock@master/htmlcov/index.html
     :alt: Coverage Status
 
 *A python package with command-line tools for contextually classifying variable/transient astronomical sources. Sherlock mines a library of historical and on-going astronomical survey data in an attempt to identify sources transient/variable events, and predict their classifications based on the associated crossmatched data*.

diff --git a/blog-post.rst b/blog-post.rst
@@ -5,8 +5,8 @@ sherlock
     :target: http://qub-sherlock.readthedocs.io/en/latest/?badge
     :alt: Documentation Status
 
-.. image:: https://cdn.rawgit.com/thespacedoctor/sherlock/master/coverage.svg
-    :target: https://cdn.rawgit.com/thespacedoctor/sherlock/master/htmlcov/index.html
+.. image:: https://cdn.jsdelivr.net/gh/thespacedoctor/sherlock@master/coverage.svg
+    :target: https://cdn.jsdelivr.net/gh/thespacedoctor/sherlock@master/htmlcov/index.html
     :alt: Coverage Status
 
 *A python package with command-line tools for contextually classifying variable/transient astronomical sources. Sherlock mines a library of historical and on-going astronomical survey data in an attempt to identify sources transient/variable events, and predict their classifications based on the associated crossmatched data*.

diff --git a/docs/source/_includes/description.rst b/docs/source/_includes/description.rst
@@ -5,8 +5,8 @@ sherlock
     :target: http://qub-sherlock.readthedocs.io/en/latest/?badge
     :alt: Documentation Status
 
-.. image:: https://cdn.rawgit.com/thespacedoctor/sherlock/master/coverage.svg
-    :target: https://cdn.rawgit.com/thespacedoctor/sherlock/master/htmlcov/index.html
+.. image:: https://cdn.jsdelivr.net/gh/thespacedoctor/sherlock@master/coverage.svg
+    :target: https://cdn.jsdelivr.net/gh/thespacedoctor/sherlock@master/htmlcov/index.html
     :alt: Coverage Status
 
 *A python package with command-line tools for contextually classifying variable/transient astronomical sources. Sherlock mines a library of historical and on-going astronomical survey data in an attempt to identify sources transient/variable events, and predict their classifications based on the associated crossmatched data*.

diff --git a/sherlock/__version__.py b/sherlock/__version__.py
@@ -1 +1 @@
-__version__ = '2.0.1'
+__version__ = '2.0.2'
diff --git a/sherlock/transient_classifier.py b/sherlock/transient_classifier.py
@@ -1395,23 +1395,25 @@ def update_classification_annotations_and_summaries(
         # start_time = time.time()
         # print "COLLECTING TRANSIENTS WITH NO ANNOTATIONS"
 
-        if updatePeakMagnitudes:
-            sqlQuery = u"""
-                SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id= cm.transient_object_id and (cl.annotation is null or cl.dateLastModified is null or cl.dateLastModified > DATE_SUB(NOW(), INTERVAL 30 DAY)) order by  cl.dateLastModified asc limit 100000
-            """ % locals()
-        else:
-            sqlQuery = u"""
-                SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id=cm.transient_object_id and cl.summary is null
-            """ % locals()
+        if not crossmatches:
+            if updatePeakMagnitudes:
+                sqlQuery = u"""
+                    SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id= cm.transient_object_id and ((cl.classification not in ("AGN","CV","BS","VS") AND cm.dateLastModified > DATE_SUB(NOW(), INTERVAL 1 Day))  or cl.annotation is null)
+                    -- SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id= cm.transient_object_id and (cl.annotation is null or cl.dateLastModified is null or cl.dateLastModified > DATE_SUB(NOW(), INTERVAL 30 DAY)) order by  cl.dateLastModified asc limit 100000
+                """ % locals()
+            else:
+                sqlQuery = u"""
+                    SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id=cm.transient_object_id and cl.summary is null
+                """ % locals()
 
-        rows = readquery(
-            log=self.log,
-            sqlQuery=sqlQuery,
-            dbConn=self.transientsDbConn,
-            quiet=False
-        )
+            rows = readquery(
+                log=self.log,
+                sqlQuery=sqlQuery,
+                dbConn=self.transientsDbConn,
+                quiet=False
+            )
 
-        if crossmatches:
+        else:
             rows = crossmatches
 
         # print "FINISHED COLLECTING TRANSIENTS WITH NO ANNOTATIONS/GENERATING ANNOTATIONS: %d" % (time.time() - start_time,)
@@ -1424,12 +1426,15 @@ def update_classification_annotations_and_summaries(
             log=self.log
         )
 
+        if self.verbose == 0 and cl:
+            return
+
         for row in rows:
 
             annotation, summary, sep = self.generate_match_annotation(
                 match=row, updatePeakMagnitudes=updatePeakMagnitudes)
 
-            if cl and row["rank"] == 1:
+            if cl and self.verbose and row["rank"] == 1:
                 print annotation
 
             update = {
@@ -1523,7 +1528,9 @@ def update_peak_magnitudes(
         WHERE
             IFNULL(direct_distance_modulus,
                     distance_modulus) IS NOT NULL
-                AND t.id = s.transient_object_id;""" % locals()
+            AND (s.association_type not in ("AGN","CV","BS","VS") or s.transientAbsMag is null)
+                AND t.id = s.transient_object_id
+                AND (s.dateLastModified!=s.dateCreated and s.dateLastModified > DATE_SUB(NOW(), INTERVAL 1 DAY));""" % locals()
 
         writequery(
             log=self.log,
@@ -1780,6 +1787,8 @@ def generate_match_annotation(
         """
         self.log.debug('starting the ``generate_match_annotation`` method')
 
+        if "catalogue_object_subtype" not in match:
+            match["catalogue_object_subtype"] = None
         catalogue = match["catalogue_table_name"]
         objectId = match["catalogue_object_id"]
         objectType = match["catalogue_object_type"]