Merge branch 'main' of https://github.com/lsst-uk/lasair4 into main

docs/source/concepts/lightcurve.md

@@ -1,14 +1,13 @@
 # Lightcurve
 
-A lightcurve is a record of the brightness of an astrophysical object with time, so it is a collection of 
-(time, brightness) pairs. For LSST the brightness is measured as flux in nanoJanskies (nJ), and for ZTF it
-is measured with the traditional magnitude system. Note that the values in the lightcurve are *difference*
-fluxes, as defined in [Objects and Sources](objects_sources.html).
-
-Since each source brightness is measured for a specific optical filter, there may be 
-several lightcurves for a given object, for example the g-lightcurve and r-lightcurve will
-be derived from the detections in the g filter and r filter respectively.
-
-The nature of the lightcurve informs us of the underlying astrophysics. Variable stars can be characterised
-by the shape of their lightcurves, and explosive transients such as supernovae can be distinguished by the 
-rise and fall rates of their lightcurves.
+A lightcurve is a record of the brightness of an astrophysical object with time, so it is a collection of (time, brightness) pairs. For LSST the brightness of sources are calibrated and provided in flux, in the units of nanoJanskies (nJ). In ZTF the brightness measurement is provided as an AB magnitude.Note that the values in the lightcurve are *difference* fluxes, as defined in [Objects and Sources](objects_sources.html).
+
+Since each source brightness is measured for a specific optical filter, there may be several lightcurves for a given object, for example the g-lightcurve and r-lightcurve will be derived from the detections in the g filter and r filter respectively.
+
+When a source is detected at a significance of 5-sigma or more in the difference image then an alert is issued. This is defined as 5-sigma above the noise in the difference image, within a point-spread-function (PSF) aperture. 
+This is the "unforced" photometry plotted in the object pages. Another important measurement computed from
+the difference image is called the "forced" photometry. After an object has been identified then a photometric measurement (based on a PSF model) is forced at the position of the object on all images, irrespective of whether or not the object exists at 5-sigma significance. What is being measured is a "difference" in flux compared to the reference image. This flux can be positive or negative and can be simply calibrated in a physical flux unit such as a
+nanoJansky.  More information about forced photometry and ZTF can be found 
+[here](https://arxiv.org/abs/2305.16279).
+
+The nature of the lightcurve informs us of the underlying astrophysics. Variable stars can be characterised by the shape and periodicity of their lightcurves, and explosive transients such as supernovae can be distinguished by the rise and fall rates of their lightcurves.

docs/source/core_functions/rest-api.md

@@ -10,7 +10,7 @@ The Lasair API uses either HTTP GET or POST. Arguments can be passed in the quer
 The examples below show how to drive the API with either GET URL, POST curl or python with the 'lasair' package. The URL should be pasted into a web browser. The curl script pasted into a terminal window, and the python code copied into a file and executed as a python program.
 
 #### Sample Notebooks
-There is an [accompanying set of jupyter notebooks](../python-notebooks.html)
+There is an [accompanying set of jupyter notebooks](python-notebooks.md)
  that show how to use the API.
 
 #### Throttling of API Usage

pipeline/filter/check_alerts_areas.py

@@ -89,7 +89,7 @@ def check_alerts_against_areas(alertlist, arealist):
         hits += check_alerts_against_area(alertlist, area)
     return hits
 
-def fetch_alerts(msl):
+def fetch_alerts(msl, jd=None, limit=None, offset=None):
     """ fetch_alerts.
     Get all the alerts from the local cache to check againstr watchlist
 
@@ -99,6 +99,10 @@ def fetch_alerts(msl):
     cursor = msl.cursor(buffered=True, dictionary=True)
 
     query = 'SELECT objectId, ramean, decmean from objects'
+    if jd:
+        query += ' WHERE jd>%f ' % jd
+    if limit:
+        query += ' LIMIT %d OFFSET %d' % (limit, offset)
     cursor.execute(query)
     objlist = []
     ralist = []

pipeline/filter/run_active_queries.py

@@ -176,7 +176,7 @@ def run_query(query, msl, annotator=None, objectId=None):
         sqlquery_real = query_for_object(sqlquery_real, objectId)
 
     # in any case, 10 second timeout and limit the output
-    sqlquery_real = ('SET STATEMENT max_statement_time=10 FOR %s LIMIT %d' % (sqlquery_real, limit))
+    sqlquery_real = ('SET STATEMENT max_statement_time=%d FOR %s LIMIT %d' % (settings.MAX_STATEMENT_TIME, sqlquery_real, limit))
 
     cursor = msl.cursor(buffered=True, dictionary=True)
     n = 0
@@ -303,14 +303,14 @@ def send_email(email, topic, message, message_html=''):
     msg = MIMEMultipart('alternative')
 
     msg['Subject'] = 'Lasair query ' + topic
-    msg['From']    = 'donotreply@%s' % settings.LASAIR_URL
+    msg['From']    = 'lasair@lsst.ac.uk'
     msg['To']      = email
 
     msg.attach(MIMEText(message, 'plain'))
     if len(message_html) > 0:
         msg.attach(MIMEText(message_html, 'html'))
     s = smtplib.SMTP('localhost')
-    s.sendmail('donotreply@%s' % settings.LASAIR_URL, email, msg.as_string())
+    s.sendmail('lasair@lsst.ac.uk', email, msg.as_string())
     s.quit()
 
 def dispose_kafka(query_results, topic):

pipeline/ingest/ingest.py

@@ -198,7 +198,7 @@ def handle_alert(alert, image_store, producer, topic_out, cassandra_session):
             producer.produce(topic_out, json.dumps(alert_noimages))
         except Exception as e:
             log.error("ERROR in ingest/ingest: Kafka production failed for %s" % topic_out)
-            log.error("ERROR:", e)
+            log.error(str(e))
             sys.stdout.flush()
             return (0,0,0)   # ingest failed
     return (ncandidate, nnoncandidate, nforcedphot)
@@ -298,6 +298,7 @@ def run_ingest(args):
     producer_conf = {
         'bootstrap.servers': '%s' % settings.KAFKA_SERVER,
         'client.id': 'client-1',
+        'message.max.bytes': 10000000,
     }
     producer = Producer(producer_conf)
     log.info('Producing to   %s' % settings.KAFKA_SERVER)
@@ -330,7 +331,7 @@ def run_ingest(args):
             bytes_io = io.BytesIO(msg.value())
             msg = fastavro.reader(bytes_io)
         except:
-            log.error('ERROR in ingest/ingest: ', msg.value())
+            log.error('ERROR in ingest/ingest: ', str(msg.value()))
             break
 
         for alert in msg:

pipeline/sherlock/wrapper.py

@@ -5,7 +5,7 @@
 republishes on the output topic.
 """
 
-__version__ = "0.6.2"
+__version__ = "0.6.7"
 
 import warnings
 import json
@@ -266,6 +266,7 @@ def produce(conf, log, alerts):
     # set up Kafka
     settings = {
         'bootstrap.servers': conf['broker'],
+        'message.max.bytes': 10000000,
     }
     p = Producer(settings, logger=log)
 

utility/run_area.py

@@ -24,14 +24,20 @@ def run_area(msl, ar_id):
     area = {'ar_id':ar_id, 'moc':moc}
     print('Found MOC for area %d' % ar_id)
 
-    alertlist = fetch_alerts(msl)
-    print('Found %d alerts' % len(alertlist['obj']))
+# runs out of memory with the whole database
+    npage = 1000000
+    for ipage in range(100):
+        alertlist = fetch_alerts(msl, limit=npage, offset=ipage*npage)
+        nalert = len(alertlist['obj'])
+        print('Found %d alerts' % nalert)
+        if nalert == 0:
+            break
 
-    hits = check_alerts_against_area(alertlist, area)
-    print('Found %d hits' % len(hits))
+        hits = check_alerts_against_area(alertlist, area)
+        print('Found %d hits' % len(hits))
 
-    insert_area_hits(msl, hits)
-    print('Inserted into database')
+        insert_area_hits(msl, hits)
+        print('Inserted into database')
 
 if __name__ == "__main__":
     lasairLogging.basicConfig(stream=sys.stdout)