Skip to content

Commit

Permalink
Cutouts + light curves in anomaly notifications (#154)
Browse files Browse the repository at this point in the history 
* Cutouts + light curves in anomaly notifications (#153)

* ZTF DR OID added

* docstring

* docstring refinement

* flake8 check

* ZTF DR OID ---> DR OID (<1'')

* '' -> "

* Moving the variables inside the if

* cutouts+lc images in notifications

* Hide token in status_check

* Add secrets to the env

* plot legend

* correction

---------

Co-authored-by: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Co-authored-by: Timofei Pshenichniy <timofei.psheno@gmail.com>
  • Loading branch information
@JulienPeloton @Knispel2
3 people committed Jul 17, 2023
1 parent fa949a8 commit 2216fe4
Show file tree
Hide file tree
Showing 3 changed files with 262 additions and 39 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/run_test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ jobs:
echo `python -V`
- name: Run test suites
env:
ANOMALY_SLACK_TOKEN: ${{ secrets.ANOMALY_SLACK_TOKEN }}
ANOMALY_TG_TOKEN: ${{ secrets.ANOMALY_TG_TOKEN }}
run: |
pip install fink-utils --upgrade
./run_tests.sh
Expand Down
17 changes: 11 additions & 6 deletions fink_filters/filter_anomaly_notification/filter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,24 +124,29 @@ def anomaly_notification_(
oid = filter_utils.get_OID(row.ra, row.dec)
t1a = f'ID: [{row.objectId}](https://fink-portal.org/{row.objectId})'
t1b = f'ID: <https://fink-portal.org/{row.objectId}|{row.objectId}>'
t_oid_1a = f'DR OID (<1"): [{oid}](https://ztf.snad.space/view/{oid})'
t_oid_1b = f'DR OID (<1"): <https://ztf.snad.space/view/{oid}|{oid}>'
t_oid_1a = f"DR OID (<1''): [{oid}](https://ztf.snad.space/view/{oid})"
t_oid_1b = f"DR OID (<1''): <https://ztf.snad.space/view/{oid}|{oid}>"
t2_ = f'GAL coordinates: {round(gal.l.deg, 6)}, {round(gal.b.deg, 6)}'
t3_ = f'UTC: {str(row.timestamp)[:-3]}'
t4_ = f'Real bogus: {round(row.rb, 2)}'
t5_ = f'Anomaly score: {round(row.anomaly_score, 2)}'
tg_data.append(f'''{t1a}
cutout, curve, cutout_perml, curve_perml = filter_utils.get_data_permalink_slack(row.objectId)
cutout_perml = f"<{cutout_perml}|{' '}>"
curve_perml = f"<{curve_perml}|{' '}>"
tg_data.append((f'''{t1a}
{t_oid_1a}
{t2_}
{t3_}
{t4_}
{t5_}''')
slack_data.append(f'''{t1b}
{t5_}''', cutout, curve))
slack_data.append(f'''==========================
{t1b}
{t_oid_1b}
{t2_}
{t3_}
{t4_}
{t5_}''')
{t5_}
{cutout_perml}{curve_perml}''')
if send_to_slack:
filter_utils.msg_handler_slack(slack_data, channel_name, med)
if send_to_tg:
Expand Down
281 changes: 248 additions & 33 deletions fink_filters/filter_anomaly_notification/filter_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,105 @@
import time
import requests

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt

from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError

import io


def get_data_permalink_slack(ztf_id):
'''
Loads cutout and light curve via the Fink API and copies them to the Slack server
Parameters
----------
ztf_id : str
unique identifier for this object
Returns
-------
cutout : BytesIO stream
cutout image in png format
curve : BytesIO stream
light curve picture
cutout_perml : str
Link to the cutout image uploaded to the Slack server
curve_perml : str
Link to the light curve image uploaded to the Slack server
'''
cutout = get_cutout(ztf_id)
curve = get_curve(ztf_id)
slack_client = WebClient(os.environ['ANOMALY_SLACK_TOKEN'])
try:
curve.seek(0)
cutout.seek(0)
result = slack_client.files_upload_v2(
file_uploads=[
{
"file": cutout,
"title": "cutout"
},
{
"file": curve,
"title": "light curve"
}
]
)
time.sleep(3)
curve.seek(0)
cutout.seek(0)
except SlackApiError as e:
if e.response["ok"] is False:
print(e.response['error'])
requests.post(
"https://api.telegram.org/bot" + os.environ['ANOMALY_TG_TOKEN'] + "/sendMessage",
data={
"chat_id": "@fink_test",
"text": e.response["error"]
},
timeout=25
)
return cutout, curve, None, None
return cutout, curve, result['files'][0]['permalink'], result['files'][1]['permalink']


def status_check(res):
'''
Checks whether the request was successful.
In case of an error, sends information about the error to the @fink_test telegram channel
Parameters
----------
res : Response object
Returns
-------
result : bool
True : The request was successful
False: The request was executed with an error
'''
if res.status_code != 200:
url = "https://api.telegram.org/bot"
url += os.environ['ANOMALY_TG_TOKEN']
method = url + "/sendMessage"
time.sleep(8)
requests.post(
method,
data={
"chat_id": "@fink_test",
"text": str(res.status_code)
},
timeout=25
)
return False
return True

def msg_handler_slack(slack_data, channel_name, med):
'''
Expand All @@ -43,7 +139,19 @@ def msg_handler_slack(slack_data, channel_name, med):
slack_data = [f'Median anomaly score overnight: {med}'] + slack_data
try:
for slack_obj in slack_data:
slack_client.chat_postMessage(channel=channel_name, text=slack_obj)
slack_client.chat_postMessage(
channel=channel_name,
text=slack_obj,
blocks=[
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": slack_obj
}
}
]
)
time.sleep(3)
except SlackApiError as e:
if e.response["ok"] is False:
Expand All @@ -53,7 +161,7 @@ def msg_handler_slack(slack_data, channel_name, med):
"chat_id": "@fink_test",
"text": e.response["error"]
},
timeout=8
timeout=25
)

def msg_handler_tg(tg_data, channel_id, med):
Expand All @@ -65,7 +173,14 @@ def msg_handler_tg(tg_data, channel_id, med):
Parameters
----------
tg_data: list
List of lines. Each item is a separate notification
List of tuples. Each item is a separate notification.
Content of the tuple:
text_data : str
Notification text
cutout : BytesIO stream
cutout image in png format
curve : BytesIO stream
light curve picture
channel_id: string
Channel id in Telegram
med: float
Expand All @@ -77,28 +192,44 @@ def msg_handler_tg(tg_data, channel_id, med):
'''
url = "https://api.telegram.org/bot"
url += os.environ['ANOMALY_TG_TOKEN']
method = url + "/sendMessage"
tg_data = [f'Median anomaly score overnight: {med}'] + tg_data
for tg_obj in tg_data:
method = url + "/sendMediaGroup"
res = requests.post(
url + '/sendMessage',
data={
"chat_id": channel_id,
"text": f'Median anomaly score overnight: {med}',
"parse_mode": "markdown"
},
timeout=25
)
status_check(res)
time.sleep(8)
for text_data, cutout, curve in tg_data:
res = requests.post(
method,
data={
"chat_id": channel_id,
"text": tg_obj,
"parse_mode": "markdown"
params={
"chat_id": "@fink_test",
"media": f'''[
{{
"type" : "photo",
"media": "attach://second",
"caption" : "{text_data}",
"parse_mode": "markdown"
}},
{{
"type" : "photo",
"media": "attach://first"
}}
]'''
},
timeout=8
files={
"second": cutout,
"first": curve,
},
timeout=25
)
if res.status_code != 200:
res = requests.post(
method,
data={
"chat_id": "@fink_test",
"text": str(res.status_code)
},
timeout=8
)
time.sleep(3)
status_check(res)
time.sleep(8)


def get_OID(ra, dec):
Expand All @@ -118,22 +249,106 @@ def get_OID(ra, dec):
-------
out: str
ZTF DR OID
or
None, if nothing was found
'''
r = requests.get(
url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1'
)
if r.status_code != 200:
url = "https://api.telegram.org/bot"
url += os.environ['ANOMALY_TG_TOKEN']
method = url + "/sendMessage"
requests.post(method, data={
"chat_id": "@fink_test",
"text": str(r.status_code)
}, timeout=8)
url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1')
if not status_check(r):
return None
oids = [key for key, _ in r.json().items()]
if oids:
return oids[0]
return None


def get_cutout(ztf_id):
'''
The function loads cutout image via Fink API
Parameters
----------
ztf_id : str
unique identifier for this object
Returns
-------
out : BytesIO stream
cutout image in png format
'''
r = requests.post(
'https://fink-portal.org/api/v1/cutouts',
json={
'objectId': ztf_id,
'kind': 'Difference',
},
timeout=25
)
status_check(r)
return io.BytesIO(r.content)

def get_curve(ztf_id):
'''
The function loads light curve image via Fink API
Parameters
----------
ztf_id : str
unique identifier for this object
Returns
-------
out : BytesIO stream
light curve picture
'''
r = requests.post(
'https://fink-portal.org/api/v1/objects',
json={
'objectId': ztf_id,
'withupperlim': 'True'
}
)
status_check(r)

# Format output in a DataFrame
pdf = pd.read_json(io.BytesIO(r.content))

plt.figure(figsize=(15, 6))

colordic = {1: 'C0', 2: 'C1'}
filter_dict = {1: 'g band', 2: 'r band'}

for filt in np.unique(pdf['i:fid']):
maskFilt = pdf['i:fid'] == filt

# The column `d:tag` is used to check data type
maskValid = pdf['d:tag'] == 'valid'
plt.errorbar(
pdf[maskValid & maskFilt]['i:jd'].apply(lambda x: x - 2400000.5),
pdf[maskValid & maskFilt]['i:magpsf'],
pdf[maskValid & maskFilt]['i:sigmapsf'],
ls='', marker='o', color=colordic[filt], label=filter_dict[filt]
)

maskUpper = pdf['d:tag'] == 'upperlim'
plt.plot(
pdf[maskUpper & maskFilt]['i:jd'].apply(lambda x: x - 2400000.5),
pdf[maskUpper & maskFilt]['i:diffmaglim'],
ls='', marker='^', color=colordic[filt], markerfacecolor='none'
)

maskBadquality = pdf['d:tag'] == 'badquality'
plt.errorbar(
pdf[maskBadquality & maskFilt]['i:jd'].apply(lambda x: x - 2400000.5),
pdf[maskBadquality & maskFilt]['i:magpsf'],
pdf[maskBadquality & maskFilt]['i:sigmapsf'],
ls='', marker='v', color=colordic[filt]
)

plt.gca().invert_yaxis()
plt.legend()
plt.xlabel('Modified Julian Date')
plt.ylabel('Difference magnitude')

buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
return buf

0 comments on commit 2216fe4

Please sign in to comment.