diff --git a/extensions/skyportal/skyportal/handlers/api/alert.py b/extensions/skyportal/skyportal/handlers/api/alert.py index f3b31ae5..1cf2e872 100644 --- a/extensions/skyportal/skyportal/handlers/api/alert.py +++ b/extensions/skyportal/skyportal/handlers/api/alert.py @@ -1,26 +1,79 @@ from astropy.io import fits from astropy.visualization import ZScaleInterval +import base64 import bson.json_util as bj import gzip import io +from marshmallow.exceptions import ValidationError import matplotlib.colors as mplc import matplotlib.pyplot as plt import numpy as np import os +import pandas as pd import pathlib -import requests +import tornado.escape +import tornado.httpclient import traceback from baselayer.app.access import auth_or_token +from baselayer.log import make_log from ..base import BaseHandler from ...models import ( DBSession, + Group, + GroupStream, + Obj, Stream, StreamUser, + Source, ) +from .photometry import PhotometryHandler +from .thumbnail import ThumbnailHandler -s = requests.Session() +log = make_log("alert") + + +c = tornado.httpclient.AsyncHTTPClient() + + +def make_thumbnail(a, ttype, ztftype): + + cutout_data = a[f'cutout{ztftype}']['stampData'] + with gzip.open(io.BytesIO(cutout_data), 'rb') as f: + with fits.open(io.BytesIO(f.read())) as hdu: + # header = hdu[0].header + data_flipped_y = np.flipud(hdu[0].data) + # fixme: png, switch to fits eventually + buff = io.BytesIO() + plt.close('all') + fig = plt.figure() + fig.set_size_inches(4, 4, forward=False) + ax = plt.Axes(fig, [0., 0., 1., 1.]) + ax.set_axis_off() + fig.add_axes(ax) + + # remove nans: + img = np.array(data_flipped_y) + img = np.nan_to_num(img) + + if ztftype != 'Difference': + img[img <= 0] = np.median(img) + plt.imshow(img, cmap="bone", norm=mplc.LogNorm(), origin='lower') + else: + plt.imshow(img, cmap="bone", origin='lower') + plt.savefig(buff, dpi=42) + + buff.seek(0) + plt.close('all') + + thumb = { + "obj_id": a["objectId"], + "data": base64.b64encode(buff.read()).decode("utf-8"), + "ttype": ttype, + } + + return thumb class ZTFAlertHandler(BaseHandler): @@ -30,7 +83,7 @@ def get_user_streams(self): DBSession() .query(Stream) .join(StreamUser) - .filter(StreamUser.user_id == self.current_user.id) + .filter(StreamUser.user_id == self.associated_user_object.id) .all() ) if streams is None: @@ -38,8 +91,23 @@ def get_user_streams(self): return streams + async def query_kowalski(self, query: dict, timeout=7): + base_url = f"{self.cfg['app.kowalski.protocol']}://" \ + f"{self.cfg['app.kowalski.host']}:{self.cfg['app.kowalski.port']}" + headers = {"Authorization": f"Bearer {self.cfg['app.kowalski.token']}"} + + resp = await c.fetch( + os.path.join(base_url, 'api/queries'), + method='POST', + body=tornado.escape.json_encode(query), + headers=headers, + request_timeout=timeout + ) + + return resp + @auth_or_token - def get(self, objectId: str = None): + async def get(self, objectId: str = None): """ --- single: @@ -107,11 +175,15 @@ def get(self, objectId: str = None): "candidate.jd": 1, "candidate.programid": 1, "candidate.fid": 1, + "candidate.ra": 1, + "candidate.dec": 1, "candidate.magpsf": 1, "candidate.sigmapsf": 1, "candidate.rb": 1, "candidate.drb": 1, "candidate.isdiffpos": 1, + "coordinates.l": 1, + "coordinates.b": 1, } }, ] @@ -121,19 +193,14 @@ def get(self, objectId: str = None): # } } - base_url = f"{self.cfg['app.kowalski.protocol']}://" \ - f"{self.cfg['app.kowalski.host']}:{self.cfg['app.kowalski.port']}" - headers = {"Authorization": f"Bearer {self.cfg['app.kowalski.token']}"} + candid = self.get_query_argument('candid', None) + if candid: + query["query"]["pipeline"][0]["$match"]["candid"] = int(candid) - resp = s.post( - os.path.join(base_url, 'api/queries'), - json=query, - headers=headers, - timeout=7 - ) + resp = await self.query_kowalski(query=query) - if resp.status_code == requests.codes.ok: - alert_data = bj.loads(resp.text).get('data') + if resp.code == 200: + alert_data = tornado.escape.json_decode(resp.body).get('data') return self.success(data=alert_data) else: return self.error(f"Failed to fetch data for {objectId} from Kowalski") @@ -142,10 +209,341 @@ def get(self, objectId: str = None): _err = traceback.format_exc() return self.error(f'failure: {_err}') + @auth_or_token + async def post(self, objectId): + """ + --- + description: Save ZTF objectId from Kowalski as source in SkyPortal + requestBody: + content: + application/json: + schema: + allOf: + - type: object + properties: + candid: + type: integer + description: "alert candid to use to pull thumbnails. defaults to latest alert" + minimum: 1 + group_ids: + type: array + items: + type: integer + description: "group ids to save source to. defaults to all user groups" + minItems: 1 + responses: + 200: + content: + application/json: + schema: Success + 400: + content: + application/json: + schema: Error + """ + streams = self.get_user_streams() + + # allow access to public data only by default + selector = {1} + + for stream in streams: + if "ztf" in stream.name.lower(): + selector.update(set(stream.altdata.get("selector", []))) + + selector = list(selector) + + data = self.get_json() + candid = data.get("candid", None) + group_ids = data.pop("group_ids", None) + + try: + query = { + "query_type": "aggregate", + "query": { + "catalog": "ZTF_alerts_aux", + "pipeline": [ + { + "$match": { + "_id": objectId + } + }, + { + "$project": { + "_id": 1, + "cross_matches": 1, + "prv_candidates": { + "$filter": { + "input": "$prv_candidates", + "as": "item", + "cond": { + "$in": [ + "$$item.programid", + selector + ] + } + } + }, + } + }, + { + "$project": { + "_id": 1, + "prv_candidates.magpsf": 1, + "prv_candidates.sigmapsf": 1, + "prv_candidates.diffmaglim": 1, + "prv_candidates.programid": 1, + "prv_candidates.fid": 1, + "prv_candidates.rb": 1, + "prv_candidates.ra": 1, + "prv_candidates.dec": 1, + "prv_candidates.candid": 1, + "prv_candidates.jd": 1, + } + } + ] + } + } + + resp = await self.query_kowalski(query=query) + + if resp.code == 200: + alert_data = tornado.escape.json_decode(resp.body).get('data', list(dict())) + if len(alert_data) > 0: + alert_data = alert_data[0] + else: + return self.error(f"{objectId} not found on Kowalski") + else: + return self.error(f"Failed to fetch data for {objectId} from Kowalski") + + # grab and append most recent candid as it should not be in prv_candidates + query = { + "query_type": "aggregate", + "query": { + "catalog": "ZTF_alerts", + "pipeline": [ + { + "$match": { + "objectId": objectId, + "candidate.programid": {"$in": selector} + } + }, + { + "$project": { + # grab only what's going to be rendered + "_id": 0, + "candidate.candid": {"$toString": "$candidate.candid"}, + "candidate.programid": 1, + "candidate.jd": 1, + "candidate.fid": 1, + "candidate.rb": 1, + "candidate.drb": 1, + "candidate.ra": 1, + "candidate.dec": 1, + "candidate.magpsf": 1, + "candidate.sigmapsf": 1, + "candidate.diffmaglim": 1, + } + }, + { + "$sort": { + "candidate.jd": -1 + } + }, + { + "$limit": 1 + } + ] + } + } + + resp = await self.query_kowalski(query=query) + + if resp.code == 200: + latest_alert_data = tornado.escape.json_decode(resp.body).get('data', list(dict())) + if len(latest_alert_data) > 0: + latest_alert_data = latest_alert_data[0] + else: + return self.error(f"Failed to fetch data for {objectId} from Kowalski") + + if len(latest_alert_data) > 0: + candids = {a.get('candid', None) for a in alert_data['prv_candidates']} + if latest_alert_data['candidate']["candid"] not in candids: + alert_data['prv_candidates'].append(latest_alert_data['candidate']) + + df = pd.DataFrame.from_records(alert_data["prv_candidates"]) + w = df["candid"] == str(candid) + + if candid is None or sum(w) == 0: + candids = {int(can) for can in df["candid"] if not pd.isnull(can)} + candid = max(candids) + alert = df.loc[df["candid"] == str(candid)].to_dict(orient="records")[0] + else: + alert = df.loc[w].to_dict(orient="records")[0] + + # post source + drb = alert.get('drb') + rb = alert.get('rb') + score = drb if drb is not None and not np.isnan(drb) else rb + alert_thin = { + "id": objectId, + "ra": alert.get('ra'), + "dec": alert.get('dec'), + "score": score, + "altdata": { + "passing_alert_id": candid, + }, + } + + schema = Obj.__schema__() + user_group_ids = [g.id for g in self.associated_user_object.groups if not g.single_user_group] + user_accessible_group_ids = [g.id for g in self.associated_user_object.accessible_groups] + if not user_group_ids: + return self.error( + "You must belong to one or more groups before you can add sources." + ) + if (group_ids is not None) and (len(set(group_ids) - set(user_accessible_group_ids)) > 0): + forbidden_groups = list(set(group_ids) - set(user_accessible_group_ids)) + return self.error( + "Insufficient group access permissions. Not a member of " + f"group IDs: {forbidden_groups}." + ) + try: + group_ids = [ + int(_id) + for _id in group_ids + if int(_id) in user_accessible_group_ids + ] + except Exception: + group_ids = user_group_ids + if not group_ids: + return self.error( + "Invalid group_ids field. Please specify at least " + "one valid group ID that you belong to." + ) + try: + obj = schema.load(alert_thin) + except ValidationError as e: + return self.error( + 'Invalid/missing parameters: ' f'{e.normalized_messages()}' + ) + groups = Group.query.filter(Group.id.in_(group_ids)).all() + if not groups: + return self.error( + "Invalid group_ids field. Please specify at least " + "one valid group ID that you belong to." + ) + + # check that all groups have access to same streams as user + for group in groups: + group_streams = ( + DBSession() + .query(Stream) + .join(GroupStream) + .filter(GroupStream.group_id == group.id) + .all() + ) + if group_streams is None: + group_streams = [] + + group_stream_selector = {1} + + for stream in group_streams: + if "ztf" in stream.name.lower(): + group_stream_selector.update(set(stream.altdata.get("selector", []))) + + if not set(selector).issubset(group_stream_selector): + return self.error(f"Cannot save to group {group.name}: " + "insufficient group alert stream permissions") + + DBSession().add(obj) + DBSession().add_all([Source(obj=obj, group=group) for group in groups]) + DBSession().commit() + + # post photometry + ztf_filters = {1: 'ztfg', 2: 'ztfr', 3: 'ztfi'} + df['ztf_filter'] = df['fid'].apply(lambda x: ztf_filters[x]) + df['magsys'] = "ab" + df['mjd'] = df['jd'] - 2400000.5 + + photometry = { + "obj_id": objectId, + "group_ids": group_ids, + "instrument_id": 1, # placeholder + "mjd": df.mjd.tolist(), + "mag": df.magpsf.tolist(), + "magerr": df.sigmapsf.tolist(), + "limiting_mag": df.diffmaglim.tolist(), + "magsys": df.magsys.tolist(), + "filter": df.ztf_filter.tolist(), + "ra": df.ra.tolist(), + "dec": df.dec.tolist(), + } + + photometry_handler = PhotometryHandler(request=self.request, application=self.application) + photometry_handler.request.body = tornado.escape.json_encode(photometry) + try: + photometry_handler.post() + except Exception: + log(f"Failed to post photometry of {objectId}") + # do not return anything yet + self.clear() + + # post cutouts + for ttype, ztftype in [('new', 'Science'), ('ref', 'Template'), ('sub', 'Difference')]: + query = { + "query_type": "find", + "query": { + "catalog": "ZTF_alerts", + "filter": { + "candid": candid, + "candidate.programid": { + "$in": selector + } + }, + "projection": { + "_id": 0, + "objectId": 1, + f"cutout{ztftype}": 1 + } + }, + "kwargs": { + "limit": 1, + } + } + + resp = await self.query_kowalski(query=query) + + if resp.code == 200: + cutout = bj.loads(bj.dumps(tornado.escape.json_decode(resp.body).get('data', list(dict()))[0])) + else: + cutout = dict() + + thumb = make_thumbnail(cutout, ttype, ztftype) + + try: + thumbnail_handler = ThumbnailHandler(request=self.request, application=self.application) + thumbnail_handler.request.body = tornado.escape.json_encode(thumb) + thumbnail_handler.post() + except Exception as e: + log(f"Failed to post thumbnails of {objectId} | {candid}") + log(str(e)) + self.clear() + + # todo: notify Kowalski so that it puts this objectId on tracking list + # (to post new photometry to SP when new alerts arrive) + + self.push_all(action="skyportal/FETCH_SOURCES") + self.push_all(action="skyportal/FETCH_RECENT_SOURCES") + return self.success(data={"id": objectId}) + + except Exception: + _err = traceback.format_exc() + return self.error(f'failure: {_err}') + class ZTFAlertAuxHandler(ZTFAlertHandler): @auth_or_token - def get(self, objectId: str = None): + async def get(self, objectId: str = None): """ --- single: @@ -226,6 +624,8 @@ def get(self, objectId: str = None): "prv_candidates.diffmaglim": 1, "prv_candidates.programid": 1, "prv_candidates.fid": 1, + "prv_candidates.ra": 1, + "prv_candidates.dec": 1, "prv_candidates.candid": 1, "prv_candidates.jd": 1, } @@ -234,19 +634,10 @@ def get(self, objectId: str = None): } } - base_url = f"{self.cfg['app.kowalski.protocol']}://" \ - f"{self.cfg['app.kowalski.host']}:{self.cfg['app.kowalski.port']}" - headers = {"Authorization": f"Bearer {self.cfg['app.kowalski.token']}"} - - resp = s.post( - os.path.join(base_url, 'api/queries'), - json=query, - headers=headers, - timeout=7 - ) + resp = await self.query_kowalski(query=query) - if resp.status_code == requests.codes.ok: - alert_data = bj.loads(resp.text).get('data', list(dict())) + if resp.code == 200: + alert_data = tornado.escape.json_decode(resp.body).get('data', list(dict())) if len(alert_data) > 0: alert_data = alert_data[0] else: @@ -268,13 +659,17 @@ def get(self, objectId: str = None): "$project": { # grab only what's going to be rendered "_id": 0, - "candidate.candid": 1, + "candidate.candid": {"$toString": "$candidate.candid"}, "candidate.programid": 1, "candidate.jd": 1, "candidate.fid": 1, + "candidate.ra": 1, + "candidate.dec": 1, "candidate.magpsf": 1, "candidate.sigmapsf": 1, "candidate.diffmaglim": 1, + "coordinates.l": 1, + "coordinates.b": 1, } }, { @@ -289,15 +684,10 @@ def get(self, objectId: str = None): } } - resp = s.post( - os.path.join(base_url, 'api/queries'), - json=query, - headers=headers, - timeout=7 - ) + resp = await self.query_kowalski(query=query) - if resp.status_code == requests.codes.ok: - latest_alert_data = bj.loads(resp.text).get('data', list(dict())) + if resp.code == 200: + latest_alert_data = tornado.escape.json_decode(resp.body).get('data', list(dict())) if len(latest_alert_data) > 0: latest_alert_data = latest_alert_data[0] else: @@ -317,7 +707,7 @@ def get(self, objectId: str = None): class ZTFAlertCutoutHandler(ZTFAlertHandler): @auth_or_token - def get(self, objectId: str = None): + async def get(self, objectId: str = None): """ --- summary: Serve ZTF alert cutout as fits or png @@ -442,19 +832,10 @@ def get(self, objectId: str = None): } } - base_url = f"{self.cfg['app.kowalski.protocol']}://" \ - f"{self.cfg['app.kowalski.host']}:{self.cfg['app.kowalski.port']}" - headers = {"Authorization": f"Bearer {self.cfg['app.kowalski.token']}"} - - resp = s.post( - os.path.join(base_url, 'api/queries'), - json=query, - headers=headers, - timeout=7 - ) + resp = await self.query_kowalski(query=query) - if resp.status_code == requests.codes.ok: - alert = bj.loads(resp.text).get('data', list(dict()))[0] + if resp.code == 200: + alert = bj.loads(bj.dumps(tornado.escape.json_decode(resp.body).get('data', list(dict()))[0])) else: alert = dict() @@ -507,9 +888,7 @@ def get(self, objectId: str = None): ax.imshow(img, origin='lower', cmap=cmap, vmin=limits[0], vmax=limits[1]) elif scaling == 'arcsinh': ax.imshow(np.arcsinh(img - np.median(img)), cmap=cmap, origin='lower') - plt.savefig(buff, dpi=42) - buff.seek(0) plt.close('all') self.set_header("Content-Type", 'image/png') diff --git a/extensions/skyportal/static/js/components/Filter.jsx b/extensions/skyportal/static/js/components/Filter.jsx index 5d76c3d7..bf736673 100644 --- a/extensions/skyportal/static/js/components/Filter.jsx +++ b/extensions/skyportal/static/js/components/Filter.jsx @@ -117,7 +117,7 @@ const Filter = () => { useEffect(() => { const fetchFilterVersion = async () => { const data = await dispatch(filterVersionActions.fetchFilterVersion(fid)); - if ((data.status === "error") && !(data.message.includes("not found"))) { + if (data.status === "error" && !data.message.includes("not found")) { setFilterVersionLoadError(data.message); if (filterVersionLoadError.length > 1) { dispatch(showNotification(filterVersionLoadError, "error")); @@ -127,7 +127,7 @@ const Filter = () => { if (loadedId !== fid) { fetchFilterVersion(); } - }, [fid, loadedId, dispatch]); + }, [fid, loadedId, dispatch, filterVersionLoadError]); const group_id = useSelector((state) => state.filter.group_id); @@ -142,7 +142,7 @@ const Filter = () => { } }; if (group_id) fetchGroup(); - }, [group_id, dispatch]); + }, [group_id, dispatch, groupLoadError]); const filter = useSelector((state) => state.filter); const filter_v = useSelector((state) => state.filter_v); diff --git a/extensions/skyportal/static/js/components/SaveAlertButton.jsx b/extensions/skyportal/static/js/components/SaveAlertButton.jsx new file mode 100644 index 00000000..02324fb4 --- /dev/null +++ b/extensions/skyportal/static/js/components/SaveAlertButton.jsx @@ -0,0 +1,221 @@ +import React, { useState, useEffect, useRef } from "react"; +import PropTypes from "prop-types"; +import { useDispatch } from "react-redux"; +import Dialog from "@material-ui/core/Dialog"; +import DialogContent from "@material-ui/core/DialogContent"; +import DialogTitle from "@material-ui/core/DialogTitle"; +import Checkbox from "@material-ui/core/Checkbox"; +import Button from "@material-ui/core/Button"; +import ButtonGroup from "@material-ui/core/ButtonGroup"; +import FormControlLabel from "@material-ui/core/FormControlLabel"; +import ArrowDropDownIcon from "@material-ui/icons/ArrowDropDown"; +import ClickAwayListener from "@material-ui/core/ClickAwayListener"; +import Grow from "@material-ui/core/Grow"; +import Paper from "@material-ui/core/Paper"; +import Popper from "@material-ui/core/Popper"; +import MenuItem from "@material-ui/core/MenuItem"; +import MenuList from "@material-ui/core/MenuList"; +import { useForm, Controller } from "react-hook-form"; + +import * as alertActions from "../ducks/alert"; +import * as sourceActions from "../ducks/source"; +import FormValidationError from "./FormValidationError"; + +const SaveAlertButton = ({ alert, userGroups }) => { + const [isSubmitting, setIsSubmitting] = useState(false); + // Dialog logic: + + const dispatch = useDispatch(); + const [dialogOpen, setDialogOpen] = useState(false); + + const { handleSubmit, errors, reset, control, getValues } = useForm(); + + useEffect(() => { + reset({ + group_ids: [] + }); + }, [reset, userGroups, alert]); + + const handleClickOpenDialog = () => { + setDialogOpen(true); + }; + + const handleCloseDialog = () => { + setDialogOpen(false); + }; + + const validateGroups = () => { + const formState = getValues({ nest: true }); + return formState.group_ids.filter((value) => Boolean(value)).length >= 1; + }; + + const onSubmitGroupSelectSave = async (data) => { + setIsSubmitting(true); + data.id = alert.id; + const groupIDs = userGroups.map((g) => g.id); + const selectedGroupIDs = groupIDs.filter((ID, idx) => data.group_ids[idx]); + + data.payload = {candid: alert.candid, group_ids: selectedGroupIDs}; + + const result = await dispatch(alertActions.saveAlertAsSource(data)); + if (result.status === "error") { + setIsSubmitting(false); + } else { + setDialogOpen(false); + reset(); + await dispatch(sourceActions.fetchSource(alert.id)); + } + }; + + // Split button logic (largely copied from + // https://material-ui.com/components/button-group/#split-button): + + const options = ["Select groups & save as a source"]; + // const options = ["Select groups & save as a source", "Select filters & save as a candidate"]; + + const [splitButtonMenuOpen, setSplitButtonMenuOpen] = useState(false); + const anchorRef = useRef(null); + const [selectedIndex, setSelectedIndex] = useState(0); + + const handleClickMainButton = async () => { + if (selectedIndex === 0) { + handleClickOpenDialog(); + } + }; + + const handleMenuItemClick = (event, index) => { + setSelectedIndex(index); + setSplitButtonMenuOpen(false); + }; + + const handleToggleSplitButtonMenu = () => { + setSplitButtonMenuOpen((prevOpen) => !prevOpen); + }; + + const handleCloseSplitButtonMenu = (event) => { + if (anchorRef.current && anchorRef.current.contains(event.target)) { + return; + } + setSplitButtonMenuOpen(false); + }; + + return ( +
+ + + + + + {({ TransitionProps, placement }) => ( + + + + + {options.map((option, index) => ( + handleMenuItemClick(event, index)} + > + {option} + + ))} + + + + + )} + + + + Select one or more groups: + +
+ {errors.group_ids && ( + + )} + {userGroups.map((userGroup, idx) => ( + + } + label={userGroup.name} + /> + ))} +
+
+ +
+ +
+ +
+ ); +}; +SaveAlertButton.propTypes = { + alert: PropTypes.shape({ + id: PropTypes.string, + group_ids: PropTypes.arrayOf(PropTypes.number), + }).isRequired, + userGroups: PropTypes.arrayOf( + PropTypes.shape({ + id: PropTypes.number, + name: PropTypes.string, + }) + ).isRequired, +}; + +export default SaveAlertButton; diff --git a/extensions/skyportal/static/js/components/ZTFAlert.jsx b/extensions/skyportal/static/js/components/ZTFAlert.jsx index 13248c22..46707742 100644 --- a/extensions/skyportal/static/js/components/ZTFAlert.jsx +++ b/extensions/skyportal/static/js/components/ZTFAlert.jsx @@ -1,18 +1,15 @@ import React, { useEffect, useState, Suspense } from "react"; import { useDispatch, useSelector } from "react-redux"; +import { useHistory } from "react-router-dom"; import Button from "@material-ui/core/Button"; -import SaveIcon from "@material-ui/icons/Save"; import PropTypes from "prop-types"; -import { withStyles, makeStyles, useTheme } from "@material-ui/core/styles"; -import Table from "@material-ui/core/Table"; -import TableBody from "@material-ui/core/TableBody"; -import TableCell from "@material-ui/core/TableCell"; -import TableContainer from "@material-ui/core/TableContainer"; -import TableHead from "@material-ui/core/TableHead"; -import TablePagination from "@material-ui/core/TablePagination"; -import TableRow from "@material-ui/core/TableRow"; -import TableSortLabel from "@material-ui/core/TableSortLabel"; +import { + makeStyles, + useTheme, + createMuiTheme, + MuiThemeProvider, +} from "@material-ui/core/styles"; import Paper from "@material-ui/core/Paper"; import Grid from "@material-ui/core/Grid"; import Accordion from "@material-ui/core/Accordion"; @@ -21,19 +18,21 @@ import AccordionDetails from "@material-ui/core/AccordionDetails"; import ExpandMoreIcon from "@material-ui/icons/ExpandMore"; import Typography from "@material-ui/core/Typography"; import CircularProgress from "@material-ui/core/CircularProgress"; +import Chip from "@material-ui/core/Chip"; +import OpenInNewIcon from "@material-ui/icons/OpenInNew"; -import ThumbnailList from "./ThumbnailList"; +import MUIDataTable from "mui-datatables"; import ReactJson from "react-json-view"; -import * as Actions from "../ducks/alert"; +import SaveAlertButton from "./SaveAlertButton"; +import ThumbnailList from "./ThumbnailList"; -const VegaPlot = React.lazy(() => import("./VegaPlotZTFAlert")); +import { ra_to_hours, dec_to_hours } from "../units"; +import SharePage from "./SharePage"; -const StyledTableCell = withStyles(() => ({ - body: { - fontSize: "0.875rem", - }, -}))(TableCell); +import * as Actions from "../ducks/alert"; + +const VegaPlotZTFAlert = React.lazy(() => import("./VegaPlotZTFAlert")); const useStyles = makeStyles((theme) => ({ root: { @@ -45,22 +44,11 @@ const useStyles = makeStyles((theme) => ({ whitish: { color: "#f0f0f0", }, - visuallyHidden: { - border: 0, - clip: "rect(0 0 0 0)", - height: 1, - margin: -1, - overflow: "hidden", - padding: 0, - position: "absolute", - top: 20, - width: 1, - }, - margin_bottom: { - "margin-bottom": "2em", + itemPaddingBottom: { + paddingBottom: "0.5rem", }, - margin_left: { - "margin-left": "2em", + saveAlertButton: { + margin: "0.5rem 0", }, image: { padding: theme.spacing(1), @@ -80,188 +68,106 @@ const useStyles = makeStyles((theme) => ({ paddingBottom: "0.625rem", color: theme.palette.text.primary, }, -})); - -function createRows( - id, - jd, - fid, - mag, - emag, - rb, - drb, - isdiffpos, - programid, - alert_actions -) { - return { - id, - jd, - fid, - mag, - emag, - rb, - drb, - isdiffpos, - programid, - alert_actions, - }; -} -const columns = [ - { - id: "id", - label: "candid", - numeric: false, - disablePadding: false, - }, - { - id: "jd", - numeric: true, - disablePadding: false, - label: "JD", - align: "left", - format: (value) => value.toFixed(5), - }, - { - id: "fid", - numeric: true, - disablePadding: false, - label: "fid", - align: "left", - }, - { - id: "mag", - numeric: true, - disablePadding: false, - label: "mag", - align: "left", - format: (value) => value.toFixed(3), + accordionHeading: { + fontSize: "1.25rem", + fontWeight: theme.typography.fontWeightRegular, }, - { - id: "emag", - numeric: true, - disablePadding: false, - label: "e_mag", - align: "left", - format: (value) => value.toFixed(3), - }, - { - id: "rb", - numeric: true, - disablePadding: false, - label: "rb", - align: "left", - format: (value) => value.toFixed(5), + + source: { + padding: "1rem", + display: "flex", + flexDirection: "row", }, - { - id: "drb", - numeric: true, - disablePadding: false, - label: "drb", - align: "left", - format: (value) => value.toFixed(5), + column: { + display: "flex", + flexFlow: "column nowrap", + verticalAlign: "top", + flex: "0 2 100%", + minWidth: 0, }, - { - id: "isdiffpos", - numeric: false, - disablePadding: false, - label: "isdiffpos", - align: "left", + columnItem: { + margin: "0.5rem 0", }, - { - id: "programid", - numeric: true, - disablePadding: false, - label: "programid", - align: "left", + name: { + fontSize: "200%", + fontWeight: "900", + color: "darkgray", + paddingBottom: "0.25em", + display: "inline-block", }, - { - id: "alert_actions", - numeric: false, - disablePadding: false, - label: "actions", - align: "right", + alignRight: { + display: "inline-block", + verticalAlign: "super", }, -]; - -function descendingComparator(a, b, orderBy) { - if (b[orderBy] < a[orderBy]) { - return -1; - } - if (b[orderBy] > a[orderBy]) { - return 1; - } - return 0; -} +})); -function getComparator(order, orderBy) { - return order === "desc" - ? (a, b) => descendingComparator(a, b, orderBy) - : (a, b) => -descendingComparator(a, b, orderBy); +function isString(x) { + return Object.prototype.toString.call(x) === "[object String]"; } -function stableSort(array, comparator) { - const stabilizedThis = array.map((el, index) => [el, index]); - stabilizedThis.sort((a, b) => { - const order = comparator(a[0], b[0]); - if (order !== 0) return order; - return a[1] - b[1]; +const getMuiTheme = (theme) => + createMuiTheme({ + overrides: { + MUIDataTableBodyCell: { + root: { + padding: `${theme.spacing(0.25)}px 0px ${theme.spacing( + 0.25 + )}px ${theme.spacing(1)}px`, + }, + }, + }, }); - return stabilizedThis.map((el) => el[0]); -} -function EnhancedTableHead(props) { - const { classes, order, orderBy, onRequestSort } = props; - const createSortHandler = (property) => (event) => { - onRequestSort(event, property); +const ZTFAlert = ({ route }) => { + const objectId = route.id; + const dispatch = useDispatch(); + const history = useHistory(); + + // figure out if this objectId has been saved as Source. + const [savedSource, setsavedSource] = useState(false); + const [checkedIfSourceSaved, setsCheckedIfSourceSaved] = useState(false); + + // not using API/source duck as that would throw an error if source does not exist + const fetchInit = { + credentials: "same-origin", + headers: { + "Content-Type": "application/json", + }, + method: "GET", }; - return ( - - - {columns.map((headCell) => ( - - - {headCell.label} - {orderBy === headCell.id ? ( - - {order === "desc" ? "sorted descending" : "sorted ascending"} - - ) : null} - - - ))} - - - ); -} + const loadedSourceId = useSelector((state) => state?.source?.id); -EnhancedTableHead.propTypes = { - classes: PropTypes.shape({ - visuallyHidden: PropTypes.any, - }).isRequired, - onRequestSort: PropTypes.func.isRequired, - order: PropTypes.oneOf(["asc", "desc"]).isRequired, - orderBy: PropTypes.string.isRequired, -}; + useEffect(() => { + const fetchSource = async () => { + const response = await fetch(`/api/sources/${objectId}`, fetchInit); + + let json = ""; + try { + json = await response.json(); + } catch (error) { + throw new Error(`JSON decoding error: ${error}`); + } -function isString(x) { - return Object.prototype.toString.call(x) === "[object String]"; -} + if (json.status === "success") { + setsavedSource(true); + } + setsCheckedIfSourceSaved(true) + }; -const ZTFAlert = ({ route }) => { - const objectId = route.id; - const dispatch = useDispatch(); + if (!checkedIfSourceSaved) { + fetchSource(); + } + }, [objectId, dispatch, fetchInit]); + + const userAccessibleGroups = useSelector( + (state) => state.groups.userAccessible + ); + + const userAccessibleGroupIds = useSelector((state) => + state.groups.userAccessible?.map((a) => a.id) + ); const theme = useTheme(); const darkTheme = theme.palette.type === "dark"; @@ -288,31 +194,26 @@ const ZTFAlert = ({ route }) => { const [jd, setJd] = useState(0); const alert_data = useSelector((state) => state.alert_data); + + const makeRow = (alert) => { + return { + candid: alert?.candid, + jd: alert?.candidate.jd, + fid: alert?.candidate.fid, + mag: alert?.candidate.magpsf, + emag: alert?.candidate.sigmapsf, + rb: alert?.candidate.rb, + drb: alert?.candidate.drb, + isdiffpos: alert?.candidate.isdiffpos, + programid: alert?.candidate.programid, + alert_actions: "show thumbnails", + }; + }; + let rows = []; if (alert_data !== null && !isString(alert_data)) { - rows = alert_data.map((a) => - createRows( - a.candid, - a.candidate.jd, - a.candidate.fid, - a.candidate.magpsf, - a.candidate.sigmapsf, - a.candidate.rb, - a.candidate.drb, - a.candidate.isdiffpos, - a.candidate.programid, - - ) - ); + rows = alert_data?.map((a) => makeRow(a)); } const alert_aux_data = useSelector((state) => state.alert_aux_data); @@ -355,25 +256,6 @@ const ZTFAlert = ({ route }) => { }, [dispatch, isCached, route.id, objectId]); const classes = useStyles(); - const [order, setOrder] = React.useState("desc"); - const [orderBy, setOrderBy] = React.useState("jd"); - const [page, setPage] = React.useState(0); - const [rowsPerPage, setRowsPerPage] = React.useState(10); - - const handleRequestSort = (event, property) => { - const isAsc = orderBy === property && order === "asc"; - setOrder(isAsc ? "desc" : "asc"); - setOrderBy(property); - }; - - const handleChangePage = (event, newPage) => { - setPage(newPage); - }; - - const handleChangeRowsPerPage = (event) => { - setRowsPerPage(+event.target.value); - setPage(0); - }; const thumbnails = [ { @@ -391,10 +273,136 @@ const ZTFAlert = ({ route }) => { id: 2, public_url: `/api/alerts/ztf/${objectId}/cutout?candid=${candid}&cutout=difference&file_format=png`, }, + // { + // type: "sdss", + // id: 3, + // public_url: `http://skyserver.sdss.org/dr12/SkyserverWS/ImgCutout/getjpeg?ra=${alert_data.filter((a) => a.candid === candid)[0].candidate.ra}&dec=${alert_data.filter((a) => a.candid === candid)[0].candidate.dec}&scale=0.3&width=200&height=200&opt=G&query=&Grid=on` + // }, + // { + // type: "dr8", + // id: 4, + // public_url: `http://legacysurvey.org/viewer/jpeg-cutout?ra=${alert_data.filter((a) => a.candid === candid)[0].candidate.ra}&dec=${alert_data.filter((a) => a.candid === candid)[0].candidate.dec}&size=200&layer=dr8&pixscale=0.262&bands=grz` + // }, + ]; + + const options = { + selectableRows: "none", + elevation: 1, + sortOrder: { + name: "jd", + direction: "desc", + }, + }; + + const columns = [ + { + name: "candid", + label: "candid", + options: { + filter: false, + sort: true, + sortDescFirst: true, + }, + }, + { + name: "jd", + label: "JD", + options: { + filter: false, + sort: true, + sortDescFirst: true, + customBodyRender: (value, tableMeta, updateValue) => value.toFixed(5), + }, + }, + { + name: "fid", + label: "fid", + options: { + filter: true, + sort: true, + }, + }, + { + name: "mag", + label: "mag", + options: { + filter: false, + sort: true, + customBodyRender: (value, tableMeta, updateValue) => value.toFixed(3), + }, + }, + { + name: "emag", + label: "e_mag", + options: { + filter: false, + sort: true, + customBodyRender: (value, tableMeta, updateValue) => value.toFixed(3), + }, + }, + { + name: "rb", + label: "rb", + options: { + filter: false, + sort: true, + sortDescFirst: true, + customBodyRender: (value, tableMeta, updateValue) => value.toFixed(5), + }, + }, + { + name: "drb", + label: "drb", + options: { + filter: false, + sort: true, + sortDescFirst: true, + customBodyRender: (value, tableMeta, updateValue) => value.toFixed(5), + }, + }, + { + name: "isdiffpos", + label: "isdiffpos", + options: { + filter: true, + sort: true, + }, + }, + { + name: "programid", + label: "programid", + options: { + filter: true, + sort: true, + }, + }, + { + name: "alert_actions", + label: "actions", + options: { + filter: false, + sort: false, + customBodyRender: (value, tableMeta, updateValue) => ( + + ), + }, + }, ]; if (alert_data === null) { - return
; + return ( +
+ +
+ ); } if (isString(alert_data) || isString(alert_aux_data)) { return
Failed to fetch alert data, please try again later.
; @@ -410,133 +418,151 @@ const ZTFAlert = ({ route }) => { } if (alert_data.length > 0) { return ( -
- - {objectId} - - - - - } - aria-controls="panel-content" - id="panel-header" - > - - Photometry and cutouts - - - - - - Loading plot...
}> - - - - - {candid > 0 && ( - +
+
+
+ +
+
{objectId}
+
+ {savedSource || loadedSourceId ? ( +
+ history.push(`/source/${objectId}`)} + onDelete={() => window.open(`/source/${objectId}`, "_blank")} + deleteIcon={} + color="primary" + /> +
+ ) : ( +
+ +
+
+ +
+
+ )} + {candid > 0 && ( + <> + candid: +   + {candid} +
+ Position (J2000): +   + {alert_data.filter((a) => a.candid === candid)[0].candidate.ra}, +   + {alert_data.filter((a) => a.candid === candid)[0].candidate.dec} +   (α,δ= + {ra_to_hours( + alert_data.filter((a) => a.candid === candid)[0].candidate.ra + )} + ,   + {dec_to_hours( + alert_data.filter((a) => a.candid === candid)[0].candidate.dec )} + )   (l,b= + {alert_data + .filter((a) => a.candid === candid)[0] + .coordinates.l.toFixed(6)} + ,   + {alert_data + .filter((a) => a.candid === candid)[0] + .coordinates.b.toFixed(6)} + ) + + )} +
+ + + } + aria-controls="panel-content" + id="panel-header" + > + + Photometry and cutouts + + + + + + }> + + + + + {candid > 0 && ( + a.candid === candid)[0].candidate.ra} + dec={alert_data.filter((a) => a.candid === candid)[0].candidate.dec} + thumbnails={thumbnails} + displayTypes={["new", "ref", "sub"]} + size="10rem" + /> + )} + - - - - - - - - + + + + + - - {stableSort(rows, getComparator(order, orderBy)) - .slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage) - .map((row) => ( - - {columns.map((column) => { - const value = row[column.id]; - return ( - - {column.format && typeof value === "number" - ? column.format(value) - : value} - - ); - })} - - ))} - -
- - - - - - } - aria-controls="panel-content" - id="panel-header" + + + + - Cross-matches - - - - - -
+ } + aria-controls="panel-content" + id="panel-header" + > + + Cross-matches + + + + + + + + ); } return
Error rendering page...
; diff --git a/extensions/skyportal/static/js/ducks/alert.js b/extensions/skyportal/static/js/ducks/alert.js index f4d63e77..c501f946 100644 --- a/extensions/skyportal/static/js/ducks/alert.js +++ b/extensions/skyportal/static/js/ducks/alert.js @@ -11,6 +11,9 @@ export const FETCH_AUX_OK = "skyportal/FETCH_AUX_OK"; export const FETCH_AUX_ERROR = "skyportal/FETCH_AUX_ERROR"; export const FETCH_AUX_FAIL = "skyportal/FETCH_AUX_FAIL"; +export const SAVE_ALERT = "skyportal/SAVE_ALERT"; +export const SAVE_ALERT_OK = "skyportal/SAVE_ALERT_OK"; + export function fetchAlertData(id) { return API.GET(`/api/alerts/ztf/${id}`, FETCH_ALERT); } @@ -18,6 +21,10 @@ export function fetchAlertData(id) { export const fetchAuxData = (id) => API.GET(`/api/alerts/ztf/${id}/aux`, FETCH_AUX); +export function saveAlertAsSource({ id, payload }) { + return API.POST(`/api/alerts/ztf/${id}`, SAVE_ALERT, payload); +} + const alertDataReducer = (state = null, action) => { switch (action.type) { case FETCH_ALERT_OK: { diff --git a/extensions/skyportal/static/js/ducks/kowalski_filter.js b/extensions/skyportal/static/js/ducks/kowalski_filter.js index 1e6b5d45..a95ba6d8 100644 --- a/extensions/skyportal/static/js/ducks/kowalski_filter.js +++ b/extensions/skyportal/static/js/ducks/kowalski_filter.js @@ -28,11 +28,15 @@ export function fetchFilterVersion(id) { } export function addFilterVersion({ filter_id, pipeline }) { - return API.POST(`/api/filters/${filter_id}/v`, ADD_FILTER_VERSION, { pipeline }); + return API.POST(`/api/filters/${filter_id}/v`, ADD_FILTER_VERSION, { + pipeline, + }); } export function editActiveFilterVersion({ filter_id, active }) { - return API.PATCH(`/api/filters/${filter_id}/v`, EDIT_ACTIVE_FILTER_VERSION, { active }); + return API.PATCH(`/api/filters/${filter_id}/v`, EDIT_ACTIVE_FILTER_VERSION, { + active, + }); } export function editActiveFidFilterVersion({ filter_id, active_fid }) { diff --git a/fritz b/fritz index 6e3e6e17..3cdd62da 100755 --- a/fritz +++ b/fritz @@ -489,6 +489,12 @@ if __name__ == "__main__": p_run.add_argument( "--init", action="store_true", help="Initialize Fritz" ) + p_run.add_argument( + "--repo", type=str, default="origin", help="Remote repository to pull from at init" + ) + p_run.add_argument( + "--branch", type=str, default="master", help="Branch on the remote repository at init" + ) p_run.add_argument( "--dev", action="store_true", help="Run Fritz in dev mode" ) diff --git a/kowalski b/kowalski index 81b7267b..0d63b464 160000 --- a/kowalski +++ b/kowalski @@ -1 +1 @@ -Subproject commit 81b7267bcdea414a44f6e336f28b832f50fd7906 +Subproject commit 0d63b464177193fd0ff6293c120f17ba7bac1681 diff --git a/skyportal b/skyportal index 8fea7b78..32ed5697 160000 --- a/skyportal +++ b/skyportal @@ -1 +1 @@ -Subproject commit 8fea7b78198e66f2c1c2b4492bb06829a36208f6 +Subproject commit 32ed5697b6da241884ca90f382b22b3e539d0079