scheduler/ops_rehearsal_scheduler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65d25835-60a4-47f9-9019-e2977f9db0c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from astropy.time import Time\n",
    "\n",
    "import matplotlib.pylab as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "849229a7-afe8-4bd5-bcaf-64f58893aedd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# There ought to be a way to set the environment variable for this kernel\n",
    "import os\n",
    "os.environ[\"RUBIN_SIM_DATA_DIR\"] = \"/home/b/bechtol/rubin-user/rubin_sim_data\"\n",
    "os.getenv(\"RUBIN_SIM_DATA_DIR\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90adf812-0137-479b-aafa-81a77f34826c",
   "metadata": {},
   "source": [
    "You can set environment variable in the kernel either by adding it to your .user_setups file (https://rsp.lsst.io/guides/notebooks/configuration/notebook-user-setups.html) if you installed rubin_sim into the standard kernel via pip,  or you can just symlink the actual directory to \"~/rubin_sim_data\" in the shell.\n",
    "\n",
    "If you're using a non-standard kernel, you can set the env variable there too, but you would probably want to add a kernel-helper.sh file, sourced in your kernel.json file.  (more like https://confluence.lsstcorp.org/display/LSCO/Running+rubin_sim+and+the+LSST+stack+together+on+the+RSP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2b46172-80aa-4a8a-a36e-ac99f6046d09",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rubin_scheduler.scheduler.model_observatory import ModelObservatory\n",
    "from rubin_scheduler.scheduler.basis_functions import BaseBasisFunction, M5DiffBasisFunction, NotTwilightBasisFunction, NObsHighAmBasisFunction, FilterLoadedBasisFunction, OnceInNightBasisFunction, AvoidFastRevists\n",
    "from rubin_scheduler.scheduler.detailers import DitherDetailer, CameraRotDetailer\n",
    "from rubin_scheduler.scheduler.surveys import FieldSurvey\n",
    "from rubin_scheduler.scheduler.schedulers import CoreScheduler\n",
    "from rubin_scheduler.scheduler import  sim_runner\n",
    "from rubin_scheduler.utils import ddf_locations\n",
    "from rubin_scheduler.scheduler.utils import SchemaConverter\n",
    "from rubin_scheduler.site_models import SeeingData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "099777ce-3f78-4e68-a966-bfd798ca264d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note that can only set this date after downloading the full set sky brightness data\n",
    "#mjd_start = Time('2024-04-01 00:00:00.000', format='iso').mjd\n",
    "mjd_start = Time('2024-03-21 00:00:00.000', format='iso').mjd\n",
    "mjd_start"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb2edcdb-e222-4b22-9ca7-00be1a49c62e",
   "metadata": {},
   "source": [
    "In the cell below, turn off observatory downtime and clouds, but keep a realistic distribution of seeing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6c11925-ec5e-4c42-b1e3-86b5374cc4e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We need the start date of the survey, so let's load up our model observatory and get that from the conditions\n",
    "nside = 32  # Specify the HEALpix resolution\n",
    "mo = ModelObservatory(nside=nside, ideal_conditions=True, mjd_start=mjd_start)\n",
    "mo.seeing_data = SeeingData(Time(mjd_start, format='mjd'))\n",
    "conditions = mo.return_conditions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67feaf9d-4ffc-4206-967e-78bbf44751fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conditions.mjd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f0dfde0-f6cf-4850-8428-93d31f2c9473",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conditions.mounted_filters = ['g', 'r', 'i']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29f9a8da-3bd3-4a53-a232-63065092a0ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conditions.mounted_filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b833863-094a-45f4-8ba5-de57c15c25e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#dir(conditions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "726d8c9c-e0d4-4c3d-a32a-047f79dd95d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conditions.current_filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d299a8f-679d-4b17-9b7f-5f30880e64a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conditions.night"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb6a1c9f-f08e-455c-9048-20c21770192c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conditions.mjd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbaa3cca-570e-4bf4-95a6-3d732c5806b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conditions.moon_ra, conditions.moon_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cde35e8-abb6-4d6a-8bf2-11fb473a9055",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conditions.rot_tel_pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66bc4a41-3ea4-4b9e-9da6-e5540a608047",
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.nanmax(conditions.airmass)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef952238-774e-4bef-a6cc-4b5710518bea",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(conditions.site.latitude_rad, conditions.site.longitude_rad)\n",
    "#print(conditions.site.latitude, conditions.site.longitude)\n",
    "#assert np.radians(conditions.site.longitude) == conditions.site.longitude_rad\n",
    "#assert np.radians(conditions.site.latitude) == conditions.site.latitude_rad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b51f3fc-99fb-4245-b6a2-50145006556b",
   "metadata": {},
   "source": [
    "# Field Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "184ee62a-32de-4b48-aebc-c2914334e002",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "# Start with a set of fields chosen to be uniformly spaced in RA so that some field should be visible throughout the night all year\n",
    "#ra_array = np.arange(0., 360., 30.)\n",
    "ra_array = np.arange(0., 360., 60.) + 5\n",
    "dec_array = np.tile(-25., len(ra_array))\n",
    "#dec_array = np.tile(-30., len(ra_array))\n",
    "print(ra_array)\n",
    "print(dec_array)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d70969f4-eac9-42f1-95c4-ea57105e8bb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "targets = (\n",
    "    (95., -25.), # High stellar densty, low extinction\n",
    "    (125., -15.), # High stellar densty, low extinction\n",
    "    #(360 - 145., -25.),\n",
    "    (216, -12.5), # DEEP Solar Systen\n",
    "    (360 - 110., 0.), # High stellar densty, low extinction\n",
    "    #(219.80, -0.600), # DESI, GAMA, HSC DR2, KiDS-N\n",
    "    #(360 - 53., -25),\n",
    "    (270.891667, -30.033889), # Baade's Window\n",
    "    (310, -19), # DEEP Solar System\n",
    "    (9.45, -44.0),\n",
    "    (35.708333, -4.75),\n",
    "    (53.125, -28.1), # ECDFS\n",
    "    (150.1, 2.1819444444444445), # COSMOS\n",
    "    (58.9, -49.315), # EDFS_a\n",
    "    (63.6, -47.6), # EDFS_b\n",
    ")\n",
    "\"\"\"\n",
    "\"\"\"\n",
    "# 2 January 2024\n",
    "targets = (\n",
    "    (95., -25.), # High stellar densty, low extinction\n",
    "    (125., -15.), # High stellar densty, low extinction\n",
    "    (179.60, 0.000), # DESI, GAMA, HSC DR2, KiDS-N\n",
    "    #(360 - 145., -25.),\n",
    "    (360 - 135., -40.), # High stellar densty, low extinction\n",
    "    (216, -12.5), # DEEP Solar Systen\n",
    "    (360 - 110., 2.), # High stellar densty, low extinction\n",
    "    #(219.80, -0.600), # DESI, GAMA, HSC DR2, KiDS-N\n",
    "    #(360 - 53., -25),\n",
    "    #(270.891667, -30.033889), # Baade's Window\n",
    "    (300., -41.), # High stellar densty, low extinction \n",
    "    (280., -48.), # High stellar densty, low extinction \n",
    "    (310, -19), # DEEP Solar System\n",
    "    (9.45, -44.0), # ELAIS-S1 LSST DDF\n",
    "    (35.708333, -4.75), # LSST DDF\n",
    "    (53.125, -28.1), # ECDFS\n",
    "    (150.1, 2.1819444444444445), # COSMOS\n",
    "    (58.9, -49.315), # EDFS_a\n",
    "    (63.6, -47.6), # EDFS_b\n",
    ")\n",
    "\n",
    "\n",
    "ra_array, dec_array = zip(*targets)\n",
    "print(ra_array)\n",
    "print(dec_array)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b973d64-878c-4da3-b29b-e867739e2d38",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "ra_array = []\n",
    "dec_array = []\n",
    "for key in ['ELAISS1', 'XMM_LSS', 'ECDFS', 'COSMOS', 'EDFS_a']:\n",
    "    ra_array.append(ddf_locations[key][0])\n",
    "    dec_array.append(ddf_locations[key][1])\n",
    "\n",
    "for key in ['A0', 'B0']:\n",
    "    ra_array.append(deep_locations[key][0])\n",
    "    dec_array.append(deep_locations[key][1])\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c313006f-972c-4bb1-a7ff-bb9372e5d5dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "fields = {}\n",
    "for ra, dec in zip(ra_array, dec_array):\n",
    "    name = f\"{int(ra)}{int(dec)}\"\n",
    "    fields[name] = {\"RA\": ra, \"dec\": dec}\n",
    "print(fields)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a98448b6-98c6-4225-91c8-3e9bbc5baf67",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ops_rehearsal_fields\n",
    "import importlib\n",
    "importlib.reload(ops_rehearsal_fields)\n",
    "\n",
    "name_fields, ra_fields, dec_fields = zip(*ops_rehearsal_fields.fields)\n",
    "\n",
    "fields = {}\n",
    "for name, ra, dec in zip(name_fields, ra_fields, dec_fields):\n",
    "    fields[name] = {\"RA\": ra, \"dec\": dec}\n",
    "for key in fields:\n",
    "    print(key, fields[key][\"RA\"], fields[key][\"dec\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abb5d558-5498-40eb-9a30-4aa35400ba89",
   "metadata": {},
   "outputs": [],
   "source": [
    "import healpy as hp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52ac0fe0-7ae1-43fc-9afc-639f1b1a529a",
   "metadata": {},
   "outputs": [],
   "source": [
    "m_stellar_density = np.load('/home/b/bechtol/rubin_sim_data/maps/StarMaps/starDensity_g_nside_64.npz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d4718a3-03b3-442a-a7b6-2e4af78a7f29",
   "metadata": {},
   "outputs": [],
   "source": [
    "hp.mollview(np.log10(m_stellar_density['starDensity'][:,20]), title='log10(stellar density)')\n",
    "hp.projscatter(ra_fields, dec_fields, lonlat=True, color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4dfc1a2c-e9de-494f-be21-8ac7d83be8bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#m_extinction = np.load('/home/b/bechtol/rubin_sim_data/maps/DustMaps/dust_nside_32.npz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39b817cb-b442-4ef1-b465-26741838758e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#hp.mollview(m_extinction['ebvMap'], min=0, max=0.2, title='E(B-V)')\n",
    "#hp.projscatter(ra_fields, dec_fields, lonlat=True, color='red')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60ab2a1d-8577-4d0a-8d62-3120ff0fd294",
   "metadata": {},
   "source": [
    "# Basis Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77d5554d-b9bd-4793-8bf8-241d70fcf1bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rubin_scheduler.scheduler import features\n",
    "from rubin_scheduler.scheduler.utils import IntRounded\n",
    "\n",
    "class AvoidFastRevistsBasisFunction(BaseBasisFunction):\n",
    "    \"\"\"Marks targets as unseen if they are in a specified time window\n",
    "    in order to avoid fast revisits.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    filtername: `str` ('r')\n",
    "        The name of the filter for this target map.\n",
    "    gap_min : `float` (25.)\n",
    "        Minimum time for the gap (minutes).\n",
    "    nside: `int` (default_nside)\n",
    "        The healpix resolution.\n",
    "    penalty_val : `float` (np.nan)\n",
    "        The reward value to use for regions to penalize.\n",
    "        Will be masked if set to np.nan (default).\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, survey_name=None, gap_min=30.0, penalty_val=np.nan):\n",
    "        super(AvoidFastRevistsBasisFunction, self).__init__()\n",
    "\n",
    "        self.penalty_val = penalty_val\n",
    "\n",
    "        self.gap_min = IntRounded(gap_min / 60.0 / 24.0)\n",
    "\n",
    "        self.survey_features = dict()\n",
    "        self.survey_features[\"LastObservation\"] = features.LastObservation(survey_name)\n",
    "\n",
    "    def _calc_value(self, conditions, indx=None):\n",
    "        diff = IntRounded(conditions.mjd - self.survey_features[\"LastObservation\"].feature['mjd'])\n",
    "        if diff < self.gap_min:\n",
    "            result = self.penalty_val\n",
    "        else:\n",
    "            result = 0.\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a19b547-94ee-4c71-91ae-6ad88e23c229",
   "metadata": {},
   "outputs": [],
   "source": [
    "#bfs = AvoidFastRevistsBasisFunction(survey_name='test')\n",
    "#conditions.mjd - bfs.survey_features[\"LastObservation\"].feature['mjd'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09fd5d5b-f614-4d65-a844-5b9e77907863",
   "metadata": {},
   "outputs": [],
   "source": [
    "class RisingBasisFunction(BaseBasisFunction):\n",
    "    \"\"\"Reward fields that are rising.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    RA : float\n",
    "        The RA of the point in the sky (degrees)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, RA, weight=0.1):\n",
    "        super(RisingBasisFunction, self).__init__()\n",
    "        self.ra_hours = RA * 24 / 360.0\n",
    "        self.weight = weight\n",
    "\n",
    "    # Probably not needed\n",
    "    def check_feasibility(self, conditions):\n",
    "        result = True\n",
    "        return result\n",
    "\n",
    "    def _calc_value(self, conditions, indx=None):\n",
    "        hour_angle = conditions.lmst - self.ra_hours\n",
    "        return -1 * self.weight * hour_angle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef13cc2f-fdc2-4669-81dd-5b2da429d9ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "#from astropy.coordinates import SkyCoord, angular_separation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46124d44-8483-4687-8ea4-629f5e830214",
   "metadata": {},
   "outputs": [],
   "source": [
    "#ra_1, dec_1 = 180., 0.\n",
    "#ra_2, dec_2 = 181., 1.\n",
    "\n",
    "#angsep = angular_separation(np.radians(ra_1), np.radians(dec_1), np.radians(ra_2), np.radians(dec_2))\n",
    "#print(np.degrees(angsep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4ced779-7d5e-48ff-a848-b7be6b5e4608",
   "metadata": {},
   "outputs": [],
   "source": [
    "from astropy.coordinates import angular_separation\n",
    "\n",
    "class AvoidMoonBasisFunction(BaseBasisFunction):\n",
    "    \"\"\"Only execute if angular separation to the Moon is above a specified threshold.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    ra : float\n",
    "        The RA of the point in the sky (degrees)\n",
    "    dec : float\n",
    "        The declination of the point in the sky (degrees)\n",
    "    min_moon_dist : float\n",
    "        The minimum angular separation to the moon (degrees)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, ra, dec, min_moon_dist=20.):\n",
    "        super(AvoidMoonBasisFunction, self).__init__()\n",
    "        self.ra = ra\n",
    "        self.dec = dec\n",
    "        self.min_moon_dist = min_moon_dist\n",
    "\n",
    "    def check_feasibility(self, conditions):\n",
    "        moon_dist = np.degrees(\n",
    "            angular_separation(\n",
    "                np.radians(self.ra), np.radians(self.dec), conditions.moon_ra, conditions.moon_dec\n",
    "            )\n",
    "        )\n",
    "        if moon_dist > self.min_moon_dist:\n",
    "            result = True\n",
    "        else:\n",
    "            result = False\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "376a0620-912d-400d-8773-4a9b2cb88843",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rubin_scheduler.utils import approx_ra_dec2_alt_az\n",
    "\n",
    "class MaximumAirmassBasisFunction(BaseBasisFunction):\n",
    "    \"\"\"Only execute if the airmass is a below a specified threshold.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    ra : float\n",
    "        The RA of the point in the sky (degrees)\n",
    "    dec : float\n",
    "        The declination of the point in the sky (degrees)\n",
    "    max_airmass : float\n",
    "        The maximum airmass (degrees)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, ra, dec, max_airmass=2.0):\n",
    "        super(MaximumAirmassBasisFunction, self).__init__()\n",
    "        self.ra = ra\n",
    "        self.dec = dec\n",
    "        self.max_airmass = max_airmass\n",
    "\n",
    "    def check_feasibility(self, conditions):\n",
    "        alt, az = approx_ra_dec2_alt_az(\n",
    "            self.ra,\n",
    "            self.dec,\n",
    "            conditions.site.latitude,\n",
    "            conditions.site.longitude,\n",
    "            conditions.mjd,\n",
    "        )\n",
    "        \n",
    "        airmass = 1. / np.cos(np.radians(90. - alt))\n",
    "        \n",
    "        if airmass < self.max_airmass:\n",
    "            result = True\n",
    "        else:\n",
    "            result = False\n",
    "\n",
    "        #import pdb; pdb.set_trace()\n",
    "        \n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad8dbe47-9f82-453a-a3e9-1a05764d516e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rubin_scheduler.scheduler.detailers import BaseDetailer\n",
    "from rubin_scheduler.utils import _approx_ra_dec2_alt_az, _approx_altaz2pa\n",
    "\n",
    "class CameraRotPerObservationListDetailer(BaseDetailer):\n",
    "    \"\"\"\n",
    "    Randomly set the camera rotation for each observation list.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    max_rot : `float` (90.)\n",
    "        The maximum amount to offset the camera (degrees)\n",
    "    min_rot : `float` (90)\n",
    "        The minimum to offset the camera (degrees)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, max_rot=90.0, min_rot=-90.0, seed=42, per_visit_rot=0.): # nnights=7305):\n",
    "        self.survey_features = {}\n",
    "\n",
    "        self.current_night = -1\n",
    "        self.max_rot = np.radians(max_rot)\n",
    "        self.min_rot = np.radians(min_rot)\n",
    "        self.range = self.max_rot - self.min_rot\n",
    "        self.seed = seed\n",
    "        self.per_visit_rot = per_visit_rot\n",
    "        #self.rng = np.random.default_rng(seed)\n",
    "        #self.offsets = self.rng.random(100 * nnights)\n",
    "        self.offset = None\n",
    "\n",
    "    def _generate_offsets_orig(self, n_offsets, night, mjd):\n",
    "        #print(mjd)\n",
    "        #print(np.asarray(mjd).item() % 100)\n",
    "        mjd_hash = round(100 * (np.asarray(mjd).item() % 100))\n",
    "        #print(mjd_hash)\n",
    "        rng = np.random.default_rng(mjd_hash * self.seed)\n",
    "        #print(night)\n",
    "        #print(type(night))\n",
    "        #print(mjd)\n",
    "        #print(type(mjd))\n",
    "        #index = 1\n",
    "        #index = round(\n",
    "        #    (100 * night) + (100. * (mjd % 100))\n",
    "        #)\n",
    "        #self.offset = self.offsets[index] * self.range + self.min_rot\n",
    "        self.offset = (rng.random() * self.range) + self.min_rot\n",
    "        offsets = np.ones(n_offsets) * self.offset\n",
    "        return offsets\n",
    "\n",
    "    def _generate_offsets_filter_change(self, filter_list, mjd, initial_offset):\n",
    "        \"\"\"Generate a random camera rotation for each filter change.\n",
    "        \"\"\"\n",
    "        \n",
    "        mjd_hash = round(100 * (np.asarray(mjd).item() % 100))\n",
    "        rng = np.random.default_rng(mjd_hash * self.seed)\n",
    "        \n",
    "        offsets = np.zeros(len(filter_list))\n",
    "        offset = np.asarray(initial_offset).item()\n",
    "        offsets[0] = offset\n",
    "        \n",
    "        for ii in range(1, len(offsets)):            \n",
    "            if filter_list[ii] != filter_list[ii - 1]:\n",
    "                # Filter change\n",
    "                offset = (rng.random() * self.range) + self.min_rot\n",
    "            else:\n",
    "                # If no filter change, apply a small a small rotation\n",
    "                offset += np.radians(self.per_visit_rot)\n",
    "            offsets[ii] = offset\n",
    "\n",
    "        return offsets\n",
    "        \n",
    "    def __call__(self, observation_list, conditions):\n",
    "        # Generate offsets in camera rotator\n",
    "        #offsets = self._generate_offsets_orig(len(observation_list), conditions.night, conditions.mjd)\n",
    "        #import pdb; pdb.set_trace()\n",
    "        #offsets = self._generate_offsets(observation_list, conditions.night, conditions.mjd)\n",
    "\n",
    "        filter_list = [np.asarray(obs['filter']).item() for obs in observation_list]\n",
    "        offsets = self._generate_offsets_filter_change(filter_list, conditions.mjd, conditions.rot_tel_pos)\n",
    "        \n",
    "        for i, obs in enumerate(observation_list):\n",
    "            alt, az = _approx_ra_dec2_alt_az(\n",
    "                obs[\"RA\"],\n",
    "                obs[\"dec\"],\n",
    "                conditions.site.latitude_rad,\n",
    "                conditions.site.longitude_rad,\n",
    "                conditions.mjd,\n",
    "            )\n",
    "            obs_pa = _approx_altaz2pa(alt, az, conditions.site.latitude_rad)\n",
    "            obs[\"rotSkyPos\"] = (offsets[i] - obs_pa) % (2.0 * np.pi)\n",
    "            obs[\"rotTelPos\"] = offsets[i]\n",
    "\n",
    "        return observation_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8aaa69a7-664b-4c5d-98d9-16ae1c749069",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing non-repeatability of simulations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56318f41-f6eb-47e7-a12b-4ee8506f09d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#filter_list = ['g', 'g', 'r', 'r', 'i', 'i']\n",
    "#mjd = [0.001, 0.002, 0.003, 0.004]\n",
    "#initial_offset = 45.\n",
    "\n",
    "#detailer = CameraRotPerObservationListDetailer()\n",
    "#detailer._generate_offsets_filter_change(filter_list, conditions.mjd, conditions.rot_tel_pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22c883f5-2612-43d7-aef7-c6186d761eb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#detailer = DitherDetailer()\n",
    "#detailer._generate_offsets(10, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5888b63a-fc3b-4603-a801-8b134afbde31",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-03T15:48:23.739018Z",
     "iopub.status.busy": "2024-01-03T15:48:23.738809Z",
     "iopub.status.idle": "2024-01-03T15:48:23.741447Z",
     "shell.execute_reply": "2024-01-03T15:48:23.741069Z",
     "shell.execute_reply.started": "2024-01-03T15:48:23.739003Z"
    }
   },
   "source": [
    "# Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01e9b2fd-cf06-4dd9-bd92-17244e0a25ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basis functions\n",
    "#bfs = []\n",
    "\n",
    "sun_alt_limit = -12.0 # deg\n",
    "\n",
    "bfs = [\n",
    "    M5DiffBasisFunction(filtername='r', nside=nside),\n",
    "    NotTwilightBasisFunction(sun_alt_limit=sun_alt_limit),\n",
    "    #AvoidFastRevistsBasisFunction(filtername='r', nside=nside, gap_min=25.0, penalty_val=np.nan),\n",
    "    #FilterLoadedBasisFunction(filternames=['g', 'r', 'i']),\n",
    "    #OnceInNightBasisFunction(notes=['dense_dithered']),\n",
    "    #NObsHighAmBasisFunction(),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8a58a7c-a36f-416c-a8ab-0df9ea61b903",
   "metadata": {},
   "outputs": [],
   "source": [
    "#bfs[2].check_feasibility(conditions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42483de3-bef7-4895-a4b5-3963517c2984",
   "metadata": {},
   "outputs": [],
   "source": [
    "#rng = np.random.default_rng(42)\n",
    "#for ii in range(0, 10):\n",
    "#    print(rng.random())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff3981b4-e218-40e7-b906-37a9aeb52bda",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add in detailer\n",
    "#detailers = None\n",
    "detailers = [\n",
    "    DitherDetailer(max_dither=0.2, per_night=False),\n",
    "    CameraRotPerObservationListDetailer(\n",
    "        max_rot=45.0, \n",
    "        min_rot=-45.0, \n",
    "        seed=42, \n",
    "        per_visit_rot=1.,\n",
    "    ),\n",
    "    #CameraRotDetailer(per_night=True),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2eeee58-dcf1-4f2b-877a-15c6b2d14ba5",
   "metadata": {},
   "outputs": [],
   "source": [
    "nvis_master = [20, 20, 20]\n",
    "sequence = \"gri\"\n",
    "exptime = 30\n",
    "u_exptime = 30\n",
    "reward_value = None\n",
    "nexp = 1 # 1 --> single 30 second exposure\n",
    "\n",
    "min_moon_dist = 20. # deg\n",
    "max_airmass = 2.0\n",
    "\n",
    "surveys = []\n",
    "\n",
    "for survey_name in fields.keys():\n",
    "\n",
    "    bfs_survey = copy.deepcopy(bfs)\n",
    "    bfs_survey.append(RisingBasisFunction(RA=fields[survey_name][\"RA\"]))\n",
    "    bfs_survey.append(AvoidFastRevistsBasisFunction(survey_name=survey_name))\n",
    "    bfs_survey.append(AvoidMoonBasisFunction(\n",
    "        ra=fields[survey_name][\"RA\"], \n",
    "        dec=fields[survey_name][\"dec\"], \n",
    "        min_moon_dist=min_moon_dist)\n",
    "    )\n",
    "    bfs_survey.append(MaximumAirmassBasisFunction(\n",
    "        ra=fields[survey_name][\"RA\"], \n",
    "        dec=fields[survey_name][\"dec\"],\n",
    "        max_airmass=max_airmass)\n",
    "    )\n",
    "    \n",
    "    print(survey_name)\n",
    "    #reward_value = 100 if fields[survey_name][\"RA\"] == 150. else 0\n",
    "    #reward_value = 100.\n",
    "    #reward_value = np.random.random()\n",
    "    #reward_value = None\n",
    "    surveys.append(\n",
    "        #DeepDrillingSurvey(\n",
    "        FieldSurvey(\n",
    "            bfs_survey,\n",
    "            fields[survey_name][\"RA\"],\n",
    "            fields[survey_name][\"dec\"],\n",
    "            sequence=sequence,\n",
    "            nvis=nvis_master,\n",
    "            exptime=exptime,\n",
    "            u_exptime=u_exptime,\n",
    "            survey_name=survey_name,\n",
    "            reward_value=reward_value,\n",
    "            nside=nside,\n",
    "            nexp=nexp,\n",
    "            detailers=detailers,\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91442eca-8786-44ed-b53d-0c64b3e1de6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#surveys[0].basis_functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad63a7db-fd45-4b37-8093-4c4d8ecf7d2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "scheduler = CoreScheduler(surveys, nside=nside)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "480b46b2-7e5d-481d-a7e9-5ef75e42dfa6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note the deep copy used here. The function sim_runner modifies the observatory\n",
    "new_mo, new_scheduler, observations = sim_runner(copy.deepcopy(mo), copy.deepcopy(scheduler), survey_length=21., verbose=True)\n",
    "del new_mo\n",
    "del new_scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acea5445-35be-4d08-b74b-51c745aecebf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.hstack(observations))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f8318ba-6dfe-4b99-bf30-407f4e1e0722",
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.testing.assert_almost_equal(df['RA'].iloc[-1], 2.1821887017065236)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b98edc68-385d-4590-997b-7d67b004c466",
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.testing.assert_almost_equal(df['RA'].iloc[-1], 2.1812680654192746)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f70402fe-a9de-4df4-8e55-1c0745911eb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2.179457\n",
    "# 60390.002539 \t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c00d01d-4962-42c1-be39-c396774b3f1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc44d2f2-e85a-4507-b08b-d284d8f77c59",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Observability\n",
    "assert np.all(observations['airmass'] < max_airmass)\n",
    "assert np.all(np.degrees(observations['moonDist']) > min_moon_dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b2a029a-c879-4a3a-b689-14f624d79dd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forbid 2 x 15 sec snaps\n",
    "assert np.all(observations['nexp'] == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b26323fd-0aac-436c-b74c-92b04d16e181",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.sum(observations['airmass'] > 2.0))\n",
    "print(np.max(observations['airmass']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd83c984-8ce4-4b65-a475-6186a9b18410",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.min(np.degrees(observations['moonDist'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af5c9581-feca-474c-8657-f3800b9ac545",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(observations['note'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "560315db-a36b-43f7-8137-3be1af77f9bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(observations['filter'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d82ea40-9d1d-43b1-bac7-ea3c1f7396ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert np.std(observations['FWHM_500']) > 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bf216a7-2f23-4469-98c7-ce484b99ae7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.std(observations['FWHM_500']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "582e0513-6463-43e1-aa5c-c325248e6f29",
   "metadata": {},
   "source": [
    "# Quick Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7570a0c7-1da9-48d4-824b-eb6de894d018",
   "metadata": {},
   "outputs": [],
   "source": [
    "f2c = {'u': 'purple', 'g': 'blue', 'r': 'green',\n",
    "       'i': 'cyan', 'z': 'orange', 'y': 'red'}\n",
    "\n",
    "plt.figure(dpi=200)\n",
    "for filtername in f2c:\n",
    "    in_filt = np.where(observations['filter'] == filtername)[0]\n",
    "    if in_filt.size > 0:\n",
    "        plt.plot(observations['mjd'][in_filt], np.degrees(observations['alt'][in_filt]), \n",
    "                 'o', markersize=1, color=f2c[filtername], label=filtername)\n",
    "plt.legend()\n",
    "plt.xlabel('MJD')\n",
    "plt.ylabel('Altitude (degrees)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8016213-2408-495a-9d87-c22c819a31fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "f2c = {'u': 'purple', 'g': 'blue', 'r': 'green',\n",
    "       'i': 'cyan', 'z': 'orange', 'y': 'red'}\n",
    "\n",
    "plt.figure(dpi=200)\n",
    "for filtername in f2c:\n",
    "    in_filt = np.where(observations['filter'] == filtername)[0]\n",
    "    if in_filt.size > 0:\n",
    "        plt.plot(observations['mjd'][in_filt], observations['airmass'][in_filt], \n",
    "                 'o', markersize=1, color=f2c[filtername], label=filtername)\n",
    "plt.legend()\n",
    "plt.xlabel('MJD')\n",
    "plt.ylabel('Airmass')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc09baa2-2164-4c02-a31c-7003371f40ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=200)\n",
    "for filtername in f2c:\n",
    "    in_filt = np.where(observations['filter'] == filtername)[0]\n",
    "    if in_filt.size > 0:\n",
    "        plt.plot(observations['mjd'][in_filt], observations['slewtime'][in_filt], \n",
    "                 'o', markersize=1, color=f2c[filtername], label=filtername)\n",
    "plt.xlim(observations['mjd'][0], observations['mjd'][500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "105740f3-7c55-47b5-992a-e42904443962",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=200)\n",
    "plt.scatter(observations['mjd'], np.degrees(observations['rotSkyPos']), c=np.degrees(observations['RA']), s=1)\n",
    "plt.colorbar(label='RA (deg)')\n",
    "\n",
    "plt.xlabel('MJD')\n",
    "plt.ylabel('rotSkyPos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f15e586b-6550-48e4-938a-e1552d4a0a51",
   "metadata": {},
   "outputs": [],
   "source": [
    "f2c = {'u': 'purple', 'g': 'blue', 'r': 'green',\n",
    "       'i': 'cyan', 'z': 'orange', 'y': 'red'}\n",
    "\n",
    "plt.figure(dpi=200)\n",
    "#plt.scatter(observations['mjd'], np.degrees(observations['rotTelPos']), c=np.degrees(observations['RA']), s=1)\n",
    "#plt.colorbar(label='RA (deg)')\n",
    "\n",
    "for filtername in f2c:\n",
    "    in_filt = np.where(observations['filter'] == filtername)[0]\n",
    "    if in_filt.size > 0:\n",
    "        plt.plot(observations['mjd'][in_filt], np.degrees(observations['rotTelPos'][in_filt]), \n",
    "                 'o', markersize=1, color=f2c[filtername], label=filtername)\n",
    "plt.legend()\n",
    "\n",
    "plt.xlabel('MJD')\n",
    "plt.ylabel('rotTelPos')\n",
    "plt.xlim(observations['mjd'][0], observations['mjd'][500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dafe6ff-4e22-4d93-9e99-41cfb5eaba64",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=200)\n",
    "for filtername in f2c:\n",
    "    in_filt = np.where(observations['filter'] == filtername)[0]\n",
    "    if in_filt.size > 0:\n",
    "        plt.plot(np.degrees(observations['RA'][in_filt]), observations['airmass'][in_filt], \n",
    "                 'o', markersize=1, color=f2c[filtername], label=filtername)\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel('RA (deg)')\n",
    "plt.ylabel('Airmass')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "281bfdcf-2723-4412-8dc5-8420538994cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=200)\n",
    "for filtername in f2c:\n",
    "    in_filt = np.where(observations['filter'] == filtername)[0]\n",
    "    if in_filt.size > 0:\n",
    "        plt.plot(observations['mjd'][in_filt], np.degrees(observations['RA'][in_filt]), \n",
    "                 'o', markersize=1, color=f2c[filtername], label=filtername)\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel('MJD')\n",
    "plt.ylabel('RA (deg)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44597818-9288-46c3-8a36-bca3ea26408d",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=200)\n",
    "#for filtername in f2c:\n",
    "#    in_filt = np.where(observations['filter'] == filtername)[0]\n",
    "#    if in_filt.size > 0:\n",
    "#        plt.plot(observations['mjd'][in_filt], np.degrees(observations['moonDist'][in_filt]), \n",
    "#                 'o', markersize=1, color=f2c[filtername], label=filtername)\n",
    "\n",
    "plt.scatter(observations['mjd'], np.degrees(observations['moonDist']), c=observations['moonPhase'], s=1)\n",
    "plt.colorbar(label='moonPhase')\n",
    "\n",
    "#plt.legend()\n",
    "plt.xlabel('MJD')\n",
    "plt.ylabel('moonDist (deg)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6123bd97-3089-4e8c-aef0-0ac895511067",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=200)\n",
    "for filtername in f2c:\n",
    "    in_filt = np.where(observations['filter'] == filtername)[0]\n",
    "    if in_filt.size > 0:\n",
    "        plt.plot(np.degrees(observations['RA'][in_filt]), np.degrees(observations['rotSkyPos'][in_filt]), \n",
    "                 'o', markersize=1, color=f2c[filtername], label=filtername)\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel('RA (deg)')\n",
    "plt.ylabel('rotSkyPos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "652f0893-d646-456d-890a-859c2ff862ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "gap = (np.diff(observations['mjd']) * 24. * 3600.)\n",
    "\n",
    "plt.figure()\n",
    "plt.yscale('log')\n",
    "bins = np.linspace(0., 300., 101)\n",
    "plt.hist(gap, bins=bins)\n",
    "plt.ylabel('Counts')\n",
    "plt.xlabel('Time Gap (s)')\n",
    "\n",
    "selection = gap > 300.\n",
    "for index in np.nonzero(selection)[0]:\n",
    "    print(gap[index], observations['mjd'][index + 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c75d649-09ab-451c-b475-bf09cd72e704",
   "metadata": {},
   "source": [
    "# Write the output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e89d779c-15ff-40d9-9af5-6937b71c1e89",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import sqlite3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a30bc7e6-d8ed-4b73-8315-24a5ba8a103b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# open up a connection to a new database\n",
    "#conn = sqlite3.connect('ops_rehearsal_apr_2024.db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad631592-7ada-42a5-8272-ac85bda62abe",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.to_sql('observations', conn, index=False, if_exists='replace')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbe4d442-ffcc-469b-9ee1-0515caa71e83",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Request to fill the target column with the field name\n",
    "observations['target'] = observations['note']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "555dcfa2-2ed2-4384-b132-1c78cebe353c",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert np.all(observations['target'] == observations['note'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cae8b51-de1e-4de8-b44d-8b810b7c385b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert observation array to the standard opsim schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9396a136-33a4-4212-bd17-aca47ed006d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'ops_rehearsal_apr_2024_v3.db'\n",
    "SchemaConverter().obs2opsim(observations, filename=filename, info=None, delete_past=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbeacf1a-daed-4d7a-bc60-a3924c6f17f9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dade36e2-a6be-4a3a-ab28-2868770d54ae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7554878-e980-4cae-a58d-813141bd4a0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check Consistency with Previous Runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0e27a72-348d-47e6-acbb-0952f7c059db",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conn_new = sqlite3.connect('ops_rehearsal_apr_2024.db')\n",
    "#observations_new = pd.read_sql('select * from observations;', conn_new)\n",
    "\n",
    "#conn_old = sqlite3.connect('ops_rehearsal_apr_2024.db.BACKUP')\n",
    "#conn_old = sqlite3.connect('/sdf/data/rubin/shared/ops-rehearsal-3/scheduler_sims/ops_rehearsal_apr_2024.db')\n",
    "#observations_old = pd.read_sql('select * from observations;', conn_old)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68f2f61d-2145-43cb-a206-9eddf849ca35",
   "metadata": {},
   "outputs": [],
   "source": [
    "#observations_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b41d9d7-0e63-4443-b1ca-2a47f0898a15",
   "metadata": {},
   "outputs": [],
   "source": [
    "#observations_old"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3385c147-085a-4ccb-9d75-941bf25e2d4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#observations_new['RA'].values == observations_old['RA'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "185cb5fb-4eec-46d3-86ba-44ebd4a2eb1f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe8c4f99-1489-4182-839e-a69e8c16af75",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a42b51f9-9fc9-4583-ae7b-e6b2c8ad8e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "conn_new = sqlite3.connect(filename)\n",
    "observations_new = pd.read_sql('select * from observations;', conn_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73488bfa-e2a5-47dd-b8cb-580fdb94e5f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "observations_new.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d069e8ed-472f-40d9-b18c-5ebda4070338",
   "metadata": {},
   "outputs": [],
   "source": [
    "observations_new['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a117958b-3e9a-48fc-a465-7ec5d3d1c8f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert np.all(observations_new['numExposures'] == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ec52e66-53b1-4092-a76b-76aea95d6243",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rubin-sim",
   "language": "python",
   "name": "rubin-sim"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}