demo/demo_hsc_config.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from astropy.utils.data import download_file\n",
    "\n",
    "from IPython.display import Image\n",
    "from IPython.core.display import HTML \n",
    "    \n",
    "from unagi import config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup HSC archive\n",
    "\n",
    "* Here we show you how to config and setup a HSC archive to use\n",
    "* HSC-SSP has public and internal archive for data release\n",
    "    - To use the [public data release](https://hsc-release.mtk.nao.ac.jp\") (PDR), you need to register an account.\n",
    "    - And the [internal data release](https://hscdata.mtk.nao.ac.jp) (IDR) is only available to HSC-SSP members. \n",
    "\n",
    "### Credential\n",
    "\n",
    "For both archives, you need to have a `username` and a `password` to use it. There are three ways to pass these information to `unagi`. \n",
    "1. You can type in your credential information everytime.\n",
    "2. Put your `username` and `password` in a plain ASCII file and pass it as a configuration file. \n",
    "3. Setup environment variables. \n",
    "    - For PDR, please use: `SSP_PDR_USR` and `SSP_PDR_PWD`\n",
    "    - For IDR, please use: `SSP_IDR_USR` and `SSP_IDR_PWD`\n",
    "    - To permanently save your username and password to environment variables, you can open `~/.bash_profile` (or `~/.bashrc`) and write `export SSP_PDR_USR='xxx'`, `export SSP_PDR_PWD='yyy'`. To make them take effect, do `$. ~/.bash_profile` (or `$. ~/.bashrc`).\n",
    "    \n",
    "### Data Release: `DR`\n",
    "\n",
    "You can find the available `DR`s here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pdr1', 'pdr2', 'dr1', 'dr2', 'dr3']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config.AVAILABLE_DRS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# SQL search: https://hsc-release.mtk.nao.ac.jp/datasearch/api/catalog_jobs/\n",
      "# Image cutout: https://hsc-release.mtk.nao.ac.jp/das_cutout/pdr2/cgi-bin/cutout?\n",
      "# Image files: https://hsc-release.mtk.nao.ac.jp/archive/filetree/\n",
      "# PSF models: https://hsc-release.mtk.nao.ac.jp/archive/filetree/\n",
      "# DAS search: https://hsc-release.mtk.nao.ac.jp/das_search/\n",
      "# Coverage maps: https://hsc-release.mtk.nao.ac.jp/rsrc/pdr2/koike/survey-area/fig/\n",
      "# Information about the DR: https://hsc-release.mtk.nao.ac.jp/rsrc/pdr2/koike/survey-area/info/\n"
     ]
    }
   ],
   "source": [
    "# For example, here we setup the configuration object for pdr2\n",
    "pdr2_config = config.Server(dr='pdr2')\n",
    "\n",
    "# Here is some key URLs to use this archive\n",
    "print(\"# SQL search: {}\".format(pdr2_config.cat_url))\n",
    "print(\"# Image cutout: {}\".format(pdr2_config.img_url))\n",
    "print(\"# Image files: {}\".format(pdr2_config.file_url))\n",
    "print(\"# PSF models: {}\".format(pdr2_config.file_url))\n",
    "print(\"# DAS search: {}\".format(pdr2_config.das_url))\n",
    "print(\"# Coverage maps: {}\".format(pdr2_config.map_url))\n",
    "print(\"# Information about the DR: {}\".format(pdr2_config.txt_url))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z', 'HSC-Y', 'NB0387', 'NB0816', 'NB0921']\n",
      "['g', 'r', 'i', 'z', 'y', 'nb387', 'nb816', 'nb921']\n"
     ]
    }
   ],
   "source": [
    "# List of available filters\n",
    "print(pdr2_config.filter_list)\n",
    "\n",
    "# Short \"Nickname\" of the filters\n",
    "print(pdr2_config.filter_list_short)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['any', 'pdr2_dud', 'pdr2_wide', 'pdr2_cosmos_wide_depth_best', 'pdr2_cosmos_wide_depth_median', 'pdr2_cosmos_wide_depth_worst']\n",
      "# Default Rerun: pdr2_wide\n",
      "# Default Wide-layer Rerun: pdr2_wide\n",
      "# Default Deep-layer Rerun: pdr2_dud\n",
      "# Default UltraDeep-layer Rerun: pdr2_dud\n"
     ]
    }
   ],
   "source": [
    "# Within each DR, SSP could have multiple datasets with different sky coverage, imaging depth, and \n",
    "# were reduced by different versions of the HSC pipeline. \n",
    "# Each of these dataset is called a `Rerun`\n",
    "\n",
    "# List of available reruns\n",
    "print(pdr2_config.rerun_list)\n",
    "\n",
    "# A default `rerun` has been selected for each layer\n",
    "print(\"# Default Rerun: {}\".format(pdr2_config.rerun_default))\n",
    "print(\"# Default Wide-layer Rerun: {}\".format(pdr2_config.wide_default))\n",
    "print(\"# Default Deep-layer Rerun: {}\".format(pdr2_config.deep_default))\n",
    "print(\"# Default UltraDeep-layer Rerun: {}\".format(pdr2_config.udeep_default))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['DUD_COSMOS', 'DUD_DEEP2-3', 'DUD_ELAIS-N1', 'DUD_XMM-LSS', 'WIDE_WIDE01', 'WIDE_WIDE02', 'WIDE_WIDE03', 'WIDE_WIDE04', 'WIDE_WIDE05', 'WIDE_WIDE06', 'WIDE_WIDE07']\n"
     ]
    }
   ],
   "source": [
    "# HSC data are seperated into a bunch of small, isolated fields (for now)\n",
    "\n",
    "# List of available fields\n",
    "print(pdr2_config.field_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abbr', 'field_map', 'file', 'filter_available', 'name', 'patch_info', 'type']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# More detailed information of each field is kept in `field_table`\n",
    "pdr2_config.field_table.colnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://hsc-release.mtk.nao.ac.jp/rsrc/pdr2/koike/survey-area/fig/tracts_patches_W_w01_HSC-R.png\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# e.g. If you want to check out the coverage of the `WIDE_WIDE01` field in i-band\n",
    "\n",
    "w01_flag = pdr2_config.field_table['name'] == 'WIDE_WIDE01' \n",
    "w01_i_map = pdr2_config.field_table['field_map'][w01_flag][0]\n",
    "\n",
    "Image(url=w01_i_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The RA, Dec information of all the `Tract` and `Patch` in a field is kept in this file\n",
    "w01_i_patches = pdr2_config.field_table['patch_info'][w01_flag][0]\n",
    "\n",
    "w01_i_info = [line.rstrip('\\n') for line in open(download_file(w01_i_patches), 'r')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['************ Tract *************', 'Tract: 8994  Center (RA, Dec): (14.8760330579 , -2.23140495868)', 'Tract: 8994  Corner0 (RA, Dec): (15.7170438263 , -3.07096790218)', 'Tract: 8994  Corner1 (RA, Dec): (14.0349755707 , -3.07096786549)', 'Tract: 8994  Corner2 (RA, Dec): (14.0359357788 , -1.39131562101)', 'Tract: 8994  Corner3 (RA, Dec): (15.7160836715 , -1.39131563762)', '************ Patches *************', 'Tract: 8994  Patch: 0,0  Center (RA, Dec): (15.6235801635 , -2.97775302135)', 'Tract: 8994  Patch: 0,0      Corner0 (RA, Dec): (15.7170438263 , -3.07096790218)', 'Tract: 8994  Patch: 0,0      Corner1 (RA, Dec): (15.5302074436 , -3.0710983265)']\n"
     ]
    }
   ],
   "source": [
    "print(w01_i_info[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rerun\n",
    "\n",
    "* Most of the times, you only need data from a single `rerun`. In that case, you can use the `Rerun` object instead.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DUD_COSMOS',\n",
       " 'DUD_DEEP2-3',\n",
       " 'DUD_ELAIS-N1',\n",
       " 'DUD_XMM-LSS',\n",
       " 'WIDE_WIDE01',\n",
       " 'WIDE_WIDE02',\n",
       " 'WIDE_WIDE03',\n",
       " 'WIDE_WIDE04',\n",
       " 'WIDE_WIDE05',\n",
       " 'WIDE_WIDE06',\n",
       " 'WIDE_WIDE07']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pdr2_wide = config.Rerun(dr='pdr2', rerun='pdr2_wide')\n",
    "\n",
    "pdr2_wide.field_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pdr2_cosmos_good = config.Rerun(dr='pdr2', rerun='pdr2_cosmos_wide_depth_best')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test the Internal Data Release Server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['DUD_COSMOS' 'DUD_DEEP2-3' 'DUD_ELAIS-N1' 'DUD_XMM-LSS' 'WIDE_AEGIS'\n",
      " 'WIDE_SPRING' 'WIDE_AUTUMN' 'WIDE_HECTOMAP']\n"
     ]
    }
   ],
   "source": [
    "dr3_config = config.Server(dr='dr3')\n",
    "print(dr3_config.field_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  },
  "nav_menu": {},
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}