Notebooks/3_Obtain_SEDs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Obtaining SEDs of Seyfert galaxies\n",
    "In this file we obtain the photometry for each of the galaxies. First, we upload the Python packages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:16:19.252607Z",
     "start_time": "2021-07-26T15:16:18.979616Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: Qt5Agg\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:16:20.451243Z",
     "start_time": "2021-07-26T15:16:19.534804Z"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "from os import path\n",
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "\n",
    "import astropy.units as u\n",
    "from astropy.table import Table, vstack, unique\n",
    "\n",
    "from astroquery.ned import Ned\n",
    "from astroquery.simbad import Simbad\n",
    "from astroquery.vizier import Vizier\n",
    "from astroquery.ned.core import RemoteServiceError, TableParseError\n",
    "\n",
    "import requests\n",
    "from requests import Request, Session\n",
    "\n",
    "from numpy import unique as uniq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the SEDs\n",
    "We define a class that will get the SEDs from [CDS](https://cds.u-strasbg.fr/) and [NED](https://ned.ipac.caltech.edu/), add the bibcode to the CDS tables and eliminate photometric duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:16:24.114150Z",
     "start_time": "2021-07-26T15:16:24.094078Z"
    }
   },
   "outputs": [],
   "source": [
    "class ObtainPhotometry:\n",
    "    \"\"\"This class allows us to download the SEDs from NED and SIMBAD\"\"\"\n",
    "\n",
    "    def __init__(self, name, url):\n",
    "        self.name = name\n",
    "        self.cds_url = url\n",
    "        self.tmp_cds = '../Data/Interim/CDSVotables/'\n",
    "        self.tmp_ned = '../Data/Interim/NEDVotables/'\n",
    "        Path(self.tmp_cds).mkdir(parents=True, exist_ok=True)\n",
    "        Path(self.tmp_ned).mkdir(parents=True, exist_ok=True)\n",
    "        print('Working on galaxy ', self.name)\n",
    "        self.obtain_vot()\n",
    "        self.read_votable()\n",
    "        self.add_bibcode_cds()\n",
    "        self.check_both_tables()\n",
    "\n",
    "    def check_vot(self):\n",
    "        \"\"\"Check that the content of the votable is ok\"\"\"\n",
    "        vot_bad = True\n",
    "        while vot_bad:\n",
    "            try:\n",
    "                self.cds_table = Table.read(self.tmp_cds+self.name+'.vot',\n",
    "                                            format='votable')\n",
    "                print('Votable checked')\n",
    "                vot_bad = False\n",
    "            except ValueError:\n",
    "                print('Error in votable, deleting content-length')\n",
    "                ses = Session()\n",
    "                req_p = Request('POST', self.cds_url)\n",
    "                prepped = req_p.prepare()\n",
    "                del prepped.headers['content-length']\n",
    "                ses.send(prepped)\n",
    "                self.request_url()\n",
    "\n",
    "    def request_url(self):\n",
    "        \"\"\"Request content (votable) from url\"\"\"\n",
    "        req = requests.get(self.cds_url,\n",
    "                           headers={'cache-control': 'private, max-age=0, no-cache',\n",
    "                                    \"Pragma\": \"no-cache\"})\n",
    "        if req.status_code == 200:\n",
    "            with open(self.tmp_cds+self.name+'.vot', 'wb') as f_vot:\n",
    "                f_vot.write(req.content)\n",
    "                f_vot.close()\n",
    "\n",
    "    def obtain_vot(self):\n",
    "        \"\"\"Function to obtain the VOTables\"\"\"\n",
    "        print(datetime.now(), 'Get SEDs')\n",
    "        if path.exists(self.tmp_cds+self.name+'.vot'):\n",
    "            print('Reading CDS file')\n",
    "        else:\n",
    "            #             time.sleep(4)  # This is used to avoid being flag by the server\n",
    "            print(self.cds_url)\n",
    "            self.request_url()\n",
    "            self.check_vot()\n",
    "        print(datetime.now(), 'Got CDS SED')\n",
    "        self.ned_flag = False\n",
    "        if path.exists(self.tmp_ned+self.name+'.vot'):\n",
    "            print('Reading NED file')\n",
    "        else:\n",
    "            #             time.sleep(0.5)\n",
    "            other_ids = Simbad.query_objectids(self.name)\n",
    "            for gal_id in other_ids['ID']:\n",
    "                try:\n",
    "                    ned_tab = Ned.get_table(gal_id, table='photometry')\n",
    "                    # Use just photometry by using the units in Jy\n",
    "                    ned_tab[ned_tab['NED Units'] == 'Jy'].write(self.tmp_ned+self.name+'.vot',\n",
    "                                                                 format='votable')\n",
    "                    print(datetime.now(), 'Got NED SED')\n",
    "                    break\n",
    "                except (RemoteServiceError, TableParseError):\n",
    "                    if gal_id == other_ids['ID'][-1]:\n",
    "                        self.ned_flag = True\n",
    "                    continue\n",
    "                except IndexError:\n",
    "                    self.ned_flag = True\n",
    "\n",
    "    def read_votable(self):\n",
    "        \"\"\"Function to read the VOTables\"\"\"\n",
    "        self.cds_table = Table.read(self.tmp_cds+self.name+'.vot',\n",
    "                                    format='votable')\n",
    "        if self.ned_flag:\n",
    "            print('No NED Table')\n",
    "            self.ned_table = Table(names=['Refcode', 'Flux_Density', 'Observed_Passband',\n",
    "                                          'Frequency', 'NED_Uncertainty'], masked=True)\n",
    "        else:\n",
    "            self.ned_table = Table.read(self.tmp_ned+self.name+'.vot',\n",
    "                                        format='votable')\n",
    "\n",
    "    def plot_sed(self):\n",
    "        \"\"\"In case we want to plot the SED\"\"\"\n",
    "        plt.scatter(self.cds_table['sed_freq'].to(u.micron, equivalencies=u.spectral()),\n",
    "                    self.cds_table['sed_flux'], label='CDS')\n",
    "        plt.scatter(self.ned_table['Frequency'].to(u.micron, equivalencies=u.spectral()),\n",
    "                    self.ned_table['Flux_Density'], marker='*', label='NED')\n",
    "        plt.loglog()\n",
    "        plt.xlabel('Wavelength [um]')\n",
    "        plt.ylabel('Flux [Jy]')\n",
    "        plt.legend()\n",
    "\n",
    "    def add_bibcode_cds(self):\n",
    "        \"\"\"Adding the Bibcode to the CDS tables\"\"\"\n",
    "        self.cds_table['Bibcode'] = np.array(['Empty']*len(self.cds_table),\n",
    "                                             dtype='object')\n",
    "        for tabindx, tabinfo in enumerate(self.cds_table['_tabname']):\n",
    "            try:\n",
    "                time.sleep(0.001)  # This is used to avoid being flag by the server\n",
    "                name_cat = tabinfo.rpartition('/')[0]\n",
    "                search = Vizier.query_constraints(catalog='METAcat', name=name_cat)\n",
    "                self.cds_table['Bibcode'][tabindx] = search[0][0]['bibcode']\n",
    "            except IndexError:\n",
    "                print('There is an error at ', tabindx, tabinfo)\n",
    "\n",
    "    def check_both_tables(self):\n",
    "        \"\"\"Check the rows of CDS and NED and remove all duplicates in the photometric bands\"\"\"\n",
    "        to_rem = []\n",
    "        for u_refcode in np.unique(self.ned_table['Refcode']):\n",
    "            l_cds = np.where(self.cds_table['Bibcode'] == u_refcode)[0]#.decode('utf-8'))[0]\n",
    "            l_ned = np.where(self.ned_table['Refcode'] == u_refcode)[0]\n",
    "            if len(l_cds) > 0 and len(l_ned) > 0:\n",
    "                print('Duplicate!')\n",
    "                for lcds in l_cds:\n",
    "                    for lned in l_ned:\n",
    "                        flx_ned = str(self.ned_table[lned]['Flux_Density'])\n",
    "                        flx_cds = str(self.cds_table[lcds]['sed_flux'])\n",
    "                        if flx_ned == flx_cds:\n",
    "                            print('Deleting NED filter ',\n",
    "                                  self.ned_table[lned]['Observed_Passband'],\n",
    "                                  ' with Bibcode ',\n",
    "                                  self.ned_table[lned]['Refcode'])\n",
    "                            to_rem.append(lned)\n",
    "        self.ned_table.remove_rows(to_rem)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, we define a class to clean the Photometry and ignore empty values in the tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:16:25.989720Z",
     "start_time": "2021-07-26T15:16:25.947460Z"
    }
   },
   "outputs": [],
   "source": [
    "class CleanPhotometry:\n",
    "    \"\"\"Clean the photometry in CDS and NED tables\"\"\"\n",
    "    def __init__(self, table_cds, table_ned):\n",
    "        print('Cleaning')\n",
    "        self.cds_table = table_cds\n",
    "        self.ned_table = table_ned\n",
    "        self.bib_codes = np.array([])\n",
    "        self.join_tables()\n",
    "\n",
    "    def get_bibcodes(self, bibcode):\n",
    "        \"\"\"Get the bibcodes for the tables to analyse them later.\"\"\"\n",
    "        bib = np.array(bibcode)\n",
    "        self.bib_codes = np.unique(np.concatenate((self.bib_codes, bib)))\n",
    "\n",
    "    def clean_cds_to(self, table):\n",
    "        \"\"\"Remove rows with masked (empty) and null (0) values, then average the values.\"\"\"\n",
    "        table.remove_rows(np.where(table['sed_eflux'].mask)[0])\n",
    "        table.remove_rows(np.where(table['sed_eflux'] == 0.0)[0])\n",
    "        self.get_bibcodes(table['Bibcode'])\n",
    "        values, counts = uniq(table['sed_flux'], return_counts=True)\n",
    "        if len(values) > 0:\n",
    "            avg = np.average(values, weights=counts)\n",
    "            std = np.sqrt(np.sum(table['sed_eflux']**2))\n",
    "            return(avg, std)\n",
    "        return(np.nan, np.nan)\n",
    "\n",
    "    def clean_ned_to(self, table):\n",
    "        \"\"\"Remove rows with masked (empty) and string values, then average the values.\"\"\"\n",
    "        table.remove_rows(np.where(table['Flux_Density'].mask)[0])\n",
    "        table.remove_rows(np.where(table['NED_Uncertainty'] == '')[0])\n",
    "        table.remove_rows(np.where(table['NED_Uncertainty'] == '+/-...')[0])\n",
    "        self.get_bibcodes(table['Refcode'])\n",
    "        values, counts = uniq(table['Flux_Density'], return_counts=True)\n",
    "        table['NED_Uncertainty'] = [float(j.split('+/-')[-1]) for j in table['NED_Uncertainty']]\n",
    "        if len(values) > 0:\n",
    "            avg = np.average(values, weights=counts)\n",
    "            std = np.sqrt(np.sum(table['NED_Uncertainty']**2))\n",
    "            return(avg, std)\n",
    "        return(np.nan, np.nan)\n",
    "\n",
    "    def filters_ned(self, table_filter):\n",
    "        \"\"\" Use only the information that we need from NED \"\"\"\n",
    "        data_tab = []\n",
    "        band = [np.where(self.ned_table['Observed_Passband'] == filt)[0] for filt in table_filter]\n",
    "        for bandinx, bandif in enumerate(band):\n",
    "            avg, std = self.clean_ned_to(self.ned_table[bandif])\n",
    "            if np.isnan(avg):\n",
    "                continue\n",
    "            to_um = self.ned_table[bandif]['Frequency'].to(u.micron,\n",
    "                                                           equivalencies=u.spectral())\n",
    "            data_tab.append([table_filter[bandinx], np.mean(to_um), avg*u.Jy, std*u.Jy])\n",
    "        return(Table(np.array(data_tab),\n",
    "                     names=['Filter', 'Wave', 'Flux', 'F_er'],\n",
    "                     dtype=('U32', 'float64', 'float64', 'float64')))\n",
    "\n",
    "    def filters_cds(self, table_filter):\n",
    "        \"\"\"Use only the information that we need from CDS\"\"\"\n",
    "        data_tab = []\n",
    "        band = [np.where(self.cds_table['sed_filter'] == filt)[0] for filt in table_filter]\n",
    "        for bandinx, bandif in enumerate(band):\n",
    "            avg, std = self.clean_cds_to(self.cds_table[bandif])\n",
    "            if np.isnan(avg):\n",
    "                continue\n",
    "            to_um = self.cds_table[bandif]['sed_freq'].to(u.micron,\n",
    "                                                          equivalencies=u.spectral())\n",
    "            data_tab.append([table_filter[bandinx], np.mean(to_um), avg*u.Jy, std*u.Jy])\n",
    "        return(Table(np.array(data_tab),\n",
    "                     names=['Filter', 'Wave', 'Flux', 'F_er'],\n",
    "                     dtype=('U32', 'float64', 'float64', 'float64')))\n",
    "\n",
    "    def join_tables(self):\n",
    "        \"\"\"Finally, we join the tables and remove those whose errors are too high\"\"\"\n",
    "        self.final_tab = vstack([self.filters_cds(CDSFilters), self.filters_ned(NEDFilters)])\n",
    "        self.final_tab.remove_rows(np.where(self.final_tab['F_er']/self.final_tab['Flux'] >= 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we load the SMB-VCV sample and define the filters in CDS and NED that will be used to create the relations between them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:16:29.259070Z",
     "start_time": "2021-07-26T15:16:29.058201Z"
    }
   },
   "outputs": [],
   "source": [
    "Sample = Table.read('../Data/Final/VCV_SMB_otype.txt', format='ascii')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-19T14:14:21.014582Z",
     "start_time": "2021-01-19T14:14:21.007443Z"
    }
   },
   "source": [
    "## Select filters/bands\n",
    "Here we create a list for the bands we want to use from the CDS and NED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:16:30.781119Z",
     "start_time": "2021-07-26T15:16:30.770308Z"
    }
   },
   "outputs": [],
   "source": [
    "CDSFilters = ['GALEX:FUV', 'GALEX:NUV',\n",
    "              \"SDSS:u'\", \"SDSS:g'\", \"SDSS:r'\", \"SDSS:i'\", \"SDSS:z'\",\n",
    "              'SDSS:u', 'SDSS:g', 'SDSS:r', 'SDSS:i', 'SDSS:z',\n",
    "              '2MASS:J', '2MASS:H', '2MASS:Ks',\n",
    "              ':=3.6um', ':=4.5um', ':=5.8um', ':=8um',\n",
    "              'WISE:W1', 'WISE:W2', 'WISE:W3', 'WISE:W4',\n",
    "              'IRAS:12', 'IRAS:25', 'IRAS:60', 'IRAS:100',\n",
    "              'Spitzer/MIPS:24', 'Spitzer/MIPS:70', 'Spitzer/MIPS:160',\n",
    "              'Herschel/PACS:70', 'Herschel/PACS:100', 'Herschel/PACS:160',\n",
    "              'Herschel/SPIRE:250', 'Herschel/SPIRE:350', 'Herschel/SPIRE:500',\n",
    "              ':=250um', ':=350um', ':=500um',\n",
    "              ':=1.4GHz', ':=21cm', ':=1.5GHz', ':=20cm', ':=5GHz', ':=6cm']\n",
    "\n",
    "NEDFilters = ['2-10 keV (XMM)', '0.5-2 keV (XMM)',\n",
    "              'FUV (GALEX)', 'NUV (GALEX)',\n",
    "              'u (SDSS) AB', 'g (SDSS) AB', 'r (SDSS) AB', 'i (SDSS) AB', 'z (SDSS) AB',\n",
    "              'J (2MASS) AB', 'H (2MASS) AB', 'Ks (2MASS) AB',\n",
    "              'W1 (WISE)', 'W2 (WISE)', 'W3 (WISE)', 'W4 (WISE)',\n",
    "              '3.6 microns (IRAC)', '4.5 microns (IRAC)',\n",
    "              '5.8 microns (IRAC)', '8.0 microns (IRAC)',\n",
    "              '12 microns (IRAS)', '25 microns (IRAS)',\n",
    "              '60 microns (IRAS)', '100 microns (IRAS)',\n",
    "              '24 microns (MIPS)', '70 microns (MIPS)', '160 microns (MIPS)',\n",
    "              '70 microns (PACS)', '100 microns (PACS)', '160 microns (PACS)',\n",
    "              '250 microns (SPIRE)', '350 microns (SPIRE)', '500 microns (SPIRE)',\n",
    "              '4.89 GHz (VLA)', '1.46 GHz (VLA)', '1.4GHz']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-19T14:15:48.942889Z",
     "start_time": "2021-01-19T14:15:48.938294Z"
    }
   },
   "source": [
    "## Save SEDs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With the information presented before and the selected bands, we look galaxy by galaxy for the SED in NED and CDS. Then, we clean the Photometry and write the final file per galaxy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T12:32:01.070667Z",
     "start_time": "2021-07-20T12:32:01.021039Z"
    }
   },
   "outputs": [],
   "source": [
    "INIT_URL = 'http://vizier.u-strasbg.fr/viz-bin/sed?-c='\n",
    "urls = [INIT_URL+str(Sample['ra'][row])+'%20' +\n",
    "        str(Sample['dec'][row])+'&-c.rs=5' for row in range(len(Sample))]\n",
    "Sample['cds_url'] = urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T12:33:30.380954Z",
     "start_time": "2021-07-20T12:33:27.794900Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on galaxy  CLASXS 427\n",
      "2021-07-20 14:33:27.802167 Get CDS SED\n",
      "Reading CDS file\n",
      "2021-07-20 14:33:27.802387 Got CDS SED\n",
      "Reading NED file\n",
      "Cleaning\n",
      "305 Finish\n",
      "Working on galaxy  SDSS J020655.71+010826.6\n",
      "2021-07-20 14:33:28.995553 Get CDS SED\n",
      "Reading CDS file\n",
      "2021-07-20 14:33:28.995735 Got CDS SED\n",
      "Reading NED file\n",
      "Duplicate!\n",
      "Deleting NED filter  1.5 GHz (VLA)  with Bibcode  2016MNRAS.460.4433H\n",
      "Cleaning\n",
      "306 Finish\n",
      "Working on galaxy  [VV2006c] J165738.3+403432\n",
      "2021-07-20 14:33:29.866388 Get CDS SED\n",
      "Reading CDS file\n",
      "2021-07-20 14:33:29.866574 Got CDS SED\n",
      "No NED Table\n",
      "Cleaning\n",
      "307 Finish\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n",
      "/Users/users/ramos/.local/lib/python3.8/site-packages/numpy/ma/core.py:4075: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  check = compare(sdata, odata)\n"
     ]
    }
   ],
   "source": [
    "# We show an example of the last galaxies from the positon 17725\n",
    "IND = 305  # It should be zero for all, here is 17725 as an example\n",
    "SED_PATH = '../Data/Interim/SEDs/'\n",
    "Path(SED_PATH).mkdir(parents=True, exist_ok=True)\n",
    "for Galaxy, cds_url in Sample['main_id','cds_url'][IND:IND+3]:\n",
    "    if path.exists(SED_PATH+Galaxy+'_Phot.txt'):\n",
    "        print(IND, 'Already exist')\n",
    "        IND += 1\n",
    "        continue\n",
    "    InitTabl = ObtainPhotometry(Galaxy, cds_url)\n",
    "    cleaned = CleanPhotometry(InitTabl.cds_table, InitTabl.ned_table)\n",
    "    SED = cleaned.final_tab\n",
    "    BIBCODES = ';'.join(cleaned.bib_codes)\n",
    "    with open(\"../Data/Interim/Bibcodes_SED.csv\", \"a+\") as file_bibcodes:\n",
    "        file_bibcodes.writelines([Galaxy+',', BIBCODES+'\\n'])\n",
    "        file_bibcodes.close()\n",
    "    SED.write(SED_PATH+Galaxy+'_Phot.txt', format='ascii')\n",
    "    print(IND, 'Finish')\n",
    "    IND += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to know how many references where used at this point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:13:38.591744Z",
     "start_time": "2021-07-26T15:13:38.137575Z"
    }
   },
   "outputs": [],
   "source": [
    "All_Data = Table.read('../Data/Interim/Bibcodes_SED.csv', format='ascii.csv',\n",
    "                      names=('Name','References'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:13:39.415232Z",
     "start_time": "2021-07-26T15:13:39.409216Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38161"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(All_Data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:13:41.302553Z",
     "start_time": "2021-07-26T15:13:41.216898Z"
    }
   },
   "outputs": [],
   "source": [
    "All_Data.remove_rows(All_Data['References'].mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:13:42.505300Z",
     "start_time": "2021-07-26T15:13:42.286477Z"
    }
   },
   "outputs": [],
   "source": [
    "All_Data = unique(All_Data, keys=['Name'], keep='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:13:43.024660Z",
     "start_time": "2021-07-26T15:13:43.016154Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18883"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(All_Data) ## Number of galaxies with references"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:15:50.702971Z",
     "start_time": "2021-07-26T15:13:45.008470Z"
    }
   },
   "outputs": [],
   "source": [
    "unique_ref = []\n",
    "for indx, data_bib in enumerate(All_Data):\n",
    "    unique_ref = np.concatenate((data_bib['References'].split(';'),\n",
    "                                 unique_ref))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:15:50.809366Z",
     "start_time": "2021-07-26T15:15:50.712179Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "462"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(unique_ref, return_counts=True)[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We create a .py script with the code available in this Jupyter notebook (located in [Additionals](../Additionals/3_Obtain_SEDs.py)) to meet the time requirements of the url request (Important if experimenting problems with retrieving times). Sometimes when there are some problems with the VOTables (when you lost connection and the saved file is truncated) is better to try to download the CDS file manually and add it into the folder. Keep in mind some limitations on the notebook server (i.e. iopub_msg_rate_limit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Notebook info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-26T15:16:36.876766Z",
     "start_time": "2021-07-26T15:16:36.794075Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Andres Ramos\n",
      "\n",
      "Python implementation: CPython\n",
      "Python version       : 3.8.3\n",
      "IPython version      : 7.16.1\n",
      "\n",
      "Compiler    : GCC 7.3.0\n",
      "OS          : Linux\n",
      "Release     : 3.10.0-1160.25.1.el7.x86_64\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 8\n",
      "Architecture: 64bit\n",
      "\n",
      "Specific Python packages\n",
      "astroquery: 0.4.1\n",
      "\n",
      "sys       : 3.8.3 (default, Jul  2 2020, 16:21:59) \n",
      "[GCC 7.3.0]\n",
      "autopep8  : 1.5.7\n",
      "numpy     : 1.19.5\n",
      "requests  : 2.24.0\n",
      "re        : 2.2.1\n",
      "astropy   : 4.2\n",
      "logging   : 0.5.1.2\n",
      "json      : 2.0.9\n",
      "matplotlib: 3.2.2\n",
      "\n",
      "Watermark: 2.1.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a \"Andres Ramos\" -d -v -m\n",
    "print('Specific Python packages')\n",
    "%watermark -iv -w --packages astroquery"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": "3",
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}