diff --git a/README.md b/README.md index 24d1f24..ad1bbc7 100644 --- a/README.md +++ b/README.md @@ -18,27 +18,18 @@ To improve cross-match query execution speed, we install `SQLxMatch` in a SQL Se The advantage of this `in-database` remote cross-match, compared to other `in-memory` local cross-match software libraries, is that the users leverage the remote database server's own (and potentially bigger) computing/memory/storage resources to filter and cross-match the full catalogs right away, having only a relatively small-sized cross-match output table returned to them. This can be faster and more efficient than having users to download the full catalogs into their own computers (if they have enough storage), and then load them in python for filtering and running the cross-match, for instance. - - ## **Documentation** - Instructions on how to install and operate `SQLxMatch` can found under the [docs](https://github.com/sciserver/sqlxmatch/tree/main/docs) folder. - ## **Examples** Example Jupyter Notebooks and demos can be found under [demo](https://github.com/sciserver/sqlxmatch/tree/main/demo) folder. - ## **Citation** Taghizadeh-Popp, M. and Dobos, L. (2023) “SQLxMatch: In-Database Spatial Cross-Match of Astronomical Catalogs”. Zenodo. doi: 10.5281/zenodo.10142771. - - - - [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10142770.svg)](https://doi.org/10.5281/zenodo.10142770) ## **License** diff --git a/demo/SQLxMatch_CasJobs_demo.ipynb b/demo/SQLxMatch_CasJobs_demo.ipynb index 3ef8b70..839ddf6 100644 --- a/demo/SQLxMatch_CasJobs_demo.ipynb +++ b/demo/SQLxMatch_CasJobs_demo.ipynb @@ -39,7 +39,7 @@ "\n", "\n", "\n", - "### The `SQLxMatch` stored procedure\n", + "### The SQLxMatch stored procedure\n", "\n", "The first input parameters are the names of 2 catalog tables, views, or temporary tables located in the CasJobs `xmatch` database context. This context already contains several table views from specific astronomical catalogs, and are named with the following format: `CatalogName_TableName`.\n", "\n", @@ -69,17 +69,21 @@ "
  • @output_table NVARCHAR(128) or SYSNAME: If NOT NULL, this procedure will insert the output results into the table @output_table (of format 'server.database.schema.table', 'database.schema.table', 'database.table', or simply 'table'), which must already exist and be visible within the scope of the procedure. If set to null, the output results will be simply returned as a table resultset. Takes a default value of NULL.\n", "
  • @only_nearest BIT: If set to 0 (default value), then all matches within a distance @radius to an object are returned. If set to 1, only the closest match to an object is returned.\n", "
  • @sort_by_separation BIT: If set to 1, then the output will be sorted by the 'id1' and 'sep' columns. If set to 0 (default value), no particular ordering is applied.\n", - "
  • @radec_in_output BIT : If set to 1, then the output table will contain as well the (RA, Dec) values of each object.\n", + "
  • @include_sep_rank BIT: If set to 1, then the output table will include an extra column named 'sep_rank', denoting the rank of all matches to an object when sorted by angular separation. If set to 0 (default value), this column is not included.\n", + "
  • @include_radec BIT : If set to 1, then the output table will contain the original (RA, Dec) columns from both input tables, as ra1, dec1, ra2, and dec2.\n", "
  • @print_messages BIT : If set to 1, then time-stamped messages will be printed as the different sections in this procedure are completed.\n", "\n", "\n", "RETURNS:
    \n", " " + "
  • TABLE (id1, id2, sep), where id1 and id2 are the unique object identifier columns in @table1 and @table2, respectively, and sep (FLOAT) is the angular separation between objects in arseconds. \n", + "\n", + "
  • TABLE (id1, id2, sep, ra1, dec1, ra2, dec2) when @include_radec=1\n", + "\n", + "
  • TABLE (id1, id2, sep, sep_rank) when @include_sep_rank=1\n", + "\n", + "
  • TABLE (id1, id2, sep, sep_rank, ra1, dec1, ra2, dec2) when @include_radec=1 and @include_sep_rank = 1.\n", + " \n" ] }, { @@ -99,11 +103,11 @@ "id": "556ced78-8c2f-4e11-988f-d95c892ca965", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:18.197370Z", - "iopub.status.busy": "2023-11-16T02:06:18.196878Z", - "iopub.status.idle": "2023-11-16T02:06:19.251214Z", - "shell.execute_reply": "2023-11-16T02:06:19.249531Z", - "shell.execute_reply.started": "2023-11-16T02:06:18.197256Z" + "iopub.execute_input": "2023-11-17T17:36:04.102718Z", + "iopub.status.busy": "2023-11-17T17:36:04.102114Z", + "iopub.status.idle": "2023-11-17T17:36:05.125328Z", + "shell.execute_reply": "2023-11-17T17:36:05.123187Z", + "shell.execute_reply.started": "2023-11-17T17:36:04.102566Z" }, "tags": [] }, @@ -132,11 +136,11 @@ "id": "ead71a6c-c121-479e-a4d3-6d7b41e9af89", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:19.254016Z", - "iopub.status.busy": "2023-11-16T02:06:19.253595Z", - "iopub.status.idle": "2023-11-16T02:06:21.165238Z", - "shell.execute_reply": "2023-11-16T02:06:21.163286Z", - "shell.execute_reply.started": "2023-11-16T02:06:19.253967Z" + "iopub.execute_input": "2023-11-17T17:36:05.128806Z", + "iopub.status.busy": "2023-11-17T17:36:05.128289Z", + "iopub.status.idle": "2023-11-17T17:36:06.826945Z", + "shell.execute_reply": "2023-11-17T17:36:06.825162Z", + "shell.execute_reply.started": "2023-11-17T17:36:05.128751Z" }, "tags": [] }, @@ -212,35 +216,35 @@ " ...\n", " \n", " \n", - " 47\n", + " 50\n", " VVDS\n", " The VIMOS VLT Deep Survey\n", " \\r\\n A total of 11 564 objects have been ...\n", " http://cesam.lam.fr/vvdsproject/index.html\n", " \n", " \n", - " 48\n", + " 51\n", " WiggleZ\n", " WiggleZ Dark Energy Survey Data Release 1\n", " \\r\\n The WiggleZ Dark Energy Survey is a ...\n", " http://wigglez.swin.edu.au/site/index.html\n", " \n", " \n", - " 49\n", + " 52\n", " WISE\n", " \\tThe WISE All-Sky data Release\n", " \\n NASA's Wide-field Infrared Survey Expl...\n", " http://wise2.ipac.caltech.edu/docs/release/all...\n", " \n", " \n", - " 50\n", + " 53\n", " WMAP\n", " \\tNine-year WMAP point source catalogs\n", " \\r\\n Nine-year WMAP point source catalogs\n", " https://heasarc.gsfc.nasa.gov/W3Browse/radio-c...\n", " \n", " \n", - " 51\n", + " 54\n", " zCOSMOS\n", " zCOSMOS-bright DR2 catalog\n", " \\r\\n zCOSMOS is a large-redshift survey t...\n", @@ -248,7 +252,7 @@ " \n", " \n", "\n", - "

    52 rows × 4 columns

    \n", + "

    55 rows × 4 columns

    \n", "" ], "text/plain": [ @@ -259,11 +263,11 @@ "3 CHANDRA The Chandra Source Catalog, Release 1.1 \n", "4 CNOC2 The Canadian Network for Observational Cosmol... \n", ".. ... ... \n", - "47 VVDS The VIMOS VLT Deep Survey \n", - "48 WiggleZ WiggleZ Dark Energy Survey Data Release 1 \n", - "49 WISE \\tThe WISE All-Sky data Release \n", - "50 WMAP \\tNine-year WMAP point source catalogs \n", - "51 zCOSMOS zCOSMOS-bright DR2 catalog \n", + "50 VVDS The VIMOS VLT Deep Survey \n", + "51 WiggleZ WiggleZ Dark Energy Survey Data Release 1 \n", + "52 WISE \\tThe WISE All-Sky data Release \n", + "53 WMAP \\tNine-year WMAP point source catalogs \n", + "54 zCOSMOS zCOSMOS-bright DR2 catalog \n", "\n", " remarks \\\n", "0 \\r\\n The ASAS-3 Catalog of Variable Stars... \n", @@ -272,11 +276,11 @@ "3 \\r\\n The first official release of the CS... \n", "4 \\r\\n The Canadian Network for Observation... \n", ".. ... \n", - "47 \\r\\n A total of 11 564 objects have been ... \n", - "48 \\r\\n The WiggleZ Dark Energy Survey is a ... \n", - "49 \\n NASA's Wide-field Infrared Survey Expl... \n", - "50 \\r\\n Nine-year WMAP point source catalogs \n", - "51 \\r\\n zCOSMOS is a large-redshift survey t... \n", + "50 \\r\\n A total of 11 564 objects have been ... \n", + "51 \\r\\n The WiggleZ Dark Energy Survey is a ... \n", + "52 \\n NASA's Wide-field Infrared Survey Expl... \n", + "53 \\r\\n Nine-year WMAP point source catalogs \n", + "54 \\r\\n zCOSMOS is a large-redshift survey t... \n", "\n", " url \n", "0 http://www.astrouw.edu.pl/asas/?page=main \n", @@ -285,13 +289,13 @@ "3 http://cxc.cfa.harvard.edu/csc/ \n", "4 http://www.astro.utoronto.ca/~cnoc/cnoc2.html \n", ".. ... \n", - "47 http://cesam.lam.fr/vvdsproject/index.html \n", - "48 http://wigglez.swin.edu.au/site/index.html \n", - "49 http://wise2.ipac.caltech.edu/docs/release/all... \n", - "50 https://heasarc.gsfc.nasa.gov/W3Browse/radio-c... \n", - "51 http://vizier.cfa.harvard.edu/viz-bin/VizieR?-... \n", + "50 http://cesam.lam.fr/vvdsproject/index.html \n", + "51 http://wigglez.swin.edu.au/site/index.html \n", + "52 http://wise2.ipac.caltech.edu/docs/release/all... \n", + "53 https://heasarc.gsfc.nasa.gov/W3Browse/radio-c... \n", + "54 http://vizier.cfa.harvard.edu/viz-bin/VizieR?-... \n", "\n", - "[52 rows x 4 columns]" + "[55 rows x 4 columns]" ] }, "execution_count": 2, @@ -321,11 +325,11 @@ "id": "5be240a8-716b-4b38-aefd-45a61ff3d9e0", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:21.170985Z", - "iopub.status.busy": "2023-11-16T02:06:21.170300Z", - "iopub.status.idle": "2023-11-16T02:06:22.829568Z", - "shell.execute_reply": "2023-11-16T02:06:22.826593Z", - "shell.execute_reply.started": "2023-11-16T02:06:21.170937Z" + "iopub.execute_input": "2023-11-17T17:36:06.832406Z", + "iopub.status.busy": "2023-11-17T17:36:06.832000Z", + "iopub.status.idle": "2023-11-17T17:36:08.599315Z", + "shell.execute_reply": "2023-11-17T17:36:08.597605Z", + "shell.execute_reply.started": "2023-11-17T17:36:06.832336Z" }, "tags": [] }, @@ -361,34 +365,10 @@ " \n", " \n", " 0\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " view\n", - " GOODS-N IRS 16 micron Photometry Catalog\n", - " dbo\n", - " \n", - " \n", - " 1\n", - " SPITZER\n", - " SPITZER_goodsnMIPS24micron\n", - " view\n", - " GOODS-N MIPS 24 micron Photometry Catalog\n", - " dbo\n", - " \n", - " \n", - " 2\n", - " SPITZER\n", - " SPITZER_goodssIRS16micron\n", - " view\n", - " GOODS-S IRS 16 micron Photometry Catalog\n", - " dbo\n", - " \n", - " \n", - " 3\n", - " SPITZER\n", - " SPITZER_goodssMIPS24micron\n", + " IRAS\n", + " IRAS_PhotoObj\n", " view\n", - " GOODS-S MIPS 24 micron Photometry Catalog\n", + " The main PhotoObj table for the IRAS catalog\n", " dbo\n", " \n", " \n", @@ -396,17 +376,11 @@ "" ], "text/plain": [ - " catalog_name table_name table_type \\\n", - "0 SPITZER SPITZER_goodsnIRS16micron view \n", - "1 SPITZER SPITZER_goodsnMIPS24micron view \n", - "2 SPITZER SPITZER_goodssIRS16micron view \n", - "3 SPITZER SPITZER_goodssMIPS24micron view \n", + " catalog_name table_name table_type \\\n", + "0 IRAS IRAS_PhotoObj view \n", "\n", - " description schema_name \n", - "0 GOODS-N IRS 16 micron Photometry Catalog dbo \n", - "1 GOODS-N MIPS 24 micron Photometry Catalog dbo \n", - "2 GOODS-S IRS 16 micron Photometry Catalog dbo \n", - "3 GOODS-S MIPS 24 micron Photometry Catalog dbo " + " description schema_name \n", + "0 The main PhotoObj table for the IRAS catalog dbo " ] }, "execution_count": 3, @@ -415,7 +389,7 @@ } ], "source": [ - "sql = \"SELECT * FROM Tables WHERE catalog_name = 'SPITZER'\"\n", + "sql = \"SELECT * FROM Tables WHERE catalog_name = 'IRAS'\"\n", "cj.executeQuery(sql, context='xmatch')" ] }, @@ -444,11 +418,11 @@ "id": "bd484de1-c066-499e-bdb3-141d8b0b087b", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:22.832326Z", - "iopub.status.busy": "2023-11-16T02:06:22.831896Z", - "iopub.status.idle": "2023-11-16T02:06:24.898115Z", - "shell.execute_reply": "2023-11-16T02:06:24.896220Z", - "shell.execute_reply.started": "2023-11-16T02:06:22.832277Z" + "iopub.execute_input": "2023-11-17T17:36:08.601840Z", + "iopub.status.busy": "2023-11-17T17:36:08.601390Z", + "iopub.status.idle": "2023-11-17T17:36:10.458256Z", + "shell.execute_reply": "2023-11-17T17:36:10.456486Z", + "shell.execute_reply.started": "2023-11-17T17:36:08.601790Z" }, "tags": [] }, @@ -492,44 +466,44 @@ " \n", " \n", " 0\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " cx\n", - " Cartesian X (J2000)\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " objID\n", + " unique object identifier\n", " \n", - " pos.eq.x;pos.frame=j2000\n", + " meta.id\n", " \n", - " float\n", + " bigint\n", " 8\n", - " 53\n", + " 19\n", " 0\n", " 1\n", " dbo\n", " \n", " \n", " 1\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " cy\n", - " Cartesian Y (J2000)\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " name\n", + " IRAS Source Name\n", " \n", - " pos.eq.y;pos.frame=j2000\n", + " meta.id\n", " \n", - " float\n", - " 8\n", - " 53\n", + " varchar\n", + " 11\n", + " 0\n", " 0\n", " 2\n", " dbo\n", " \n", " \n", " 2\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " cz\n", - " Cartesian Z (J2000)\n", - " \n", - " pos.eq.z;pos.frame=j2000\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " ra\n", + " J2000 right ascension\n", + " deg\n", + " pos.eq.ra;pos.frame=j2000\n", " \n", " float\n", " 8\n", @@ -540,32 +514,32 @@ " \n", " \n", " 3\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " htmid\n", - " HTM ID (J2000)\n", - " \n", - " pos.eq.HTM; pos.frame=j2000\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " dec\n", + " J2000 declination\n", + " deg\n", + " pos.eq.dec;pos.frame=j2000\n", " \n", - " bigint\n", + " float\n", " 8\n", - " 19\n", + " 53\n", " 0\n", " 4\n", " dbo\n", " \n", " \n", " 4\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " zoneid\n", - " Zone ID (J2000)\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " cx\n", + " Cartesian coordinate x\n", " \n", - " pos.eq.zone;pos.frame=j2000\n", + " pos.eq.x;pos.frame=j2000\n", " \n", - " int\n", - " 4\n", - " 10\n", + " float\n", + " 8\n", + " 53\n", " 0\n", " 5\n", " dbo\n", @@ -587,144 +561,144 @@ " ...\n", " \n", " \n", - " 37\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " ebmag\n", - " HST B magnitude uncertainty\n", - " mag\n", - " stat.error;phot.mag;em.opt.B\n", + " 54\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " mhcon\n", + " Possible number of HCONs, 99 for NULL\n", " \n", - " real\n", - " 4\n", - " 24\n", + " meta.number\n", + " \n", + " smallint\n", + " 2\n", + " 5\n", " 0\n", - " 38\n", + " 55\n", " dbo\n", " \n", " \n", - " 38\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " evmag\n", - " HST V magnitude uncertainty\n", - " mag\n", - " stat.error;phot.mag;em.opt.V\n", + " 55\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " fcor_12\n", + " Flux correction factor applied (times 1000),99...\n", + " \n", + " stat.param;phot.flux;em.IR.IRAS.12\n", " \n", - " real\n", + " int\n", " 4\n", - " 24\n", + " 10\n", " 0\n", - " 39\n", + " 56\n", " dbo\n", " \n", " \n", - " 39\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " eimag\n", - " HST I magnitude uncertainty\n", - " mag\n", - " stat.error;phot.mag;em.opt.I\n", + " 56\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " fcor_25\n", + " Flux correction factor applied (times 1000),99...\n", " \n", - " real\n", + " stat.param;phot.flux;em.IR.IRAS.25\n", + " \n", + " int\n", " 4\n", - " 24\n", + " 10\n", " 0\n", - " 40\n", + " 57\n", " dbo\n", " \n", " \n", - " 40\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " ezmag\n", - " HST z magnitude uncertainty\n", - " mag\n", - " stat.error;phot.mag;em.opt.SDDS.z\n", + " 57\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " fcor_60\n", + " Flux correction factor applied (times 1000),99...\n", + " \n", + " stat.param;phot.flux;em.IR.IRAS.60\n", " \n", - " real\n", + " int\n", " 4\n", - " 24\n", + " 10\n", " 0\n", - " 41\n", + " 58\n", " dbo\n", " \n", " \n", - " 41\n", - " SPITZER\n", - " SPITZER_goodsnIRS16micron\n", - " icls\n", - " Number of I-band sources within 1 arcsec\n", - " mag\n", - " meta.number;src;em.opt.I\n", + " 58\n", + " IRAS\n", + " IRAS_PhotoObj\n", + " fcor_100\n", + " Flux correction factor applied (times 1000),99...\n", " \n", - " tinyint\n", - " 1\n", - " 3\n", + " stat.param;phot.flux;em.IR.IRAS.100\n", + " \n", + " int\n", + " 4\n", + " 10\n", " 0\n", - " 42\n", + " 59\n", " dbo\n", " \n", " \n", "\n", - "

    42 rows × 13 columns

    \n", + "

    59 rows × 13 columns

    \n", "" ], "text/plain": [ - " catalog_name table_name column_name \\\n", - "0 SPITZER SPITZER_goodsnIRS16micron cx \n", - "1 SPITZER SPITZER_goodsnIRS16micron cy \n", - "2 SPITZER SPITZER_goodsnIRS16micron cz \n", - "3 SPITZER SPITZER_goodsnIRS16micron htmid \n", - "4 SPITZER SPITZER_goodsnIRS16micron zoneid \n", - ".. ... ... ... \n", - "37 SPITZER SPITZER_goodsnIRS16micron ebmag \n", - "38 SPITZER SPITZER_goodsnIRS16micron evmag \n", - "39 SPITZER SPITZER_goodsnIRS16micron eimag \n", - "40 SPITZER SPITZER_goodsnIRS16micron ezmag \n", - "41 SPITZER SPITZER_goodsnIRS16micron icls \n", + " catalog_name table_name column_name \\\n", + "0 IRAS IRAS_PhotoObj objID \n", + "1 IRAS IRAS_PhotoObj name \n", + "2 IRAS IRAS_PhotoObj ra \n", + "3 IRAS IRAS_PhotoObj dec \n", + "4 IRAS IRAS_PhotoObj cx \n", + ".. ... ... ... \n", + "54 IRAS IRAS_PhotoObj mhcon \n", + "55 IRAS IRAS_PhotoObj fcor_12 \n", + "56 IRAS IRAS_PhotoObj fcor_25 \n", + "57 IRAS IRAS_PhotoObj fcor_60 \n", + "58 IRAS IRAS_PhotoObj fcor_100 \n", "\n", - " description unit \\\n", - "0 Cartesian X (J2000) \n", - "1 Cartesian Y (J2000) \n", - "2 Cartesian Z (J2000) \n", - "3 HTM ID (J2000) \n", - "4 Zone ID (J2000) \n", - ".. ... ... \n", - "37 HST B magnitude uncertainty mag \n", - "38 HST V magnitude uncertainty mag \n", - "39 HST I magnitude uncertainty mag \n", - "40 HST z magnitude uncertainty mag \n", - "41 Number of I-band sources within 1 arcsec mag \n", + " description unit \\\n", + "0 unique object identifier \n", + "1 IRAS Source Name \n", + "2 J2000 right ascension deg \n", + "3 J2000 declination deg \n", + "4 Cartesian coordinate x \n", + ".. ... ... \n", + "54 Possible number of HCONs, 99 for NULL \n", + "55 Flux correction factor applied (times 1000),99... \n", + "56 Flux correction factor applied (times 1000),99... \n", + "57 Flux correction factor applied (times 1000),99... \n", + "58 Flux correction factor applied (times 1000),99... \n", "\n", - " ucd utype datatype size precision scale \\\n", - "0 pos.eq.x;pos.frame=j2000 float 8 53 0 \n", - "1 pos.eq.y;pos.frame=j2000 float 8 53 0 \n", - "2 pos.eq.z;pos.frame=j2000 float 8 53 0 \n", - "3 pos.eq.HTM; pos.frame=j2000 bigint 8 19 0 \n", - "4 pos.eq.zone;pos.frame=j2000 int 4 10 0 \n", - ".. ... ... ... ... ... ... \n", - "37 stat.error;phot.mag;em.opt.B real 4 24 0 \n", - "38 stat.error;phot.mag;em.opt.V real 4 24 0 \n", - "39 stat.error;phot.mag;em.opt.I real 4 24 0 \n", - "40 stat.error;phot.mag;em.opt.SDDS.z real 4 24 0 \n", - "41 meta.number;src;em.opt.I tinyint 1 3 0 \n", + " ucd utype datatype size precision \\\n", + "0 meta.id bigint 8 19 \n", + "1 meta.id varchar 11 0 \n", + "2 pos.eq.ra;pos.frame=j2000 float 8 53 \n", + "3 pos.eq.dec;pos.frame=j2000 float 8 53 \n", + "4 pos.eq.x;pos.frame=j2000 float 8 53 \n", + ".. ... ... ... ... ... \n", + "54 meta.number smallint 2 5 \n", + "55 stat.param;phot.flux;em.IR.IRAS.12 int 4 10 \n", + "56 stat.param;phot.flux;em.IR.IRAS.25 int 4 10 \n", + "57 stat.param;phot.flux;em.IR.IRAS.60 int 4 10 \n", + "58 stat.param;phot.flux;em.IR.IRAS.100 int 4 10 \n", "\n", - " column_index schema_name \n", - "0 1 dbo \n", - "1 2 dbo \n", - "2 3 dbo \n", - "3 4 dbo \n", - "4 5 dbo \n", - ".. ... ... \n", - "37 38 dbo \n", - "38 39 dbo \n", - "39 40 dbo \n", - "40 41 dbo \n", - "41 42 dbo \n", + " scale column_index schema_name \n", + "0 0 1 dbo \n", + "1 0 2 dbo \n", + "2 0 3 dbo \n", + "3 0 4 dbo \n", + "4 0 5 dbo \n", + ".. ... ... ... \n", + "54 0 55 dbo \n", + "55 0 56 dbo \n", + "56 0 57 dbo \n", + "57 0 58 dbo \n", + "58 0 59 dbo \n", "\n", - "[42 rows x 13 columns]" + "[59 rows x 13 columns]" ] }, "execution_count": 4, @@ -733,7 +707,7 @@ } ], "source": [ - "sql = \"SELECT * FROM Columns WHERE catalog_name = 'SPITZER' and table_name = 'SPITZER_goodsnIRS16micron' ORDER BY column_index\"\n", + "sql = \"SELECT * FROM Columns WHERE table_name = 'IRAS_PhotoObj' ORDER BY column_index\"\n", "cj.executeQuery(sql, context='xmatch')" ] }, @@ -752,11 +726,11 @@ "id": "fa28b430-8353-4ddc-99be-07f1f4288a8e", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:24.900326Z", - "iopub.status.busy": "2023-11-16T02:06:24.899949Z", - "iopub.status.idle": "2023-11-16T02:06:24.912028Z", - "shell.execute_reply": "2023-11-16T02:06:24.909680Z", - "shell.execute_reply.started": "2023-11-16T02:06:24.900279Z" + "iopub.execute_input": "2023-11-17T17:36:10.460832Z", + "iopub.status.busy": "2023-11-17T17:36:10.460374Z", + "iopub.status.idle": "2023-11-17T17:36:10.468721Z", + "shell.execute_reply": "2023-11-17T17:36:10.467134Z", + "shell.execute_reply.started": "2023-11-17T17:36:10.460783Z" }, "tags": [] }, @@ -778,7 +752,7 @@ "source": [ "### Cross-matching two small catalogs - Pandas DataFrame output\n", "\n", - "Here we cross-match 2 local tables (the `PhotoObjAll` tables in the `FUSE` and `FIRST` catalogs). \n", + "Here we cross-match 2 local infrared catalog tables: `IRAS_PhotoObj` and `AKARI_FIS`.\n", "\n", "We can retrieve the cross-match output table as a Pandas dataframe by using the syncronous `executeQuery` function in CasJobs, as this cross-match takes less time than its `1 minute timeout`. " ] @@ -789,11 +763,11 @@ "id": "7816e30a-0190-41bd-b38e-cde897a15faa", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:24.916295Z", - "iopub.status.busy": "2023-11-16T02:06:24.915580Z", - "iopub.status.idle": "2023-11-16T02:06:30.473359Z", - "shell.execute_reply": "2023-11-16T02:06:30.471394Z", - "shell.execute_reply.started": "2023-11-16T02:06:24.916216Z" + "iopub.execute_input": "2023-11-17T17:36:10.470812Z", + "iopub.status.busy": "2023-11-17T17:36:10.470393Z", + "iopub.status.idle": "2023-11-17T17:36:17.020050Z", + "shell.execute_reply": "2023-11-17T17:36:17.018116Z", + "shell.execute_reply.started": "2023-11-17T17:36:10.470766Z" }, "tags": [] }, @@ -802,9 +776,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "There are 84 matches.\n", - "CPU times: user 38.7 ms, sys: 4.59 ms, total: 43.3 ms\n", - "Wall time: 5.52 s\n" + "There are 97 matches.\n", + "CPU times: user 29.8 ms, sys: 2.27 ms, total: 32.1 ms\n", + "Wall time: 6.52 s\n" ] }, { @@ -836,33 +810,33 @@ " \n", " \n", " 0\n", - " 748\n", - " 420609\n", - " 21.133745\n", + " 125607\n", + " 3251473\n", + " 0.881630\n", " \n", " \n", " 1\n", - " 1061\n", - " 753859\n", - " 2.683405\n", + " 217378\n", + " 3393112\n", + " 0.566069\n", " \n", " \n", " 2\n", - " 39\n", - " 406505\n", - " 29.820647\n", + " 170663\n", + " 3122486\n", + " 0.681755\n", " \n", " \n", " 3\n", - " 39\n", - " 406467\n", - " 2.344853\n", + " 27009\n", + " 3096636\n", + " 0.563840\n", " \n", " \n", " 4\n", - " 39\n", - " 406486\n", - " 4.631321\n", + " 135385\n", + " 3268209\n", + " 0.771273\n", " \n", " \n", " ...\n", @@ -871,55 +845,55 @@ " ...\n", " \n", " \n", - " 79\n", - " 752\n", - " 671376\n", - " 0.425636\n", + " 92\n", + " 95744\n", + " 3074376\n", + " 0.352167\n", " \n", " \n", - " 80\n", - " 1259\n", - " 449080\n", - " 0.187065\n", + " 93\n", + " 121214\n", + " 3231044\n", + " 0.205401\n", " \n", " \n", - " 81\n", - " 85\n", - " 566564\n", - " 0.612966\n", + " 94\n", + " 33112\n", + " 3137825\n", + " 0.621349\n", " \n", " \n", - " 82\n", - " 82\n", - " 329941\n", - " 1.084348\n", + " 95\n", + " 92458\n", + " 3207524\n", + " 0.741632\n", " \n", " \n", - " 83\n", - " 31\n", - " 463142\n", - " 1.271296\n", + " 96\n", + " 108160\n", + " 3231002\n", + " 0.428036\n", " \n", " \n", "\n", - "

    84 rows × 3 columns

    \n", + "

    97 rows × 3 columns

    \n", "" ], "text/plain": [ - " id1 id2 sep\n", - "0 748 420609 21.133745\n", - "1 1061 753859 2.683405\n", - "2 39 406505 29.820647\n", - "3 39 406467 2.344853\n", - "4 39 406486 4.631321\n", - ".. ... ... ...\n", - "79 752 671376 0.425636\n", - "80 1259 449080 0.187065\n", - "81 85 566564 0.612966\n", - "82 82 329941 1.084348\n", - "83 31 463142 1.271296\n", + " id1 id2 sep\n", + "0 125607 3251473 0.881630\n", + "1 217378 3393112 0.566069\n", + "2 170663 3122486 0.681755\n", + "3 27009 3096636 0.563840\n", + "4 135385 3268209 0.771273\n", + ".. ... ... ...\n", + "92 95744 3074376 0.352167\n", + "93 121214 3231044 0.205401\n", + "94 33112 3137825 0.621349\n", + "95 92458 3207524 0.741632\n", + "96 108160 3231002 0.428036\n", "\n", - "[84 rows x 3 columns]" + "[97 rows x 3 columns]" ] }, "execution_count": 6, @@ -931,10 +905,10 @@ "%%time\n", "\n", "sql = f\"\"\"\n", - "EXECUTE SQLxMatch @table1='FUSE_PhotoObjAll', @table2='FIRST_PhotoObjAll', @radius={radius}\n", + "EXECUTE SQLxMatch @table1='IRAS_PhotoObj', @table2='AKARI_FIS', @radius=1\n", "\n", - "-- Note that we could be more explicit by specifying the columns names, but that was not needed:\n", - "--EXECUTE SQLxMatch @table1='FUSE_PhotoObjAll', @id_col1='objid', @ra_col1='ra', @dec_col1='dec', @table2='FIRST_PhotoObjAll', @radius={radius}\n", + "-- Note that we could be more explicit by specifying the columns names, but that was not needed.\n", + "--EXECUTE SQLxMatch @table1='IRAS_PhotoObj', @id_col1='objid', @ra_col1='ra', @dec_col1='dec', @table2='AKARI_FIS', @radius={radius}\n", "\"\"\"\n", "\n", "df = cj.executeQuery(sql, context=\"xmatch\")\n", @@ -949,9 +923,12 @@ "source": [ "### Cross-matching two filtered catalogs - Pandas DataFrame output\n", "\n", - "Here we use 2 local tables (the `PhotoObjAll` tables in the `BestDR17` and `GalexDR3` catalogs), which we filter by `ra` and `dec` in rectangular regions and store in local temporary tables. These temporary tables are then passed as input to the `sp_xmatch` procedure.\n", + "Here we use 2 local tables (the `PhotoObjAll` tables in the `BestDR17` and `GalexDR6` catalogs), which we filter by `ra` and `dec` in rectangular regions and store in local temporary tables. These temporary tables are then passed as input to the `SQLxMatch` procedure.\n", + "\n", + "Note that we can also store the (`cx`, `cy`, `cz`) columns in the temporary tables in order to avaoid having the code to compute them later internally on the fly.\n", "\n", - "Note that we can also store the (`cx`, `cy`, `cz`) columns in the temporary tables in order to avaoid having the code to compute them later internally on the fly." + "Also, note that time we also give `@include_radec`=1, `@include_sep_rank`=1, `@sort_by_separation`=1 as input parameters. \n", + "\n" ] }, { @@ -960,11 +937,11 @@ "id": "6db61e21-b49a-4ea1-83e7-18f372d7acb8", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:30.476586Z", - "iopub.status.busy": "2023-11-16T02:06:30.476001Z", - "iopub.status.idle": "2023-11-16T02:06:34.889511Z", - "shell.execute_reply": "2023-11-16T02:06:34.887782Z", - "shell.execute_reply.started": "2023-11-16T02:06:30.476527Z" + "iopub.execute_input": "2023-11-17T17:36:17.022509Z", + "iopub.status.busy": "2023-11-17T17:36:17.022058Z", + "iopub.status.idle": "2023-11-17T17:36:25.781444Z", + "shell.execute_reply": "2023-11-17T17:36:25.779510Z", + "shell.execute_reply.started": "2023-11-17T17:36:17.022418Z" }, "tags": [] }, @@ -974,8 +951,8 @@ "output_type": "stream", "text": [ "There are 14482 matches.\n", - "CPU times: user 118 ms, sys: 24.5 ms, total: 142 ms\n", - "Wall time: 4.39 s\n" + "CPU times: user 145 ms, sys: 57.4 ms, total: 202 ms\n", + "Wall time: 8.72 s\n" ] }, { @@ -1002,95 +979,168 @@ " id1\n", " id2\n", " sep\n", + " sep_rank\n", + " ra1\n", + " dec1\n", + " ra2\n", + " dec2\n", " \n", " \n", " \n", " \n", " 0\n", - " 1237667430101877625\n", - " 6387874658268480342\n", - " 24.713197\n", + " 1237667322710261954\n", + " 6387874658268479704\n", + " 1.798951\n", + " 1\n", + " 160.008407\n", + " 25.004219\n", + " 160.008487\n", + " 25.003724\n", " \n", " \n", " 1\n", - " 1237667430101877625\n", - " 6387874658268480335\n", - " 24.360351\n", + " 1237667322710261954\n", + " 6387874659342225269\n", + " 12.009139\n", + " 2\n", + " 160.008407\n", + " 25.004219\n", + " 160.011864\n", + " 25.003073\n", " \n", " \n", " 2\n", - " 1237667430101877799\n", - " 6387874658268480335\n", - " 21.183498\n", + " 1237667322710261955\n", + " 6387874658268479704\n", + " 1.792355\n", + " 1\n", + " 160.008402\n", + " 25.004216\n", + " 160.008487\n", + " 25.003724\n", " \n", " \n", " 3\n", - " 1237667430101877631\n", - " 6387874658268480335\n", - " 23.689764\n", + " 1237667322710261955\n", + " 6387874659342225269\n", + " 12.022925\n", + " 2\n", + " 160.008402\n", + " 25.004216\n", + " 160.011864\n", + " 25.003073\n", " \n", " \n", " 4\n", - " 1237667430101877644\n", - " 6387874658268480316\n", - " 28.674751\n", + " 1237667322710261983\n", + " 6387874658268479748\n", + " 23.739031\n", + " 1\n", + " 160.027793\n", + " 25.022443\n", + " 160.034873\n", + " 25.020916\n", " \n", " \n", " ...\n", " ...\n", " ...\n", " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", " \n", " \n", " 14477\n", - " 1237667323247329512\n", - " 6387874658268481830\n", - " 27.112988\n", + " 1237667430638814217\n", + " 6387874658270578860\n", + " 27.791207\n", + " 1\n", + " 160.041967\n", + " 25.474943\n", + " 160.033920\n", + " 25.477554\n", " \n", " \n", " 14478\n", - " 1237667323247329953\n", - " 6387874658268481830\n", - " 13.102262\n", + " 1237667430638814217\n", + " 6387874658269533411\n", + " 29.379765\n", + " 2\n", + " 160.041967\n", + " 25.474943\n", + " 160.042697\n", + " 25.466809\n", " \n", " \n", " 14479\n", - " 1237667323247263986\n", - " 6387874658268481805\n", - " 16.845794\n", + " 1237667430638814224\n", + " 6387874658268481825\n", + " 23.994318\n", + " 1\n", + " 160.036649\n", + " 25.495353\n", + " 160.029969\n", + " 25.498194\n", " \n", " \n", " 14480\n", - " 1237667430638878978\n", - " 6387874658268481805\n", - " 16.813032\n", + " 1237667430638814224\n", + " 6387874658270578846\n", + " 26.040727\n", + " 2\n", + " 160.036649\n", + " 25.495353\n", + " 160.037995\n", + " 25.488222\n", " \n", " \n", " 14481\n", - " 1237667323247264396\n", - " 6387874658268481811\n", - " 11.216161\n", + " 1237667430638878978\n", + " 6387874658268481805\n", + " 16.813032\n", + " 1\n", + " 160.173026\n", + " 25.499917\n", + " 160.173691\n", + " 25.495285\n", " \n", " \n", "\n", - "

    14482 rows × 3 columns

    \n", + "

    14482 rows × 8 columns

    \n", "" ], "text/plain": [ - " id1 id2 sep\n", - "0 1237667430101877625 6387874658268480342 24.713197\n", - "1 1237667430101877625 6387874658268480335 24.360351\n", - "2 1237667430101877799 6387874658268480335 21.183498\n", - "3 1237667430101877631 6387874658268480335 23.689764\n", - "4 1237667430101877644 6387874658268480316 28.674751\n", - "... ... ... ...\n", - "14477 1237667323247329512 6387874658268481830 27.112988\n", - "14478 1237667323247329953 6387874658268481830 13.102262\n", - "14479 1237667323247263986 6387874658268481805 16.845794\n", - "14480 1237667430638878978 6387874658268481805 16.813032\n", - "14481 1237667323247264396 6387874658268481811 11.216161\n", + " id1 id2 sep sep_rank \\\n", + "0 1237667322710261954 6387874658268479704 1.798951 1 \n", + "1 1237667322710261954 6387874659342225269 12.009139 2 \n", + "2 1237667322710261955 6387874658268479704 1.792355 1 \n", + "3 1237667322710261955 6387874659342225269 12.022925 2 \n", + "4 1237667322710261983 6387874658268479748 23.739031 1 \n", + "... ... ... ... ... \n", + "14477 1237667430638814217 6387874658270578860 27.791207 1 \n", + "14478 1237667430638814217 6387874658269533411 29.379765 2 \n", + "14479 1237667430638814224 6387874658268481825 23.994318 1 \n", + "14480 1237667430638814224 6387874658270578846 26.040727 2 \n", + "14481 1237667430638878978 6387874658268481805 16.813032 1 \n", "\n", - "[14482 rows x 3 columns]" + " ra1 dec1 ra2 dec2 \n", + "0 160.008407 25.004219 160.008487 25.003724 \n", + "1 160.008407 25.004219 160.011864 25.003073 \n", + "2 160.008402 25.004216 160.008487 25.003724 \n", + "3 160.008402 25.004216 160.011864 25.003073 \n", + "4 160.027793 25.022443 160.034873 25.020916 \n", + "... ... ... ... ... \n", + "14477 160.041967 25.474943 160.033920 25.477554 \n", + "14478 160.041967 25.474943 160.042697 25.466809 \n", + "14479 160.036649 25.495353 160.029969 25.498194 \n", + "14480 160.036649 25.495353 160.037995 25.488222 \n", + "14481 160.173026 25.499917 160.173691 25.495285 \n", + "\n", + "[14482 rows x 8 columns]" ] }, "execution_count": 7, @@ -1109,7 +1159,7 @@ "SELECT objid, ra, dec, cx, cy, cz INTO #temp2 \n", "FROM GalexGR6_PhotoObjAll WHERE ra BETWEEN {ra1} AND {ra2} AND dec BETWEEN {dec1} AND {dec2}\n", "\n", - "EXECUTE SQLxMatch @table1='#temp1', @table2='#temp2', @radius={radius}\n", + "EXECUTE SQLxMatch @table1='#temp1', @table2='#temp2', @radius={radius}, @include_radec=1, @include_sep_rank=1, @sort_by_separation=1\n", "\"\"\"\n", "\n", "df = cj.executeQuery(sql, context=\"xmatch\")\n", @@ -1143,11 +1193,11 @@ "id": "85ee81f3-4a05-4db2-a191-26e3b16bf925", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:34.892065Z", - "iopub.status.busy": "2023-11-16T02:06:34.891534Z", - "iopub.status.idle": "2023-11-16T02:06:36.517757Z", - "shell.execute_reply": "2023-11-16T02:06:36.515931Z", - "shell.execute_reply.started": "2023-11-16T02:06:34.892017Z" + "iopub.execute_input": "2023-11-17T17:36:25.784040Z", + "iopub.status.busy": "2023-11-17T17:36:25.783641Z", + "iopub.status.idle": "2023-11-17T17:36:27.475685Z", + "shell.execute_reply": "2023-11-17T17:36:27.473606Z", + "shell.execute_reply.started": "2023-11-17T17:36:25.783989Z" }, "tags": [] }, @@ -1158,7 +1208,7 @@ "sql = f\"\"\"\n", "IF EXISTS (select * from sys.objects WHERE object_id = OBJECT_ID(N'{mydb_output_table_name}') AND TYPE = 'U')\n", "DROP TABLE {mydb_output_table_name} \n", - "CREATE TABLE {mydb_output_table_name}(id1 bigint, id2 bigint, sep float, ra1 float, dec1 float, ra2 float, dec2 float)\n", + "CREATE TABLE {mydb_output_table_name}(id1 BIGINT, id2 BIGINT, sep FLOAT, sep_rank INTEGER, ra1 BIGINT, dec1 FLOAT, ra2 FLOAT, dec2 FLOAT)\n", "\"\"\"\n", "df = cj.executeQuery(sql, context=\"mydb\")" ] @@ -1179,11 +1229,11 @@ "id": "7597e6c7-5293-49cd-be7c-1b4875f14ea7", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:36.520397Z", - "iopub.status.busy": "2023-11-16T02:06:36.520003Z", - "iopub.status.idle": "2023-11-16T02:06:53.297130Z", - "shell.execute_reply": "2023-11-16T02:06:53.295273Z", - "shell.execute_reply.started": "2023-11-16T02:06:36.520349Z" + "iopub.execute_input": "2023-11-17T17:36:27.478255Z", + "iopub.status.busy": "2023-11-17T17:36:27.477869Z", + "iopub.status.idle": "2023-11-17T17:36:44.297308Z", + "shell.execute_reply": "2023-11-17T17:36:44.294953Z", + "shell.execute_reply.started": "2023-11-17T17:36:27.478205Z" }, "tags": [] }, @@ -1192,7 +1242,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 145 ms, sys: 22.9 ms, total: 168 ms\n", + "Job Message: Query Complete\n", + "CPU times: user 136 ms, sys: 19.8 ms, total: 156 ms\n", "Wall time: 16.8 s\n" ] } @@ -1208,10 +1259,10 @@ "FROM GALEXGR6_PhotoObjAll WHERE ra BETWEEN {ra1} AND {ra2} AND dec BETWEEN {dec1} AND {dec2}\n", "\n", "-- Creating temporary output table\n", - "CREATE TABLE #out(id1 bigint, id2 bigint, sep float, ra1 float, dec1 float, ra2 float, dec2 float)\n", + "CREATE TABLE #out(id1 BIGINT, id2 BIGINT, sep FLOAT, sep_rank INTEGER, ra1 BIGINT, dec1 FLOAT, ra2 FLOAT, dec2 FLOAT)\n", "\n", "-- Executing cross-match that fills output table.\n", - "EXECUTE SQLxMatch @table1='#temp1', @table2='#temp2', @radius={radius}, @output_table='#out', @radec_in_output=1\n", + "EXECUTE SQLxMatch @table1='#temp1', @table2='#temp2', @radius={radius}, @output_table='#out', @include_radec=1, @include_sep_rank=1\n", "\n", "-- Filling up table in MyDB:\n", "INSERT INTO mydb.{mydb_output_table_name}\n", @@ -1221,7 +1272,8 @@ "job_id = cj.submitJob(sql, context=\"xmatch\")\n", "\n", "# this line will make the code wait until the job is done, if desired:\n", - "job_description = cj.waitForJob(job_id)" + "job_description = cj.waitForJob(job_id)\n", + "print(f\"Job Message: {job_description.get('Message')}\")" ] }, { @@ -1239,11 +1291,11 @@ "id": "2843127a-f8fd-4596-b514-973a85b43e42", "metadata": { "execution": { - "iopub.execute_input": "2023-11-16T02:06:53.299629Z", - "iopub.status.busy": "2023-11-16T02:06:53.299209Z", - "iopub.status.idle": "2023-11-16T02:06:55.879003Z", - "shell.execute_reply": "2023-11-16T02:06:55.877149Z", - "shell.execute_reply.started": "2023-11-16T02:06:53.299581Z" + "iopub.execute_input": "2023-11-17T17:36:44.300412Z", + "iopub.status.busy": "2023-11-17T17:36:44.299961Z", + "iopub.status.idle": "2023-11-17T17:36:46.684560Z", + "shell.execute_reply": "2023-11-17T17:36:46.682578Z", + "shell.execute_reply.started": "2023-11-17T17:36:44.300337Z" }, "tags": [] }, @@ -1279,6 +1331,7 @@ " id1\n", " id2\n", " sep\n", + " sep_rank\n", " ra1\n", " dec1\n", " ra2\n", @@ -1288,53 +1341,58 @@ " \n", " \n", " 0\n", - " 1237667430102008414\n", - " 6387874666859466785\n", - " 160.348991\n", - " 25.105292\n", - " 160.347935\n", - " 25.105733\n", - " 3.789954\n", + " 1237667322710261954\n", + " 6387874658268479704\n", + " 1.798951\n", + " 1\n", + " 160\n", + " 25.004219\n", + " 160.008487\n", + " 25.003724\n", " \n", " \n", " 1\n", - " 1237667430102008414\n", - " 6387874666858417103\n", - " 160.348991\n", - " 25.105292\n", - " 160.352474\n", - " 25.105610\n", - " 11.411807\n", + " 1237667322710261954\n", + " 6387874659342225269\n", + " 12.009139\n", + " 2\n", + " 160\n", + " 25.004219\n", + " 160.011864\n", + " 25.003073\n", " \n", " \n", " 2\n", - " 1237667430102008414\n", - " 6387874658269533031\n", - " 160.348991\n", - " 25.105292\n", - " 160.352479\n", - " 25.105861\n", - " 11.553421\n", + " 1237667322710261955\n", + " 6387874658268479704\n", + " 1.792355\n", + " 1\n", + " 160\n", + " 25.004216\n", + " 160.008487\n", + " 25.003724\n", " \n", " \n", " 3\n", - " 1237667430102008414\n", - " 2486974493255277081\n", - " 160.348991\n", - " 25.105292\n", - " 160.352513\n", - " 25.105603\n", - " 11.535380\n", + " 1237667322710261955\n", + " 6387874659342225269\n", + " 12.022925\n", + " 2\n", + " 160\n", + " 25.004216\n", + " 160.011864\n", + " 25.003073\n", " \n", " \n", " 4\n", - " 1237667430102008414\n", - " 6387874658268480014\n", - " 160.348991\n", - " 25.105292\n", - " 160.353779\n", - " 25.104910\n", - " 15.667722\n", + " 1237667322710261983\n", + " 6387874658268479748\n", + " 23.739031\n", + " 1\n", + " 160\n", + " 25.022443\n", + " 160.034873\n", + " 25.020916\n", " \n", " \n", " ...\n", @@ -1345,90 +1403,96 @@ " ...\n", " ...\n", " ...\n", + " ...\n", " \n", " \n", " 14477\n", - " 1237667323247329512\n", - " 6387874658268481830\n", - " 160.424081\n", - " 25.492595\n", - " 160.426413\n", - " 25.499826\n", - " 27.112988\n", + " 1237667430638814217\n", + " 6387874658270578860\n", + " 27.791207\n", + " 1\n", + " 160\n", + " 25.474943\n", + " 160.033920\n", + " 25.477554\n", " \n", " \n", " 14478\n", - " 1237667323247329953\n", - " 6387874658268481830\n", - " 160.425375\n", - " 25.496309\n", - " 160.426413\n", - " 25.499826\n", - " 13.102262\n", + " 1237667430638814217\n", + " 6387874658269533411\n", + " 29.379765\n", + " 2\n", + " 160\n", + " 25.474943\n", + " 160.042697\n", + " 25.466809\n", " \n", " \n", " 14479\n", - " 1237667323247263986\n", - " 6387874658268481805\n", - " 160.172991\n", - " 25.499922\n", - " 160.173691\n", - " 25.495285\n", - " 16.845794\n", + " 1237667430638814224\n", + " 6387874658268481825\n", + " 23.994318\n", + " 1\n", + " 160\n", + " 25.495353\n", + " 160.029969\n", + " 25.498194\n", " \n", " \n", " 14480\n", + " 1237667430638814224\n", + " 6387874658270578846\n", + " 26.040727\n", + " 2\n", + " 160\n", + " 25.495353\n", + " 160.037995\n", + " 25.488222\n", + " \n", + " \n", + " 14481\n", " 1237667430638878978\n", " 6387874658268481805\n", - " 160.173026\n", + " 16.813032\n", + " 1\n", + " 160\n", " 25.499917\n", " 160.173691\n", " 25.495285\n", - " 16.813032\n", - " \n", - " \n", - " 14481\n", - " 1237667323247264396\n", - " 6387874658268481811\n", - " 160.245373\n", - " 25.499857\n", - " 160.246752\n", - " 25.497001\n", - " 11.216161\n", " \n", " \n", "\n", - "

    14482 rows × 7 columns

    \n", + "

    14482 rows × 8 columns

    \n", "" ], "text/plain": [ - " id1 id2 sep ra1 \\\n", - "0 1237667430102008414 6387874666859466785 160.348991 25.105292 \n", - "1 1237667430102008414 6387874666858417103 160.348991 25.105292 \n", - "2 1237667430102008414 6387874658269533031 160.348991 25.105292 \n", - "3 1237667430102008414 2486974493255277081 160.348991 25.105292 \n", - "4 1237667430102008414 6387874658268480014 160.348991 25.105292 \n", - "... ... ... ... ... \n", - "14477 1237667323247329512 6387874658268481830 160.424081 25.492595 \n", - "14478 1237667323247329953 6387874658268481830 160.425375 25.496309 \n", - "14479 1237667323247263986 6387874658268481805 160.172991 25.499922 \n", - "14480 1237667430638878978 6387874658268481805 160.173026 25.499917 \n", - "14481 1237667323247264396 6387874658268481811 160.245373 25.499857 \n", + " id1 id2 sep sep_rank ra1 \\\n", + "0 1237667322710261954 6387874658268479704 1.798951 1 160 \n", + "1 1237667322710261954 6387874659342225269 12.009139 2 160 \n", + "2 1237667322710261955 6387874658268479704 1.792355 1 160 \n", + "3 1237667322710261955 6387874659342225269 12.022925 2 160 \n", + "4 1237667322710261983 6387874658268479748 23.739031 1 160 \n", + "... ... ... ... ... ... \n", + "14477 1237667430638814217 6387874658270578860 27.791207 1 160 \n", + "14478 1237667430638814217 6387874658269533411 29.379765 2 160 \n", + "14479 1237667430638814224 6387874658268481825 23.994318 1 160 \n", + "14480 1237667430638814224 6387874658270578846 26.040727 2 160 \n", + "14481 1237667430638878978 6387874658268481805 16.813032 1 160 \n", "\n", - " dec1 ra2 dec2 \n", - "0 160.347935 25.105733 3.789954 \n", - "1 160.352474 25.105610 11.411807 \n", - "2 160.352479 25.105861 11.553421 \n", - "3 160.352513 25.105603 11.535380 \n", - "4 160.353779 25.104910 15.667722 \n", - "... ... ... ... \n", - "14477 160.426413 25.499826 27.112988 \n", - "14478 160.426413 25.499826 13.102262 \n", - "14479 160.173691 25.495285 16.845794 \n", - "14480 160.173691 25.495285 16.813032 \n", - "14481 160.246752 25.497001 11.216161 \n", + " dec1 ra2 dec2 \n", + "0 25.004219 160.008487 25.003724 \n", + "1 25.004219 160.011864 25.003073 \n", + "2 25.004216 160.008487 25.003724 \n", + "3 25.004216 160.011864 25.003073 \n", + "4 25.022443 160.034873 25.020916 \n", + "... ... ... ... \n", + "14477 25.474943 160.033920 25.477554 \n", + "14478 25.474943 160.042697 25.466809 \n", + "14479 25.495353 160.029969 25.498194 \n", + "14480 25.495353 160.037995 25.488222 \n", + "14481 25.499917 160.173691 25.495285 \n", "\n", - "[14482 rows x 7 columns]" + "[14482 rows x 8 columns]" ] }, "execution_count": 10, @@ -1438,7 +1502,7 @@ ], "source": [ "sql = f\"\"\"\n", - "SELECT * FROM mydb.{mydb_output_table_name}\n", + "SELECT * FROM mydb.{mydb_output_table_name} ORDER BY id1, sep\n", "\"\"\"\n", "df = cj.executeQuery(sql, context=\"mydb\")\n", "print(f\"There are {df.shape[0]} matches.\")\n", diff --git a/docs/README.md b/docs/README.md index 9fc8597..339262c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -106,7 +106,8 @@ Taghizadeh-Popp, M. and Dobos, L. (2023) “SQLxMatch: In-Database Spatial Cross
  • @output_table NVARCHAR(128) or SYSNAME: If NOT NULL, this procedure will insert the output results into the table @output_table (of format 'server.database.schema.table', 'database.schema.table', 'database.table', or simply 'table'), which must already exist and be visible within the scope of the procedure. If set to null, the output results will be simply returned as a table resultset. Takes a default value of NULL.
  • @only_nearest BIT: If set to 0 (default value), then all matches within a distance @radius to an object are returned. If set to 1, only the closest match to an object is returned.
  • @sort_by_separation BIT: If set to 1, then the output will be sorted by the 'id1' and 'sep' columns. If set to 0 (default value), no particular ordering is applied. -
  • @radec_in_output BIT : If set to 1, then the output table will contain as well the (RA, Dec) values of each object. +
  • @include_sep_rank BIT: If set to 1, then the output table will include an extra column named 'sep_rank', denoting the rank of all matches to an object when sorted by angular separation. If set to 0 (default value), this column is not included. +
  • @include_radec BIT : If set to 1, then the output table will contain the original (RA, Dec) columns from both input tables, as ra1, dec1, ra2, and dec2.
  • @print_messages BIT : If set to 1, then time-stamped messages will be printed as the different sections in this procedure are completed. @@ -115,7 +116,14 @@ Taghizadeh-Popp, M. and Dobos, L. (2023) “SQLxMatch: In-Database Spatial Cross
  • TABLE (id1, id2, sep), where id1 and id2 are the unique object identifier columns in @table1 and @table2, respectively, and sep (FLOAT) is the angular separation between objects in arseconds. or -
  • TABLE (id1, id2, sep, ra1, dec1, ra2, dec2) when @radec_in_output=1, where ra1, dec1, ra2 and dec2 (all FLOAT) are the coordinates of the objects in @table1 and @table2, respectively. +
  • TABLE (id1, id2, sep, ra1, dec1, ra2, dec2) when @include_radec=1 + + or +
  • TABLE (id1, id2, sep, sep_rank) when @include_sep_rank=1 + + + or +
  • TABLE (id1, id2, sep, sep_rank, ra1, dec1, ra2, dec2) when @include_radec=1 and @include_sep_rank = 1. diff --git a/sql/sqlserver/catalog/CreateCatalogTableViews.sql b/sql/sqlserver/catalog/CreateCatalogTableViews.sql index 6ce368f..cd00614 100644 --- a/sql/sqlserver/catalog/CreateCatalogTableViews.sql +++ b/sql/sqlserver/catalog/CreateCatalogTableViews.sql @@ -64,15 +64,15 @@ AS BEGIN USE [?]; SELECT ''create view '' + QUOTENAME( REPLACE(''?'', ''SkyNode_'', '''') + ''_'' + name) + - '' AS SELECT * FROM '' + QUOTENAME(''?'') + ''.dbo.'' + QUOTENAME(name) + '' '' AS cmd + '' AS SELECT * FROM '' + QUOTENAME(''?'') + ''.'' + schema_name + ''.'' + QUOTENAME(name) + '' '' AS cmd FROM ( - SELECT name - FROM sys.tables + SELECT name, SCHEMA_NAME(schema_id) as schema_name + FROM sys.tables UNION ALL - SELECT name + SELECT name, SCHEMA_NAME(schema_id) as schema_name FROM sys.views ) AS q - WHERE ''?'' like ''skynode_%'' + WHERE ''?'' like ''skynode_%'' AND ''?'' != ''SkyNode_g'' AND ''?'' NOT LIKE ''%_STAT'' AND ''?'' NOT LIKE ''SkyNode_Test%'' ORDER BY cmd ' INSERT INTO #AllTables @@ -81,14 +81,14 @@ AS BEGIN ----------------------------------------------------------------------------------------------------- -- Create now the views one-by-one with a cursor - DECLARE cur2 Cursor For SELECT * FROM #AllTables + DECLARE cur2 Cursor For SELECT * FROM #AllTables ORDER BY cmd OPEN cur2 FETCH NEXT FROM cur2 INTO @sql WHILE @@FETCH_status = 0 BEGIN BEGIN TRY - PRINT(@sql) - EXECUTE @sql + --PRINT(@sql) + EXECUTE(@sql) END TRY BEGIN CATCH PRINT 'Error: ' + @sql @@ -105,6 +105,6 @@ GO -------------------------------------------------------------------------------------------------------------------------------------------- -- Create all views to all skynode tables (and views) ---EXECUTE spDropAllViews +EXECUTE spDropAllViews -EXECUTE spCreateSkynodeViews \ No newline at end of file +EXECUTE spCreateSkynodeViews diff --git a/sql/sqlserver/cross-match/Install_SQLxMatch.sql b/sql/sqlserver/cross-match/Install_SQLxMatch.sql index bf41b09..595982e 100644 --- a/sql/sqlserver/cross-match/Install_SQLxMatch.sql +++ b/sql/sqlserver/cross-match/Install_SQLxMatch.sql @@ -38,12 +38,12 @@ CREATE FUNCTION fAlpha(@theta FLOAT, @decMin FLOAT, @decMax FLOAT, @zoneHeight F --/H Returns the value of alpha, which is a modified search radius along the RA direction. --/U ------------------------------------------------------------ --/T Parameters:
    ---/T
  • @theta float: value of theta (input search radius) in degrees. ---/T
  • @@decMin float: value of minimum declination in degrees. ---/T
  • @decMax float: value of maximum declination in degrees. ---/T
  • @zoneHeight float: value of the zone height in degrees. ---/T
  • returns alpha float: value of alpha. -RETURNS float +--/T
  • @theta FLOAT: value of theta (input search radius) in degrees. +--/T
  • @@decMin FLOAT: value of minimum declination in degrees. +--/T
  • @decMax FLOAT: value of maximum declination in degrees. +--/T
  • @zoneHeight FLOAT: value of the zone height in degrees. +--/T
  • returns alpha FLOAT: value of alpha. +RETURNS FLOAT AS BEGIN DECLARE @dec FLOAT @@ -77,7 +77,7 @@ CREATE PROCEDURE spPrintMessage @initial_datetime DATETIME, @message NVARCHAR(ma --/H Prints a text message together with the current time and the time elapsed since an initial time. --/U ------------------------------------------------------------ --/T Parameters:
    ---/T
  • @initial_datetime DATETIME: initial time from which to calculate the time elapsed. +--/T
  • @initial_datetime DATETIME: initial time FROM which to calculate the time elapsed. --/T
  • @message NVARCHAR(max): text to be printed. AS BEGIN DECLARE @current_datetime DATETIME = SYSDATETIME() @@ -192,7 +192,7 @@ AS BEGIN SET @temp_table_name = 'tempdb.' + QUOTENAME(@schema_name) + '.' + QUOTENAME(@table_name) - DECLARE @sql nvarchar(max) = N' + DECLARE @sql NVARCHAR(max) = N' SELECT @numrows = count(*) FROM ( SELECT c.name as col_name FROM ' + @server_name + N'.' + @db_name + N'.sys.tables t @@ -223,7 +223,7 @@ AS BEGIN --PRINT @sql DECLARE @numrows BIGINT - EXECUTE sp_executesql @sql, N'@table_name SYSNAME, @schema_name SYSNAME, @temp_table_name SYSNAME, @column SYSNAME, @numrows bigint OUTPUT', + EXECUTE sp_executesql @sql, N'@table_name SYSNAME, @schema_name SYSNAME, @temp_table_name SYSNAME, @column SYSNAME, @numrows BIGINT OUTPUT', @table_name=@table_name, @schema_name=@schema_name, @temp_table_name=@temp_table_name, @column=@column, @numrows=@numrows OUTPUT IF @numrows > 0 @@ -278,7 +278,7 @@ AS BEGIN SET @temp_table_name = 'tempdb.' + QUOTENAME(@schema_name) + '.' + QUOTENAME(@table_name) - DECLARE @sql nvarchar(max) = N' + DECLARE @sql NVARCHAR(max) = N' SELECT @col_type=col_type, @col_length=col_length, @col_precision=col_precision, @col_scale=col_scale FROM ( SELECT y.name as col_type, c.max_length as col_length, c.precision as col_precision, c.scale as col_scale FROM ' + @server_name + N'.' + @db_name + N'.sys.tables t @@ -313,7 +313,7 @@ AS BEGIN --PRINT @sql DECLARE @numrows BIGINT - EXECUTE sp_executesql @sql, N'@table_name SYSNAME, @schema_name SYSNAME, @temp_table_name SYSNAME, @column SYSNAME, @col_type nvarchar(max) OUTPUT, @col_length BIGINT OUTPUT, @col_precision BIGINT OUTPUT, @col_scale BIGINT OUTPUT', + EXECUTE sp_executesql @sql, N'@table_name SYSNAME, @schema_name SYSNAME, @temp_table_name SYSNAME, @column SYSNAME, @col_type NVARCHAR(max) OUTPUT, @col_length BIGINT OUTPUT, @col_precision BIGINT OUTPUT, @col_scale BIGINT OUTPUT', @table_name=@table_name, @schema_name=@schema_name, @temp_table_name=@temp_table_name, @column=@column, @col_type=@col_type OUTPUT, @col_length=@col_length OUTPUT, @col_precision=@col_precision OUTPUT, @col_scale=@col_scale OUTPUT END GO @@ -333,7 +333,7 @@ CREATE PROCEDURE spGetQuotedPath(@table SYSNAME, @quoted_path SYSNAME OUTPUT) --/U ------------------------------------------------------------ --/T Parameters:
    --/T
  • @table SYSNAME: input table. Can be any of these formats: 'server.database.schema.table', 'database.schema.table', 'database.table', or simply 'table'. ---/T
  • @quoted_path float: output path to the input table, with quotes. +--/T
  • @quoted_path FLOAT: output path to the input table, with quotes. AS BEGIN DECLARE @server_name SYSNAME, @db_name SYSNAME, @@ -444,7 +444,7 @@ IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'spBuildCatalo DROP PROCEDURE spBuildCatalog GO -CREATE PROCEDURE spBuildCatalog(@target_table SYSNAME, @reference_table SYSNAME, @zoneHeight float = 30, @id_col SYSNAME = 'objid', @ra_col SYSNAME = 'ra', @dec_col SYSNAME = 'dec', @max_catalog_rows BIGINT = null) +CREATE PROCEDURE spBuildCatalog(@target_table SYSNAME, @reference_table SYSNAME, @zoneHeight FLOAT = 30, @id_col SYSNAME = 'objid', @ra_col SYSNAME = 'ra', @dec_col SYSNAME = 'dec', @max_catalog_rows BIGINT = null) ---------------------------------------------------------------- --/H Populates the @target_table table with a modified version of the inpute @reference_table table, in a format ready to be used by the SQLxMatch procedure. --/T The input @reference_table is expected to have at least a unique identifier column, as well as an RA (Right Ascension) and Dec (Declination) columns. @@ -454,11 +454,11 @@ CREATE PROCEDURE spBuildCatalog(@target_table SYSNAME, @reference_table SYSNAME, --/T Parameters:
    --/T
  • @target_table SYSNAME: target table to be populated. Can be any of these formats: 'server.database.schema.table', 'database.schema.table', or 'database.table', or simply 'table'. --/T
  • @reference_table SYSNAME: reference table. Can be any of these formats: 'server.database.schema.table', 'database.schema.table', 'database.table', or simply 'table'. ---/T
  • @zoneHeight float: If @table does not contain a column named 'zoneid', then it will calculate this column using zones of this height (in units of arcsec). Takes a default value of 30 arsec. +--/T
  • @zoneHeight FLOAT: If @table does not contain a column named 'zoneid', then it will calculate this column using zones of this height (in units of arcsec). Takes a default value of 30 arsec. --/T
  • @id_col SYSNAME: name of column that uniquely identifies an object. Defaults to 'objid'. --/T
  • @ra_col SYSNAME: name of column containing the RA (Right Ascension) value of an object. Defaults to 'ra'. --/T
  • @dec_col SYSNAME: name of column containing the Dec (Declination) value of an object. Defaults to 'dec'. ---/T
  • @max_catalog_rows bigint: default value of null. If set, the procedure will return the TOP @max_catalog_rows rows, with no special ordering. +--/T
  • @max_catalog_rows BIGINT: default value of null. If set, the procedure will return the TOP @max_catalog_rows rows, with no special ordering. AS BEGIN SET NOCOUNT ON @@ -512,15 +512,15 @@ AS BEGIN SET @has_cxcycz = @has_columns & @has_column EXECUTE spHasColumn @table=@reference_table, @column='zoneid', @has_column=@has_zoneid OUTPUT - DECLARE @select_top nvarchar(max) = N'INSERT INTO ' + @target_quoted_path + ' WITH (TABLOCKX) SELECT ' + DECLARE @select_top NVARCHAR(max) = N'INSERT INTO ' + @target_quoted_path + ' WITH (TABLOCKX) SELECT ' IF @max_catalog_rows is not null - SET @select_top = @select_top + N' TOP ' + CAST(@max_catalog_rows as nvarchar) + N' ' + SET @select_top = @select_top + N' TOP ' + CAST(@max_catalog_rows as NVARCHAR) + N' ' - DECLARE @sql nvarchar(max) = N'DECLARE @d2r float = PI()/180.0; ' + @select_top + DECLARE @sql NVARCHAR(max) = N'DECLARE @d2r FLOAT = PI()/180.0; ' + @select_top IF @has_zoneid = 1 SET @sql = @sql + N'zoneid, ' ELSE - SET @sql = @sql + 'CONVERT(INT,FLOOR((' + @dec_col + N' + 90.0)/' + CAST(@zoneHeight as nvarchar) + N')) as zoneid, ' + SET @sql = @sql + 'CONVERT(INT,FLOOR((' + @dec_col + N' + 90.0)/' + CAST(@zoneHeight as NVARCHAR) + N')) as zoneid, ' SET @sql = @sql + @id_col + N' as objid, ' + @ra_col + N' as ra, ' + @dec_col + N' as dec, ' @@ -536,7 +536,7 @@ AS BEGIN --PRINT @sql - EXEC sp_executesql @sql, N'@zoneHeight float', @zoneHeight=@zoneHeight + EXEC sp_executesql @sql, N'@zoneHeight FLOAT', @zoneHeight=@zoneHeight END GO @@ -561,12 +561,13 @@ CREATE PROCEDURE SQLxMatch @ra_col2 SYSNAME = 'ra', @dec_col1 SYSNAME = 'dec', @dec_col2 SYSNAME = 'dec', - @max_catalog_rows1 BIGINT = null, - @max_catalog_rows2 BIGINT = null, - @output_table SYSNAME = null, - @only_nearest bit = 0, + @max_catalog_rows1 BIGINT = NULL, + @max_catalog_rows2 BIGINT = NULL, + @output_table SYSNAME = NULL, + @only_nearest BIT = 0, @sort_by_separation BIT = 0, - @radec_in_output BIT = 0, + @include_sep_rank BIT = 0, + @include_radec BIT = 0, @print_messages BIT = 0 ---------------------------------------------------------------- --/H Runs a spatial crossmatch between 2 catalogs of objects, using the Zones algorithm. Returns a table with the object IDs and angular separation between matching objects. @@ -577,22 +578,26 @@ CREATE PROCEDURE SQLxMatch --/T Parameters:
    --/T
  • @table1 SYSNAME: name of first input catalog. Can be any of these formats: 'server.database.schema.table', 'database.schema.table', 'database.table', or simply 'table' --/T
  • @table2 SYSNAME: name of first input catalog. Can be any of these formats: 'server.database.schema.table', 'database.schema.table', 'database.table', or simply 'table' ---/T
  • @radius float: search radius around each object, in arcseconds. Takes a default value of 10 arcseconds. +--/T
  • @radius FLOAT: search radius around each object, in arcseconds. Takes a default value of 10 arcseconds. --/T
  • @id_col1 SYSNAME: name of the column defining a unique object identifier in catalog @table1. Takes a default value of 'objid'. --/T
  • @id_col2 SYSNAME: name of the column defining a unique object identifier in catalog @table2. Takes a default value of 'objid'. --/T
  • @ra_col1 SYSNAME: name of the column containing the Right Ascention (RA) in degrees of objects in catalog @table1. Takes a default value of 'ra'. --/T
  • @ra_col2 SYSNAME: name of the column containing the Right Ascention (RA) in degrees of objects in catalog @table2. Takes a default value of 'ra'. --/T
  • @dec_col1 SYSNAME: name of the column containing the Declination (Dec) in degrees of objects in catalog @table1. Takes a default value of 'dec'. --/T
  • @dec_col2 SYSNAME: name of the column containing the Declination (Dec) in degrees of objects in catalog @table2. Takes a default value of 'dec'. ---/T
  • @max_catalog_rows1 bigint: default value of null. If set, the procedure will use only the TOP @max_catalog_rows1 rows in catalog @table1, with no special ordering. ---/T
  • @max_catalog_rows2 bigint: default value of null. If set, the procedure will use only the TOP @max_catalog_rows2 rows in catalog @table2, with no special ordering. +--/T
  • @max_catalog_rows1 BIGINT: default value of null. If set, the procedure will use only the TOP @max_catalog_rows1 rows in catalog @table1, with no special ordering. +--/T
  • @max_catalog_rows2 BIGINT: default value of null. If set, the procedure will use only the TOP @max_catalog_rows2 rows in catalog @table2, with no special ordering. --/T
  • @output_table SYSNAME: If NOT NULL, this procedure will insert the output results into the table @output_table (of format 'server.database.schema.table', 'database.schema.table', 'database.table', or simply 'table'), which must already exist and be visbile within the scope of the procedure. If set to null, the output results will be simply returned as a table resultset. Takes a default value of null. ---/T
  • @only_nearest bit: If set to 0 (default value), then all matches within a distance @radius to an object are returned. If set to 1, only the closest match to an object is returned. ---/T
  • @sort_by_separation bit: If set to 1, then the output will be sorted by the 'id1' and 'sep' columns. If set to 0 (default value), no particular ordering is applied. ---/T
  • @radec_in_output bit: If set to 1, then the output table will contain as well the (RA, Dec) values of each object. ---/T
  • @print_messages bit: If set to 1, then time-stamped messages will be printed as the different sections in this procedure are completed. ---/T
  • RETURNS TABLE(id1, id2, sep), where id1 and id2 are the unique object identifier columns in @table1 and @table2, respectively, and sep (float) is the angular separation between objetcs in arseconds. ---/T
  • In the case @radec_in_output=1, the procedure returns extra columns in the form of TABLE(id1, id2, sep, ra1, dec1, ra2, dec2), where ra1, dec1, ra2 and dec2 (float) are the coordinates of the objets in @table1 and @table2, respectively. +--/T
  • @only_nearest BIT: If set to 0 (default value), then all matches within a distance @radius to an object are returned. If set to 1, only the closest match to an object is returned. +--/T
  • @sort_by_separation BIT: If set to 1, then the output will be sorted by the 'id1' and 'sep' columns. If set to 0 (default value), no particular ordering is applied. +--/T
  • @include_sep_rank BIT: If set to 1, then the output table will include an extra column named 'sep_rank', denoting the rank of all matches to an object when sorted by angular separation. If set to 0 (default value), this column is not included. +--/T
  • @include_radec BIT: If set to 1, then the output table will contain the original (RA, Dec) columns from both input tables, as ra1, dec1, ra2, and dec2. +--/T
  • @print_messages BIT: If set to 1, then time-stamped messages will be printed as the different sections in this procedure are completed. +--/T
  • RETURNS TABLE(id1, id2, sep), where id1 and id2 are the unique object identifier columns in @table1 and @table2, respectively, and sep (FLOAT) is the angular separation between objetcs in arseconds. +--/T
  • In the case @include_radec=1, the procedure returns TABLE(id1, id2, sep, ra1, dec1, ra2, dec2). +--/T
  • In the case @include_sep_rank=1, the procedure returns TABLE(id1, id2, sep, sep_rank). +--/T
  • In the case @include_sep_rank=1 and @include_radec=1, the procedure returns TABLE(id1, id2, sep, sep_rank, ra1, dec1, ra2, dec2). + AS BEGIN @@ -600,30 +605,28 @@ AS BEGIN IF @print_messages = 1 BEGIN - PRINT 'Laszlo' DECLARE @initial_datetime datetime = SYSDATETIME() EXECUTE spPrintMessage @initial_datetime , N'START xmatch procedure' END -- Enforce maximum allowed radius: - DECLARE @max_radius float = 10 * 60.0 -- maximum allowed search radius in arcsec - DECLARE @theta float = @radius/3600.0 -- in degrees + DECLARE @max_radius FLOAT = 10 * 60.0 -- maximum allowed search radius in arcsec + DECLARE @theta FLOAT = @radius/3600.0 -- in degrees IF @theta > @max_radius/ 3600 BEGIN - DECLARE @max_radius_str nvarchar(128) = CAST(@max_radius as nvarchar(128)) -- RAISERROR won't allow float data types like @max_radius as arguments. + DECLARE @max_radius_str NVARCHAR(128) = CAST(@max_radius as NVARCHAR(128)) -- RAISERROR won't allow FLOAT data types like @max_radius as arguments. RAISERROR(N'Input search radius @radius surpassed maxiumum limit of %s arcsec.', 16, 1, @max_radius_str) RETURN END - -- Define variables -- If a zoneid column is provided in both input @table1 and @table2 tables, - -- then we assume that they are calculated from a standard value of @DefaultZoneHeight. + -- then we assume that they are calculated FROM a standard value of @DefaultZoneHeight. - DECLARE @zoneHeight float = 10.0 / 3600.0 -- in degrees - DECLARE @DefaultZoneHeight float = 4.0 / 3600.0 -- 4 arcsec was chosen as a standard value for creating the zoneid columns in the public catalogs. + DECLARE @zoneHeight FLOAT = 10.0 / 3600.0 -- in degrees + DECLARE @DefaultZoneHeight FLOAT = 4.0 / 3600.0 -- 4 arcsec was chosen as a standard value for creating the zoneid columns in the public catalogs. DECLARE @has_zones bit = 1 DECLARE @has_column bit @@ -719,6 +722,7 @@ AS BEGIN -- Setting up query that returns cross-match table . DECLARE @rank_column NVARCHAR(256) = N'' + DECLARE @sep_rank_column NVARCHAR(256) = N'' DECLARE @order_clause NVARCHAR(256) = N'' DECLARE @radec_columns NVARCHAR(256) = N'' DECLARE @dec_columns NVARCHAR(256) = N'' @@ -727,27 +731,35 @@ AS BEGIN -- Set output for radec input option. - IF @radec_in_output = 1 + IF @include_radec= 1 BEGIN SET @radec_columns = N', ra1, dec1, ra2, dec2 ' SET @dec_columns = N', t1.dec AS dec1, t2.dec AS dec2 ' END + + -- Set sep_rank column name + IF @include_sep_rank = 1 + BEGIN + SET @sep_rank_column = N', sep_rank' + END - -- Set output for only-nearest-object input option. - IF @only_nearest = 1 + -- Set output for only-nearest-object and @include_sep_rank input options. + IF @only_nearest = 1 OR @include_sep_rank = 1 BEGIN SET @rank_column = N', RANK() OVER (PARTITION BY id1 ORDER BY sep) AS sep_rank' END - SET @sql = 'SELECT id1, id2' + @radec_columns + N', sep' + @rank_column + N' FROM wrap' -- wrap is the common table expression defined below. + SET @sql = 'SELECT id1, id2, sep' + @rank_column + @radec_columns + N' FROM wrap' -- wrap is the common table expression defined below. - IF @only_nearest = 1 + IF @only_nearest = 1 OR @include_sep_rank = 1 BEGIN - SET @sql = N'SELECT * FROM ( ' + @sql + N' ) AS q WHERE sep_rank = 1' + SET @sql = N'SELECT id1, id2, sep' + @sep_rank_column + @radec_columns + N' FROM ( ' + @sql + N' ) AS q ' + IF @only_nearest = 1 + SET @sql = @sql + ' WHERE sep_rank = 1' END -- Set output for sort-by-separation input option. - IF @sort_by_separation = 1 + IF @sort_by_separation = 1 AND @only_nearest = 0 BEGIN SET @sql = @sql + N' ORDER BY id1, sep' END @@ -798,7 +810,7 @@ AS BEGIN ) ' + @sql - --print @sql + print @sql EXECUTE sp_executesql @sql, N'@dist2 FLOAT', @dist2=@dist2 diff --git a/sql/sqlserver/metadata/Step02-InstallMetadataProcedures.sql b/sql/sqlserver/metadata/Step02-InstallMetadataProcedures.sql index 4bf8bf9..ecb4a30 100644 --- a/sql/sqlserver/metadata/Step02-InstallMetadataProcedures.sql +++ b/sql/sqlserver/metadata/Step02-InstallMetadataProcedures.sql @@ -45,7 +45,7 @@ AS BEGIN DECLARE @SQL NVARCHAR(max) = N' USE [?]; DECLARE @database SYSNAME = DB_NAME(); INSERT INTO #tapcatalogs - SELECT + SELECT DISTINCT @database, REPLACE(@database, ''SkyNode_'', ''''), CASE WHEN [meta.summary] IS NULL THEN N'''' ELSE CAST([meta.summary] AS NVARCHAR(max)) END AS summary, @@ -116,7 +116,7 @@ AS BEGIN IF EXISTS(SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES where TABLE_NAME = ''tables'' AND TABLE_SCHEMA=''TAP_SCHEMA'') -- try first using metadata in TAP_SCHEMA BEGIN INSERT INTO #taptables - SELECT + SELECT DISTINCT t.TABLE_SCHEMA [schema_name], REPLACE(@database, ''SkyNode_'', '''') + ''_'' + t.TABLE_NAME table_name, ''view'' [table_type], COALESCE(ts.description, N'''') description, ts.utype, NULL as [table_index] @@ -126,7 +126,7 @@ AS BEGIN ELSE IF EXISTS(SELECT name from sys.tables where name = ''dbobjects'') -- try using metadata in dbobjects table BEGIN INSERT INTO #taptables - SELECT + SELECT DISTINCT ta.[schema_name], REPLACE(@database, ''SkyNode_'', '''') + ''_'' + ta.table_name as table_name, ''view'' [table_type], COALESCE(do.description, N'''') description, NULL utype, NULL as [table_index] @@ -142,7 +142,7 @@ AS BEGIN ELSE BEGIN INSERT INTO #taptables - SELECT + SELECT DISTINCT ''dbo'' as ''schema_name'', REPLACE(@database, ''SkyNode_'', '''') + ''_'' + table_name as ''table_name'', ''view'' as table_type, @@ -217,21 +217,21 @@ AS BEGIN IF EXISTS(SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES where TABLE_NAME = ''columns'' AND TABLE_SCHEMA=''TAP_SCHEMA'') -- try first using metadata in TAP_SCHEMA BEGIN INSERT INTO #tapcolumns - SELECT + SELECT DISTINCT c.TABLE_SCHEMA schema_name, REPLACE(@database, ''SkyNode_'', '''') + ''_'' + c.TABLE_NAME as table_name, c.COLUMN_NAME, COALESCE(tc.description, N'''') description, COALESCE(tc.unit, N'''') unit, COALESCE(tc.ucd, N'''') ucd, COALESCE(tc.utype, N'''') utype, - c.DATA_TYPE as datatype, c.CHARACTER_MAXIMUM_LENGTH as size, c.CHARACTER_MAXIMUM_LENGTH arraysize, c.NUMERIC_PRECISION precision, c.NUMERIC_SCALE scale, 1 principal,0 as indexed, 0 as std, ic.ordinal_position column_index + c.DATA_TYPE as datatype, c.CHARACTER_MAXIMUM_LENGTH as size, c.CHARACTER_MAXIMUM_LENGTH arraysize, c.NUMERIC_PRECISION precision, c.NUMERIC_SCALE scale, 1 principal,0 as indexed, 0 as std, c.ordinal_position column_index FROM INFORMATION_SCHEMA.COLUMNS c - LEFT JOIN TAP_SCHEMA.columns AS tc ON c.COLUMN_NAME = tc.COLUMN_NAME and c.TABLE_NAME = tc.TABLE_NAME and c.TABLE_SCHEMA = tc.schema_name + LEFT JOIN TAP_SCHEMA.columns AS tc ON c.COLUMN_NAME = tc.COLUMN_NAME AND c.TABLE_NAME = tc.TABLE_NAME ORDER BY table_name, column_index END ELSE IF EXISTS(SELECT name from sys.tables where name = ''DBColumns'') -- try using metadata in DBColumns table BEGIN INSERT INTO #tapcolumns - SELECT + SELECT DISTINCT c.TABLE_SCHEMA schema_name, REPLACE(@database, ''SkyNode_'', '''') + ''_'' + c.TABLE_NAME as table_name, c.COLUMN_NAME, COALESCE(dc.description, N'''') description, COALESCE(dc.unit, N'''') unit, COALESCE(dc.ucd, N'''') ucd, N'''' utype, - c.DATA_TYPE as datatype, c.CHARACTER_MAXIMUM_LENGTH as size, c.CHARACTER_MAXIMUM_LENGTH arraysize, c.NUMERIC_PRECISION precision, c.NUMERIC_SCALE scale, 1 principal,0 as indexed, 0 as std, ic.ordinal_position column_index + c.DATA_TYPE as datatype, c.CHARACTER_MAXIMUM_LENGTH as size, c.CHARACTER_MAXIMUM_LENGTH arraysize, c.NUMERIC_PRECISION precision, c.NUMERIC_SCALE scale, 1 principal,0 as indexed, 0 as std, c.ordinal_position column_index FROM INFORMATION_SCHEMA.COLUMNS c LEFT JOIN DBColumns AS dc ON dc.name = c.COLUMN_NAME AND dc.tablename = c.TABLE_NAME ORDER BY table_name, column_index @@ -239,13 +239,13 @@ AS BEGIN ELSE BEGIN INSERT INTO #tapcolumns - SELECT - [schema_name], REPLACE(@database, ''SkyNode_'', '''') + ''_'' + table_name as table_name, column_name, + SELECT DISTINCT + schema_name, REPLACE(@database, ''SkyNode_'', '''') + ''_'' + table_name as table_name, column_name, COALESCE(CAST([meta.summary] AS NVARCHAR(max)), N'''') as description, COALESCE(CAST([meta.unit] AS NVARCHAR(max)), N'''') as unit, COALESCE(CAST([meta.quantity] AS NVARCHAR(max)), N'''') as ucd, N'''' as utype, datatype, size, arraysize, precision, scale, 1 as principal, 0 as indexed, 0 as std, column_index FROM ( - SELECT + SELECT table_name, c.name column_name, col.DATA_TYPE datatype, c.max_length size, c.max_length arraysize, c.precision, c.scale, col.ordinal_position column_index, col.TABLE_SCHEMA schema_name, ep.name property_name, ep.value property_value FROM sys.columns c JOIN sys.all_objects o ON c.object_id = o.object_id