Glimpse360 (#46)

* initial commit of glimpse360_create_table.sql and glimpse360_tbl.c

* peewee model for GLIMPSE360in catalogdb.py

* comment for glimpse360_tbl.c

* initial commit of glimpse360_create_indexes.sql, glimpse360_alter_table_pkey.sql, and glimpse360_alter_table_fkey.sql

* update glimpse360_alter_table_pkey.sql

* update peewee model for GLIMPSE360 in catalogdb.py

* initial commit of glimpse360_load.sql

* update glimpse360_load.sql

* update glimpse360_load.sql

* initial commit of glimpse360_update_table_null.sql

* edit glimpse360_update_table_null.sql

python/sdssdb/peewee/sdss5db/catalogdb.py

@@ -812,6 +812,19 @@ class Meta:
         print_fields = ['designation']
 
 
+class GLIMPSE360(CatalogdbModel):
+
+    pkey = BigIntegerField(primary_key=True)
+
+    twomass = ForeignKeyField(TwoMassPSC, field='pts_key',
+                              column_name='tmass_cntr',
+                              backref='glimpse360_targets')
+
+    class Meta:
+        table_name = 'glimpse360'
+        print_fields = ['designation']
+
+
 class BHM_eFEDS_Veto(CatalogdbModel):
 
     pk = BigIntegerField(primary_key=True)

schema/sdss5db/catalogdb/GLIMPSE360/glimpse360_alter_table_fkey.sql

@@ -0,0 +1,7 @@
+\o glimpse360_alter_table_fkey.out
+
+ALTER TABLE catalogdb.glimpse360
+    ADD FOREIGN KEY (tmass_cntr)
+    REFERENCES catalogdb.twomass_psc(pts_key);
+
+\o

schema/sdss5db/catalogdb/GLIMPSE360/glimpse360_alter_table_pkey.sql

@@ -0,0 +1,3 @@
+\o glimpse360_alter_table_pkey.out
+alter table catalogdb.glimpse360 add column pkey bigserial primary key;
+\o

schema/sdss5db/catalogdb/GLIMPSE360/glimpse360_create_indexes.sql

@@ -0,0 +1,5 @@
+\o glimpse360_create_indexes.out
+create index on catalogdb.glimpse360(q3c_ang2ipix(ra,dec));
+-- column tmass_cntr is a foreign key
+create index on catalogdb.glimpse360(tmass_cntr);
+\o

schema/sdss5db/catalogdb/GLIMPSE360/glimpse360_create_table.sql

@@ -0,0 +1,86 @@
+-- The column names are from line 10 of 
+-- and the column types are from line 11 of
+-- GLM360A_l108-169.tbl
+-- the char sizes are from table 8 page 26 of
+-- http://www.astro.wisc.edu/sirtf/glimpse360_dataprod_v1.2.pdf
+
+create table catalogdb.glimpse360 (
+designation char(26),
+tmass_designation char(16),
+tmass_cntr integer,
+l double precision,
+b double precision,
+dl double precision,
+db double precision,
+ra double precision,
+dec double precision,
+dra double precision,
+ddec double precision,
+csf integer,
+mag_J real,
+dJ_m real,
+mag_H real,
+dH_m real,
+mag_Ks real,
+dKs_m real,
+mag3_6 real,
+d3_6m real,
+mag4_5 real,
+d4_5m real,
+mag5_8 real,
+d5_8m real,
+mag8_0 real,
+d8_0m real,
+f_J real,
+df_J real,
+f_H real,
+df_H real,
+f_Ks real,
+df_Ks real,
+f3_6 real,
+df3_6 real,
+f4_5 real,
+df4_5 real,
+f5_8 real,
+df5_8 real,
+f8_0 real,
+df8_0 real,
+rms_f3_6 real,
+rms_f4_5 real,
+rms_f5_8 real,
+rms_f8_0 real,
+sky_3_6 real,
+sky_4_5 real,
+sky_5_8 real,
+sky_8_0 real,
+sn_J real,
+sn_H real,
+sn_Ks real,
+sn_3_6 real,
+sn_4_5 real,
+sn_5_8 real,
+sn_8_0 real,
+dens_3_6 real,
+dens_4_5 real,
+dens_5_8 real,
+dens_8_0 real,
+m3_6 integer,
+m4_5 integer,
+m5_8 integer,
+m8_0 integer,
+n3_6 integer,
+n4_5 integer,
+n5_8 integer,
+n8_0 integer,
+sqf_J integer,
+sqf_H integer,
+sqf_Ks integer,
+sqf_3_6 integer,
+sqf_4_5 integer,
+sqf_5_8 integer,
+sqf_8_0 integer,
+mf3_6 integer,
+mf4_5 integer,
+mf5_8 integer,
+mf8_0 integer
+)

schema/sdss5db/catalogdb/GLIMPSE360/glimpse360_load.sql

@@ -0,0 +1,3 @@
+\o glimpse360_load.out
+\copy catalogdb.glimpse360 from '/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/GLIMPSE360/GLM360A_l108-169.tbl.csv' delimiter ',';
+\o

schema/sdss5db/catalogdb/GLIMPSE360/glimpse360_tbl.c

@@ -0,0 +1,220 @@
+// Program glimpse360_tbl.c
+// Aim: Read a GLIMPSE360 tbl data file and output a CSV file
+//
+// column names and types are from line 10 and line 11 of
+// GLM360A_l108-169.tbl
+//
+// Use the below command to get the above .tbl file:
+// wget
+// https://irsa.ipac.caltech.edu/data/SPITZER/GLIMPSE/catalogs/GLIMPSE360/GLM360A_l108-169.tbl.gz
+//
+// See Table 8 page 26 for types of first and second columns:
+// http://www.astro.wisc.edu/sirtf/glimpse360_dataprod_v1.2.pdf
+// first column is char(26)
+// second column is char(16)
+//
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int main(int argc, char *argv[]) {
+
+    const int NUM_HEADER_LINES = 13;
+
+    // This is used in fgets()
+    const int MAX_CHAR_HEADER_LINE = 1024;
+    char header_line[MAX_CHAR_HEADER_LINE];
+
+    // +1 for '\0'
+    // +1 for the space between fields in the .tbl file
+    char designation[26 + 1 + 1];
+    char tmass_designation[16 + 1 + 1];
+    int tmass_cntr;
+    double l;
+    double b;
+    double dl;
+    double db;
+    double ra;
+    double dec;
+    double dra;
+    double ddec;
+    int csf;
+    float mag_J;
+    float dJ_m;
+    float mag_H;
+    float dH_m;
+    float mag_Ks;
+    float dKs_m;
+    float mag3_6;
+    float d3_6m;
+    float mag4_5;
+    float d4_5m;
+    float mag5_8;
+    float d5_8m;
+    float mag8_0;
+    float d8_0m;
+    float f_J;
+    float df_J;
+    float f_H;
+    float df_H;
+    float f_Ks;
+    float df_Ks;
+    float f3_6;
+    float df3_6;
+    float f4_5;
+    float df4_5;
+    float f5_8;
+    float df5_8;
+    float f8_0;
+    float df8_0;
+    float rms_f3_6;
+    float rms_f4_5;
+    float rms_f5_8;
+    float rms_f8_0;
+    float sky_3_6;
+    float sky_4_5;
+    float sky_5_8;
+    float sky_8_0;
+    float sn_J;
+    float sn_H;
+    float sn_Ks;
+    float sn_3_6;
+    float sn_4_5;
+    float sn_5_8;
+    float sn_8_0;
+    float dens_3_6;
+    float dens_4_5;
+    float dens_5_8;
+    float dens_8_0;
+    int m3_6;
+    int m4_5;
+    int m5_8;
+    int m8_0;
+    int n3_6;
+    int n4_5;
+    int n5_8;
+    int n8_0;
+    int sqf_J;
+    int sqf_H;
+    int sqf_Ks;
+    int sqf_3_6;
+    int sqf_4_5;
+    int sqf_5_8;
+    int sqf_8_0;
+    int mf3_6;
+    int mf4_5;
+    int mf5_8;
+    int mf8_0;
+
+    FILE *fptr;
+    FILE *fptr_try;
+    FILE *fptr_out;
+    int i;
+    long long num_rows;
+    char *fgets_ptr;
+    int c1;
+
+    if (argc != 3) {
+        printf("usage: glimpse360_tbl input_file output_file\n");
+        exit(1);
+    }
+
+    fptr = fopen(argv[1], "rb");
+    if (fptr == NULL) {
+        printf("error: could not open input file %s\n", argv[1]);
+        exit(1);
+    }
+
+    printf("info:input_file=%s\n", argv[1]);
+
+    fptr_try = fopen(argv[2], "r");
+    if (fptr_try != NULL) {
+        printf("error: output file exists %s\n", argv[2]);
+        fclose(fptr_try);
+        exit(1);
+    }
+
+    fptr_out = fopen(argv[2], "w");
+    if (fptr_out == NULL) {
+        printf("error: could not open output file %s\n", argv[2]);
+        exit(1);
+    }
+
+    printf("info:output_file=%s\n", argv[2]);
+
+    // skip header lines
+    for (i = 1; i <= NUM_HEADER_LINES; i++) {
+        fgets(header_line, MAX_CHAR_HEADER_LINE, fptr);
+        printf("%d:%s\n", i, header_line);
+    }
+
+    num_rows = 0;
+    // start loop to read lines
+    while (1) {
+        // +1 for '\0'
+        // +1 for the space between fields in the .tbl file
+        fgets_ptr = fgets(designation, 26 + 1 + 1, fptr);
+
+        if (fgets_ptr == NULL) {
+            break;
+        }
+
+        fgets(tmass_designation, 16 + 1 + 1, fptr);
+
+        // The format string below does not have %26s for designation
+        // and %16s for tmass_designation since we have the above two fgets()
+
+        fscanf(fptr,
+               "%d %le %le %le %le %le %le %le %le %d %e %e %e %e %e %e %e %e "
+               "%e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %e "
+               "%e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %e %d %d %d "
+               "%d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
+               &tmass_cntr, &l, &b, &dl, &db, &ra, &dec, &dra, &ddec, &csf,
+               &mag_J, &dJ_m, &mag_H, &dH_m, &mag_Ks, &dKs_m, &mag3_6, &d3_6m,
+               &mag4_5, &d4_5m, &mag5_8, &d5_8m, &mag8_0, &d8_0m, &f_J, &df_J,
+               &f_H, &df_H, &f_Ks, &df_Ks, &f3_6, &df3_6, &f4_5, &df4_5, &f5_8,
+               &df5_8, &f8_0, &df8_0, &rms_f3_6, &rms_f4_5, &rms_f5_8,
+               &rms_f8_0, &sky_3_6, &sky_4_5, &sky_5_8, &sky_8_0, &sn_J, &sn_H,
+               &sn_Ks, &sn_3_6, &sn_4_5, &sn_5_8, &sn_8_0, &dens_3_6, &dens_4_5,
+               &dens_5_8, &dens_8_0, &m3_6, &m4_5, &m5_8, &m8_0, &n3_6, &n4_5,
+               &n5_8, &n8_0, &sqf_J, &sqf_H, &sqf_Ks, &sqf_3_6, &sqf_4_5,
+               &sqf_5_8, &sqf_8_0, &mf3_6, &mf4_5, &mf5_8, &mf8_0);
+
+        // read remaining characters in the current line
+        // till we get to the next line
+        c1 = 'a';
+        while ((c1 != EOF) && (c1 != '\n')) {
+            c1 = fgetc(fptr);
+        }
+
+        // completed row read so increment num_rows
+        num_rows = num_rows + 1;
+
+        // write CSV data
+
+        fprintf(fptr_out,
+                "%s,%s,%d,%.16e,%.16e,%.16e,%.16e,%.16e,%.16e,%.16e,%.16e,%d,"
+                "%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,"
+                "%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,"
+                "%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,"
+                "%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,%.8e,"
+                "%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d",
+                designation, tmass_designation, tmass_cntr, l, b, dl, db, ra,
+                dec, dra, ddec, csf, mag_J, dJ_m, mag_H, dH_m, mag_Ks, dKs_m,
+                mag3_6, d3_6m, mag4_5, d4_5m, mag5_8, d5_8m, mag8_0, d8_0m, f_J,
+                df_J, f_H, df_H, f_Ks, df_Ks, f3_6, df3_6, f4_5, df4_5, f5_8,
+                df5_8, f8_0, df8_0, rms_f3_6, rms_f4_5, rms_f5_8, rms_f8_0,
+                sky_3_6, sky_4_5, sky_5_8, sky_8_0, sn_J, sn_H, sn_Ks, sn_3_6,
+                sn_4_5, sn_5_8, sn_8_0, dens_3_6, dens_4_5, dens_5_8, dens_8_0,
+                m3_6, m4_5, m5_8, m8_0, n3_6, n4_5, n5_8, n8_0, sqf_J, sqf_H,
+                sqf_Ks, sqf_3_6, sqf_4_5, sqf_5_8, sqf_8_0, mf3_6, mf4_5, mf5_8,
+                mf8_0);
+
+        fprintf(fptr_out, "\n");
+    }
+    // end loop to read lines
+
+    fclose(fptr);
+    fclose(fptr_out);
+    return 0;
+}

schema/sdss5db/catalogdb/GLIMPSE360/glimpse360_update_table_null.sql

@@ -0,0 +1,11 @@
+-- update table catalogdb.glimpse360 and set field to NULL based on
+-- page 26 table 8 of the below document
+-- http://www.astro.wisc.edu/sirtf/glimpse360_dataprod_v1.2.pdf
+
+\o glimpse360_update_table_null.out
+
+update catalogdb.glimpse360 set tmass_cntr=NULL where tmass_cntr=0 ;
+
+update catalogdb.glimpse360 set tmass_designation=NULL where tmass_designation like'%null%' ;
+
+\o