Minor error checking added.

Classifying transient .dat files instead of combined .fits files in analyse_all_ozdes_atels.py.

dash/analyse_all_ozdes_atels.py

@@ -1,4 +1,5 @@
 import os
+import glob
 import dash
 import pandas as pd
 import numpy as np
@@ -23,48 +24,41 @@ def read_all_atels(atelTextFile):
 
 
 def main(spectraDir, atelTextFile, saveMatchesFilename):
+    filenames, knownRedshifts, wikiClassifications = [], [], []
 
     # Store all the file paths to the objects in this directory
-    allFilePaths = os.listdir(spectraDir)
+    allFilePaths = glob.glob('%s/*.dat' % spectraDir)
 
     # Get filenames and corresponding redshifts
     atels = read_all_atels(atelTextFile)
-    for filename in allFilePaths:
-        name = filename.replace('.fits', '')
-        i = np.where(atels['Name'] == name)[0]
-        if len(i) > 1:
-            print("Something wrong with:", name)
-        atels.at[i, 'Filename'] = os.path.join(spectraDir, filename)
-
-    # Check how many observed runs
-    count = 0
-    for filename in atels.Filename:
-        hdulist = afits.open(filename)
-        run = hdulist[3].header['SOURCEF'].split('/')[0]
-        for i in range(3, len(hdulist)):
-            try:
-                run_new = hdulist[i].header['SOURCEF'].split('/')[0]
-                if run_new != run:
-                    count += 1
-                    print(count, len(atels.Filename), filename, i, run, run_new)
-                    break
-            except KeyError:
-                pass
+    for i, row in atels.iterrows():
+        count = 0
+        for filePath in allFilePaths:
+            name = os.path.basename(filePath).replace('.dat', '').split('_')[0]
+            if row.Name == name:
+                filenames.append(filePath)
+                knownRedshifts.append(row.Redshift)
+                wikiClassifications.append("{} {}".format(row.Type, row.Phase))
+                count += 1
+                # break  # Uncomment the break to only classify the last dated spectrum for each object instead of classifying all dates.
+        print(count)
+        if count == 0:
+            print(row.Name)
 
     # Classify and print the best matches
-    classification = dash.Classify(atels.Filename.values, atels.Redshift.values, classifyHost=False, rlapScores=True, smooth=6)
+    classification = dash.Classify(filenames, knownRedshifts, classifyHost=False, rlapScores=True, smooth=6)
     bestFits, redshifts, bestTypes, rlapFlag, matchesFlag = classification.list_best_matches(n=5, saveFilename=saveMatchesFilename)
 
     print("{0:17} | {1:5} | {2:8} | {3:10} | {4:6} | {5:10} | {6:10}".format("Name", "  z  ", "DASH_Fit", "  Age ", "Prob.", "Flag", "Wiki Fit"))
-    for i in range(len(atels)):
-        print("{0:17} | {1:5} | {2:8} | {3:10} | {4:6} | {5:10} | {6:10}".format(os.path.basename(atels.Filename[i]), round(atels.Redshift[i], 3), bestTypes[i][0], bestTypes[i][1], bestTypes[i][2], matchesFlag[i].replace(' matches',''), "{} {}".format(atels['Type'][i], atels['Phase'][i])))
+    for i in range(len(filenames)):
+        print("{0:17} | {1:5} | {2:8} | {3:10} | {4:6} | {5:10} | {6:10}".format('_'.join([filenames[i].split('/')[-1].split('_')[0], filenames[i].split('/')[-1].split('_')[3]]) , redshifts[i], bestTypes[i][0], bestTypes[i][1], bestTypes[i][2], matchesFlag[i].replace(' matches',''), wikiClassifications[i]))
 
     # Plot one of the matches
     classification.plot_with_gui(indexToPlot=7)
 
 
 if __name__=='__main__':
-    main(spectraDir='/Users/danmuth/PycharmProjects/DASH/templates/OzDES_data/180413/',
+    main(spectraDir='/Users/danmuth/PycharmProjects/DASH/templates/OzDES_data/transients_all/',
          atelTextFile='/Users/danmuth/PycharmProjects/DASH/templates/OzDES_data/all_atels.txt',
          saveMatchesFilename='DASH_matches_all_atels.txt')
 

dash/create_arrays.py

@@ -13,6 +13,7 @@
     from imblearn import over_sampling
     IMBLEARN_EXISTS = True
 except ImportError:
+    print("Need to install imblearn `pip install imblearn` to use smote.")
     IMBLEARN_EXISTS = False
 
 

dash/preprocessing.py

@@ -153,6 +153,11 @@ def two_col_input_spectrum(self, wave, flux, z):
         wave = wave[mask]
         flux = flux[mask]
 
+        if not wave.any():
+            raise Exception("The spectrum {0} with redshift {1} is out of the wavelength range {2}A to {3}A, "
+                            "and cannot be classified. Please remove this object or change the input redshift of this"
+                            " spectrum.".format(self.filename, z, int(self.w0), int(self.w1)))
+
         fluxNorm = (flux - min(flux)) / (max(flux) - min(flux))
 
         return wave, fluxNorm

templates/OzDES_data/all_atels.txt

@@ -179,8 +179,6 @@ DES15X2mpm |02:22:25.40|-06:08:59.5 |2015 Nov 05|  24.0   |2015 Dec 14|  0.235 |
 DES15X3mpq |02:24:52.26|-03:39:55.3 |2015 Nov 13|  22.4   |2015 Dec 12|  0.188 |  II  |+1 month|a
 DES15E1neh |00:26:40.77|-42:57:50.6 |2015 Nov 29|  22.6   |2015 Dec 13|  0.390 |  Ia? |   max  |
 
-DES15E2mlf |00:41:33.40|-43:27:17.2 |2015 Nov 07|  24.1   |2015 Dec 06 and 15 (Gemini-S), 2015 Dec 12 and 14 (AAT)|  1.86  | SLSN |  max  |a,b
-
 DES15S2mpg |02:43:52.74|-01:13:45.4 |2015 Nov 12|  22.1   |2015 Dec 03|  0.186 |  Ia  |  max   |a
 DES15S2mpl |02:46:53.90|-01:09:52.4 |2015 Nov 12|  22.2   |2015 Dec 03|  0.257 |  Ia  |+1 week |a
 DES15S1lyi |02:48:51.28|-00:09:07.9 |2015 Oct 29|  22.6   |2015 Dec 03|  0.359 |  Ia? |+3 weeks|a