Permalink
Browse files

revised for multiple sample dates

  • Loading branch information...
1 parent 99b4500 commit e66b3c25a0b5f752937a07be69b46cb011266a5a @aflock committed Apr 1, 2012
Showing with 37 additions and 54 deletions.
  1. +10 −7 sem2/get_samples.py
  2. +27 −47 sem2/nearest_neighbors.py
View
@@ -6,15 +6,15 @@
Author: AFlock
Description: retrieve samples from a diff'd image, pickle them.
'''
-
import os, pickle
+import datetime
import pyfits as pft
import numpy as np
import matplotlib.cm as cm
import matplotlib.pyplot as plt
import scipy.interpolate as intpl
from optparse import OptionParser
-from mpl_toolkits.axes_grid1 import ImageGrid
+#from mpl_toolkits.axes_grid1 import ImageGrid
data_dir = "/misc/vlgscratch1/FergusGroup/abf277/hst"
xshift = 0.1
@@ -147,15 +147,15 @@ def main(options, args):
if pixel > 200 and in_bounds:
slice_coords = ((row_num-p/2, row_num+p/2+1), (p_num-p/2, p_num+p/2+1))
#print "CR found %s", pixel
- """
+ """#{{{
fig = plt.figure(1, (1., 3.))
grid = ImageGrid(fig, 111, nrows_ncols = (1, 3), axes_pad=0.1)
im_s = match_file_data[slice_coords[0][0]:slice_coords[0][1],
slice_coords[1][0]:slice_coords[1][1]]
im_u = subtracted_image[slice_coords[0][0]:slice_coords[0][1],
slice_coords[1][0]:slice_coords[1][1]]
- """
+ """#}}}
if sample_count < 1000: #only want to save 1000
print slice_coords, sample_count
@@ -180,9 +180,12 @@ def main(options, args):
cr_coords.append((row_num, p_num))
#Now we want 1000 CR-negative samples -> get the highest pixels that were NOT marked as CRs
+ #first construct a false image with the Cosmic rays subtracted out.
+ #this is essentially the opposite of "subtracted image", thus it is original-subtracted
+ no_cr_img = original_file_data - subtracted_image
pixel_list = []
negative_samples = []
- for row_num, row in enumerate(original_file_data):
+ for row_num, row in enumerate(no_cr_img):
for p_num, pixel in enumerate(row):
if (row_num, p_num) not in cr_coords:
print "add to poss neg", row_num, p_num
@@ -221,8 +224,8 @@ def main(options, args):
print "pickling %s negative samples" % len(negative_samples)
print "pickling %s positive samples" % len(positive_samples)
- pickle.dump(negative_samples, open("%s/samples/negative.p" % data_dir, "wb"))
- pickle.dump(positive_samples, open("%s/samples/positive.p" % data_dir, "wb"))
+ pickle.dump(negative_samples, open("%s/samples/negative_%s.p" % (data_dir, datetime.datetime.now().strftime("%Y-%m-%d")), "wb"))
+ pickle.dump(positive_samples, open("%s/samples/positive_%s.p" % (data_dir,datetime.datetime.now().strftime("%Y-%m-%d")), "wb"))
View
@@ -8,29 +8,34 @@
#imports
import numpy as np
-from random import shuffle
import pickle
+from random import shuffle
+from optparse import OptionParser
#load samples
data_dir = "/misc/vlgscratch1/FergusGroup/abf277/hst"
-neg_samples = pickle.load(open("%s/samples/negative.p" % data_dir,"rb"))
-pos_samples = pickle.load(open("%s/samples/positive.p" % data_dir,"rb"))
-samples = neg_samples + pos_samples
-#sanitize for incorrectly shaped samples
-for s in samples:
- if s['x'].size is not 81:
- print s['x'].size
- print "removing sample : \n", s
- samples.remove(s)
-shuffle(samples)
-num_s = len(samples)
-training_set = samples[:num_s/2]
-remainder = samples[num_s/2:]
-validation_set = remainder[:len(remainder)/6]
-test_set = remainder[len(remainder)/6:]
-def main():
+training_set = []
+def main(options, args):
+ print "options", options
+ sd = options.sd
+ neg_samples = pickle.load(open("%s/samples/negative_%s.p" % (data_dir, sd),"rb"))
+ pos_samples = pickle.load(open("%s/samples/positive_%s.p" % (data_dir, sd),"rb"))
+ samples = neg_samples + pos_samples
+ #sanitize for incorrectly shaped samples
+ for s in samples:
+ if s['x'].size is not 81:
+ print s['x'].size
+ print "removing sample : \n", s
+ samples.remove(s)
+ shuffle(samples)
+ num_s = len(samples)
+ global training_set
+ training_set = samples[:num_s/2]
+ remainder = samples[num_s/2:]
+ validation_set = remainder[:len(remainder)/6]
+ test_set = remainder[len(remainder)/6:]
"""runs nearest neighbors """
k_scores = {}
print "length of validation set : %s" % len(validation_set)
@@ -72,36 +77,11 @@ def nearest_neighbors(sample, k):
if __name__ == '__main__':
#DATA_FILE = open("./data.p", "rb")
#samples = pickle.load(DATA_FILE)
- samples = [#{{{
- (np.array([1,2,3,1,5,6,0,1,204,6,23,6,2,1,4,6,78,2,1,12,4,5,7,3,1,21,344,2,1,2,3,45,6,67]), 1),
- (np.array([1,2,3,1,24,6,0,1,2015,6,23,6,2,1,4,6,78,2,1,12,4,5,7,3,1,21,34,2,1,2,3,5,6,67]), 0),
- (np.array([1,5,3,-1,5,9,0,2,5,31,253,6,2,169,4,6,78,2,1,12,4,5,7,3,1,21,34,2,1,24,3,-8,6,67]), 1),
- (np.array([1,2,3,1,5,-1101,10,1,156,6,23,6,7,881,37,6,78,-8,1,12,4,5,7,3,1,21,34,2,1,2,-24,5,6,67]), 0),
- (np.array([1,2,3,23,5,6,0,1,218,6,23,6,2,1,4,6,78,2,1,12,4,5,27,3,1,21,34,2,1,-94,17,5,6,67]), 1),
- (np.array([11,2,31,1,5,16,0,1,2063,6,20,6,4,4,4,6,78,2,1,12,4,5,7,17,1,21,34,2,1,2,3,5,17,42]), 0),
- (np.array([1,2,19,1,5,24,1,1,2054,6,93,6,2,1,-6,6,78,2,1,12,4,55,7,6,1,21,34,2,14,2,31,5,6,98]), 1),
- (np.array([1,28,3,-15,5,6,30,109,404,6,23,6,2,1,4,6,78,2,1,12,4,42,8,3,11,2,34,2,17,3,-19,-14,6,67]), 0),
- (np.array([1,2,3,1,5,6,0,1,204,6,23,6,2,1,4,6,78,2,1,12,4,5,7,3,1,21,34,2,1,2,3,5,6,67]), 1),
- (np.array([1,2,24,1,-7,6,0,1,204,6,23,6,2,1,4,6,78,2,1,12,4,5,75,3,516,21,34,2,1,2,3,16,-15,67]), 0),
- (np.array([1,2,8,1,-10,14,0,1,143,6,23,6,19,1,4,6,78,11,1,124,4,16,7,3,41,22,34,2,1,-9,3,5,6,51]), 1),
- (np.array([1,2,3,17,5,6,0,1,204,6,23,6,2,1,4,16,78,2,1,12,4,5,7,3,51,21,34,2,1,2,3,-9,6,67]), 0),
- (np.array([1,-7,3,1,-16,6,20,1,204,6,23,6,2,1,4,6,78,2,1,12,4,5,75,3,-10,21,34,2,1,2,-17,5,6,67]), 0),
- (np.array([42,2,3,1,-10,14,0,1,143,6,23,6,19,1,4,6,78,11,1,12,4,16,7,3,1,22,34,2,1,-9,3,5,6,51]), 1),
- (np.array([1,2,109,1,-25,414,0,1,1434,6,23,6,19,1,4,6,78,11,1,-24,4,16,7,3,1,22,34,2,1,-9,3,5,6,51]), 1),
- (np.array([1,3,4,1,-10,14,0,1,205,6,3323,6,19,1,4,6,78,11,26,412,4,156,7,43,1,22,344,2,1,-9,3,5,6,51]), 1),
- (np.array([4,2,63,1,-40,14,0,1,170,6,23,66,619,1,4,6,78,11,-14,12,4,16,7,3,1,22,34,2,1,-9,3,5,6,51]), 1),
- (np.array([-9,62,3,1,-160,14,0,1,143,6,23,6,19,1,4,71,78,610,1,12,-42,166,7,3,1,22,34,2,1,-9,3,5,6,51]), 0),
- (np.array([16,2,3,1,-10,164,-1,-16,108,0,23,6,19,1,4,28,78,11,16,12,16,16,467,3,1,22,34,2,1,-9,3,5,6,51]), 1),
- (np.array([17,7,3,1,-10,146,0,1,143,71,23,-35,19,1,4,66,78,11,16,12,55,16,7,3,1,22,34,2,1,-9,3,5,6,51]), 0),
- (np.array([1,2,19,1,-25,14,0,1,143,6,23,6,19,15,4,6,78,11,1,12,4,16,7,3,1,22,34,2,1,-9,3,5,6,591]), 1),
- (np.array([156,2,19,6,-25,14,0,1,143,56,23,6,19,51,4,6,78,45,1,12,4,16,36,3,1,22,344,42,1,-9,3,5,6,51]), 0),
- (np.array([1,2,19,15,-15,14,0,1,1447,6,23,6,19,1,4,6,78,45,1,12,4,16,7,3,1,22,34,2,1,-9,3,5,6,51]), 1),
- (np.array([51,2,19,1,-254,14,0,1,143,24,28,6,19,1,4,6,78,1144,1,12,4,16,7494,3,1,22,34,2,1,-99,3,5,6,51]), 0),
- (np.array([1,2,19,1,-24,14,0,1,143,6,23,16,19,1,44,6,78,151,1,12,4,16,7,43,1,22,344,2,1,-9,3,95,6,51]), 1),
- (np.array([1,2,19,1,-2,14,0,1,143,46,23,-34,19,1,544,6,785,11,1,12,-8,16,7,3,91,22,34,2,1,-9,3,5,6,51]), 1),
- (np.array([1,2,19,51,-25,145,0,1,143,65,28,6,19,51,4,6,78,1551,1,12,4,16,7,3,1,22,34,2,1,-9,3,95,6,51]), 1)
- ]#}}}
+ parser = OptionParser()
+ parser.add_option("-d", "--sample_date", dest="sd",
+ help ="which date of samples we want to use", default="2012-04-01")
+ (options, args) = parser.parse_args()
print "start"
- main()
+ main(options, args)

0 comments on commit e66b3c2

Please sign in to comment.