Index: /branches/alston_branches/nn.20200129/Pan_STARRS_main.py
===================================================================
--- /branches/alston_branches/nn.20200129/Pan_STARRS_main.py	(revision 41245)
+++ /branches/alston_branches/nn.20200129/Pan_STARRS_main.py	(revision 41245)
@@ -0,0 +1,411 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Jan  6 14:08:50 2020
+
+@author: jacka
+"""
+
+import numpy as np
+from astropy.io import fits
+import matplotlib.pyplot as plt
+import pandas as pd
+import os
+import glob
+import time
+import datetime
+
+def training_set(filenames, savefiles = True):
+    
+    training_data = np.zeros([len(filenames), 2], dtype=np.ndarray)
+    counter1 = 0
+    counter2 = 0
+    counter3 = 0
+    counter4 = 0
+    counter5 = 0
+    counter6 = 0
+    counter7 = 0
+    counter8 = 0
+    counter9 = 0
+    counter10 = 0
+            
+    for i in range(len(filenames)):
+        
+        #opens files
+        hdu_list = fits.open(filenames[i])
+        image_data = np.nan_to_num(np.arcsinh(hdu_list[0].data)/5) #set nans to -ve?
+        
+        #shows image and takes input
+        plt.imshow(image_data, cmap = 'gray')
+        plt.show(block=False)
+        identifier = raw_input()
+        print(i)
+        
+        #only allows certain numbers defined by rules and defines undo function
+        rules = [identifier != '0', identifier != '1', identifier != '2', identifier != '3', identifier != '4', identifier != '5', identifier != '6', identifier != '7', identifier != '8', identifier != '9', identifier != 'b']
+        while identifier == 'b':       
+            new_identifier = raw_input('type correct value')
+            new_output = np.zeros([10,1])
+            new_output[int(new_identifier)] = 1
+            training_data[i-1][1] = new_output
+            os.unlink(filenames[i])
+            
+            if saved_identifier == '0':
+                counter1 -= 1
+                
+            if saved_identifier == '1':
+                counter2 -=1
+                
+            if saved_identifier == '2':
+                counter3 -=1
+                
+            if saved_identifier == '3':
+                counter4 -=1
+                
+            if saved_identifier == '4':
+                counter5 -=1
+                
+            if saved_identifier == '5':
+                counter6 -=1
+                
+            if saved_identifier == '6':
+                counter7 -=1
+                
+            if saved_identifier == '7':
+                counter8 -=1
+                
+            if saved_identifier == '8':
+                counter9 -=1
+                
+            if saved_identifier == '9':
+                counter10 -=1
+            
+            if new_identifier == '0':
+                hdu_list2.writeto('Temp Train/star' + str(counter1) + '.fits', clobber = True)
+                counter1 += 1
+                
+            if new_identifier == '1':
+                hdu_list2.writeto('Temp Train/corrupted_star' + str(counter2) + '.fits', clobber = True)
+                counter2 += 1
+                
+            if new_identifier == '2':
+                hdu_list2.writeto('Temp Train/double_star' + str(counter3) + '.fits', clobber = True)
+                counter3 += 1
+                
+            if new_identifier == '3':
+                hdu_list2.writeto('Temp Train/ghost' + str(counter4) + '.fits', clobber = True)
+                counter4 += 1
+
+            if new_identifier == '4':
+                hdu_list2.writeto('Temp Train/eliptical' + str(counter5) + '.fits', clobber = True)
+                counter5 += 1
+                
+            if new_identifier == '5':
+                hdu_list2.writeto('Temp Train/spiral' + str(counter6) + '.fits', clobber = True)
+                counter6 += 1
+                
+            if new_identifier == '6':
+                hdu_list2.writeto('Temp Train/missing_slice' + str(counter7) + '.fits', clobber = True)
+                counter7 += 1
+
+            if new_identifier == '7':
+                hdu_list2.writeto('Temp Train/hot_pixel' + str(counter8) + '.fits', clobber = True)
+                counter8 += 1
+                
+            if new_identifier == '8':
+                hdu_list2.writeto('Temp Train/mostly_blank' + str(counter9) + '.fits', clobber = True)
+                counter9 += 1
+                
+            if new_identifier == '9':
+                hdu_list2.writeto('Temp Train/other_object' + str(counter10) + '.fits', clobber = True)
+                counter10 += 1      
+            
+            identifier = raw_input('now type current value')
+            
+        while all(rules):
+            identifier = raw_input('wrong format, please try again')
+            rules = [identifier != '0', identifier != '1', identifier != '2', identifier != '3', identifier != '4', identifier != '5', identifier != '6', identifier != '7', identifier != '8', identifier != '9', identifier != 'b']
+        
+    
+        #saves data as fits files
+        if identifier == '0':
+            hdu_list.writeto('Temp Train/star' + str(counter1) + '.fits', clobber = True)
+            counter1 += 1
+            
+        if identifier == '1':
+            hdu_list.writeto('Temp Train/corrupted_star' + str(counter2) + '.fits', clobber = True)
+            counter2 += 1
+            
+        if identifier == '2':
+            hdu_list.writeto('Temp Train/double_star' + str(counter3) + '.fits', clobber = True)
+            counter3 += 1
+            
+        if identifier == '3':
+            hdu_list.writeto('Temp Train/ghost' + str(counter4) + '.fits', clobber = True)
+            counter4 += 1
+
+        if identifier == '4':
+            hdu_list.writeto('Temp Train/eliptical' + str(counter5) + '.fits', clobber = True)
+            counter5 += 1
+            
+        if identifier == '5':
+            hdu_list.writeto('Temp Train/spiral' + str(counter6) + '.fits', clobber = True)
+            counter6 += 1
+            
+        if identifier == '6':
+            hdu_list.writeto('Temp Train/missing_slice' + str(counter7) + '.fits', clobber = True)
+            counter7 += 1
+
+        if identifier == '7':
+            hdu_list.writeto('Temp Train/hot_pixel' + str(counter8) + '.fits', clobber = True)
+            counter8 += 1
+            
+        if identifier == '8':
+            hdu_list.writeto('Temp Train/mostly_blank' + str(counter9) + '.fits', clobber = True)
+            counter9 += 1
+            
+        if identifier == '9':
+            hdu_list.writeto('Temp Train/other_object' + str(counter10) + '.fits', clobber = True)
+            counter10 += 1 
+        
+        #generates output data for neural network
+        output = np.zeros([10,1])
+        output[int(identifier)] = 1
+        
+        #generates input data for neural network
+        flattened_array = np.ravel(image_data)
+        flattened_array = flattened_array.reshape([len(flattened_array), 1])
+        
+        training_data[i][0] = flattened_array
+        training_data[i][1] = output
+        
+        hdu_list2 = hdu_list
+        saved_identifier = identifier
+        #save data to csv file
+    now = str(datetime.datetime.now())
+    np.save('training_data' + now + '.txt', training_data)
+
+    return training_data
+
+def testing_set(filenames):
+    test_data = np.zeros([len(filenames), 2], dtype=np.ndarray)
+    counter1 = 0
+    counter2 = 0
+    counter3 = 0
+    counter4 = 0
+    counter5 = 0
+    counter6 = 0
+    counter7 = 0
+    counter8 = 0
+    counter9 = 0
+    counter10 = 0
+            
+    for i in range(len(filenames)):
+        
+        #opens files
+        hdu_list = fits.open(filenames[i])
+        image_data = np.nan_to_num(np.arcsinh(hdu_list[0].data)/5)
+        
+        #shows image and taks input
+        plt.imshow(image_data, cmap = 'gray')
+        plt.show(block=False)
+        identifier = raw_input()
+        print(i)
+        
+        #only allows certain numbers defined by rules and defines undo function
+        rules = [identifier != '0', identifier != '1', identifier != '2', identifier != '3', identifier != '4', identifier != '5', identifier != '6', identifier != '7', identifier != '8', identifier != '9', identifier != 'b']
+        while identifier == 'b':
+            new_identifier = raw_input('type correct value')
+            test_data[i-1][1] = int(new_identifier)
+            os.unlink(filenames[i])
+            
+            if saved_identifier == '0':
+                counter1 -= 1
+                
+            if saved_identifier == '1':
+                counter2 -=1
+                
+            if saved_identifier == '2':
+                counter3 -=1
+                
+            if saved_identifier == '3':
+                counter4 -=1
+                
+            if saved_identifier == '4':
+                counter5 -=1
+                
+            if saved_identifier == '5':
+                counter6 -=1
+                
+            if saved_identifier == '6':
+                counter7 -=1
+                
+            if saved_identifier == '7':
+                counter8 -=1
+                
+            if saved_identifier == '8':
+                counter9 -=1
+                
+            if saved_identifier == '9':
+                counter10 -=1
+            
+            if new_identifier == '0':
+                hdu_list2.writeto('Temp Test/Temp Test/star' + str(counter1) + '.fits', clobber = True)
+                counter1 += 1
+                
+            if new_identifier == '1':
+                hdu_list2.writeto('Temp Test/corrupted_star' + str(counter2) + '.fits', clobber = True)
+                counter2 += 1
+                
+            if new_identifier == '2':
+                hdu_list2.writeto('Temp Test/double_star' + str(counter3) + '.fits', clobber = True)
+                counter3 += 1
+                
+            if new_identifier == '3':
+                hdu_list2.writeto('Temp Test/ghost' + str(counter4) + '.fits', clobber = True)
+                counter4 += 1
+
+            if new_identifier == '4':
+                hdu_list2.writeto('Temp Test/eliptical' + str(counter5) + '.fits', clobber = True)
+                counter5 += 1
+                
+            if new_identifier == '5':
+                hdu_list2.writeto('Temp Test/spiral' + str(counter6) + '.fits', clobber = True)
+                counter6 += 1
+                
+            if new_identifier == '6':
+                hdu_list2.writeto('Temp Test/missing_slice' + str(counter7) + '.fits', clobber = True)
+                counter7 += 1
+
+            if new_identifier == '7':
+                hdu_list2.writeto('Temp Test/hot_pixel' + str(counter8) + '.fits', clobber = True)
+                counter8 += 1
+                
+            if new_identifier == '8':
+                hdu_list2.writeto('Temp Test/mostly_blank' + str(counter9) + '.fits', clobber = True)
+                counter9 += 1
+                
+            if new_identifier == '9':
+                hdu_list2.writeto('Temp Test/other_object' + str(counter10) + '.fits', clobber = True)
+                counter10 += 1                
+            identifier = raw_input('now type current value')
+            
+        while all(rules):
+            identifier = raw_input('wrong format, please try again')
+            rules = [identifier != '0', identifier != '1', identifier != '2', identifier != '3', identifier != '4', identifier != '5', identifier != '6', identifier != '7', identifier != '8', identifier != '9', identifier != 'b']
+
+
+        #saves data as fits files
+        if identifier == '0':
+            hdu_list.writeto('Temp Test/star' + str(counter1) + '.fits', clobber = True)
+            counter1 += 1
+            
+        if identifier == '1':
+            hdu_list.writeto('Temp Test/corrupted_star' + str(counter2) + '.fits', clobber = True)
+            counter2 += 1
+            
+        if identifier == '2':
+            hdu_list.writeto('Temp Test/double_star' + str(counter3) + '.fits', clobber = True)
+            counter3 += 1
+            
+        if identifier == '3':
+            hdu_list.writeto('Temp Test/ghost' + str(counter4) + '.fits', clobber = True)
+            counter4 += 1
+
+        if identifier == '4':
+            hdu_list.writeto('Temp Test/eliptical' + str(counter5) + '.fits', clobber = True)
+            counter5 += 1
+            
+        if identifier == '5':
+            hdu_list.writeto('Temp Test/spiral' + str(counter6) + '.fits', clobber = True)
+            counter6 += 1
+            
+        if identifier == '6':
+            hdu_list.writeto('Temp Test/missing_slice' + str(counter7) + '.fits', clobber = True)
+            counter7 += 1
+
+        if identifier == '7':
+            hdu_list.writeto('Temp Test/hot_pixel' + str(counter8) + '.fits', clobber = True)
+            counter8 += 1
+            
+        if identifier == '8':
+            hdu_list.writeto('Temp Test/mostly_blank' + str(counter9) + '.fits', clobber = True)
+            counter9 += 1
+            
+        if identifier == '9':
+            hdu_list.writeto('Temp Test/other_object' + str(counter10) + '.fits', clobber = True)
+            counter10 += 1 
+            
+        #generates input data for neural network
+        flattened_array = np.ravel(image_data)
+        flattened_array = flattened_array.reshape([len(flattened_array), 1])
+        
+        test_data[i][0] = flattened_array
+        test_data[i][1] = int(identifier)
+        
+        hdu_list2 = hdu_list
+        saved_identifier = identifier
+        #save data to csv file
+    now = str(datetime.datetime.now())
+    np.save('test_data' + now + '.txt', test_data)
+    
+    return test_data
+
+def filenames(foldername):
+    return glob.glob(foldername + r'/*.fits')
+
+def prelim_junk_removal(filenames):
+    good = []
+    bad = []
+    for i in filenames:
+        hdu_list = fits.open(i)
+        image_data = np.nan_to_num(hdu_list[0].data)
+        if np.mean(image_data) > 0:
+            good.append(image_data)
+            plt.imshow(image_data, cmap = 'gray')
+            plt.show(block=False)
+        else:
+            bad.append(image_data)
+            plt.imshow(image_data, cmap = 'gray')
+            plt.show(block=False)
+    return good, bad
+
+def plot_histogram(filenames, binss = 500):
+    means = []
+    counter = 0
+    for i in filenames:
+            counter +=1
+            print(counter)
+            hdu_list = fits.open(i)
+            image_data = np.nan_to_num(hdu_list[0].data)
+            image_data -= np.min(image_data)
+            mean = np.mean(image_data)
+            means.append(mean)
+            print(mean)
+    means = np.log(means)
+    n, bins, patches = plt.hist(means, binss, facecolor='blue')
+    plt.title('Histogram of Adjusted Pixel Counts')
+    plt.xlabel('log(Adjusted Count)')
+    plt.ylabel('Frequency')
+    plt.show()
+        
+    
+
+def image_displayer(foldername, t = False):
+    filenames = glob.glob(foldername + r'/*.fits')
+    counter = 0
+    for i in filenames:
+        print(i)
+        counter += 1
+        hdu_list = fits.open(i)
+        image_data = hdu_list[0].data
+        plt.imshow(image_data, cmap = 'gray')
+        plt.show(block=False)
+        if t == True:
+            time.sleep(0.5)
+        
+#need to scale data with log10 done
+#exclude well known unsaturated objects m < -13
+#list of galaxies for training set
+#use stamp.dat to generate training set with many stars   
+#make fuctions more versatile done
+