Index: /branches/alston_branches/nn.20200129/Pan_STARRS_brightness_2.py =================================================================== --- /branches/alston_branches/nn.20200129/Pan_STARRS_brightness_2.py (revision 41266) +++ /branches/alston_branches/nn.20200129/Pan_STARRS_brightness_2.py (revision 41266) @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Jan 6 14:08:50 2020 + +@author: jacka +""" + +import numpy as np +from astropy.io import fits +import matplotlib.pyplot as plt +import pandas as pd +import os +import glob +import time +import datetime +import network + +def training_set(filenames1, filenames2, filenames3, savefiles = True): + + training_data = np.zeros([len(filenames1)+len(filenames2)+len(filenames3), 2], dtype=np.ndarray) + + for i in range(len(filenames1)): + hdu_list = fits.open(filenames1[i]) + image_data = np.nan_to_num(np.arcsinh(hdu_list[0].data)/5) #set nans to -ve? + flattened_array = np.ravel(image_data) + flattened_array = flattened_array.reshape([len(flattened_array), 1]) + output = np.zeros([3,1]) + output[0] = 1 + training_data[i][0] = flattened_array + training_data[i][1] = output + + for i in range(len(filenames2)): + hdu_list = fits.open(filenames2[i]) + image_data = np.nan_to_num(np.arcsinh(hdu_list[0].data)/5) #set nans to -ve? + flattened_array = np.ravel(image_data) + flattened_array = flattened_array.reshape([len(flattened_array), 1]) + output = np.zeros([3,1]) + output[1] = 1 + training_data[i+len(filenames1)][0] = flattened_array + training_data[i+len(filenames1)][1] = output + + for i in range(len(filenames3)): + hdu_list = fits.open(filenames3[i]) + image_data = np.nan_to_num(np.arcsinh(hdu_list[0].data)/5) #set nans to -ve? + flattened_array = np.ravel(image_data) + flattened_array = flattened_array.reshape([len(flattened_array), 1]) + output = np.zeros([3,1]) + output[2] = 1 + training_data[i+len(filenames1)+len(filenames2)][0] = flattened_array + training_data[i+len(filenames1)+len(filenames2)][1] = output + + now = str(datetime.datetime.now()) + np.save('training_data' + now + '.txt', training_data) + + return training_data + +def testing_set(filenames1, filenames2, filenames3): + test_data = np.zeros([len(filenames1)+len(filenames2)+len(filenames3), 2], dtype=np.ndarray) + + for i in range(len(filenames1)): + hdu_list = fits.open(filenames1[i]) + image_data = np.nan_to_num(np.arcsinh(hdu_list[0].data)/5) #set nans to -ve? + flattened_array = np.ravel(image_data) + flattened_array = flattened_array.reshape([len(flattened_array), 1]) + test_data[i][0] = flattened_array + test_data[i][1] = 0 + + for i in range(len(filenames2)): + hdu_list = fits.open(filenames2[i]) + image_data = np.nan_to_num(np.arcsinh(hdu_list[0].data)/5) #set nans to -ve? + flattened_array = np.ravel(image_data) + flattened_array = flattened_array.reshape([len(flattened_array), 1]) + test_data[i+len(filenames1)][0] = flattened_array + test_data[i+len(filenames1)][1] = 1 + + for i in range(len(filenames3)): + hdu_list = fits.open(filenames3[i]) + image_data = np.nan_to_num(np.arcsinh(hdu_list[0].data)/5) #set nans to -ve? + flattened_array = np.ravel(image_data) + flattened_array = flattened_array.reshape([len(flattened_array), 1]) + test_data[i+len(filenames1)+len(filenames2)][0] = flattened_array + test_data[i+len(filenames1)+len(filenames2)][1] = 2 + now = str(datetime.datetime.now()) + np.save('test_data' + now + '.txt', test_data) + + return test_data + +def image_sorter(filenames, destination_folder, numbers, n): + counter = 0 + for i in range(len(filenames)): + counter += 1 + print(str(counter) + '/' + str(len(filenames))) + hdu_list = fits.open(filenames[i]) + image_data = np.nan_to_num(np.arcsinh(hdu_list[0].data)/5) + plt.imshow(image_data, cmap = 'gray') + plt.show(block=False) + identifier = raw_input() + if identifier == '1': + hdu_list.writeto(destination_folder + '/star/star' + numbers[i] + 'xy' + str(n) + '.fits', clobber = True) + if identifier == '2': + hdu_list.writeto(destination_folder + '/other/other' + numbers[i] + 'xy' + str(n) + '.fits', clobber = True) + +def filenames(foldername): + return glob.glob(foldername + r'/*.fits') + +def plot_histogram(filenames, binss = 500): + means = [] + counter = 0 + for i in filenames: + counter +=1 + print(counter) + hdu_list = fits.open(i) + image_data = np.nan_to_num(hdu_list[0].data) + image_data -= np.min(image_data) + mean = np.mean(image_data) + means.append(mean) + print(mean) + means = np.log(means) + n, bins, patches = plt.hist(means, binss, facecolor='blue') + plt.title('Histogram of Adjusted Pixel Counts') + plt.xlabel('log(Adjusted Count)') + plt.ylabel('Frequency') + plt.show() + + + +def image_displayer(filenames, t = False): + #filenames = glob.glob(foldername + r'/*.fits') + counter = 0 + for i in filenames: + print(i) + counter += 1 + hdu_list = fits.open(i) + image_data = hdu_list[0].data + plt.imshow(np.log(image_data), cmap = 'gray') + plt.show(block=False) + if t == True: + time.sleep(0.5) + +def data_extractor(filepath): + hdu_list = fits.open(filepath) + data = [hdu_list[3*i+3].data for i in range(len(hdu_list)/3)] + names = [hdu_list[3*i+3].header[-1] for i in range(len(hdu_list)/3)] + empty_array = np.ndarray.tolist(np.zeros(len(data))) + for i in range(len(data)): + element = np.ndarray.tolist(np.zeros(8)) + psf_perfect = [hdu_list[3*i+3].data[j][34] for j in range(len(hdu_list[3*i+3].data))] + mask_present = [hdu_list[3*i+3].data[j][53] & 0x2000 for j in range(len(hdu_list[3*i+3].data))] #whack hexadecimal shit + kron_flux = [hdu_list[3*i+3].data[j][46] for j in range(len(hdu_list[3*i+3].data))] + element[0] = names[i] + element[1] = [hdu_list[3*i+3].data[j][0] for j in range(len(hdu_list[3*i+3].data)) if psf_perfect[j] > 0.9995 and mask_present[j] == 0 and str(kron_flux[j]) != 'nan'] #object id + element[2] = [hdu_list[3*i+3].data[j][7] for j in range(len(hdu_list[3*i+3].data)) if psf_perfect[j] > 0.9995 and mask_present[j] == 0 and str(kron_flux[j]) != 'nan'] #inst mag + element[3] = [hdu_list[3*i+3].data[j][8] for j in range(len(hdu_list[3*i+3].data)) if psf_perfect[j] > 0.9995 and mask_present[j] == 0 and str(kron_flux[j]) != 'nan'] #inst mag sig + element[4] = [hdu_list[3*i+3].data[j][33] for j in range(len(hdu_list[3*i+3].data)) if psf_perfect[j] > 0.9995 and mask_present[j] == 0 and str(kron_flux[j]) != 'nan'] #psf qf + element[5] = [hdu_list[3*i+3].data[j][34] for j in range(len(hdu_list[3*i+3].data)) if psf_perfect[j] > 0.9995 and mask_present[j] == 0 and str(kron_flux[j]) != 'nan'] #psf qf perfect + element[6] = [-2.5*np.log10(hdu_list[3*i+3].data[j][46]) for j in range(len(hdu_list[3*i+3].data)) if psf_perfect[j] > 0.9995 and mask_present[j] == 0 and str(kron_flux[j]) != 'nan'] #kron flux + element[7] = [hdu_list[3*i+3].data[j][53] for j in range(len(hdu_list[3*i+3].data)) if psf_perfect[j] > 0.9995 and mask_present[j] == 0 and str(kron_flux[j]) != 'nan'] #flags + empty_array[i] = element + return empty_array + +def file_displayer(data): + for i in data: + folder_name = r'/home/jalston/Documents/Research Project/Raw Data/128.171.123.254:22281/ps1.nng.20191219/o8836g0103o.1563924/o8836g0103o.1563924.' + i[0][0:4] + '.stamps/' + for j in range(len(i[2])): + print(i[0], i[1][j], i[2][j], i[3][j], i[4][j], i[5][j], i[6][j], i[7][j]) + if i[1][j] < 10: + filename = 'obj.0000' + str(i[1][j]) + '.fits' + elif i[1][j] < 100: + filename = 'obj.000' + str(i[1][j]) + '.fits' + elif i[1][j] < 1000: + filename = 'obj.00' + str(i[1][j]) + '.fits' + else: + filename = 'obj.0' + str(i[1][j]) + '.fits' + path = folder_name + filename + hdu_list = fits.open(path) + data = hdu_list[0].data + plt.imshow(np.log(data), cmap = 'gray') + plt.show(block=False) + print(np.argmax(data),np.argmin(data)) + stopper = str(raw_input()) + +def file_saver(data): + for i in data: + folder_name = r'/home/jalston/Documents/Research Project/Raw Data/128.171.123.254:22281/ps1.nng.20191219/o8836g0103o.1563924/o8836g0103o.1563924.' + i[0][0:4] + '.stamps/' + for j in range(len(i[2])): + random_num = random.randint(0, 10) + if i[1][j] < 10: + filename = 'obj.0000' + str(i[1][j]) + '.fits' + elif i[1][j] < 100: + filename = 'obj.000' + str(i[1][j]) + '.fits' + elif i[1][j] < 1000: + filename = 'obj.00' + str(i[1][j]) + '.fits' + else: + filename = 'obj.0' + str(i[1][j]) + '.fits' + path = folder_name + filename + hdu_list = fits.open(path) + image = hdu_list[0].data + snr = i[3][j] + if random_num == 5: + if 0 < snr < 0.01: + hdu_list.writeto(r'/home/jalston/Documents/Research Project/Main code/brightness/testing/good/' + i[0][0:4] + filename) + elif 0.01 < snr < 0.03: + hdu_list.writeto(r'/home/jalston/Documents/Research Project/Main code/brightness/testing/mid/' + i[0][0:4] + filename) + elif 0.03 < snr < 0.1: + hdu_list.writeto(r'/home/jalston/Documents/Research Project/Main code/brightness/testing/bad/' + i[0][0:4] + filename) + else: + if 0 < snr < 0.01: + hdu_list.writeto(r'/home/jalston/Documents/Research Project/Main code/brightness/training/good/' + i[0][0:4] + filename) + elif 0.01 < snr < 0.03: + hdu_list.writeto(r'/home/jalston/Documents/Research Project/Main code/brightness/training/mid/' + i[0][0:4] + filename) + elif 0.03 < snr < 0.1: + hdu_list.writeto(r'/home/jalston/Documents/Research Project/Main code/brightness/training/bad/' + i[0][0:4] + filename) + print(snr) + +def network_results(smf_data, filenames, testing_data, network): + identifiers = [[i[-18:-14], i[-10:-5]] for i in filenames] + index_1 = [] + index_2 = [] + counter = 1 + for k in identifiers: + for i in range(len(smf_data)): + for j in range(len(smf_data[i][1])): + if k[0] == smf_data[i][0][0:4] and int(k[1]) == smf_data[i][1][j]: + index_1.append(i) + index_2.append(j) + sig = [smf_data[index_1[i]][3][index_2[i]] for i in range(len(index_2))] + outputs = [net.feedforward(i[0]) for i in testing_data] + return sig, outputs + + +#need to scale data with log10 done +#exclude well known unsaturated objects m < -13 +#list of galaxies for training set +#use stamp.dat to generate training set with many stars +#make fuctions more versatile done +