Commit d8acbc79 authored by jonas's avatar jonas
Browse files

imgdirx bool + scaling str dataf

parent ae3bc606
...@@ -39,14 +39,17 @@ if __name__ == "__main__": ...@@ -39,14 +39,17 @@ if __name__ == "__main__":
path = 'stp-144/' path = 'stp-144/'
files = ['solo_L0_phi-hrt-ilam_20210421T120003_V202106080929C_0144210101.fits', 'solo_L0_phi-hrt-ilam_20210424T120003_V202106141014C_0144240101.fits', files = ['solo_L0_phi-hrt-ilam_20201117T170209_V202106300922C_0051170001.fits']
"""
#['solo_L0_phi-hrt-ilam_20210421T120003_V202106080929C_0144210101.fits', 'solo_L0_phi-hrt-ilam_20210424T120003_V202106141014C_0144240101.fits',
'solo_L0_phi-hrt-ilam_20210425T120002_V202106141020C_0144250101.fits', 'solo_L0_phi-hrt-ilam_20210426T120002_V202106162118C_0144260101.fits', 'solo_L0_phi-hrt-ilam_20210425T120002_V202106141020C_0144250101.fits', 'solo_L0_phi-hrt-ilam_20210426T120002_V202106162118C_0144260101.fits',
'solo_L0_phi-hrt-ilam_20210427T120002_V202106162052C_0144270101.fits', 'solo_L0_phi-hrt-ilam_20210427T120002_V202106171444C_0144270101.fits', 'solo_L0_phi-hrt-ilam_20210427T120002_V202106162052C_0144270101.fits', 'solo_L0_phi-hrt-ilam_20210427T120002_V202106171444C_0144270101.fits',
'solo_L0_phi-hrt-ilam_20210427T120002_V202106171517C_0144270101.fits'] 'solo_L0_phi-hrt-ilam_20210427T120002_V202106171517C_0144270101.fits']
"""
url = 'https://www2.mps.mpg.de/services/proton/phi/fm/attic/' url = 'https://www2.mps.mpg.de/services/proton/phi/fm/attic/'
download_folder = '../fits_files/' download_folder = '/data/slam/home/sinjan/fits_files/'
for file in files: for file in files:
......
...@@ -33,6 +33,6 @@ for file in text_split: ...@@ -33,6 +33,6 @@ for file in text_split:
else: else:
subprocess.call(["wget", "--user", f"{username}","--password", f"{password}", f"https://www2.mps.mpg.de/services/proton/phi/imgdb/{file}"]) subprocess.call(["wget", "--user", f"{username}","--password", f"{password}", f"https://www2.mps.mpg.de/services/proton/phi/imgdb/{file}"])
subprocess.call(["gunzip", f"{file}"]) subprocess.call(["gunzip", f"{file}"])
subprocess.call(["chmod", "a+r", f"{file}"]) subprocess.call(["chmod", "a+r", f"{file.split('.gz')[0]}"])
print("Download and unpacking complete") print("Download and unpacking complete")
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T112613_V202106111548C_0024150001.fits.gz l1/2020-11-17/solo_L1_phi-hrt-ilam_20201117T170209_V202107090920C_0051170001.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T114945_V202106111548C_0024150005.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T125349_V202106111549C_0024150006.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T125349_V202106111549C_0064151006.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T132752_V202106111549C_0024150010.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T151154_V202106111549C_0024150002.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T151455_V202106111549C_0024150003.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T163923_V202106111549C_0024150004.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T165722_V202106111549C_0024150012.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T174538_V202106111549C_0024150020.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T182538_V202106111550C_0024150021.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T190538_V202106111550C_0024150022.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T194538_V202106111550C_0024150023.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T202538_V202106111551C_0024150024.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T210538_V202106111551C_0024150025.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T214539_V202106111551C_0024150026.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T222538_V202106111552C_0024150027.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T230538_V202106111552C_0024150028.fits.gz
l1/2020-04-17/solo_L1_phi-hrt-ilam_20200417T234538_V202106111552C_0024150029.fits.gz
...@@ -11,24 +11,24 @@ from utils import * ...@@ -11,24 +11,24 @@ from utils import *
def get_data(path, scaling = True, bit_convert_scale = True): def get_data(path, scaling = True, bit_convert_scale = True):
"""load science data from path""" """load science data from path"""
try: #try:
data, header = load_fits(path) data, header = load_fits(path)
if bit_convert_scale: #conversion from 24.8bit to 32bit if bit_convert_scale: #conversion from 24.8bit to 32bit
data /= 256. data /= 256.
if scaling: if scaling:
accu = header['ACCACCUM']*header['ACCROWIT']*header['ACCCOLIT'] #getting the number of accu from header accu = header['ACCACCUM']*header['ACCROWIT']*header['ACCCOLIT'] #getting the number of accu from header
data /= accu data /= accu
printc(f"Dividing by number of accumulations: {accu}",color=bcolors.OKGREEN) printc(f"Dividing by number of accumulations: {accu}",color=bcolors.OKGREEN)
return data, header return data, header
except Exception: #except Exception:
printc("ERROR, Unable to open fits file: {}",path,color=bcolors.FAIL) # printc("ERROR, Unable to open fits file: {}",path,color=bcolors.FAIL)
...@@ -192,17 +192,14 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -192,17 +192,14 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
wve_axis_arr[scan], voltagesData_arr[scan], tuning_constant_arr[scan], cpos_arr[scan] = fits_get_sampling(data_f[scan],verbose = True) wve_axis_arr[scan], voltagesData_arr[scan], tuning_constant_arr[scan], cpos_arr[scan] = fits_get_sampling(data_f[scan],verbose = True)
if scale_data: if scale_data: #not for commissioning data
#if hdr_arr[scan]['BITPIX'] == 16:
# print(f"This scan: {data_f[scan]} has a bits per pixel of: 16 \nPerforming the extra scaling")
data_arr[scan] *= 81920/127 #conversion factor if 16 bits data_arr[scan] *= 81920/127 #conversion factor if 16 bits
if 'IMGDIRX' in hdr_arr[scan] and hdr_arr[scan]['IMGDIRX'] == 'YES': if 'IMGDIRX' in hdr_arr[scan] and hdr_arr[scan]['IMGDIRX'] == 'YES':
print("This scan has been flipped in the Y axis to conform to orientation standards.") print("This scan has been flipped in the Y axis to conform to orientation standards.")
#-------- #--------
# test if the scans have different sizes # test if the scans have different sizes
#-------- #--------
...@@ -256,6 +253,7 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -256,6 +253,7 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
# test if the scans have same IMGDIRX keyword # test if the scans have same IMGDIRX keyword
#-------- #--------
if all('IMGDIRX' in hdr for hdr in hdr_arr): if all('IMGDIRX' in hdr for hdr in hdr_arr):
header_imgdirx_exists = True
first_imgdirx = hdr_arr[0]['IMGDIRX'] first_imgdirx = hdr_arr[0]['IMGDIRX']
result = all(hdr['IMGDIRX'] == first_imgdirx for hdr in hdr_arr) result = all(hdr['IMGDIRX'] == first_imgdirx for hdr in hdr_arr)
if (result): if (result):
...@@ -265,7 +263,8 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -265,7 +263,8 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
print("The scans have different IMGDIRX keywords! Please fix \n Ending Process") print("The scans have different IMGDIRX keywords! Please fix \n Ending Process")
exit() exit()
else: else:
print("Not all the scans contain the 'IMGDIRX' keyword! Assuming not flipped") header_imgdirx_exists = False
print("Not all the scans contain the 'IMGDIRX' keyword! Assuming all not flipped - Proceed with caution")
elif isinstance(data_f, str): elif isinstance(data_f, str):
#case when data f is just one file #case when data f is just one file
...@@ -273,11 +272,9 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -273,11 +272,9 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
data = np.expand_dims(data, axis = -1) #so that it has the same dimensions as several scans data = np.expand_dims(data, axis = -1) #so that it has the same dimensions as several scans
data = np.moveaxis(data, 0, -2) #so that it is [y,x,24,1] data = np.moveaxis(data, 0, -2) #so that it is [y,x,24,1]
if header['BITPIX'] == 16: if scale_data: #not for April commissioning phase data
print(f"This scan: {data_f} has a bits per pixel is: 16 \n Performing the extra scaling")
data *= 81920/127 #conversion factor if 16 bits data *= 81920/127
print(f"Data shape is {data.shape}") print(f"Data shape is {data.shape}")
...@@ -301,9 +298,13 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -301,9 +298,13 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
print(f"Data PMP Set Point Temperature is {pmp_temp}") print(f"Data PMP Set Point Temperature is {pmp_temp}")
if 'IMGDIRX' in header: if 'IMGDIRX' in header:
header_imgdirx_exists = True
imgdirx_flipped = str(header['IMGDIRX']) imgdirx_flipped = str(header['IMGDIRX'])
if imgdirx_flipped == 'YES':
print("This scan has been flipped in the Y axis to conform to orientation standards.")
else: else:
header_imgdirx_exists = False
print("Scan header does not contain the 'IMGDIRX' keyword! Assuming not flipped") print("Scan header does not contain the 'IMGDIRX' keyword! Assuming not flipped")
...@@ -368,19 +369,21 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -368,19 +369,21 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
print(f"The continuum position of the flat field is at {cpos_f} index position") print(f"The continuum position of the flat field is at {cpos_f} index position")
if flat_f[-62:] == 'solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits': if cpos_f != cpos_arr[0]:
print("The flat field continuum position is not the same as the data, trying to correct.")
print("This flat needs to be rolled")
flat = np.roll(flat, 1, axis = -1)
cpos_f = cpos_arr[0] if cpos_f == 5 and cpos_arr[0] == 0:
#if continuum wavelength of the flat is not the same as the data, attempt to roll flat = np.roll(flat, 1, axis = -1)
if cpos_f != cpos_arr[0]:
print("The flat field continuum position is not the same as the data, please check your input data. \n Ending Process")
exit() elif cpos_f == 0 and cpos_arr[0] == 5:
flat = np.roll(flat, -1, axis = -1)
else:
print("Cannot reconcile the different continuum positions. \n Ending Process.")
exit()
flat_pmp_temp = str(header_flat['HPMPTSP1']) flat_pmp_temp = str(header_flat['HPMPTSP1'])
...@@ -444,8 +447,15 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -444,8 +447,15 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
start_time = time.time() start_time = time.time()
flat -= dark[..., np.newaxis, np.newaxis] flat -= dark[..., np.newaxis, np.newaxis]
if imgdirx_flipped == 'YES':
dark_copy = np.copy(dark)
dark_copy = dark_copy[:,::-1]
data -= dark_copy[start_row:start_row + data_size[0],start_col:start_col + data_size[1], np.newaxis, np.newaxis, np.newaxis]
data -= dark[start_row:start_row + data_size[0],start_col:start_col + data_size[1], np.newaxis, np.newaxis, np.newaxis] else:
data -= dark[start_row:start_row + data_size[0],start_col:start_col + data_size[1], np.newaxis, np.newaxis, np.newaxis]
printc('--------------------------------------------------------------',bcolors.OKGREEN) printc('--------------------------------------------------------------',bcolors.OKGREEN)
printc(f"------------- Dark Field correction time: {np.round(time.time() - start_time,3)} seconds",bcolors.OKGREEN) printc(f"------------- Dark Field correction time: {np.round(time.time() - start_time,3)} seconds",bcolors.OKGREEN)
...@@ -490,7 +500,7 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -490,7 +500,7 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
elif cpos_arr[0] == 5: elif cpos_arr[0] == 5:
wv_range = range(5) wv_range = range(5)
for pol in range(3,4): for pol in range(1,4):
for wv in wv_range: #not the continuum for wv in wv_range: #not the continuum
...@@ -552,6 +562,10 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -552,6 +562,10 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
start_time = time.time() start_time = time.time()
if header_imgdirx_exists:
if imgdirx_flipped == 'YES': #should be YES for any L1 data, but mistake in processing software
flat = flat[:,::-1, :, :]
try: try:
if flat_states == 6: if flat_states == 6:
...@@ -661,8 +675,9 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = ...@@ -661,8 +675,9 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
field_stop = np.where(field_stop > 0,1,0) field_stop = np.where(field_stop > 0,1,0)
if imgdirx_flipped == 'NO': if header_imgdirx_exists:
field_stop = field_stop[:,::-1] if imgdirx_flipped == 'YES': #should be YES for any L1 data, but mistake in processing software
field_stop = field_stop[:,::-1] #also need to flip the flat data after dark correction
data *= field_stop[start_row:start_row + data_size[0],start_col:start_col + data_size[1],np.newaxis, np.newaxis, np.newaxis] data *= field_stop[start_row:start_row + data_size[0],start_col:start_col + data_size[1],np.newaxis, np.newaxis, np.newaxis]
......
from hrt_pipe import phihrt_pipe from hrt_pipe import phihrt_pipe
import numpy as np import numpy as np
sciencedata_fits_filenames = ['solo_L1_phi-hrt-ilam_20200528T171109_V202106111600C_0045140102.fits'] sciencedata_fits_filenames = '/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20201117T170209_V202107090920C_0051170001.fits'#solo_L1_phi-hrt-ilam_20201117T170209_V202107060746C_0051170001.fits'#['solo_L1_phi-hrt-ilam_20200528T171109_V202106111600C_0045140102.fits']
# ['solo_L0_phi-hrt-ilam_20210421T120003_V202106080929C_0144210101.fits', 'solo_L0_phi-hrt-ilam_20210424T120003_V202106141014C_0144240101.fits', # ['solo_L0_phi-hrt-ilam_20210421T120003_V202106080929C_0144210101.fits', 'solo_L0_phi-hrt-ilam_20210424T120003_V202106141014C_0144240101.fits',
# 'solo_L0_phi-hrt-ilam_20210425T120002_V202106141020C_0144250101.fits', 'solo_L0_phi-hrt-ilam_20210426T120002_V202106162118C_0144260101.fits', # 'solo_L0_phi-hrt-ilam_20210425T120002_V202106141020C_0144250101.fits', 'solo_L0_phi-hrt-ilam_20210426T120002_V202106162118C_0144260101.fits',
...@@ -12,10 +12,10 @@ sciencedata_fits_filenames = ['solo_L1_phi-hrt-ilam_20200528T171109_V20210611160 ...@@ -12,10 +12,10 @@ sciencedata_fits_filenames = ['solo_L1_phi-hrt-ilam_20200528T171109_V20210611160
#sciencedata_fits_filenames = ['solo_L0_phi-hrt-ilam_0667414748_V202103221851C_0142230201.fits'] #sciencedata_fits_filenames = ['solo_L0_phi-hrt-ilam_0667414748_V202103221851C_0142230201.fits']
#sciencedata_fits_filenames = ['solo_L0_phi-hrt-ilam_0667414905_V202103221851C_0142230602.fits', 'solo_L0_phi-hrt-ilam_0667415054_V202103221851C_0142230603.fits', 'solo_L0_phi-hrt-ilam_0667415205_V202103221851C_0142230604.fits', 'solo_L0_phi-hrt-ilam_0667415354_V202103221851C_0142230605.fits', 'solo_L0_phi-hrt-ilam_0667415505_V202103221851C_0142230606.fits', 'solo_L0_phi-hrt-ilam_0667415654_V202103221851C_0142230607.fits', 'solo_L0_phi-hrt-ilam_0667415805_V202103221851C_0142230608.fits'] #sciencedata_fits_filenames = ['solo_L0_phi-hrt-ilam_0667414905_V202103221851C_0142230602.fits', 'solo_L0_phi-hrt-ilam_0667415054_V202103221851C_0142230603.fits', 'solo_L0_phi-hrt-ilam_0667415205_V202103221851C_0142230604.fits', 'solo_L0_phi-hrt-ilam_0667415354_V202103221851C_0142230605.fits', 'solo_L0_phi-hrt-ilam_0667415505_V202103221851C_0142230606.fits', 'solo_L0_phi-hrt-ilam_0667415654_V202103221851C_0142230607.fits', 'solo_L0_phi-hrt-ilam_0667415805_V202103221851C_0142230608.fits']
flatfield_fits_filename = '/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200417T174538_V202106111549C_0024150020.fits'#solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits' flatfield_fits_filename = '/data/slam/home/sinjan/fits_files/solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits' #solo_L0_phi-hrt-ilam_20200417T174529_V202004241516C_0024150020000.fits'#solo_L1_phi-hrt-ilam_20200417T174538_V202106111549C_0024150020.fits'#solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits'
darkfield_fits_filename = '../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits' darkfield_fits_filename = '../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits'
sciencedata_fits_filenames = ['/data/slam/home/sinjan/fits_files/' + i for i in sciencedata_fits_filenames] #sciencedata_fits_filenames = ['/data/slam/home/sinjan/fits_files/' + i for i in sciencedata_fits_filenames]
prefilter_f = '../fits_files/fitted_prefilter.fits' prefilter_f = '../fits_files/fitted_prefilter.fits'
...@@ -45,10 +45,10 @@ c_talk_params[1,0] = q_int ...@@ -45,10 +45,10 @@ c_talk_params[1,0] = q_int
c_talk_params[1,1] = u_int c_talk_params[1,1] = u_int
c_talk_params[1,2] = v_int c_talk_params[1,2] = v_int
phihrt_pipe(sciencedata_fits_filenames, flat_f = flatfield_fits_filename, dark_f = darkfield_fits_filename, scale_data = True, bit_flat = False, norm_f = True, clean_f = False, phihrt_pipe(sciencedata_fits_filenames, flat_f = flatfield_fits_filename, dark_f = darkfield_fits_filename, scale_data = True, bit_flat = True, norm_f = True, clean_f = True,
sigma = 59, flat_states = 24, norm_stokes = True, prefilter_f = None, dark_c = True, flat_c = True, sigma = 59, flat_states = 24, norm_stokes = True, prefilter_f = None, dark_c = True, flat_c = True,
fs_c = True, demod = True, ctalk_params = c_talk_params, ItoQUV = True, out_demod_file = True, fs_c = True, demod = True, ctalk_params = c_talk_params, ItoQUV = False, out_demod_file = True,
out_dir = '/data/slam/home/sinjan/hrt_pipe_results/may_2020_ctalk/', rte = 'False', out_rte_filename='') out_dir = '/data/slam/home/sinjan/hrt_pipe_results/nov_17_2020_imgdb_L1_test/', rte = 'False', out_rte_filename='')
""" """
Input Parameters: Input Parameters:
---------- ----------
......
...@@ -53,29 +53,29 @@ def fits_get_sampling(file,verbose = False): ...@@ -53,29 +53,29 @@ def fits_get_sampling(file,verbose = False):
''' '''
#print('-- Obtaining voltages......') #print('-- Obtaining voltages......')
fg_head = 3 fg_head = 3
try: #try:
with fits.open(file) as hdu_list: with fits.open(file) as hdu_list:
header = hdu_list[fg_head].data header = hdu_list[fg_head].data
j = 0 j = 0
dummy = 0 dummy = 0
voltagesData = np.zeros((6)) voltagesData = np.zeros((6))
tunning_constant = 0.0 tunning_constant = 0.0
ref_wavelength = 0.0 ref_wavelength = 0.0
for v in header: for v in header:
#print(v) #print(v)
if (j < 6): if (j < 6):
if tunning_constant == 0: if tunning_constant == 0:
tunning_constant = float(v[4])/1e9 tunning_constant = float(v[4])/1e9
if ref_wavelength == 0: if ref_wavelength == 0:
ref_wavelength = float(v[5])/1e3 ref_wavelength = float(v[5])/1e3
if np.abs(np.abs(v[2]) - np.abs(dummy)) > 5: #check that the next voltage is more than 5 from the previous, as voltages change slightly if np.abs(np.abs(v[2]) - np.abs(dummy)) > 5: #check that the next voltage is more than 5 from the previous, as voltages change slightly
#print(dummy, v[2]) #print(dummy, v[2])
voltagesData[j] = float(v[2]) voltagesData[j] = float(v[2])
dummy = voltagesData[j] dummy = voltagesData[j]
j += 1 j += 1
except Exception: #except Exception:
print("Unable to open fits file: {}",file) # print("Unable to open fits file: {}",file)
#print(voltagesData) #print(voltagesData)
d1 = voltagesData[0] - voltagesData[1] d1 = voltagesData[0] - voltagesData[1]
d2 = voltagesData[4] - voltagesData[5] d2 = voltagesData[4] - voltagesData[5]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment