Commit c2681504 authored by jonas's avatar jonas
Browse files

april 2020 L1 run + nov 2020 + feb2021 no us

parent 060f3bab
......@@ -19,3 +19,4 @@ run.py
P-MILOS/*
utils.py
file_names.txt
april_2020_flat_us_sigma.py
import json
"""
#April 2020
# CAN also do the L1 files - but must set bit_convert_scale to false, and scale data to False for the flat fields, and presumably same for the input data
science_april = ['solo_L0_phi-hrt-ilam_20200420T141752_V202004221450C_0024160030000.fits',
'solo_L0_phi-hrt-ilam_20200420T142022_V202004221451C_0024160031000.fits',
'solo_L0_phi-hrt-ilam_20200420T142252_V202004221452C_0024160032000.fits',
......@@ -23,4 +29,78 @@ input_dict = {
'dark_f' : darkfield_fits_filename
}
json.dump(input_dict, open(f"./input_jsons/april_2020.txt", "w"))
\ No newline at end of file
json.dump(input_dict, open(f"./input_jsons/april_2020.txt", "w"))
"""
"""
#April 2020 L1
# CAN also do the L1 files - but must set bit_convert_scale to false, and scale data to False for the flat fields, and presumably same for the input data
science_april = ['solo_L1_phi-hrt-ilam_20200420T141802_V202107221036C_0024160030.fits',
'solo_L1_phi-hrt-ilam_20200420T142032_V202107221036C_0024160031.fits',
'solo_L1_phi-hrt-ilam_20200420T142302_V202107221036C_0024160032.fits',
'solo_L1_phi-hrt-ilam_20200420T142532_V202107221037C_0024160033.fits',
'solo_L1_phi-hrt-ilam_20200420T142803_V202107221037C_0024160034.fits',
'solo_L1_phi-hrt-ilam_20200420T143033_V202107221037C_0024160035.fits',
'solo_L1_phi-hrt-ilam_20200420T143303_V202107221037C_0024160036.fits',
'solo_L1_phi-hrt-ilam_20200420T143533_V202107221037C_0024160037.fits',
'solo_L1_phi-hrt-ilam_20200420T143803_V202107221037C_0024160038.fits',
'solo_L1_phi-hrt-ilam_20200420T144033_V202107221038C_0024160039.fits']
flatfield_fits_filename = '/data/slam/home/sinjan/fits_files/april_avgd_2020_flat.fits' #solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits'
darkfield_fits_filename = '../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits'
science_april = ['/data/slam/home/sinjan/fits_files/' + i for i in science_april]
input_dict = {
'data_f': science_april,
'flat_f' : flatfield_fits_filename,
'dark_f' : darkfield_fits_filename
}
json.dump(input_dict, open(f"./input_jsons/april_2020_L1.txt", "w"))
"""
"""
#Nov 17 2020 L1
# CAN also do the L1 files - but must set bit_convert_scale to false, and scale data to False for the flat fields, and presumably same for the input data
science_nov = ['solo_L1_phi-hrt-ilam_20201117T170209_V202107060747C_0051170001.fits']
flatfield_fits_filename = '/data/slam/home/sinjan/fits_files/april_avgd_2020_flat.fits' #solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits'
darkfield_fits_filename = '../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits'
science_april = ['/data/slam/home/sinjan/fits_files/' + i for i in science_nov]
input_dict = {
'data_f': science_nov,
'flat_f' : flatfield_fits_filename,
'dark_f' : darkfield_fits_filename
}
json.dump(input_dict, open(f"./input_jsons/nov_2020_L1.txt", "w"))
"""
#Feb 2021 L1
science_feb = ['solo_L1_phi-hrt-ilam_20210223T170002_V202107221048C_0142230201.fits']
flatfield_fits_filename = '/data/slam/home/sinjan/fits_files/solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits' #solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits'
darkfield_fits_filename = '../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits'
science_feb = ['/data/slam/home/sinjan/fits_files/' + i for i in science_feb]
input_dict = {
'data_f': science_feb,
'flat_f' : flatfield_fits_filename,
'dark_f' : darkfield_fits_filename
}
json.dump(input_dict, open(f"./input_jsons/feb_2k_2021_L1.txt", "w"))
\ No newline at end of file
l1/2021-02-23/solo_L1_phi-hrt-ilam_20210223T170002_V202107221048C_0142230201.fits.gz
\ No newline at end of file
l1/2021-04-21/solo_L1_phi-hrt-ilam_20210421T120003_V202107221049C_0144210101.fits.gz
l1/2021-04-24/solo_L1_phi-hrt-ilam_20210424T120003_V202107221049C_0144240101.fits.gz
l1/2021-04-25/solo_L1_phi-hrt-ilam_20210425T120002_V202107221049C_0144250101.fits.gz
l1/2021-04-26/solo_L1_phi-hrt-ilam_20210426T120002_V202107221049C_0144260101.fits.gz
l1/2021-04-27/solo_L1_phi-hrt-ilam_20210427T120002_V202107221049C_0144270101.fits.gz
l1/2021-04-28/solo_L1_phi-hrt-ilam_20210428T120003_V202107221050C_0144280101.fits.gz
l1/2021-04-28/solo_L1_phi-hrt-ilam_20210428T130238_V202107221050C_0164281001.fits.gz
from posix import listdir
import numpy as np
import os.path
from astropy.io import fits
......@@ -73,9 +74,9 @@ def demod_hrt(data,pmp_temp):
def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat = True, norm_f = True, clean_f = False,
sigma = 59, flat_states = 24, prefilter_f = None,flat_c = True, dark_c = True, fs_c = True,
demod = True, norm_stokes = True, out_dir = './', out_demod_file = False, out_demod_filename = None,
def phihrt_pipe(data_f, dark_f = '', flat_f = '', L1_input = True, L1_8_generate = False, scale_data = True, accum_scaling = True,
bit_conversion = True, norm_f = True, clean_f = False, sigma = 59, flat_states = 24, prefilter_f = None,flat_c = True,
dark_c = True, fs_c = True, demod = True, norm_stokes = True, out_dir = './', out_demod_file = False, out_demod_filename = None,
ItoQUV = False, ctalk_params = None, rte = False, out_rte_filename = None, p_milos = True, config_file = True):
'''
......@@ -110,9 +111,13 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
Fits file of a HRT flatfield (ONLY ONE FILE)
** Options:
L1_input: bool, DEFAULT True
ovverides scale_data, bit_conversion, and accum_scaling, so that correct scaling for L1 data applied
L1_8_generate: bool, DEFAULT False
if True, assumes L1 input, and generates RTE output with the calibration header information
scale_data: bool, DEFAULT True
performs the accumulation scaling + conversion for flat and science (only FALSE for commissioning data)
bit_flat: bool, DEFAULT True
bit_conversion: bool, DEFAULT True
divides the scan + flat by 256 to convert from 24.8bit to 32bits
norm_f: bool, DEFAULT: True
to normalise the flat fields before applying
......@@ -171,6 +176,14 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
overall_time = time.time()
saved_args = locals()
saved_args['ctalk_params'] = ctalk_params.tolist()
if L1_input:
print("L1_input param set to True - Assuming L1 science data")
accum_scaling = True
bit_conversion = False
scale_data = True
#-----------------
# READ DATA
#-----------------
......@@ -198,7 +211,7 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
tuning_constant_arr = [0]*number_of_scans
for scan in range(number_of_scans):
data_arr[scan], hdr_arr[scan] = get_data(data_f[scan], scaling = scale_data)
data_arr[scan], hdr_arr[scan] = get_data(data_f[scan], scaling = accum_scaling, bit_convert_scale = bit_conversion)
wve_axis_arr[scan], voltagesData_arr[scan], tuning_constant_arr[scan], cpos_arr[scan] = fits_get_sampling(data_f[scan],verbose = True)
......@@ -319,7 +332,7 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
start_time = time.time()
#try:
flat, header_flat = get_data(flat_f, bit_convert_scale=bit_flat)
flat, header_flat = get_data(flat_f, scaling = accum_scaling, bit_convert_scale=bit_conversion)
print(f"Flat field shape is {flat.shape}")
......@@ -360,6 +373,10 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
flat_pmp_temp = str(header_flat['HPMPTSP1'])
print(f"Flat PMP Temperature Set Point: {flat_pmp_temp}")
if flat_f[-62:] == 'solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits':
flat[1345, 296:, 1, 2] = flat[1344, 296:, 1, 2]
flat[1346, :291, 1, 2] = flat[1345, :291, 1, 2]
printc('--------------------------------------------------------------',bcolors.OKGREEN)
printc(f"------------ Load flats time: {np.round(time.time() - start_time,3)} seconds",bcolors.OKGREEN)
......@@ -459,7 +476,7 @@ def phihrt_pipe(data_f, dark_f = '', flat_f = '', scale_data = True, bit_flat =
flat_demod, demodM = demod_hrt(flat, flat_pmp_temp)
norm_factor = norm_factor = np.mean(flat_demod[512:1536,512:1536,0,0])
norm_factor = np.mean(flat_demod[512:1536,512:1536,0,0])
flat_demod /= norm_factor
......
{"data_f": ["/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T141802_V202107221036C_0024160030.fits", "/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T142032_V202107221036C_0024160031.fits", "/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T142302_V202107221036C_0024160032.fits", "/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T142532_V202107221037C_0024160033.fits", "/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T142803_V202107221037C_0024160034.fits", "/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T143033_V202107221037C_0024160035.fits", "/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T143303_V202107221037C_0024160036.fits", "/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T143533_V202107221037C_0024160037.fits", "/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T143803_V202107221037C_0024160038.fits", "/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20200420T144033_V202107221038C_0024160039.fits"], "flat_f": "/data/slam/home/sinjan/fits_files/april_avgd_2020_flat.fits", "dark_f": "../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits"}
\ No newline at end of file
{"data_f": ["/data/slam/home/sinjan/fits_files/solo_L1_phi-hrt-ilam_20210223T170002_V202107221048C_0142230201.fits"], "flat_f": "/data/slam/home/sinjan/fits_files/solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits", "dark_f": "../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits"}
\ No newline at end of file
{"data_f": ["solo_L1_phi-hrt-ilam_20201117T170209_V202107060747C_0051170001.fits"], "flat_f": "/data/slam/home/sinjan/fits_files/april_avgd_2020_flat.fits", "dark_f": "../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits"}
\ No newline at end of file
......@@ -13,19 +13,13 @@ import json
# sciencedata_fits_filenames = ['solo_L0_phi-hrt-ilam_0667414905_V202103221851C_0142230602.fits']#['solo_L0_phi-hrt-ilam_0667414748_V202103221851C_0142230201.fits']
# #sciencedata_fits_filenames = ['solo_L0_phi-hrt-ilam_0667414905_V202103221851C_0142230602.fits', 'solo_L0_phi-hrt-ilam_0667415054_V202103221851C_0142230603.fits', 'solo_L0_phi-hrt-ilam_0667415205_V202103221851C_0142230604.fits', 'solo_L0_phi-hrt-ilam_0667415354_V202103221851C_0142230605.fits', 'solo_L0_phi-hrt-ilam_0667415505_V202103221851C_0142230606.fits', 'solo_L0_phi-hrt-ilam_0667415654_V202103221851C_0142230607.fits', 'solo_L0_phi-hrt-ilam_0667415805_V202103221851C_0142230608.fits']
# science_april = ['solo_L0_phi-hrt-ilam_20200420T142022_V202004221451C_0024160031000.fits','solo_L0_phi-hrt-ilam_20200420T142252_V202004221452C_0024160032000.fits','solo_L0_phi-hrt-ilam_20200420T142522_V202004221457C_0024160033000.fits','solo_L0_phi-hrt-ilam_20200420T142752_V202004221511C_0024160034000.fits','solo_L0_phi-hrt-ilam_20200420T143023_V202004221517C_0024160035000.fits', 'solo_L0_phi-hrt-ilam_20200420T143253_V202004221518C_0024160036000.fits','solo_L0_phi-hrt-ilam_20200420T143523_V202004221522C_0024160037000.fits', 'solo_L0_phi-hrt-ilam_20200420T143753_V202004231605C_0024160038000.fits','solo_L0_phi-hrt-ilam_20200420T144023_V202004231605C_0024160039000.fits']
# flatfield_fits_filename = '/data/slam/home/sinjan/fits_files/solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits'#solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits'#solo_L0_phi-hrt-ilam_20200417T174529_V202004241516C_0024150020000.fits' #solo_L0_phi-hrt-ilam_20200417T174529_V202004241516C_0024150020000.fits'#solo_L1_phi-hrt-ilam_20200417T174538_V202106111549C_0024150020.fits'#solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits'
# darkfield_fits_filename = '../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits'
# sciencedata_fits_filenames = ['/data/slam/home/sinjan/fits_files/' + i for i in science_april]
input_json_file = './input_jsons/april_2020.txt'
input_json_file = './input_jsons/feb_2k_2021_L1.txt'
input_dict = json.load(open(input_json_file))
data_f = input_dict['data_f']
data_f = input_dict['data_f']#[1:]
flat_f = input_dict['flat_f']
dark_f = input_dict['dark_f']
......@@ -41,13 +35,13 @@ prefilter_f = '../fits_files/fitted_prefilter.fits'
c_talk_params = np.zeros((2,3))
q_slope = -0.0141#0.0038#-0.0098#
u_slope = -0.0008#-0.0077#-0.0003#
v_slope = -0.0075#-0.0009#-0.0070#
q_slope = 0.0038#-0.0140##-0.0098#
u_slope = -0.0077#-0.0008##-0.0003#
v_slope = -0.0009#-0.0073##-0.0070#
q_int = 0.0018#-0.0056#-0.0015# #the offset, normalised to I_c
u_int = 0.0017#0.0031#0.0007#
v_int = 0.001#-0.0002#0.0006#
q_int = -0.0056#0.0016#-0.0056#-0.0015# #the offset, normalised to I_c
u_int = 0.0031#0.0016##0.0007#
v_int = -0.0002#0.0007##0.0006#
c_talk_params[0,0] = q_slope
c_talk_params[0,1] = u_slope
......@@ -59,11 +53,11 @@ c_talk_params[1,2] = v_int
#out_names = ['nov_17_0051170001']#, '0024160032000_noflat', '0024160033000_noflat', '0024160034000_noflat', '0024160035000_noflat', '0024160036000_noflat', '0024160037000_noflat', '0024160038000_noflat', '0024160039000_noflat']
phihrt_pipe(data_f, flat_f = flat_f, dark_f = dark_f, scale_data = False, bit_flat = True, norm_f = True,
clean_f = False, sigma = 59, flat_states = 6, norm_stokes = True, prefilter_f = None,
phihrt_pipe(data_f, flat_f = flat_f, dark_f = dark_f, scale_data = True, bit_conversion = True, accum_scaling = True, norm_f = True,
clean_f = False, sigma = 59, flat_states = 24, norm_stokes = True, prefilter_f = None,
dark_c = True, flat_c = True, fs_c = True, demod = True, ctalk_params = c_talk_params,
ItoQUV = False, out_demod_file = True, out_demod_filename = None,
out_dir = '/data/slam/home/sinjan/hrt_pipe_results/april_2020_6f/', rte = '-',
out_dir = '/data/slam/home/sinjan/hrt_pipe_results/stp-136_ctalk_no_us/', rte = 'RTE',
out_rte_filename=None, p_milos = False, config_file = True)
"""
Input Parameters:
......@@ -78,10 +72,19 @@ flat_f : string, DEFAULT ''
Fits file of a HRT flatfield (ONLY ONE FILE)
** Options:
L1_input: bool, DEFAULT True
ovverides scale_data, bit_conversion, and accum_scaling, so that correct scaling for L1 data applied
L1_8_generate: bool, DEFAULT False
if True, assumes L1 input, and generates RTE output with the calibration header information
scale_data: bool, DEFAULT True
performs the accumulation scaling + conversion for flat and science (only FALSE for commissioning data)
accum_scaling: bool, DEFAULT True
applies the scaling for the accumulation, (extracted from header)
bit_flat: bool, DEFAULT True
bit_conversion: bool, DEFAULT True
divides the scan + flat by 256 to convert from 24.8bit to 32bits
norm_f: bool, DEFAULT: True
......@@ -134,4 +137,7 @@ rte: str, DEFAULT: False
out_rte_filename: str or list, DEFAULT = ''
if '', takes last 10 characters of input scan filename (assumes its a DID), change if want other name(s) for each scan
config_file: bool, DEFAULT = True
if True, will generate config.txt file that writes the reduction process steps done
"""
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment