Commit 06d6a4ec authored by jonas's avatar jonas
Browse files

dload files + DID duplicate handling hrt pipe

parent 038b5708
......@@ -13,4 +13,5 @@ cmilos/*.c
cmilos/*.h
cmilos/*
_RTE.npz
*.npz
\ No newline at end of file
*.npz
.env
\ No newline at end of file
from dotenv import load_dotenv
import requests
import os
#########################################################
"""
To make this work need to create a .env file:
USER_NAME = YOUR LOGIN USERNAME (NORMALLY JUST YOUR LAST NAME)
PHIDATAPASSWORD = YOUR WINDOWS PASSWORD
"""
#########################################################
def download_file(url, dload_location = '../fits_files'):
"""
Download files from the given url and store locally
Parameters
----------
url: url of the desired fits file
"""
load_dotenv()
username = os.environ.get('USER_NAME')
password = os.environ.get('PHIDATAPASSWORD')
local_filename = url.split('/')[-1]
# NOTE the stream=True parameter below
with requests.get(url, stream=True, auth=(username,password)) as r:
r.raise_for_status()
with open(dload_location + local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
# If you have chunk encoded response uncomment if
# and set chunk_size parameter to None.
#if chunk:
f.write(chunk)
return dload_location + local_filename
if __name__ == "__main__":
path = 'stp-144/'
files = ['solo_L0_phi-hrt-ilam_20210421T120003_V202106080929C_0144210101.fits', 'solo_L0_phi-hrt-ilam_20210424T120003_V202106141014C_0144240101.fits',
'solo_L0_phi-hrt-ilam_20210425T120002_V202106141020C_0144250101.fits', 'solo_L0_phi-hrt-ilam_20210426T120002_V202106162118C_0144260101.fits',
'solo_L0_phi-hrt-ilam_20210427T120002_V202106162052C_0144270101.fits', 'solo_L0_phi-hrt-ilam_20210427T120002_V202106171444C_0144270101.fits',
'solo_L0_phi-hrt-ilam_20210427T120002_V202106171517C_0144270101.fits']
url = 'https://www2.mps.mpg.de/services/proton/phi/fm/attic/'
for file in files:
download_file(url + path + file)
......@@ -732,12 +732,36 @@ def phihrt_pipe(data_f,dark_f,flat_f,norm_f = True, clean_f = False, sigma = 59,
print(" ")
printc('Saving demodulated data to one _reduced.fits file per scan')
#checking if the science scans have the same DID - this would cause an issue for naming the output demod files
scan_name_list = [str(scan.split('.fits')[0][-10:]) for scan in data_f]
seen = set()
uniq_scan_DIDs = [x for x in scan_name_list if x in seen or seen.add(x)] #creates list of unique DIDs from the list
if uniq_scan_DIDs == scan_name_list:
print("The scan's DIDs are all unique")
else:
for x in uniq_scan_DIDs:
number = scan_name_list.count(x)
if number > 1: #if more than one
print(f"The DID: {x} is repeated {number} times.")
i = 1
for index, name in enumerate(scan_name_list):
if name == x:
scan_name_list[index] = name + f"_{i}" #add _1, _2, etc to the file name, so that when written to output file not overwriting
i += 1
print("The New DID list is: ", scan_name_list)
for count, scan in enumerate(data_f):
with fits.open(scan) as hdu_list:
hdu_list[0].data = data[:,:,:,:,count]
hdu_list.writeto(out_dir + str(scan.split('.fits')[0][-10:]) + '_reduced.fits', overwrite=True)
hdu_list.writeto(out_dir + scan_name_list[count] + '_reduced.fits', overwrite=True)
if isinstance(data_f, str):
print(" ")
......
......@@ -8,4 +8,5 @@ os
random
subprocess
statistics
operator
\ No newline at end of file
operator
dotenv
\ No newline at end of file
from hrt_pipe import phihrt_pipe
import numpy as np
sciencedata_fits_filenames = ['solo_L0_phi-hrt-ilam_0667414748_V202103221851C_0142230201.fits']#['solo_L0_phi-hrt-ilam_0667414905_V202103221851C_0142230602.fits', 'solo_L0_phi-hrt-ilam_0667415054_V202103221851C_0142230603.fits', 'solo_L0_phi-hrt-ilam_0667415205_V202103221851C_0142230604.fits', 'solo_L0_phi-hrt-ilam_0667415354_V202103221851C_0142230605.fits', 'solo_L0_phi-hrt-ilam_0667415505_V202103221851C_0142230606.fits', 'solo_L0_phi-hrt-ilam_0667415654_V202103221851C_0142230607.fits', 'solo_L0_phi-hrt-ilam_0667415805_V202103221851C_0142230608.fits']#['../fits_files/solo_L0_phi-hrt-ilam_0667414748_V202103221851C_0142230201.fits']
sciencedata_fits_filenames = ['solo_L0_phi-hrt-ilam_20210421T120003_V202106080929C_0144210101.fits', 'solo_L0_phi-hrt-ilam_20210424T120003_V202106141014C_0144240101.fits',
'solo_L0_phi-hrt-ilam_20210425T120002_V202106141020C_0144250101.fits', 'solo_L0_phi-hrt-ilam_20210426T120002_V202106162118C_0144260101.fits',
'solo_L0_phi-hrt-ilam_20210427T120002_V202106162052C_0144270101.fits', 'solo_L0_phi-hrt-ilam_20210427T120002_V202106171444C_0144270101.fits',
'solo_L0_phi-hrt-ilam_20210427T120002_V202106171517C_0144270101.fits']#['solo_L0_phi-hrt-ilam_0667414748_V202103221851C_0142230201.fits']#['solo_L0_phi-hrt-ilam_0667414905_V202103221851C_0142230602.fits', 'solo_L0_phi-hrt-ilam_0667415054_V202103221851C_0142230603.fits', 'solo_L0_phi-hrt-ilam_0667415205_V202103221851C_0142230604.fits', 'solo_L0_phi-hrt-ilam_0667415354_V202103221851C_0142230605.fits', 'solo_L0_phi-hrt-ilam_0667415505_V202103221851C_0142230606.fits', 'solo_L0_phi-hrt-ilam_0667415654_V202103221851C_0142230607.fits', 'solo_L0_phi-hrt-ilam_0667415805_V202103221851C_0142230608.fits']#['../fits_files/solo_L0_phi-hrt-ilam_0667414748_V202103221851C_0142230201.fits']
flatfield_fits_filename = '../fits_files/solo_L0_phi-hrt-flat_0667134081_V202103221851C_0162201100.fits'
darkfield_fits_filename = '../fits_files/solo_L0_phi-fdt-ilam_20200228T155100_V202002281636_0022210004_000.fits'
......@@ -27,5 +30,5 @@ c_talk_params[1,0] = q_int
c_talk_params[1,1] = u_int
c_talk_params[1,2] = v_int
phihrt_pipe(sciencedata_fits_filenames, darkfield_fits_filename, flatfield_fits_filename, norm_stokes = True, prefilter_f = None,
clean_f = True, ctalk_params = c_talk_params, ItoQUV = True, out_demod_file = True, out_dir = '/data/slam/home/sinjan/hrt_pipe_results/', rte = 'False')
\ No newline at end of file
phihrt_pipe(sciencedata_fits_filenames, darkfield_fits_filename, flatfield_fits_filename, lat_states = 24, norm_stokes = True, prefilter_f = None,
clean_f = True, ctalk_params = None, ItoQUV = False, out_demod_file = True, out_dir = '/data/slam/home/sinjan/hrt_pipe_results/stp-144/', rte = 'False')
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment