Dear Gitlab Users, for our upcoming upgrade to Gitlab v14, Gitlab will be unavailable on Thursday, 05.08.2021 from 5:00 pm to approximately 7:00 pm. Note that with v14, certain breaking changes will be introduced (https://about.gitlab.com/blog/2021/06/04/gitlab-moving-to-14-breaking-changes/).

Commit c0c9352b authored by daniel.schellhorn's avatar daniel.schellhorn
Browse files

Merge branch 'Data-IO' into 'ElserII'

Data io

See merge request !14
parents eac39412 632b53e2
......@@ -69,7 +69,7 @@ A concrete experiment class needs to override the
GetData.getData('Phase')
print('Loading data file CDI_intensity')
f = loadmat('../InputData/Phase/CDI_intensity.mat')
f = loadmat(datadir/'Phase'/'CDI_intensity.mat')
# diffraction pattern
dp = f['intensity']['img'][0, 0]
orig_res = max(dp.shape[0], dp.shape[1]) # actual data size
......
......@@ -6,6 +6,7 @@ from proxtoolbox.utils.graphics import addColorbar
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
import numpy as np
from scipy.io import loadmat
......@@ -74,7 +75,7 @@ class ART_Experiment(Experiment):
# load data
if not self.silent:
print('Loading data file ART_SheppLogan.mat ')
data_Shepp = loadmat('../InputData/CT/ART_SheppLogan.mat')
data_Shepp = loadmat(datadir/'CT'/'ART_SheppLogan.mat')
N = data_Shepp['N'].item()
p = data_Shepp['p'].item()
theta = data_Shepp['theta']
......
......@@ -5,14 +5,16 @@ from proxtoolbox.experiments.orbitaltomography.planar_molecule import PlanarMole
from proxtoolbox.utils.orbitaltomog import shifted_fft, fourier_interpolate
from proxtoolbox.utils.visualization.complex_field_visualization import complex_to_rgb
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
class DegenerateOrbital(PlanarMolecule):
@staticmethod
def getDefaultParameters():
defaultParams = {
'experiment_name': '2D ARPES',
'data_filename': '..\\Inputdata\\OrbitalTomog'
+ '\\2020_10_27_coronene_Homo1+2_ARPES_2e6counts_corrected_80x80.tif',
'data_filename': datadir/'OrbitalTomog'/'2020_10_27_coronene_Homo1+2_ARPES_2e6counts_corrected_80x80.tif',
'from_intensity_data': True,
'object': 'real',
'degeneracy': 2, # Number of degenerate states to reconstruct
......@@ -59,6 +61,10 @@ class DegenerateOrbital(PlanarMolecule):
- threshold_for_support: float, in range [0,1], fraction of the maximum at which to threshold when
determining support or support for sparsity
"""
# make sure input data can be found, otherwise download it
GetData.getData('OrbitalTomog')
super(DegenerateOrbital, self).loadData()
def createRandomGuess(self):
......
......@@ -7,6 +7,9 @@ from proxtoolbox.experiments.orbitaltomography.orbitExperiment import OrbitalTom
from proxtoolbox.utils.visualization.stack_viewer import XYZStackViewer
from proxtoolbox.utils.orbitaltomog import shifted_fft
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
class Molecule3D(OrbitalTomographyExperiment):
@staticmethod
......@@ -56,6 +59,9 @@ class Molecule3D(OrbitalTomographyExperiment):
"""
Load data and set in the correct format for reconstruction
"""
# make sure input data can be found, otherwise download it
GetData.getData('OrbitalTomog')
# TODO: copy from data processor
raise NotImplementedError
......
......@@ -6,6 +6,9 @@ from glob import glob
from proxtoolbox.utils.h5 import write_dict_to_h5
import numpy as np
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
class OrbitalTomographyExperiment(Experiment):
@staticmethod
......@@ -30,6 +33,10 @@ class OrbitalTomographyExperiment(Experiment):
self.output_ignore_keys = ['u1', 'u2', 'plots']
def loadData(self):
# make sure input data can be found, otherwise download it
GetData.getData('OrbitalTomog')
"""
Load or generate the dataset that will be used for
this experiment. Create the initial iterate.
......
......@@ -7,6 +7,10 @@ from proxtoolbox.experiments.orbitaltomography.planar_molecule import PlanarMole
from proxtoolbox.utils.orbitaltomog import shifted_fft, fourier_interpolate, bin_array, shifted_ifft
from proxtoolbox.utils.visualization.complex_field_visualization import complex_to_rgb
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
class OrbitalMomentumMicroscope(PlanarMolecule):
"""
......@@ -17,8 +21,7 @@ class OrbitalMomentumMicroscope(PlanarMolecule):
def getDefaultParameters():
defaultParams = {
'experiment_name': 'Momentum microscopy',
'data_filename': '..\\Inputdata\\OrbitalTomog'
+ '\\2020_11_05_Coronene_HOMO_near_degenerate_noisefree.tif',
'data_filename': datadir/'OrbitalTomog'/'2020_11_05_Coronene_HOMO_near_degenerate_noisefree.tif',
'from_intensity_data': False,
'object': 'real',
'constraint': 'sparse real',
......@@ -54,6 +57,10 @@ class OrbitalMomentumMicroscope(PlanarMolecule):
self.momentum_axes = (-1, -2)
def loadData(self):
# make sure input data can be found, otherwise download it
GetData.getData('OrbitalTomog')
"""
Load data and set in the correct format for reconstruction
Parameters are taken from experiment class (self) properties, which must include::
......
......@@ -7,13 +7,18 @@ from proxtoolbox.experiments.orbitaltomography.planar_molecule import PlanarMole
from proxtoolbox.utils.orbitaltomog import shifted_fft, fourier_interpolate, bin_array, shifted_ifft
from proxtoolbox.utils.visualization.complex_field_visualization import complex_to_rgb
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
class OrthogonalOrbitals(PlanarMolecule):
@staticmethod
def getDefaultParameters():
defaultParams = {
'experiment_name': '2D ARPES',
'data_filename': '..\\Inputdata\\OrbitalTomog'
'data_filename': datadir
+ '\OrbitalTomog'
+ '\\2020_10_27_coronene_Homo_stack_ARPES_2e6counts_corrected_80x80.tif',
'from_intensity_data': False,
'object': 'real',
......@@ -52,6 +57,10 @@ class OrthogonalOrbitals(PlanarMolecule):
- threshold_for_support: float, in range [0,1], fraction of the maximum at which to threshold when
determining support or support for sparsity
"""
# make sure input data can be found, otherwise download it
GetData.getData('OrbitalTomog')
# load data
if self.data_filename is None:
self.data_filename = input('Please enter the path to the datafile: ')
......
......@@ -7,6 +7,9 @@ from proxtoolbox.experiments.orbitaltomography.orbitExperiment import OrbitalTom
from proxtoolbox.utils.visualization.complex_field_visualization import complex_to_rgb
from proxtoolbox.utils.orbitaltomog import bin_array, shifted_fft, shifted_ifft, fourier_interpolate, roll_to_pos
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
class PlanarMolecule(OrbitalTomographyExperiment):
@staticmethod
......@@ -14,7 +17,7 @@ class PlanarMolecule(OrbitalTomographyExperiment):
# TODO: optimize parameters and proxoperators to get good & consistent phase retrieval using the demo
defaultParams = {
'experiment_name': 'noisy 2D ARPES', # '2D ARPES', #
'data_filename': '../InputData/OrbitalTomog/coronene_homo1.tif',
'data_filename': datadir/'OrbitalTomog'/'coronene_homo1.tif',
'from_intensity_data': False,
'object': 'real',
'constraint': 'sparse real',
......@@ -74,6 +77,9 @@ class PlanarMolecule(OrbitalTomographyExperiment):
- threshold_for_support: float, in range [0,1], fraction of the maximum at which to threshold when
determining support or support for sparsity
"""
# make sure input data can be found, otherwise download it
GetData.getData('OrbitalTomog')
# load data
if self.data_filename is None:
self.data_filename = input('Please enter the path to the datafile: ')
......
......@@ -7,6 +7,7 @@ from proxtoolbox.utils.graphics import addColorbar
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
import numpy as np
from numpy import exp, sqrt, log2, log10, ceil, floor, unravel_index, argmax, zeros
......@@ -77,7 +78,7 @@ class CDI_Experiment(PhaseExperiment):
GetData.getData('Phase')
if not self.silent:
print('Loading data file CDI_intensity')
f = loadmat('../InputData/Phase/CDI_intensity.mat')
f = loadmat(datadir/'Phase'/'CDI_intensity.mat')
# diffraction pattern
dp = f['intensity']['img'][0, 0]
orig_res = max(dp.shape[0], dp.shape[1]) # actual data size
......
......@@ -13,6 +13,11 @@ from numpy.linalg import norm
from numpy.fft import fft2, ifft2
import time
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
print(datadir)
class CDP_Experiment(PhaseExperiment):
"""
......@@ -69,6 +74,9 @@ class CDP_Experiment(PhaseExperiment):
"""
Load CDP dataset. Create the initial iterate.
"""
#make sure input data can be found, otherwise download it
GetData.getData('Phase')
# Implementation of the Wirtinger Flow (WF) algorithm presented in the paper
# "Phase Retrieval via Wirtinger Flow: Theory and algorithms"
# by E. J. Candes, X. Li, and M. Soltanolkotabi
......@@ -91,18 +99,18 @@ class CDP_Experiment(PhaseExperiment):
# make image
if debug:
if n2 == 1:
x_dict = loadMatFile('../InputData/Phase/CDP_1D_x.mat')
x_dict = loadMatFile(datadir/'Phase'/'CDP_1D_x.mat')
debug_image = x_dict['x']
masks_dict = loadMatFile('../InputData/Phase/CDP_1D_Masks.mat')
masks_dict = loadMatFile(datadir/'Phase'/'CDP_1D_Masks.mat')
debug_masks = masks_dict['Masks']
z0_dict = loadMatFile('../InputData/Phase/CDP_1D_z0.mat')
z0_dict = loadMatFile(datadir/'Phase'/'CDP_1D_z0.mat')
debug_z0 = z0_dict['z0']
elif n2 == 256:
x_dict = loadMatFile('../InputData/Phase/CDP_2D_x.mat')
x_dict = loadMatFile(datadir/'Phase'/'CDP_2D_x.mat')
debug_image = x_dict['x']
masks_dict = loadMatFile('../InputData/Phase/CDP_2D_Masks.mat')
masks_dict = loadMatFile(datadir/'Phase'/'CDP_2D_Masks.mat')
debug_masks = masks_dict['Masks']
z0_dict = loadMatFile('../InputData/Phase/CDP_2D_z0.mat')
z0_dict = loadMatFile(datadir/'Phase'/'CDP_2D_z0.mat')
debug_z0 = z0_dict['z0']
x = debug_image
else:
......
......@@ -12,6 +12,7 @@ from proxtoolbox.utils.cell import Cell, isCell
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
import matplotlib
import matplotlib.pyplot as plt
......@@ -91,7 +92,7 @@ class JWST_Experiment(PhaseExperiment):
if not self.silent:
print('Loading data.')
with open('../InputData/Phase/pupil.pmod','r') as fid:
with open(datadir/'Phase'/'pupil.pmod','r') as fid:
# read lower endian float <f
Xi_A = np.fromfile(fid, dtype='<f')
Xi_A = Xi_A.astype(np.float64)
......@@ -99,13 +100,13 @@ class JWST_Experiment(PhaseExperiment):
diversity = 3
with open('../InputData/Phase/phase_p37.pmod','r') as fid:
with open(datadir/'Phase'/'phase_p37.pmod','r') as fid:
# read lower endian float <f
temp1 = np.fromfile(fid, dtype='<f')
temp1 = temp1.astype(np.float64)
temp1 = temp1.reshape(512,512).T
with open('../InputData/Phase/phase_m37.pmod','r') as fid:
with open(datadir/'Phase'/'phase_m37.pmod','r') as fid:
# read lower endian float <f
temp2 = np.fromfile(fid, dtype='<f')
temp2 = temp2.astype(np.float64)
......
......@@ -6,6 +6,7 @@ from proxtoolbox.utils.graphics import addColorbar
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
import numpy as np
from numpy import exp, sqrt, log2, log10, ceil, floor, unravel_index, argmax, zeros
......@@ -81,13 +82,13 @@ class Krueger_Experiment(PhaseExperiment):
# parameters see below
# empty waveguide (WG) beam
WG = loadmat('../InputData/Phase/WG_beam.mat')
WG = loadmat(datadir/'Phase'/'WG_beam.mat')
WG = WG['WG']
# hologram
if not self.silent:
print('Loading data hologram_not-normalized.mat')
I_exp = loadmat('../InputData/Phase/hologram_not-normalized.mat')
I_exp = loadmat(datadir/'Phase'/'hologram_not-normalized.mat')
I_exp = I_exp['I_exp']
I_exp[np.isnan(I_exp)] = 1
......
......@@ -17,6 +17,9 @@ from matplotlib.pyplot import subplots, show, figure
import os.path
import time
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
class Phasepack_Experiment(PhaseExperiment):
'''
......@@ -63,19 +66,18 @@ class Phasepack_Experiment(PhaseExperiment):
"""
Load Phasepack dataset. Create the initial iterate.
"""
# check if data exists
# TODO: All the data corresponding to this experiment should be placed
# in a zip file on the http://vaopt.math.uni-goettingen.de/data/ website
# so that we can use the automatic data loading feature (i.e., GetData.getData("Phasepack")).
# Another possibility is to write code that downloads the data files directly from
# the original website (https://rice.app.box.com/v/TransmissionMatrices)
data_dir = "../InputData/Phase/" + self.dataset + '/'
data_dir = datadir/'Phase'/self.dataset
filenames = ['A_GS.mat', 'YH_squared_test.mat', 'XH_test.mat']
for filename in filenames:
data_path = data_dir + filename
data_path = data_dir/filename
if not os.path.isfile(data_path):
errMsg = "File " + data_path + " was not found"
errMsg = "File " + str(data_path) + " was not found"
print(errMsg)
print('*************************************************************************')
print('* INPUT DATA MISSING. Please download the phase pack input data from ')
......
......@@ -15,6 +15,7 @@ from proxtoolbox.utils.gaussian import gaussian
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
import matplotlib
import matplotlib.pyplot as plt
......
......@@ -11,6 +11,7 @@ from proxtoolbox.experiments.ptychography.ptychographyUtils import circ, \
#for downloading data
import proxtoolbox.utils.GetData as GetData
from proxtoolbox.utils.GetData import datadir
import numpy as np
from numpy import exp, sqrt, log2, log10, ceil, floor, unravel_index, argmax, zeros
......@@ -44,7 +45,7 @@ class PtychographyExperiment(Experiment):
'experiment_name' : 'Ptychography',
'object': 'complex',
'constraint': 'amplitude only',
'data_dir': '../InputData/Ptychography/',
'data_dir': datadir/'Ptychography',
'datafile': 'data_NTT_01_26210_192x192',
'farfield': True,
'noise': None,
......@@ -93,7 +94,7 @@ class PtychographyExperiment(Experiment):
def __init__(self,
data_dir = '../InputData/Ptychography/',
data_dir = datadir/'Ptychography',
datafile = 'data_NTT_01_26210_192x192',
farfield = True,
poissonfactor = 5,
......@@ -171,11 +172,11 @@ class PtychographyExperiment(Experiment):
if self.datafile is not None:
if self.data_dir is not None:
data_path = self.data_dir + self.datafile
data_path = self.data_dir/self.datafile
else:
data_path = self.datafile
if not data_path.endswith('.mat'):
data_path += '.mat'
if not str(data_path).endswith('.mat'):
data_path = self.data_dir/(self.datafile + '.mat')
if not os.path.isfile(data_path):
print('*************************************************************************')
......@@ -184,7 +185,7 @@ class PtychographyExperiment(Experiment):
print('* Save and unpack the Ptychography.tar.gz datafile in the *')
print('* ProxMatlab/InputData/Ptychography subdirectory *')
print('*************************************************************************')
errMsg = "File " + data_path + " was not found"
errMsg = "File " + str(data_path) + " was not found"
raise IOError(errMsg)
mat_dict = loadMatFile(data_path)
......@@ -223,14 +224,14 @@ class PtychographyExperiment(Experiment):
if self.datafile == 'data_NTT_01_26210_192x192':
reconst_file = 'reconstruction_data_NTT_01_26210_192x192.mat'
if self.data_dir is not None:
file_path = self.data_dir + reconst_file
file_path = self.data_dir/reconst_file
else:
file_path = reconst_file
reconst_dict = loadMatFile(file_path)
QQ = reconst_dict['P']
pos_file = 'positions_NTT_01_26210.mat'
if self.data_dir is not None:
file_path = self.data_dir + pos_file
file_path = self.data_dir/pos_file
else:
file_path = pos_file
pos_dict = loadMatFile(file_path)
......
......@@ -2,6 +2,10 @@ from pathlib import Path
import sys
import urllib.request
import tarfile
from zipfile import ZipFile
import shutil
datadir = Path(__file__).parent.parent.parent.parent / 'InputData'
datadir = Path(__file__).parent.parent.parent.parent / 'InputData'
......@@ -32,18 +36,37 @@ def getData(problemFamily):
my_file = datadir/'CT'/'ART_SheppLogan.mat'
elif problemFamily == 'Ptychography':
my_file = datadir/'Ptychography'/'gaenseliesel.png'
elif problemFamily == 'OrbitalTomog':
errMsg = 'OrbitalTomog downloader is yet a work in progress'
raise IOError(errMsg)
elif problemFamily == 'Elser':
my_file = datadir/'Elser'/'data100E'
else:
print("Invalid input in GetData.GetData. problemFamily has to be Phase, CT or Ptychography")
return -1
if not(my_file.is_file()):
print(problemFamily + " input data is missing.")
print(problemFamily + " input data is missing.")
if query_yes_no("Do you want to download the " + problemFamily + " input data?"):
urllib.request.urlretrieve(" http://vaopt.math.uni-goettingen.de/data/" + problemFamily + ".tar.gz", datadir/(problemFamily + '.tar.gz'), reporthook=dlProgress)
print("\nExtracting data...")
tar = tarfile.open(datadir/(problemFamily + '.tar.gz'), "r:gz")
tar.extractall(datadir/problemFamily)
tar.close()
if problemFamily == 'Elser':
link = 'https://github.com/veitelser/phase-retrieval-benchmarks/archive/master.zip'
urllib.request.urlretrieve(link, datadir/(problemFamily + '.zip'), reporthook=dlProgress)
print("\nExtracting data...")
with ZipFile(datadir/(problemFamily + '.zip'), 'r') as zipObj:
names = zipObj.namelist()
for name in names:
if name.startswith('phase-retrieval-benchmarks-master/data/data'):
zipObj.extract(name, datadir/problemFamily)
name_end = name.replace('phase-retrieval-benchmarks-master/data/','')
shutil.move(datadir/problemFamily/name, datadir/problemFamily/name_end)
shutil.rmtree(datadir/problemFamily/'phase-retrieval-benchmarks-master/')
else:
link = " http://vaopt.math.uni-goettingen.de/data/" + problemFamily + ".tar.gz"
urllib.request.urlretrieve(link, datadir/(problemFamily + '.tar.gz'), reporthook=dlProgress)
print("\nExtracting data...")
tar = tarfile.open(datadir/(problemFamily + '.tar.gz'), "r:gz")
tar.extractall(datadir/problemFamily)
tar.close()
if not(my_file.is_file()):
print('***************************************************************************************')
print('* Input data still missing. Please try automatic download again or manually download *')
......
......@@ -27,7 +27,7 @@ def loadMatFile(filename):
When reading v7.3 files cell objects are not currently supported.
'''
path = filename
path = str(filename)
if not path.endswith('.mat'):
path += '.mat'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment