diff --git a/src/InPartS.jl b/src/InPartS.jl index 8af14a0468c3cd5d56a708f5091ca44c4d707caa..d42e5c969f6fec0c08b7d34da5b22c0d3c0b7f9f 100644 --- a/src/InPartS.jl +++ b/src/InPartS.jl @@ -202,6 +202,9 @@ function __init__() @warn "The legacy version of InPartS was loaded alongside the current, public version. This will cause unexpected behavior." end + # set IOWARN from environment + InPartS.IOWARN[] = get(ENV, "INPARTS_IOWARN", "1") != "0" + return end diff --git a/src/export/generic.jl b/src/export/generic.jl index e5bc34c3d18551fceff366ceb7ffa5b53717fa76..b7db70262d9fcc9abdec68aab4a01795b9dffb18 100644 --- a/src/export/generic.jl +++ b/src/export/generic.jl @@ -1,5 +1,7 @@ -export SaveCallback, BackupCallback, readsim +export SaveCallback, BackupCallback, Snapshots, readsim +# this may be set from ENV by the __init__ function +const IOWARN = Ref(true) ## Datafile interface definitions """ @@ -67,11 +69,85 @@ function modetoflags(mode::String) end end +## Lazy snapshot accessor + +""" + Snapshots(path::String) +A lazy iterator to access snapshots in the InPartS data file at `path`. + +Indexing syntax `Snapshots(path)[i]` will open the file and attempt to load the `i`th +snapshot using [`readsim`](@ref). +Indexing with a unit range `Snapshots(path)[i:j]` will return a `Snapshots` struct with +modified `firstindex` and `lastindex`. Note that bounds checking may not be performed here, +as the file isn't opened and therefore the last snapshot index cannot be determined if it isn't +already known from a previous file access. Therefore, the resulting `Snapshots` may be invalid. + +Iteration is also supported, but keep in mind that the data file will be opened and closed +in every single iteration. + +Requires your data file to use zero indexed sequentially numbered snapshots (as is the case +if you used a [`SaveCallback`](@ref)). +""" +struct Snapshots + path::String + _firstsnap::Int # this is not a ref as it must be set on construction (either its 0 or you know what you want) + _lastsnap::Ref{Int} + # use -2 as magic value to distinguish empty files (numsnaps = 0) + Snapshots(path::String; firstsnap = 0, lastsnap = -2) = new(path, firstsnap, Ref(lastsnap)) +end + +# indexed access +function Base.getindex(sn::Snapshots, i::Integer) + @boundscheck if !(firstindex(sn) ≤ i ≤ lastindex(sn)) + throw(BoundsError(sn, i)) + end + readsim(sn.path, snap = i) +end + +function Base.getindex(sn::Snapshots, is::UnitRange) + # don't do a full bounds check in case `lastindex` doesn't work + @boundscheck if !(_maybe_check_bounds(sn, first(is)) && _maybe_check_bounds(sn, last(is))) + throw(BoundsError(sn, is)) + end + return Snapshots(sn.path; firstsnap = first(is), lastsnap = last(is)) +end + +function _maybe_check_bounds(sn, i) + if sn._lastsnap[] < -1 + @warn "Index may be out of range for $sn (last snap not known)" maxlog=1 + return firstindex(sn) ≤ i + else + return firstindex(sn) ≤ i ≤ lastindex(sn) + end +end + + +Base.firstindex(sn::Snapshots) = sn._firstsnap + +function Base.lastindex(sn::Snapshots) + if sn._lastsnap[] < -1 + nsnaps = numsnaps(sn.path) + sn._lastsnap[] = nsnaps - 1 + end + return sn._lastsnap[] +end + +Base.eachindex(sn) = firstindex(sn):lastindex(sn) + +# iteration +Base.iterate(sn::Snapshots, state::Int = 0) = state > lastindex(sn) ? nothing : (sn[state], state + 1) +Base.length(sn::Snapshots) = lastindex(sn) + 1 +Base.size(sn::Snapshots) = (length(sn), ) +Base.eltype(::Snapshots) = InPartS.Simulation + + +Base.show(io::IO, sn::Snapshots) = print(io, "Snapshots($(sn.path))[$(firstindex(sn)):$(sn._lastsnap[] < -1 ? "???" : lastindex(sn))]") + ## Callbacks """ - SaveCallback{backend}(trigger::AbstractCallback, filename, dumprng; [warn = true]) + SaveCallback{backend}(trigger::AbstractCallback, filename, dumprng; [warn = IOWARN[]]) Create a callback that periodically saves simulation snapshots to `filename`. # Extended help @@ -94,12 +170,12 @@ mutable struct SaveCallback{FT, CB<:AbstractCallback, FN<:Function} <: AbstractC end """ - SaveCallback([backend], filename; [interval = 1.0], [offset = 0.0], [dumprng = (s, sim) -> (s.nextsnap % 10 == 0)], [warn = true]) + SaveCallback([backend], filename; [interval = 1.0], [offset = 0.0], [dumprng = (s, sim) -> (s.nextsnap % 10 == 0)], [warn = IOWARN[]]) Create a [`SaveCallback`](@ref) that periodically saves simulation snapshots to `filename`. Uses a [`PeriodicCallback`](@ref) with the given `interval` and `offset`. """ -SaveCallback(backend, filename; interval = 1.0, offset = 0.0, dumprng = (s, sim) -> _defaultdumprng(sim.rng, s), warn = true) = +SaveCallback(backend, filename; interval = 1.0, offset = 0.0, dumprng = (s, sim) -> _defaultdumprng(sim.rng, s), warn = IOWARN[]) = SaveCallback{backend}(PeriodicCallback(sim -> true, interval; offset = offset), filename, dumprng, warn) SaveCallback(filename::String; kwargs...) = SaveCallback(backend(filename), filename; kwargs...) @@ -167,7 +243,7 @@ end """ - BackupCallback{backend}(trigger::AbstractCallback, filename; [warn = true]) + BackupCallback{backend}(trigger::AbstractCallback, filename; [warn = IOWARN[]]) Creates a callback that dumps the entire simulation state to `filename`. # Extended help @@ -184,12 +260,12 @@ struct BackupCallback{FT, CB} <: AbstractCallback end """ - BackupCallback(backend, filename; [interval = 300.0], [offset = 0.0], [warn = true]) + BackupCallback(backend, filename; [interval = 300.0], [offset = 0.0], [warn = IOWARN[]]) Creates a [`BackupCallback`](@ref) that periodically dumps the entire simulation state to `filename`. Uses a [`RealTimeCallback`](@ref) with the given `interval` and `offset`. """ -BackupCallback(backend, filename; interval = 300.0, offset = 0.0, warn = true) = +BackupCallback(backend, filename; interval = 300.0, offset = 0.0, warn = IOWARN[]) = BackupCallback{backend}(RealTimeCallback(interval, sim -> true; offset = offset), filename, warn) BackupCallback(filename::String; kwargs...) = BackupCallback(backend(filename), filename; kwargs...) @@ -223,13 +299,13 @@ finalize(cb::BackupCallback, sim; kwargs...) = cb(sim) ## Generic writing functions # writestatic """ - writestatic(simgroup, sim; [warn = true]) → snapgroup, paramgroup + writestatic(simgroup, sim; [warn = IOWARN[]]) → snapgroup, paramgroup Writes the static bits of the simulation to the output group `simgroup`. This includes the particle and parameter types, the domain and some InPartS version information. """ -function writestatic(simgroup, sim::Simulation{PTC, PT, PM}; warn = true) where {PTC, PT, PM} +function writestatic(simgroup, sim::Simulation{PTC, PT, PM}; warn = IOWARN[]) where {PTC, PT, PM} simgroup[TREEHASHNAME] = InPartS.TREE_HASH simgroup[VERSIONNAME] = string(InPartS._INPARTS_VERSION) @@ -256,7 +332,7 @@ function writestatic(simgroup, sim::Simulation{PTC, PT, PM}; warn = true) where end -function writerng(group, rng::T; complete::Bool = false, warn = true) where {T<:AbstractRNG} +function writerng(group, rng::T; complete::Bool = false, warn = IOWARN[]) where {T<:AbstractRNG} group["_type"] = string(T) group["_rng_complete"] = (T == Xoshiro) || complete # Xoshiro is always completely serialized if complete @@ -270,11 +346,11 @@ end """ - writetype(group, PT::Type; name, [warn = true]) + writetype(group, PT::Type; name, [warn = IOWARN[]]) Creates a dataset of name `name` in `group` and saves the type name of `PT`. If `PT` is a `Union`, the union types´ names will be written instead as a vector. """ -function writetype(group, PT::Type; name, prefix = "", warn = true) +function writetype(group, PT::Type; name, prefix = "", warn = IOWARN[]) if PT isa Union typenames = string.(Base.uniontypes(PT)) group[name] = prefix.*typenames @@ -286,11 +362,11 @@ end """ - writedomain(group, domain; [name = DOMAINGROUPNAME], [warn = true]) + writedomain(group, domain; [name = DOMAINGROUPNAME], [warn = IOWARN[]]) Creates a new subgroup called `name` in `DOMAINSIZENAME` and fills it withthe static properties of the domain (to wit, size & boundaries). """ -function writedomain(group, domain::Domain2D{BCX, BCY}; name = DOMAINGROUPNAME, warn = true) where {BCX, BCY} +function writedomain(group, domain::Domain2D{BCX, BCY}; name = DOMAINGROUPNAME, warn = IOWARN[]) where {BCX, BCY} wgroup = gcreate(group, name) wgroup[DOMAINSIZENAME] = domain.size wgroup[DOMAINBCNAME] = [string(BCX), string(BCY)] @@ -298,11 +374,11 @@ function writedomain(group, domain::Domain2D{BCX, BCY}; name = DOMAINGROUPNAME, end """ - writedoamin(group, domain; [name = DOMAINGROUPNAME], [warn = true]) + writedoamin(group, domain; [name = DOMAINGROUPNAME], [warn = IOWARN[]]) Creates a new subgroup called `name` in `group` and fills it withthe static properties of the domain (to wit, size & boundaries). """ -function writedomain(group, domain::Domain3D{BCX, BCY, BCZ}; name = DOMAINGROUPNAME, warn = true) where {BCX, BCY, BCZ} +function writedomain(group, domain::Domain3D{BCX, BCY, BCZ}; name = DOMAINGROUPNAME, warn = IOWARN[]) where {BCX, BCY, BCZ} dgroup = gcreate(group, name) dgroup[DOMAINSIZENAME] = domain.size dgroup[DOMAINBCNAME] = [string(BCX), string(BCY), string(BCZ)] @@ -312,7 +388,7 @@ end # writesnap (snapshots) """ - writesnap(group, sim; name = nextsnapname(group), [warn = true], kwargs...) + writesnap(group, sim; name = nextsnapname(group), [warn = IOWARN[]], kwargs...) Creates a new subgroup called `name` in `group` and saves a snapshot of the current state of the dynamical properties of `sim`. @@ -335,7 +411,7 @@ function _writesnap_common(group, sim::Simulation{PTC, PT}; name = nextsnapname(group), isunion = (PT isa Union), dumprng = !(sim.rng isa Random._GLOBAL_RNG), - warn = true + warn = IOWARN[] ) where {PTC, PT} snapgroup = gcreate(group, name) @@ -360,12 +436,12 @@ function _writesnap_common(group, sim::Simulation{PTC, PT}; end """ - writepgroup(group, particles::Vector{PT}, sim; [name = string(PT)], [warn = true]) + writepgroup(group, particles::Vector{PT}, sim; [name = string(PT)], [warn = IOWARN[]]) Tries to serialize the `particles` into a new subgroup `name` of `group`. Do not expect this to work if `PT` is a Union. """ -function writepgroup(group, particles::AbstractVector{PT}, sim::Simulation; name = string(PT), warn = true) where {PT} +function writepgroup(group, particles::AbstractVector{PT}, sim::Simulation; name = string(PT), warn = IOWARN[]) where {PT} pgroup = gcreate(group, name) ExportUtils.serialize!(pgroup, particles; sim = sim) end @@ -373,26 +449,26 @@ end # writefinal (final things) """ - writefinal(group, sim; [warn = true]) + writefinal(group, sim; [warn = IOWARN[]]) Write the information that is only available at the end of the simulation. At the moment, this includes the registrar and the parameter vector """ -function writefinal(group, sim::Simulation; warn = true) +function writefinal(group, sim::Simulation; warn = IOWARN[]) writedict(group, Dict(sim.registrar); name = REGNAME) writeparams(group[PARAMGROUPNAME], sim.params; warn) end """ - writeparams(group, params; [warn = true]) + writeparams(group, params; [warn = IOWARN[]]) Tries to serialize the `params` into a new subgroup of `group` named i, the index of the param struct . In contrast to [`writepgroup`](@ref), this will actually work for abstract/union eltypes. """ -function writeparams(group, params::AbstractVector{PT}; warn = true) where {PT<:AbstractParams} +function writeparams(group, params::AbstractVector{PT}; warn = IOWARN[]) where {PT<:AbstractParams} for (i, p) ∈ enumerate(params) if string(i) ∉ keys(group) paramgroup = gcreate(group, string(i)) @@ -403,7 +479,7 @@ function writeparams(group, params::AbstractVector{PT}; warn = true) where {PT<: end # TODO: circular references? what circular references? -function writedict(group, dict::AbstractDict; name, warn = true) +function writedict(group, dict::AbstractDict; name, warn = IOWARN[]) dgroup = gcreate(group, name) for (k, v) ∈ pairs(dict) if v isa AbstractDict @@ -420,13 +496,13 @@ end ## Convenience function for reading things """ - readsim(filename; snap, [warn = true], [kwargs...]) - readsim(f; snap, [warn = true], [kwargs...]) + readsim(filename; snap, [warn = IOWARN[]], [kwargs...]) + readsim(f; snap, [warn = IOWARN[]], [kwargs...]) Read a simulation from file and load the specified snapshot. Additional keyword arguments are passed through to [`readstatic!`](@ref) and [`readsnap`](@ref). """ -function readsim(f; snap, warn = true, kwargs...) +function readsim(f; snap, warn = IOWARN[], kwargs...) sim = readstatic(f; warn, kwargs...) try readsnap!(sim, f, snap; warn, kwargs...) @@ -451,7 +527,7 @@ end # readstatic (simulation) """ - readstatic(simgroup; [time], [warn = true]) → sim::Simulation + readstatic(simgroup; [time], [warn = IOWARN[]]) → sim::Simulation Reconstructs a [`Simulation`](@ref) from the information saved in `simgroup`. To load a specific snapshot into the `Simulation`, use [`readsnap!`](@ref). @@ -461,7 +537,7 @@ To load a specific snapshot into the `Simulation`, use [`readsnap!`](@ref). * `warn` turns on various warning messages. Defaults to `true` """ -function readstatic(simgroup; time = (0.0, 0), warn::Bool = true, version = get_version(simgroup)) +function readstatic(simgroup; time = (0.0, 0), warn::Bool = IOWARN[], version = get_version(simgroup)) if version < v"0.2.0-alpha" # legacy import for InPartS v1 files (probably only works for CG models) @@ -510,10 +586,10 @@ end """ - readdomain(group; [warn = true]) + readdomain(group; [warn = IOWARN[]]) Reconstructs a [`Domain2D`](@ref) or [`Domain3D`](@ref) from `group`. """ -function readdomain(group; warn = true) +function readdomain(group; warn = IOWARN[]) size = group[DOMAINSIZENAME] boundarystrings = group[DOMAINBCNAME] boundaries = InPartS.reconstruct_subtype.(boundarystrings, Boundary) @@ -523,14 +599,14 @@ end """ - readtype(pt; [super], [warn = true]) → T::Type + readtype(pt; [super], [warn = IOWARN[]]) → T::Type Reconstructs a type from a name as encoded with [`writetype`](@ref) ### Optional arguments * `super` limits the type reconstruction to children of a supertype. Defaults to `Any` * `mod` module in which to look for the type name. Defaults to `Main` """ -function readtype(pt; super = Any, mod = Main, warn = true) +function readtype(pt; super = Any, mod = Main, warn = IOWARN[]) #pt = simgroup[name] if pt isa Vector return Union{InPartS.reconstruct_subtype.(pt, super; mod)...} @@ -541,14 +617,14 @@ end """ - readdict(group; [maxdepth], [warn = true]) → d::Union{Missing, Dict{String, Any}} + readdict(group; [maxdepth], [warn = IOWARN[]]) → d::Union{Missing, Dict{String, Any}} Recursively reads `group` into a dictionary until depth `maxdepth`. If `maxdepth` is reached, all data is replaced by `missing` ## Optional arguments * `maxdepth` Sets the maximum recursion depth. Defaults to 32. """ -function readdict(group; maxdepth = 32, warn = true) +function readdict(group; maxdepth = 32, warn = IOWARN[]) if maxdepth == 0 warn && @warn "readdict: Maximum recursion depth reached" return missing @@ -570,8 +646,8 @@ end """ - readrng(group; [complete], [warn = true]) → rng::AbstractRNG - readrng(group, T ; [complete], [warn = true]) → rng::T + readrng(group; [complete], [warn = IOWARN[]]) → rng::AbstractRNG + readrng(group, T ; [complete], [warn = IOWARN[]]) → rng::T Reconstructs the RNG from `group`. Optionally specifiy the RNG type. ## Optional Arguments @@ -580,7 +656,7 @@ Reconstructs the RNG from `group`. Optionally specifiy the RNG type. """ readrng(group; mod = InPartS.Random, kwargs...) = readrng(group, InPartS.reconstruct_subtype(group["_type"], AbstractRNG; mod); kwargs...) -function readrng(group, T::Type{<:AbstractRNG}; complete::Bool = false, warn = true) +function readrng(group, T::Type{<:AbstractRNG}; complete::Bool = false, warn = IOWARN[]) if complete if group["_rng_complete"] return _deserialize_rng_full(group, T; warn) @@ -595,10 +671,10 @@ end """ - readparams(group; [warn = true]) → params + readparams(group; [warn = IOWARN[]]) → params Reconstructs the parameter structs from `group` """ -function readparams(group; warn = true) +function readparams(group; warn = IOWARN[]) indices = Vector{Int}() params = Vector{AbstractParams}() for n ∈ keys(group) @@ -615,7 +691,7 @@ end @deprecate readsnap(snapgroup; sim::Simulation) readsnap(snapgroup, sim) """ - readsnap(snapgroup, sim::Simulation; [warn = true]) → particles, time, step, next_id, rng[,...] + readsnap(snapgroup, sim::Simulation; [warn = IOWARN[]]) → particles, time, step, next_id, rng[,...] Loads the snapshot from `snapgroup`, using additional information from `sim`. Returns a tuple of state information that can be applied to the simulation using [`setstate`]((@ref) """ @@ -623,7 +699,7 @@ readsnap(snapgroup, sim::Simulation; kwargs...) = _readsnap_common(snapgroup, si # this function deals with particles, times, steps, registrars etc. # and can be used when overwriting readsnap for custom PTCs -function _readsnap_common(snapgroup, sim::Simulation{PTC, PT}; warn = true, ignorelist = SNAPDATANAMES) where {PTC, PT} +function _readsnap_common(snapgroup, sim::Simulation{PTC, PT}; warn = IOWARN[], ignorelist = SNAPDATANAMES) where {PTC, PT} time = snapgroup[SNAPTIMENAME] step = snapgroup[SNAPSTEPNAME] next_id = snapgroup[SNAPNEXTIDNAME] @@ -659,7 +735,7 @@ setstate!(sim::Simulation, state; kwargs...) = _setstate_common!(sim, state; kwa # this function deals with particles, times, steps, registrars etc. # and can be used when overwriting setstate! for custom PTCs -function _setstate_common!(sim::Simulation, state; warn = false) +function _setstate_common!(sim::Simulation, state; warn = IOWARN[]) particles, time, step, next_id, rng = state setparticles!(sim.particles, particles) sim.t = time @@ -675,12 +751,12 @@ end """ - readsnap!(sim::Simulation, f, snapname::String; [warn = true]) - readsnap!(sim::Simulation, f, snapid::Int; [warn = true]) - readsnap!(sim::Simulation, snapgroup; [warn = true]) + readsnap!(sim::Simulation, f, snapname::String; [warn = IOWARN[]]) + readsnap!(sim::Simulation, f, snapid::Int; [warn = IOWARN[]]) + readsnap!(sim::Simulation, snapgroup; [warn = IOWARN[]]) Loads the snapshot data from `snapgroup` into `sim`. """ -readsnap!(sim::Simulation, snapgroup; warn::Bool = true) = setstate!(sim, readsnap(snapgroup, sim; warn); warn) +readsnap!(sim::Simulation, snapgroup; warn::Bool = IOWARN[]) = setstate!(sim, readsnap(snapgroup, sim; warn); warn) function readsnap!(sim::Simulation, filename::String, snap; kwargs...) dfopen(filename) do f @@ -688,7 +764,7 @@ function readsnap!(sim::Simulation, filename::String, snap; kwargs...) end end -function readsnap!(sim::Simulation, f, snap; warn::Bool=true) +function readsnap!(sim::Simulation, f, snap; warn::Bool=IOWARN[]) if length(keys(f)) == 1 legacy_readsnap!(sim, f, snap; warn=warn) else @@ -735,22 +811,22 @@ lastfullsnap(filename::String) = dfopen(lastfullsnap, filename) ## RNG stuff -function _serialize_rng_partial!(group, @nospecialize rng::AbstractRNG; warn = true) +function _serialize_rng_partial!(group, @nospecialize rng::AbstractRNG; warn = IOWARN[]) warn && @warn "Partial serialization not implemented for $(typeof(rng))" return end -function _serialize_rng_full!(group, @nospecialize rng::AbstractRNG; warn = true) +function _serialize_rng_full!(group, @nospecialize rng::AbstractRNG; warn = IOWARN[]) warn && @warn "Full serialization not implemented for $(typeof(rng)), falling back to partial serialization" return _serialize_rng_partial!(group, rng; warn) end -function _serialize_rng_partial!(group, rng::MersenneTwister; warn = true) +function _serialize_rng_partial!(group, rng::MersenneTwister; warn = IOWARN[]) group["seed"] = rng.seed return end -function _serialize_rng_full!(group, rng::MersenneTwister; warn = true) +function _serialize_rng_full!(group, rng::MersenneTwister; warn = IOWARN[]) _serialize_rng_partial!(group, rng) group["vals"] = rng.vals group["ints"] = string.(rng.ints) @@ -761,20 +837,20 @@ function _serialize_rng_full!(group, rng::MersenneTwister; warn = true) end -function _deserialize_rng_partial(group, @nospecialize T::Type{<:AbstractRNG}; warn = true) +function _deserialize_rng_partial(group, @nospecialize T::Type{<:AbstractRNG}; warn = IOWARN[]) warn && @warn "Partial deserialization not implemented for $(T), creating new instance" return T() end _deserialize_rng_partial(group, T::Type{Random._GLOBAL_RNG}) = Random.GLOBAL_RNG -function _deserialize_rng_full(group, @nospecialize T::Type{<:AbstractRNG}; warn = true) +function _deserialize_rng_full(group, @nospecialize T::Type{<:AbstractRNG}; warn = IOWARN[]) warn && @warn "Full deserialization not implemented for $(T), falling back to partial deserialization" return _deserialize_rng_partial(group, T) end -_deserialize_rng_partial(group, ::Type{MersenneTwister}; warn = true) = MersenneTwister(group["seed"]) +_deserialize_rng_partial(group, ::Type{MersenneTwister}; warn = IOWARN[]) = MersenneTwister(group["seed"]) -function _deserialize_rng_full(group, ::Type{MersenneTwister}; warn = true) +function _deserialize_rng_full(group, ::Type{MersenneTwister}; warn = IOWARN[]) rng = _deserialize_rng_partial(group, MersenneTwister) rng.vals = group["vals"] @@ -812,7 +888,7 @@ _deserialize_rng_partial(group, T::Type{Xoshiro}; kwargs...) = _serialize_rng_fu ################################################################################ -function writeobstacles(group, sim::Simulation; warn = true) +function writeobstacles(group, sim::Simulation; warn = IOWARN[]) sov = obstacles(sim) obsgroup = gcreate(group, OBSTACLEGROUPNAME) for (i, so) ∈ enumerate(sov) @@ -822,7 +898,7 @@ function writeobstacles(group, sim::Simulation; warn = true) end end -function readobstacles(group; warn = true) +function readobstacles(group; warn = IOWARN[]) indices = Vector{Int}() params = Vector{AbstractObstacle}() for n ∈ keys(group)