diff --git a/Project.toml b/Project.toml index 7055263d67099d9c33639332f688cb37f3890ed3..95180cd5c4dccab12ab4c5d37dfad3c875c78168 100644 --- a/Project.toml +++ b/Project.toml @@ -16,6 +16,13 @@ Requires = "ae029012-a4dd-5104-9daa-d747884805df" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" +[weakdeps] +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" + +[extensions] +JLD2Ext = "JLD2" +HDF5Ext = "HDF5" [compat] julia = "1.7" diff --git a/src/export/h5wrapper.jl b/ext/HDF5Ext.jl similarity index 75% rename from src/export/h5wrapper.jl rename to ext/HDF5Ext.jl index f5fffd665ea85d3bd9dcc6e57b74b7978ea87bda..57524a36573fb1aeaca1e5c74609c72bbc417507 100644 --- a/src/export/h5wrapper.jl +++ b/ext/HDF5Ext.jl @@ -1,12 +1,15 @@ -export H5Wrapper +module HDF5Ext -using .HDF5 +using InPartS, HDF5 struct H5Wrapper{T<:Union{HDF5.File, HDF5.Group}} handle::T end +InPartS._backend(::Val{:HDF5}) = H5Wrapper + + """ H5Wrapper(filename; [kwargs...]) → H5Wrapper{HDF5.File} H5Wrapper(filename, mode::String) → H5Wrapper{HDF5.File} @@ -38,7 +41,7 @@ end # dictionary-like access function Base.getindex(h5w::H5Wrapper, key::String) item = h5w.handle[key] - return isgroup(h5w, key) ? H5Wrapper(item) : read(item) + return InPartS.isgroup(h5w, key) ? H5Wrapper(item) : read(item) end Base.setindex!(h5w::H5Wrapper, value, key::String) = setindex!(h5w.handle, value, key) Base.setindex!(h5w::H5Wrapper, value::Dict{String, Any}, key::String) = InPartS.writedict(h5w, value; name = key) @@ -48,21 +51,23 @@ Base.setindex!(h5w::H5Wrapper, value::SArray, key::String) = setindex!(h5w.handl Base.get(h5w::H5Wrapper, key::String, default) = exists(h5w.handle, key) ? h5w[key] : default # InPartS IO Backend API -dfopen(::Type{H5Wrapper}, name::String; flags...) = H5Wrapper(name; flags...) -dfclose(h5w::H5Wrapper{HDF5.File}) = close(h5w.handle) -function dfrename(::Type{H5Wrapper}, oldname, newname; force = false) +InPartS.dfopen(::Type{H5Wrapper}, name::String; flags...) = H5Wrapper(name; flags...) +InPartS.dfclose(h5w::H5Wrapper{HDF5.File}) = close(h5w.handle) +function InPartS.dfrename(::Type{H5Wrapper}, oldname, newname; force = false) cp(oldname, newname; force = force) rm(oldname) end -gcreate(h5w::H5Wrapper, name::String; kwargs...) = H5Wrapper(HDF5.create_group(h5w.handle, name)) +InPartS.gcreate(h5w::H5Wrapper, name::String; kwargs...) = H5Wrapper(HDF5.create_group(h5w.handle, name)) Base.keys(h5w::H5Wrapper) = keys(h5w.handle) Base.haskey(h5w::H5Wrapper, k::AbstractString) = haskey(h5w.handle, k) -isgroup(h5w::H5Wrapper, key) = !(h5w.handle[key] isa HDF5.Dataset) +InPartS.isgroup(h5w::H5Wrapper, key) = !(h5w.handle[key] isa HDF5.Dataset) Base.length(h5w::H5Wrapper) = length(h5w.handle) # more efficient readdict -readdict(h5w::H5Wrapper; kwargs...) = read(h5w.handle) +InPartS.readdict(h5w::H5Wrapper; kwargs...) = read(h5w.handle) + +end \ No newline at end of file diff --git a/ext/JLD2Ext.jl b/ext/JLD2Ext.jl new file mode 100644 index 0000000000000000000000000000000000000000..da0a3a0179b668e17acc0ae363ea80f585c215ef --- /dev/null +++ b/ext/JLD2Ext.jl @@ -0,0 +1,81 @@ +module JLD2Ext + +using InPartS, JLD2 + + +const JLD2Obj = Union{JLD2.JLDFile, JLD2.Group} + +struct JLD2Wrapper{T<:JLD2Obj} + _handle::T +end + +function Base.getindex(j2w::JLD2Wrapper, key) + item = j2w._handle[key] + return item isa JLD2Obj ? JLD2Wrapper(item) : item +end + +Base.get(j2w::JLD2Wrapper, key, default) = haskey(j2w, key) ? j2w[key] : default + +Base.setindex!(j2w::JLD2Wrapper, val, key) = setindex!(j2w._handle, val, key) +# For backwards compatibility with HDF5 write all SArrays as Arrays +Base.setindex!(j2w::JLD2Wrapper, value::SArray, key::String) = setindex!(j2w, collect(value), key) +# use writedict for recursive dicts +Base.setindex!(j2w::JLD2Wrapper, value::Dict{String, Any}, key::String) = InPartS.writedict(j2w, value; name = key) + +Base.keys(j2w::JLD2Wrapper) = keys(j2w._handle) +Base.haskey(j2w::JLD2Wrapper, k) = haskey(j2w._handle, k) + +# InPartS IO Backend API +InPartS._backend(::Val{:JLD2}) = JLD2Wrapper + +InPartS.dfopen(::Type{<:JLD2Wrapper}, filename::String, mode::String = "r") = JLD2Wrapper(JLD2.jldopen(filename, mode)) +InPartS.dfclose(j2w::JLD2Wrapper) = close(j2w._handle) +Base.close(j2w::JLD2Wrapper{<:JLD2.JLDFile}) = close(j2w._handle) # for lazy people + +function InPartS.dfrename(::Type{JLD2Wrapper}, oldname, newname; force = false) + cp(oldname, newname; force = force) + rm(oldname) +end + +InPartS.gcreate(g::JLD2Wrapper, name::String; kwargs...) = JLD2Wrapper(JLD2.Group(g._handle, name; kwargs...)) + +InPartS.isgroup(g::JLD2Wrapper, key) = (g[key] isa JLD2Wrapper{<:JLD2Group}) +Base.length(g::JLD2Wrapper) = length(keys(g._handle)) + + + +# more efficient readdict +# copied and modified from previous JLD2.loadtodict!(Dict{String,Any}(), g) +# to preserve nestedness +# is faster than generic version in generic.jl because reading via `g[k]` +# only happens once +function InPartS.readdict(j2w::JLD2Wrapper; kwargs...) + g = j2w._handle + d = Dict{String, Any}() + for k in keys(g) + v = g[k] + if v isa JLD2.Group + d[k] = InPartS.readdict(v) + else + d[k] = v + end + end + return d +end + +InPartS.readdict(d::Dict) = d + + + +# legacy stuff +InPartS.dfopen(::Type{<:JLD2.JLDFile}, args...; kwargs...) = InPartS.dfopen(JLD2Wrapper, args...; kwargs...) +InPartS.dfclose(df::JLD2.JLDFile) = InPartS.dfclose(JLD2Wrapper(df)) +InPartS.dfrename(df::Type{<:JLD2.JLDFile}, args...; kwargs...) = InPartS.dfrename(JLD2Wrapper, args...; kwargs...) + # fully automatic wrapping +for fun ∈ [:readdict, :readdomain, :readobstacles, :readparams, :readrng, :readsim, :readsnap, :readstatic, :readtype, :numsnaps, :lastfullsnap] + @eval InPartS.$(fun)(d::JLD2Obj, args...; kwargs...) = InPartS.$(fun)(JLD2Wrapper(d), args...; kwargs...) +end + +InPartS.readsnap!(sim::Simulation, df::JLD2.JLDFile, args...; kwargs...) = InPartS.readsnap!(sim::Simulation, JLD2Wrapper(df), args...; kwargs...) + +end \ No newline at end of file diff --git a/src/InPartS.jl b/src/InPartS.jl index 0518a437547bda7d64f479539cb21b7dd0949621..e8747044fcf4da7109efa7d71f7c714eeae21961 100644 --- a/src/InPartS.jl +++ b/src/InPartS.jl @@ -189,16 +189,20 @@ function __init__() # EXPORT BACKENDS - - @require HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" begin - include("export/h5wrapper.jl") - @info "InPartS: HDF5 export enabled" - end - @require JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" begin - include("export/jld2wrapper.jl") - @info "InPartS: JLD2 export enabled" + # fallback if no extensions available + @static if !isdefined(Base, :get_extension) + @require HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" begin + include("../ext/HDF5Ext.jl") + @info "InPartS: HDF5 export enabled" + end + + @require JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" begin + include("../ext/JLD2Ext.jl") + @info "InPartS: JLD2 export enabled" + end end + # Use Requires infrastructure to warn if legacy version is ever loaded simultaneously legacyInPartS = Base.PkgId(Base.UUID("385f2a1c-27df-41b8-9918-2c5d735168af"), "InPartS") Requires.listenpkg(legacyInPartS) do diff --git a/src/export/generic.jl b/src/export/generic.jl index b7db70262d9fcc9abdec68aab4a01795b9dffb18..a20ff6f75177fce1a35cff89720784802e259e52 100644 --- a/src/export/generic.jl +++ b/src/export/generic.jl @@ -14,15 +14,13 @@ function backend(filename::String) if !isdefined(Main, modulename) error("File appears to be a $modulename file but library is not loaded.") end - if modulename == :JLD2 - return JLD2.JLDFile - elseif modulename == :HDF5 - return H5Wrapper - else - error("This backend is not implemented") - end + # this is a hack to get around the fact that extensions cannot export copied + # so we can't use the JLD2Wrapper/H5Wrapper types from here + return _backend(Val(modulename)) end +_backend(v) = error("The backend $v is not implemented") + """ dfopen(backend, filename; kwargs...) dfopen(backend, filename, mode::String) @@ -30,9 +28,9 @@ Open the data file at `filename`. Mode specifications follow [`Base.open`](@ref). """ -dfopen(type::Type, filename::String, mode::String) = dfopen(type, filename; modetoflags(mode)...) -dfopen(@nospecialize(type::Type), filename::String; kwargs...) = error("Unknown export backend: $type") -dfopen(filename::String, mode::String="r") = dfopen(backend(filename), filename, mode) +dfopen(type::Type, filename, mode) = dfopen(type, filename; modetoflags(mode)...) +dfopen(@nospecialize(type::Type), filename; kwargs...) = error("Unknown export backend: $type") +dfopen(filename::String, mode="r") = dfopen(backend(filename), filename, mode) function dfopen(f::Function, args...; kwargs...) file = dfopen(args...; kwargs...) @@ -496,13 +494,14 @@ end ## Convenience function for reading things """ - readsim(filename; snap, [warn = IOWARN[]], [kwargs...]) - readsim(f; snap, [warn = IOWARN[]], [kwargs...]) -Read a simulation from file and load the specified snapshot. +readsim(f; snap = InPartS.lastfullsnap(f), [warn = IOWARN[]], [kwargs...]) +readsim(filename; snap, [warn = IOWARN[]], [kwargs...]) +Read a simulation from file and load the specified snapshot. Defaults to the last full snapshot +(see [`lastfullsnap`](@ref)). Additional keyword arguments are passed through to [`readstatic!`](@ref) and [`readsnap`](@ref). """ -function readsim(f; snap, warn = IOWARN[], kwargs...) +function readsim(f; snap = lastfullsnap(f), warn = IOWARN[], kwargs...) sim = readstatic(f; warn, kwargs...) try readsnap!(sim, f, snap; warn, kwargs...) @@ -921,18 +920,7 @@ A function for whenever you just need to save the ~~world~~ simulation. """ function write_sim(filename, sim::Simulation; dumprng = true) - s = FileIO.query(filename; checkfile=true) - modulename = typeof(s).parameters[1].parameters[1] - if !isdefined(InPartS, modulename) - Base.require(Main, modulename) - return Base.invokelatest(write_sim, filename, sim; dumprng) - elseif modulename == :JLD2 - FT = JLD2.JLDFile - elseif modulename == :HDF5 - FT = H5Wrapper - else - error("This backend is not implemented") - end + FT = backend(filename) dfopen(FT, filename, "w") do f writestatic(f, sim) writesnap(f[SNAPGROUPNAME], sim; name = "0", dumprng) diff --git a/src/export/jld2wrapper.jl b/src/export/jld2wrapper.jl deleted file mode 100644 index 3d640d9fef6f55083286d39570466c7c858cfee9..0000000000000000000000000000000000000000 --- a/src/export/jld2wrapper.jl +++ /dev/null @@ -1,43 +0,0 @@ -using .JLD2 - -# InPartS IO Backend API -dfopen(type::Type{JLD2.JLDFile}, filename::String, mode::String = "r") = JLD2.jldopen(filename, mode) - -dfclose(f::JLD2.JLDFile) = close(f) -function dfrename(::Type{JLD2.JLDFile}, oldname, newname; force = false) - cp(oldname, newname; force = force) - rm(oldname) -end - -gcreate(g::Union{JLD2.JLDFile, JLD2.Group}, name::String; kwargs...) = JLD2.Group(g, name; kwargs...) - -isgroup(g::Union{JLD2.JLDFile, JLD2.Group}, key) = (g[key] isa JLD2.Group) -Base.length(g::Union{JLD2.JLDFile, JLD2.Group}) = length(keys(g)) - -# For backwards compatibility with HDF5 write all SArrays as Arrays -Base.setindex!(g::JLD2.Group, value::SArray, key::String) = setindex!(g, collect(value), key) -Base.setindex!(g::JLD2.JLDFile, value::SArray, key::String) = setindex!(g, collect(value), key) -Base.setindex!(g::JLD2.Group, value::Dict{String, Any}, key::String) = InPartS.writedict(g, value; name = key) -Base.setindex!(g::JLD2.JLDFile, value::Dict{String, Any}, key::String) = InPartS.writedict(g, value; name = key) - - - -# more efficient readdict -# copied and modified from previous JLD2.loadtodict!(Dict{String,Any}(), g) -# to preserve nestedness -# is faster than generic version in generic.jl because reading via `g[k]` -# only happens once -function readdict(g::Union{JLD2.JLDFile, JLD2.Group}; kwargs...) - d = Dict{String, Any}() - for k in keys(g) - v = g[k] - if v isa JLD2.Group - d[k] = readdict(v) - else - d[k] = v - end - end - return d -end - -readdict(d::Dict) = d