From b8b59536b3fbd0cdb5833318200acb927bc6c5b8 Mon Sep 17 00:00:00 2001 From: Philip Bittihn <philip.bittihn@ds.mpg.de> Date: Mon, 26 Sep 2022 17:30:07 +0200 Subject: [PATCH 1/9] isserializable function --- src/export/generated.jl | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/export/generated.jl b/src/export/generated.jl index 545fcd6..e5a499b 100644 --- a/src/export/generated.jl +++ b/src/export/generated.jl @@ -254,6 +254,19 @@ module ExportUtils importname(::Type, ::FieldExport{S}) where S = S # if you value your life, do NOT overwrite this method!!!!! importname(t::Type, a::AdditionalExport) = Symbol(exportname(t, a)) + # + """ + isserializable(T::Type) → Bool + isserializable(item::T) → Bool + Indicates whether serialize!/deserialize were defined for the type T or item(s) of type T. The type + for which `true` is returned always corresponds to the type that can also be passed to `serialize!`. + For `VectorExport`s, this means that the relevant serializable type is `Vector{T}`, not `T` itself. + + For more information on the export system, consult the InPartS manual. + """ + isserializable(::Type) = false + isserializable(::T) where {T} = isserializable(T) + #modname(::Type) = "Main" # more defaults! """ @@ -448,6 +461,7 @@ Generates the import/export functions for the given particle and its [`InPartS.P macro genexport(type::Union{Symbol,Expr}) return quote local mod = @__MODULE__ + Base.eval(mod, _isserializable_function($(esc(type)))) Base.eval(mod, _serialize_function($(esc(type)))) Base.eval(mod, _deserialize_function($(esc(type)))) #ExportUtils.modname(::Type{mod.$type}) = "$mod" @@ -455,6 +469,22 @@ macro genexport(type::Union{Symbol,Expr}) end +## isserializable +""" + _isserializable_function(T) +Returns an expression containing the definition of [`ExportUtils.isserializable()`](@ref) +for the given type `T`. +""" +_isserializable_function(T::Type) = _isserializable_function(T, ExportStyle(T)) + +_isserializable_function(T::Type, ::ScalarExport) = quote + ExportUtils.isserializable(::Type{<:$T}) = true +end + +_isserializable_function(T::Type, ::VectorExport) = quote + ExportUtils.isserializable(::Type{Vector{ET}}) where {ET<:$T} = true +end + ## Serialization -- GitLab From 4b9b639bfeaf45f39430766290e811ce65397b0d Mon Sep 17 00:00:00 2001 From: Philip Bittihn <philip.bittihn@ds.mpg.de> Date: Mon, 26 Sep 2022 23:21:12 +0200 Subject: [PATCH 2/9] Fix inconsistent generic definitions of deserialize --- src/export/generated.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/export/generated.jl b/src/export/generated.jl index e5a499b..13d615a 100644 --- a/src/export/generated.jl +++ b/src/export/generated.jl @@ -286,8 +286,8 @@ module ExportUtils For more information on the export system, consult the InPartS manual. """ - deserialize(g, t; kwargs...) = serialize!(g, d, InPartS.ExportStyle(t); kwags...) - deserialize(::Any, @nospecialize(a::Type), @nospecialize(s::InPartS.ExportStyle); kwargs...) = error("InPartS: no $s deserializer defined for type $a") + deserialize(g, t; kwargs...) = deserialize(g, t; kwargs...) + deserialize(::Any, @nospecialize(a::Type); kwargs...) = error("InPartS: no deserializer defined for type $a") end """ -- GitLab From 802613ee4b1c3ac98f512b30951fe79b076e9cde Mon Sep 17 00:00:00 2001 From: Philip Bittihn <philip.bittihn@ds.mpg.de> Date: Tue, 27 Sep 2022 08:41:26 +0200 Subject: [PATCH 3/9] Define export system constants early, so they can be used everywhere --- src/InPartS.jl | 1 + src/export/constants.jl | 27 +++++++++++++++++++++++++++ src/export/generic.jl | 27 --------------------------- 3 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 src/export/constants.jl diff --git a/src/InPartS.jl b/src/InPartS.jl index 698bb3e..47956dc 100644 --- a/src/InPartS.jl +++ b/src/InPartS.jl @@ -116,6 +116,7 @@ include("callbacks.jl") include("evolve.jl") # IO +include("export/constants.jl") include("export/generated.jl") include("export/legacy.jl") include("export/generic.jl") diff --git a/src/export/constants.jl b/src/export/constants.jl new file mode 100644 index 0000000..6fdcc80 --- /dev/null +++ b/src/export/constants.jl @@ -0,0 +1,27 @@ + +const PTTYPENAME = "pttype" +const PMTYPENAME = "pmtype" +const PTCTYPENAME = "ptctype" +const OBSTACLEGROUPNAME = "obstacles" +const PARAMGROUPNAME = "params" +const REGNAME = "registrar" +const RNGNAME = "rng" +const SNAPGROUPNAME = "snapshots" +const SNAPTIMENAME = "time" +const SNAPSTEPNAME = "step" +const SNAPRTSNAME = "writetime" +const SNAPNEXTIDNAME = "next_id" +const STRUCTTYPE = "_type" +const TSSGROUPNAME = "timestepping" +const DOMAINGROUPNAME = "domain" +const DOMAINSIZENAME = "size" +const DOMAINBCNAME = "boundaries" + +const VERSIONNAME = "InPartS" +const TREEHASHNAME = "treehash" + +# Estimated number of snapshots in a file +# JLD2 will allocate a chunk of memory to fit links to approximately EST_NUM_SNAPSHOT +# snapshot groups. Speeds up reading files with many entries. +# Cost is a constant "large" amount of unused space (a few kb) for smaller files. +const EST_NUM_SNAPSHOTS = 2000 diff --git a/src/export/generic.jl b/src/export/generic.jl index 7e0af39..3d6432c 100644 --- a/src/export/generic.jl +++ b/src/export/generic.jl @@ -1,32 +1,5 @@ export SaveCallback, BackupCallback, readsim -const PTTYPENAME = "pttype" -const PMTYPENAME = "pmtype" -const PTCTYPENAME = "ptctype" -const OBSTACLEGROUPNAME = "obstacles" -const PARAMGROUPNAME = "params" -const REGNAME = "registrar" -const RNGNAME = "rng" -const SNAPGROUPNAME = "snapshots" -const SNAPTIMENAME = "time" -const SNAPSTEPNAME = "step" -const SNAPRTSNAME = "writetime" -const SNAPNEXTIDNAME = "next_id" -const STRUCTTYPE = "_type" -const TSSGROUPNAME = "timestepping" -const DOMAINGROUPNAME = "domain" -const DOMAINSIZENAME = "size" -const DOMAINBCNAME = "boundaries" - -const VERSIONNAME = "InPartS" -const TREEHASHNAME = "treehash" - -# Estimated number of snapshots in a file -# JLD2 will allocate a chunk of memory to fit links to approximately EST_NUM_SNAPSHOT -# snapshot groups. Speeds up reading files with many entries. -# Cost is a constant "large" amount of unused space (a few kb) for smaller files. -const EST_NUM_SNAPSHOTS = 2000 - ## Datafile interface definitions """ dfopen(backend, filename; kwargs...) -- GitLab From 3877e6aa174615d9064b2a5e4fd413504b94ea7c Mon Sep 17 00:00:00 2001 From: Philip Bittihn <philip.bittihn@ds.mpg.de> Date: Tue, 27 Sep 2022 10:52:08 +0200 Subject: [PATCH 4/9] first versions of serializetodict and deserializefromdict --- src/export/generated.jl | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/export/generated.jl b/src/export/generated.jl index 13d615a..f258598 100644 --- a/src/export/generated.jl +++ b/src/export/generated.jl @@ -222,7 +222,7 @@ This module serves as a namespace for all functions generated by the import/expo """ module ExportUtils using InPartS - import InPartS:SV, ExportItem, FieldExport, AdditionalExport + import InPartS:SV, STRUCTTYPE, ExportItem, FieldExport, AdditionalExport """ exportfield(::Type, ::ExportItem) @@ -279,6 +279,19 @@ module ExportUtils serialize!(g, d::T; kwargs...) where {T} = serialize!(g, d, InPartS.ExportStyle(T); kwargs...) serialize!(::Any, @nospecialize(a), @nospecialize(s::InPartS.ExportStyle); kwargs...) = error("InPartS: no $s serializer defined for type $(typeof(a))") + """ + serializetodict(data::T) → Dict{String, Any} + serializetodict(data::Vector{T<:AbstractParticle}; sim::Simulation) → Dict{String, Any} + Serialize the `data` of serializable type `T` into a Dict{String, Any}, including the name of the + type `T` itself, which is stored at the key "$(STRUCTTYPE)". For types with `ExportStyle(T) == VectorExport()` + (currently only `AbstractParticle`), the second form applies. The result can be deserialzed using + [`deserializefromdict`](@ref). + + For more information on the export system, consult the InPartS manual. + """ + serializetodict(d::T; kwargs...) where {T} = serialize!(Dict{String, Any}(STRUCTTYPE => string(T)), d; kwargs...) + serializetodict(d::Vector{T}; kwargs...) where {T} = serialize!(Dict{String, Any}(STRUCTTYPE => string(T)), d; kwargs...) + """ deserialize(group, type::Type{T}; [sim::Simulation]) where {T<:AbstractParticle} → particles::Vector{T} deserialize(group, type::Type{T}) where {T<:AbstractParams} → param::T @@ -288,6 +301,18 @@ module ExportUtils """ deserialize(g, t; kwargs...) = deserialize(g, t; kwargs...) deserialize(::Any, @nospecialize(a::Type); kwargs...) = error("InPartS: no deserializer defined for type $a") + + """ + deserializefromdict(dict, stype = Any; [sim::Simulation]) → Vector{T} where {T<:AbstractParticle} + deserializefromdict(dict, stype = Any) → T + The reverse operation of [`serializetodict`](@ref). Deserializes the `dict` into either a vector or + a single instance of the type T saved in `dict["$(STRUCTTYPE)"]`, depending on whether `T` + The optional `stype` argument is checked to be `T`'s supertype during reconstruction. + + For more information on the export system, consult the InPartS manual. + """ + deserializefromdict(dict, stype = Any; kwargs...) = deserialize(dict, InPartS.reconstruct_subtype(dict[STRUCTTYPE], stype); kwargs...) + end """ -- GitLab From ff7d1f012e1257b34ea9630001aa5b2362919f0f Mon Sep 17 00:00:00 2001 From: Philip Bittihn <philip.bittihn@ds.mpg.de> Date: Tue, 27 Sep 2022 11:02:08 +0200 Subject: [PATCH 5/9] deserialize: forgot to commit removal of superfluous definition --- src/export/generated.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/export/generated.jl b/src/export/generated.jl index f258598..3e26211 100644 --- a/src/export/generated.jl +++ b/src/export/generated.jl @@ -299,7 +299,6 @@ module ExportUtils For more information on the export system, consult the InPartS manual. """ - deserialize(g, t; kwargs...) = deserialize(g, t; kwargs...) deserialize(::Any, @nospecialize(a::Type); kwargs...) = error("InPartS: no deserializer defined for type $a") """ -- GitLab From c06786537b9cd236d72a0d49f0e3535b1a28258e Mon Sep 17 00:00:00 2001 From: Philip Bittihn <philip.bittihn@ds.mpg.de> Date: Tue, 27 Sep 2022 11:12:01 +0200 Subject: [PATCH 6/9] @exportrecursive: Use new dict serialization functions --- src/export/generated.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/export/generated.jl b/src/export/generated.jl index 3e26211..fa19254 100644 --- a/src/export/generated.jl +++ b/src/export/generated.jl @@ -363,7 +363,7 @@ of `type` is itself an InPartS-exportable type (produced, e.g., with @genexport) This macro produces code ```julia -@exporttransform {type} {fieldname} ExportUtils.serialize!(Dict{String, Any}("_type" => string(typeof(value))), value) ExportUtils.deserialize(data, InPartS.reconstruct_subtype(data["_type"], {fieldtype})) +@exporttransform {type} {fieldname} ExportUtils.serializetodict(value) ExportUtils.deserializefromdict(data, {fieldtype}) ``` where `fieldtype` is the automatically determined type of the field. """ @@ -371,7 +371,7 @@ macro exportrecursive(type, field::Symbol) check_type(type) local ft = fieldtype(__module__.eval(type), field) return quote - @exporttransform $(esc(type)) $(field) ExportUtils.serialize!(Dict{String, Any}("_type" => string(typeof(value))), value) ExportUtils.deserialize(data, InPartS.reconstruct_subtype(data["_type"], $(ft))) + @exporttransform $(esc(type)) $(field) ExportUtils.serializetodict(value) ExportUtils.deserializefromdict(data, $(ft)) end end -- GitLab From ed75cde3a0fc8efc36c43d35382b605d9a62449a Mon Sep 17 00:00:00 2001 From: Philip Bittihn <philip.bittihn@ds.mpg.de> Date: Tue, 27 Sep 2022 11:13:41 +0200 Subject: [PATCH 7/9] Remove outdated genexport warning from docs --- src/export/generated.jl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/export/generated.jl b/src/export/generated.jl index fa19254..a1c24ec 100644 --- a/src/export/generated.jl +++ b/src/export/generated.jl @@ -474,13 +474,7 @@ end @genexport(type<:AbstractParticle) Generates the import/export functions for the given particle and its [`InPartS.ParamType`](@ref). -!!! warning - - This macro evals generated code into `InPartS.ExportUtils`. Using it as a top-level - statement in a precompiled package will bring naught but pain and destruction. - - If you want to automatically generate export/import functions when loading a package, - consider putting `@genexport` in the [`__init__()`](@ref) function. +For more information on the export system, consult the InPartS manual. """ macro genexport(type::Union{Symbol,Expr}) return quote -- GitLab From 7da82f7e7494ff941f5d2965e780683e396cbbea Mon Sep 17 00:00:00 2001 From: Philip Bittihn <philip.bittihn@ds.mpg.de> Date: Tue, 27 Sep 2022 12:08:29 +0200 Subject: [PATCH 8/9] Remove references to "params" from serialization docs (scalar export can be any type). --- src/export/generated.jl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/export/generated.jl b/src/export/generated.jl index a1c24ec..cca64a4 100644 --- a/src/export/generated.jl +++ b/src/export/generated.jl @@ -270,9 +270,10 @@ module ExportUtils #modname(::Type) = "Main" # more defaults! """ - serialize!(group, data::Vector{<:AbstractParticle}; [sim::Simulation]) - serialize!(group, data::AbstractParams) - Serialize the `data` into `group`. + serialize!(group, data) + serialize!(group, data::Vector{<:AbstractParticle}; sim::Simulation) + Serialize the serializable `data` into `group`. For types with `ExportStyle(T) == VectorExport()` + (currently only `AbstractParticle`), the second form applies. For more information on the export system, consult the InPartS manual. """ @@ -293,9 +294,10 @@ module ExportUtils serializetodict(d::Vector{T}; kwargs...) where {T} = serialize!(Dict{String, Any}(STRUCTTYPE => string(T)), d; kwargs...) """ - deserialize(group, type::Type{T}; [sim::Simulation]) where {T<:AbstractParticle} → particles::Vector{T} - deserialize(group, type::Type{T}) where {T<:AbstractParams} → param::T - Deserializes the `group` into either a vector of `type` or a single instance of `type`. + deserialize(group, type::Type{T}) where {T} → result::T + deserialize(group, type::Type{T}; sim::Simulation) where {T<:AbstractParticle} → particles::Vector{T} + Deserializes the `group` into either a single instance of `type` or a vector of `type`. For types with + `ExportStyle(T) == VectorExport()` (currently only `AbstractParticle`), the latter, and thus, the second form, applies. For more information on the export system, consult the InPartS manual. """ -- GitLab From 87ff06d0493fae3d0363b5a5ffd1f8db816eb214 Mon Sep 17 00:00:00 2001 From: Philip Bittihn <philip.bittihn@ds.mpg.de> Date: Tue, 27 Sep 2022 12:11:06 +0200 Subject: [PATCH 9/9] further serialization docs tweaking --- src/export/generated.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/export/generated.jl b/src/export/generated.jl index cca64a4..8c80d11 100644 --- a/src/export/generated.jl +++ b/src/export/generated.jl @@ -270,9 +270,9 @@ module ExportUtils #modname(::Type) = "Main" # more defaults! """ - serialize!(group, data) - serialize!(group, data::Vector{<:AbstractParticle}; sim::Simulation) - Serialize the serializable `data` into `group`. For types with `ExportStyle(T) == VectorExport()` + serialize!(group, data::T) + serialize!(group, data::Vector{T<:AbstractParticle}; sim::Simulation) + Serialize the `data` of serializable type T into `group`. For types with `ExportStyle(T) == VectorExport()` (currently only `AbstractParticle`), the second form applies. For more information on the export system, consult the InPartS manual. @@ -294,8 +294,8 @@ module ExportUtils serializetodict(d::Vector{T}; kwargs...) where {T} = serialize!(Dict{String, Any}(STRUCTTYPE => string(T)), d; kwargs...) """ - deserialize(group, type::Type{T}) where {T} → result::T - deserialize(group, type::Type{T}; sim::Simulation) where {T<:AbstractParticle} → particles::Vector{T} + deserialize(group, type::Type{T}) → result::T + deserialize(group, type::Type{T<:AbstractParticle}; sim::Simulation) → particles::Vector{T} Deserializes the `group` into either a single instance of `type` or a vector of `type`. For types with `ExportStyle(T) == VectorExport()` (currently only `AbstractParticle`), the latter, and thus, the second form, applies. -- GitLab