From 4f5c686d6bb1715f7541d46adadf838177c8bfc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 8 Sep 2023 10:19:10 +0200 Subject: [PATCH 01/97] Update init_mtg_models.jl update doc as it now uses a cache variable --- src/mtg/init_mtg_models.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mtg/init_mtg_models.jl b/src/mtg/init_mtg_models.jl index ac472cc5..5f48479e 100644 --- a/src/mtg/init_mtg_models.jl +++ b/src/mtg/init_mtg_models.jl @@ -4,7 +4,7 @@ models::Dict{String,<:ModelList}, i=nothing; verbose=true, - attr_name=:models + attr_name=Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models")), ) initialize the components of an MTG (*i.e.* nodes) with the corresponding models. @@ -19,7 +19,7 @@ and if not found, returns an error. - `models::Dict{String,ModelList}`: a dictionary of models named by components names - `i=nothing`: the time-step to initialize. If `nothing`, initialize all the time-steps. - `verbose = true`: return information during the processes -- `attr_name = :models`: the node attribute name used to store the models, default to +- `attr_name = Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models"))`: the node attribute name used to store the models, default to Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models")) # Examples From f6f4544255e69ef91fc22f66006b1c384758f77a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 8 Sep 2023 14:47:35 +0200 Subject: [PATCH 02/97] Update Status.jl Add `refvalue()` + fix `eltype` --- src/component_models/Status.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/component_models/Status.jl b/src/component_models/Status.jl index 667cdc4e..831624b9 100644 --- a/src/component_models/Status.jl +++ b/src/component_models/Status.jl @@ -66,6 +66,8 @@ end Base.keys(::Status{names}) where {names} = names Base.values(st::Status) = getindex.(values(getfield(st, :vars))) refvalues(mnt::Status) = values(getfield(mnt, :vars)) +refvalue(mnt::Status, key::Symbol) = getfield(getfield(mnt, :vars), key) + Base.NamedTuple(mnt::Status) = NamedTuple{keys(mnt)}(values(mnt)) Base.Tuple(mnt::Status) = values(mnt) @@ -109,7 +111,7 @@ end Base.propertynames(::Status{T,R}) where {T,R} = T Base.length(mnt::Status) = length(getfield(mnt, :vars)) -Base.eltype(::Type{Status{T}}) where {T} = T +Base.eltype(::Type{Status{N,T}}) where {N,T} = eltype(T) Base.iterate(mnt::Status, iter=1) = iterate(NamedTuple(mnt), iter) From 5ada52ba7e6e0d97e1bed11c9df53e3e3a2e558b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 8 Sep 2023 14:52:04 +0200 Subject: [PATCH 03/97] Make tests pass for mtg --- test/runtests.jl | 5 +++++ test/test-mtg.jl | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 1b42ce18..a58d1009 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,6 +11,7 @@ include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimGrowthModel.jl")) include(joinpath(pkgdir(PlantSimEngine), "examples/ToyRUEGrowthModel.jl")) +include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")) @testset "Testing PlantSimEngine" begin Aqua.test_all(PlantSimEngine, ambiguities=false) @@ -48,6 +49,10 @@ include(joinpath(pkgdir(PlantSimEngine), "examples/ToyRUEGrowthModel.jl")) include("test-toy_models.jl") end + @testset "MTG" begin + include("test-mtg.jl") + end + if VERSION == v"1.8" # Error formating changed in Julia 1.8 (or was it 1.7?), so the doctest # that returns an error in PlantSimEngine.check_dimensions(models, w) diff --git a/test/test-mtg.jl b/test/test-mtg.jl index 1fe6d7af..e684b95b 100644 --- a/test/test-mtg.jl +++ b/test/test-mtg.jl @@ -25,17 +25,17 @@ meteo = Weather( @test descendants(mtg, :var1) == [nothing, nothing] @test descendants(mtg, :var2) == [nothing, var2] - to_init = init_mtg_models!(mtg, models, length(meteo)) + to_init = init_mtg_models!(mtg, models, length(meteo), attr_name=:models) @test to_init == Dict{String,Set{Symbol}}("Leaf" => Set(Symbol[:var2])) @test NamedTuple(get_node(mtg, 3)[:models][1]) == (var4=-Inf, var5=-Inf, var6=-Inf, var1=var1, var3=-Inf, var2=var2) # The following shouldn't work because var2 has only one value: - @test_throws ["Issue in function", "for node #3"] init_mtg_models!(mtg, models, 10) + @test_throws ["The attribute", "in node 3"] init_mtg_models!(mtg, models, 10) # Same with two time-steps: mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) internode = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) - leaf = Node(mtg, MultiScaleTreeGraph.NodeMTG("<", "Leaf", 1, 2)) + leaf = Node(internode, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) var1 = [15.0, 16.0] var2 = [0.3, 0.4] leaf[:var2] = var2 @@ -48,7 +48,7 @@ meteo = Weather( status=(var1=var1,) ) ) - to_init = init_mtg_models!(mtg, models, length(meteo)) + to_init = init_mtg_models!(mtg, models, length(meteo), attr_name=:models) @test NamedTuple(status(get_node(mtg, 3)[:models])[2]) == (var4=-Inf, var5=-Inf, var6=-Inf, var1=16.0, var3=-Inf, var2=0.4) end @@ -70,7 +70,7 @@ end ) ) - to_init = init_mtg_models!(mtg, models, length(meteo)) + to_init = init_mtg_models!(mtg, models, length(meteo), attr_name=:models) nsteps = length(meteo) @@ -100,7 +100,7 @@ end ) ) - to_init = init_mtg_models!(mtg, models, length(meteo)) + to_init = init_mtg_models!(mtg, models, length(meteo), attr_name=:models) attr_before_sim = deepcopy(leaf.attributes) From 957ec8f5ae03eb57cc2c37a6b47f21601a248ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 8 Sep 2023 16:39:53 +0200 Subject: [PATCH 04/97] Create RefVector.jl --- src/component_models/RefVector.jl | 127 ++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 src/component_models/RefVector.jl diff --git a/src/component_models/RefVector.jl b/src/component_models/RefVector.jl new file mode 100644 index 00000000..2a3e2884 --- /dev/null +++ b/src/component_models/RefVector.jl @@ -0,0 +1,127 @@ +""" + RefVector(field::Symbol, sts...) + RefVector(field::Symbol, sts::Vector{<:Status}) + RefVector(v::Vector{Base.RefValue{T}}) + +A vector of references to a field of a vector of structs. +This is used to efficiently pass the values between scales. + +# Arguments + +- `field`: the field of the struct to reference +- `sts...`: the structs to reference +- `sts::Vector{<:Status}`: a vector of structs to reference + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +``` + +Let's take two Status structs: + +```jldoctest mylabel +julia> status1 = Status(a = 1.0, b = 2.0, c = 3.0); +``` + +```jldoctest mylabel +julia> status2 = Status(a = 2.0, b = 3.0, c = 4.0); +``` + +We can make a RefVector of the field `a` of the structs `st1` and `st2`: + +```jldoctest mylabel +julia> rv = PlantSimEngine.RefVector(:a, status1, status2) +2-element PlantSimEngine.RefVector{Float64}: + 1.0 + 2.0 +``` + +Which is equivalent to: + +```jldoctest mylabel +julia> rv = PlantSimEngine.RefVector(:a, [status1, status2]) +2-element PlantSimEngine.RefVector{Float64}: + 1.0 + 2.0 +``` + +We can access the values of the RefVector: + +```jldoctest mylabel +julia> rv[1] +1.0 +1.0 +``` + +Updating the value in the RefVector will update the value in the original struct: + +```jldoctest mylabel +julia> rv[1] = 10.0 +10.0 +``` + +```jldoctest mylabel +julia> status1.a +10.0 +``` + +We can also make a RefVector from a vector of references: + +```jldoctest mylabel +julia> vec = [Ref(1.0), Ref(2.0), Ref(3.0)] +3-element Vector{Base.RefValue{Float64}}: + Base.RefValue{Float64}(1.0) + Base.RefValue{Float64}(2.0) + Base.RefValue{Float64}(3.0) +``` + +```jldoctest mylabel +julia> rv = PlantSimEngine.RefVector(vec) +3-element PlantSimEngine.RefVector{Float64}: + 1.0 + 2.0 + 3.0 +``` + +```jldoctest mylabel +julia> rv[1] +``` +""" +struct RefVector{T} <: AbstractVector{T} + v::Vector{Base.RefValue{T}} +end + +function Base.getindex(rv::RefVector, i::Int) + return rv.v[i][] +end + +function Base.setindex!(rv::RefVector, v, i::Int) + rv.v[i][] = v +end + +Base.size(rv::RefVector) = size(rv.v) +Base.length(rv::RefVector) = length(rv.v) +Base.eltype(::Type{RefVector{T}}) where {T} = T + +function Base.show(io::IO, rv::RefVector{T}) where {T} + print(io, "RefVector{") + print(io, T) + print(io, "}[") + for i in 1:length(rv.v) + print(io, rv.v[i][]) + if i < length(rv.v) + print(io, ", ") + end + end + print(io, "]") +end + +# A function to make a vector of values from a vector of a field from structs: +function RefVector(field::Symbol, sts...) + return RefVector(typeof(refvalue(sts[1], field))[refvalue(st, field) for st in sts]) +end + +function RefVector(field::Symbol, sts::Vector{<:Status}) + return RefVector(typeof(refvalue(sts[1], field))[refvalue(st, field) for st in sts]) +end \ No newline at end of file From 7ec5a7332ce44656518221559efb311d05e21523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 8 Sep 2023 16:40:04 +0200 Subject: [PATCH 05/97] Update PlantSimEngine.jl --- src/PlantSimEngine.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 60a6be46..9970a2bd 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -27,6 +27,7 @@ include("Abstract_model_structs.jl") # Simulation row (status): include("component_models/Status.jl") +include("component_models/RefVector.jl") # Simulation table (time-step table, from PlantMeteo): include("component_models/TimeStepTable.jl") From a2c91d73393abfd06debb0e17968dfc97a53fdcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 8 Sep 2023 16:40:14 +0200 Subject: [PATCH 06/97] Update Status.jl Fix jldoctest --- src/component_models/Status.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/component_models/Status.jl b/src/component_models/Status.jl index 831624b9..96a07722 100644 --- a/src/component_models/Status.jl +++ b/src/component_models/Status.jl @@ -12,6 +12,10 @@ so in essence, it is a stuct that stores a `NamedTuple` of the references to the A leaf with one value for all variables will make a status with one time step: +```jldoctest st1 +julia> using PlantSimEngine +``` + ```jldoctest st1 julia> st = PlantSimEngine.Status(Rₛ=13.747, sky_fraction=1.0, d=0.03, aPPFD=1500.0); ``` From 10b3ccbe0efe6b45aa4012695e74314104f21a7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 12 Sep 2023 15:05:10 +0200 Subject: [PATCH 07/97] Update Status.jl Fix error in eltype(::Status) --- src/component_models/Status.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/component_models/Status.jl b/src/component_models/Status.jl index 96a07722..ebb315cb 100644 --- a/src/component_models/Status.jl +++ b/src/component_models/Status.jl @@ -115,7 +115,7 @@ end Base.propertynames(::Status{T,R}) where {T,R} = T Base.length(mnt::Status) = length(getfield(mnt, :vars)) -Base.eltype(::Type{Status{N,T}}) where {N,T} = eltype(T) +Base.eltype(::Type{Status{N,T}}) where {N,T} = eltype.(eltype(T)) Base.iterate(mnt::Status, iter=1) = iterate(NamedTuple(mnt), iter) From 242e180f06d51983ef0fc774ff60efbdf4f8a824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 14 Sep 2023 11:10:34 +0200 Subject: [PATCH 08/97] Add toy models for multiscale models --- examples/ToyAssimModel.jl | 57 +++++++++++++++++++++++++ examples/ToyCAllocationModel.jl | 73 +++++++++++++++++++++++++++++++++ examples/ToyCDemandModel.jl | 59 ++++++++++++++++++++++++++ examples/ToySoilModel.jl | 38 +++++++++++++++++ 4 files changed, 227 insertions(+) create mode 100644 examples/ToyAssimModel.jl create mode 100644 examples/ToyCAllocationModel.jl create mode 100644 examples/ToyCDemandModel.jl create mode 100644 examples/ToySoilModel.jl diff --git a/examples/ToyAssimModel.jl b/examples/ToyAssimModel.jl new file mode 100644 index 00000000..3ea57ee3 --- /dev/null +++ b/examples/ToyAssimModel.jl @@ -0,0 +1,57 @@ +# using PlantSimEngine, PlantMeteo # Import the necessary packages, PlantMeteo is used for the meteorology + +# Defining the process: +@process "photosynthesis" verbose = false + +# Make the struct to hold the parameters, with its documentation: +""" + ToyAssimModel(A) + ToyAssimModel(; LUE=0.2, Rm_factor = 0.5, Rg_cost = 1.2) + +Computes the assimilation of a plant (= photosynthesis). + +# Arguments + +- `LUE=0.2`: the light use efficiency, in gC mol[PAR]⁻¹ + +# Inputs + +- `aPPFD`: the absorbed photosynthetic photon flux density, in mol[PAR] m⁻² d⁻¹ +- `soil_water_content`: the soil water content, in % + +# Outputs + +- `A`: the assimilation, in gC m⁻² d⁻¹ +""" +struct ToyAssimModel{T} <: AbstractPhotosynthesisModel + LUE::T +end + +# Instantiate the `struct` with keyword arguments and default values: +function ToyAssimModel(; LUE=0.2) + ToyAssimModel(LUE) +end + +# Define inputs: +function PlantSimEngine.inputs_(::ToyAssimModel) + (aPPFD=-Inf, soil_water_content=-Inf) +end + +# Define outputs: +function PlantSimEngine.outputs_(::ToyAssimModel) + (A=-Inf,) +end + +# Tells Julia what is the type of elements: +Base.eltype(::ToyAssimModel{T}) where {T} = T + +# Implement the growth model: +function PlantSimEngine.run!(::ToyAssimModel, models, status, meteo, constants, extra) + # The assimilation is simply the absorbed photosynthetic photon flux density (aPPFD) times the light use efficiency (LUE): + status.A = status.aPPFD * models.growth.LUE * status.soil_water_content +end + +# And optionally, we can tell PlantSimEngine that we can safely parallelize our model over space (objects): +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyAssimModel}) = PlantSimEngine.IsObjectIndependent() +# And also over time (time-steps): +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyAssimModel}) = PlantSimEngine.IsTimeStepIndependent() \ No newline at end of file diff --git a/examples/ToyCAllocationModel.jl b/examples/ToyCAllocationModel.jl new file mode 100644 index 00000000..1b821081 --- /dev/null +++ b/examples/ToyCAllocationModel.jl @@ -0,0 +1,73 @@ +# using PlantSimEngine, PlantMeteo # Import the necessary packages, PlantMeteo is used for the meteorology + +# Defining the process: +@process "carbon_allocation" verbose = false + +# Make the struct to hold the parameters, with its documentation: +""" + ToyCAllocationModel() + +Computes the carbon allocation to each organ of a plant. +This model should be used at the plant scale, because it first computes the carbon availaible for allocation as the minimum between the total demand +and total carbon offer, and then allocates it relative to their demand. + +# Inputs + +- `A`: the absorbed photosynthetic photon flux density, taken from the organs, in mol[PAR] m⁻² d⁻¹ + +# Outputs + +- `A`: the assimilation, in gC m⁻² d⁻¹ +""" +struct ToyCAllocationModel <: AbstractCarbon_AllocationModel end + +# Define inputs: +function PlantSimEngine.inputs_(::ToyCAllocationModel) + (A=-Inf, carbon_demand=-Inf,) +end + +# Define outputs: +function PlantSimEngine.outputs_(::ToyCAllocationModel) + (carbon_offer=-Inf, carbon_allocation=-Inf) +end + +function PlantSimEngine.run!(::ToyCAllocationModel, models, status, meteo, constants, mtg) + carbon_demand_organs = Vector{eltype(status.carbon_demand)}() + MultiScaleTreeGraph.traverse!(mtg, symbol=["Leaf", "Internode"]) do node + push!(carbon_demand_organs, node[:models].status[:carbon_demand]) + end + + carbon_demand = sum(carbon_demand_organs) + + status.carbon_offer = 0.0 + MultiScaleTreeGraph.traverse!(mtg, symbol="Leaf") do node + status.carbon_offer += node[:models].status[:A] + end + + # If the total demand is positive, we try allocating carbon: + if carbon_demand > 0.0 + # Proportion of the demand of each leaf compared to the total leaf demand: + proportion_carbon_demand = carbon_demand_organs ./ carbon_demand + + if carbon_demand <= status.carbon_offer + # If the carbon demand is lower than the offer we allocate the offer: + carbon_allocation_organs = carbon_demand + else + # Here we don't have enough carbon offer + carbon_allocation_organs = status.carbon_offer + end + carbon_allocation_organ = carbon_allocation_organs .* proportion_carbon_demand + else + # If the carbon demand is 0.0, we allocate nothing: + carbon_allocation_organs = 0.0 + carbon_allocation_organ = zeros(typeof(carbon_demand_organs[1]), length(carbon_demand_organs)) + end + + # We allocate the carbon to the organs: + MultiScaleTreeGraph.traverse!(mtg, symbol=["Leaf", "Internode"]) do organ + organ[:models].status[:carbon_allocation] = popfirst!(carbon_allocation_organ) + end +end + +# And also over time (time-steps): +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyCAllocationModel}) = PlantSimEngine.IsTimeStepIndependent() \ No newline at end of file diff --git a/examples/ToyCDemandModel.jl b/examples/ToyCDemandModel.jl new file mode 100644 index 00000000..318409b9 --- /dev/null +++ b/examples/ToyCDemandModel.jl @@ -0,0 +1,59 @@ +# using PlantSimEngine, PlantMeteo # Import the necessary packages, PlantMeteo is used for the meteorology + +# Defining the process: +@process "carbon_demand" verbose = false + +# Make the struct to hold the parameters, with its documentation: +""" + ToyCDemandModel(optimal_biomass, development_duration) + ToyCDemandModel(; optimal_biomass, development_duration) + +Computes the carbon demand of an organ depending on its biomass under optimal conditions and the duration of its development in degree days. +The model assumes that the carbon demand is linear througout the duration of the development. + +# Arguments + +- `optimal_biomass`: the biomass of the organ under optimal conditions, in gC +- `development_duration`: the duration of the development of the organ, in degree days + +# Inputs + +- `TT`: the thermal time, in degree days + +# Outputs + +- `carbon_demand`: the carbon demand, in gC +""" +struct ToyCDemandModel{T} <: AbstractCarbon_DemandModel + optimal_biomass::T + development_duration::T +end + +# Instantiate the `struct` with keyword arguments and default values: +function ToyCDemandModel(; optimal_biomass, development_duration) + ToyCDemandModel(optimal_biomass, development_duration) +end + +# Define inputs: +function PlantSimEngine.inputs_(::ToyCDemandModel) + (TT=-Inf,) +end + +# Define outputs: +function PlantSimEngine.outputs_(::ToyCDemandModel) + (carbon_demand=-Inf,) +end + +# Tells Julia what is the type of elements: +Base.eltype(::ToyCDemandModel{T}) where {T} = T + +# Implement the growth model: +function PlantSimEngine.run!(::ToyCDemandModel, models, status, meteo, constants, extra) + # The carbon demand is simply the biomass under optimal conditions divided by the duration of the development: + status.carbon_demand = status.TT * models.carbon_demand.optimal_biomass / models.carbon_demand.development_duration +end + +# And optionally, we can tell PlantSimEngine that we can safely parallelize our model over space (objects): +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyCDemandModel}) = PlantSimEngine.IsObjectIndependent() +# And also over time (time-steps): +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyCDemandModel}) = PlantSimEngine.IsTimeStepIndependent() \ No newline at end of file diff --git a/examples/ToySoilModel.jl b/examples/ToySoilModel.jl new file mode 100644 index 00000000..a744a9cb --- /dev/null +++ b/examples/ToySoilModel.jl @@ -0,0 +1,38 @@ +using Random +# Declaring the process of LAI dynamic: +@process "soil_water" verbose = false + + +""" + ToySoilWaterModel() + ToySoilWaterModel(;values=0.1:0.1:1.0,rng=MersenneTwister(1234)) + ToySoilWaterModel(values,rng) + +A toy model to compute the soil water content. The model simply take a random value in +the `values` range using the `rng` random number generator. + +# Outputs + +- `soil_water_content`: the soil water content (%). +""" +struct ToySoilWaterModel <: AbstractSoil_WaterModel + values::AbstractRange{Float64} + rng::AbstractRNG +end + +# Defining a method with keyword arguments and default values: +ToySoilWaterModel(; values=0.1:0.1:1.0, rng=MersenneTwister(1234)) = ToySoilWaterModel(values, rng) + +# Defining the inputs and outputs of the model: +PlantSimEngine.inputs_(::ToySoilWaterModel) = NamedTuple() +PlantSimEngine.outputs_(::ToySoilWaterModel) = (soil_water_content=-Inf,) + +# Implementing the actual algorithm by adding a method to the run! function for our model: +function PlantSimEngine.run!(m::ToySoilWaterModel, models, status, meteo, constants=nothing, extra=nothing) + soil_water_content = rand(m.values) +end + +# The computation of ToySoilWaterModel is independant of previous values and other objects. We can add this information as +# traits to the model to tell PlantSimEngine that it is safe to run the models in parallel: +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToySoilWaterModel}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToySoilWaterModel}) = PlantSimEngine.IsObjectIndependent() \ No newline at end of file From 127c5d797708ec5fb4310cde0ea04baada132990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 14 Sep 2023 11:11:12 +0200 Subject: [PATCH 09/97] Update ModelList.jl Remove unnecessary argument typing in parse_models --- src/component_models/ModelList.jl | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/component_models/ModelList.jl b/src/component_models/ModelList.jl index 10b52082..aa2f63ec 100644 --- a/src/component_models/ModelList.jl +++ b/src/component_models/ModelList.jl @@ -227,7 +227,7 @@ function ModelList( return model_list end -parse_models(m::Tuple) = NamedTuple([process(i) => i for i in m]) +parse_models(m) = NamedTuple([process(i) => i for i in m]) init_fun_default(x::Vector{T}) where {T} = TimeStepTable([Status(i) for i in x]) init_fun_default(x::N) where {N<:NamedTuple} = TimeStepTable([Status(x)]) @@ -261,16 +261,20 @@ function add_model_vars(x, models, type_promotion; init_fun=init_fun_default, ns # If the user gave an empty status, we initialize all variables to their default values: if x === nothing || (!Tables.istable(x) && length(x) == 0) if nsteps === nothing - return init_fun(fill(ref_vars, 1)) + return init_fun(ref_vars) else return init_fun(fill(ref_vars, nsteps)) end end - # Making a vars for each ith value in the user vars: - x_full = [] - for r in Tables.rows(x) - push!(x_full, merge(ref_vars, NamedTuple(r))) + if Tables.istable(x) + # Making a vars for each ith value in the user vars: + x_full = [merge(ref_vars, NamedTuple(x[1]))] + for r in Tables.rows(x)[2:end] + push!(x_full, merge(ref_vars, NamedTuple(r))) + end + else + x_full = merge(ref_vars, NamedTuple(x)) end return init_fun(x_full) @@ -314,7 +318,9 @@ PlantSimEngine.homogeneous_ts_kwargs((Tₗ=[25.0, 26.0], aPPFD=1000.0)) function homogeneous_ts_kwargs(kwargs::NamedTuple{N,T}, nsteps) where {N,T} length(kwargs) == 0 && return kwargs vars_vals = collect(Any, values(kwargs)) - length_vars = [length(i) for i in vars_vals] + length_vars = [isa(i, RefVector) ? 1 : length(i) for i in vars_vals] + #Note: length is 1 for RefVector because it is a vector of references to other scales, + # not a vector of values # One of the variable is given as an array, meaning this is actually several # time-steps. In this case we make an array of vars. From 23be959a23f5076c91fe53b2d0c7fb5db8173edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 14 Sep 2023 11:11:23 +0200 Subject: [PATCH 10/97] Update RefVector.jl Add mutating functions --- src/component_models/RefVector.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/component_models/RefVector.jl b/src/component_models/RefVector.jl index 2a3e2884..737660d9 100644 --- a/src/component_models/RefVector.jl +++ b/src/component_models/RefVector.jl @@ -103,6 +103,14 @@ end Base.size(rv::RefVector) = size(rv.v) Base.length(rv::RefVector) = length(rv.v) Base.eltype(::Type{RefVector{T}}) where {T} = T +Base.parent(v::RefVector) = v.v + +# Base.push!(v::RefVector, val) = push!(parent(v), val) +Base.resize!(v::RefVector, nl::Integer) = (resize!(v.parent, nl); v) +Base.push!(v::RefVector, x...) = (push!(parent(v), x...); v) +Base.pop!(v::RefVector) = pop!(parent(v)) +Base.append!(v::RefVector, items) = (append!(parent(v), items); v) +Base.empty!(v::RefVector) = (empty!(parent(v)); v) function Base.show(io::IO, rv::RefVector{T}) where {T} print(io, "RefVector{") From cf02b757f92f5afbf1851be2110b49179a629dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 14 Sep 2023 11:11:38 +0200 Subject: [PATCH 11/97] Create MultiScaleModel.jl --- src/mtg/MultiScaleModel.jl | 84 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 src/mtg/MultiScaleModel.jl diff --git a/src/mtg/MultiScaleModel.jl b/src/mtg/MultiScaleModel.jl new file mode 100644 index 00000000..ed96e494 --- /dev/null +++ b/src/mtg/MultiScaleModel.jl @@ -0,0 +1,84 @@ +""" + MultiScaleModel(model, mapping) + +A structure to make a model multi-scale. It defines a mapping between the variables of a +model and the nodes symbols from which the values are taken from. + +# Arguments + +- `model<:AbstractModel`: the model to make multi-scale +- `mapping<:Vector{Pair{Symbol,Union{AbstractString,Vector{AbstractString}}}}`: a vector of pairs of symbols and strings or vectors of strings + +The mapping can be of the form `[:variable => ["Leaf", "Internode"]]` or `[:variable => "Plant"]`. + +In the first form, the variable `variable` of the model will be taken from the `Leaf` and `Internode` nodes, and will +be available in the status as a vector of values. The order of the values in the vector is the same as the order of the nodes in the mtg. + +In the second form, the variable `variable` of the model will be taken from the `Plant` node, assuming only one node has the `Plant` symbol. +In this case the value available from the status will be a scalar. + +Note that the mapping does not make any copy of the values, it only references them. This means that if the values are updated in the status +of one node, they will be updated in the other nodes. + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +``` + +Let's take a model: + +```jldoctest mylabel +julia> model = ToyCAllocationModel() +ToyCAllocationModel() +``` + +We can make it multi-scale by defining a mapping between the variables of the model and the nodes symbols from which the values are taken from: + +For example, if the `carbon_allocation` comes from the `Leaf` and `Internode` nodes, we can define the mapping as follows: + +```jldoctest mylabel +julia> mapping = [:carbon_allocation => ["Leaf", "Internode"]] +1-element Vector{Pair{Symbol, Vector{String}}}: + :carbon_allocation => ["Leaf", "Internode"] +``` + +The mapping is a vector of pairs of symbols and strings or vectors of strings. In this case, we have only one pair to define the mapping +between the `carbon_allocation` variable and the `Leaf` and `Internode` nodes. + +We can now make the model multi-scale by passing the model and the mapping to the `MultiScaleModel` constructor : + +```jldoctest mylabel +julia> multiscale_model = MultiScaleModel(model, mapping) +MultiScaleModel{ToyCAllocationModel, String}(ToyCAllocationModel(), ["carbon_allocation" => ["Leaf", "Internode"]]) +``` + +We can access the mapping and the model: + +```jldoctest mylabel +julia> PlantSimEngine.mapping(multiscale_model) +1-element Vector{Pair{Symbol, Vector{String}}}: + :carbon_allocation => ["Leaf", "Internode"] +``` + +```jldoctest mylabel +julia> PlantSimEngine.model(multiscale_model) +ToyCAllocationModel() +``` +""" +struct MultiScaleModel{T<:AbstractModel,S<:AbstractString} + model::T + mapping::Vector{Pair{Symbol,Union{S,Vector{S}}}} +end + +function MultiScaleModel(model, mapping) + MultiScaleModel( + model, + Vector{Pair{Symbol,Union{String,Vector{String}}}}(mapping) + ) +end + +MultiScaleModel(; model, mapping) = MultiScaleModel(model, mapping) + +mapping(m::MultiScaleModel) = m.mapping +model(m::MultiScaleModel) = m.model \ No newline at end of file From 831e10cda47de18473705b7d904d1b7ffd759bf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 14 Sep 2023 11:12:08 +0200 Subject: [PATCH 12/97] import multiscale toy models in tests --- test/Project.toml | 1 + test/runtests.jl | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/test/Project.toml b/test/Project.toml index 6ae7f9b2..48535d85 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -5,6 +5,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" PlantMeteo = "4630fe09-e0fb-4da5-a846-781cb73437b6" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/runtests.jl b/test/runtests.jl index a58d1009..aee95e0c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,7 @@ using Tables, DataFrames, CSV using MultiScaleTreeGraph using PlantMeteo, Statistics using Documenter # for doctests +using Random # for ToySoilModel # Include the example dummy processes: include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) @@ -11,7 +12,12 @@ include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimGrowthModel.jl")) include(joinpath(pkgdir(PlantSimEngine), "examples/ToyRUEGrowthModel.jl")) + +# For the multiscale models: include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")) +include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")) +include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")) +include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")) @testset "Testing PlantSimEngine" begin Aqua.test_all(PlantSimEngine, ambiguities=false) From cab5994e22ccd85e5551fc12338050bfa48963d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 14 Sep 2023 11:12:26 +0200 Subject: [PATCH 13/97] Create mapping.jl [First draft] --- src/mtg/mapping.jl | 398 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100644 src/mtg/mapping.jl diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl new file mode 100644 index 00000000..9fb91c39 --- /dev/null +++ b/src/mtg/mapping.jl @@ -0,0 +1,398 @@ +""" + model(m::AbstractModel) + +Get the model of an AbstractModel (it is the model itself if it is not a MultiScaleModel). +""" +model(m::AbstractModel) = m + +# Functions to get the models from the dictionary that defines the mapping: + +""" + get_models(m) + +Get the models of a dictionary of model mapping. + +# Arguments + +- `m::Dict{String,Any}`: a dictionary of model mapping + +Returns a vector of models + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +``` + +If we just give a MultiScaleModel, we get its model as a one-element vector: + +```jldoctest mylabel +julia> models = MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + :carbon_allocation => ["Leaf", "Internode"] + ], + ); +``` + +```jldoctest mylabel +julia> get_models(models) +1-element Vector{ToyCAllocationModel}: + ToyCAllocationModel() +``` + +If we give a tuple of models, we get each model in a vector: + +```jldoctest mylabel +julia> models2 = ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ); +``` + +```jldoctest mylabel +julia> get_models(models2) +2-element Vector{AbstractModel}: + ToyAssimModel{Float64}(0.2) + ToyCDemandModel{Float64}(10.0, 200.0) +``` +""" +get_models(m) = [model(i) for i in m if !isa(i, Status)] + +# Get the models of a MultiScaleModel: +get_models(m::MultiScaleModel) = [model(m)] +# Note: it is returning a vector of models, because in this case the user provided a single MultiScaleModel instead of a vector of. + +# Get the models of an AbstractModel: +get_models(m::AbstractModel) = model(m) + +# Same, for the status (if any provided): + +""" + get_status(m) + +Get the status of a dictionary of model mapping. + +# Arguments + +- `m::Dict{String,Any}`: a dictionary of model mapping + +Returns a [`Status`](@ref) or `nothing`. + +# Examples + +See [`get_models`](@ref) for examples. +""" +function get_status(m) + st = Status[i for i in m if isa(i, Status)] + @assert length(st) <= 1 "Only one status can be provided for each organ type." + length(st) == 0 && return nothing + return first(st) +end + +get_status(m::MultiScaleModel) = nothing +get_status(m::AbstractModel) = nothing + +""" + get_mapping(m) + +Get the mapping of a dictionary of model mapping. + +# Arguments + +- `m::Dict{String,Any}`: a dictionary of model mapping + +Returns a vector of pairs of symbols and strings or vectors of strings + +# Examples + +See [`get_models`](@ref) for examples. +""" +function get_mapping(m) + mod_mapping = [mapping(i) for i in m if isa(i, MultiScaleModel)] + if length(mod_mapping) == 0 + return Pair{Symbol,String}[] + end + return reduce(vcat, mod_mapping) +end + +get_mapping(m::MultiScaleModel{T,S}) where {T,S} = mapping(m) +get_mapping(m::AbstractModel) = Pair{Symbol,String}[] + +# Functions to get the variables from the mapping: + +""" + vars_from_mapping(m) + +Get the variables that are used in the multiscale models. + +# Arguments + +- `m::Dict`: a dictionary of model mapping + +Returns a dictionary of variables (values) to organs (keys) + +See also `vars_type_from_mapping` to get the variables type. + +# Examples + +```jldoctest +vars_mapping = Dict( + ["Leaf"] => Dict(:A => PlantSimEngine.RefVector{Float64}[-Inf]), + ["Leaf", "Internode"] => Dict( + :carbon_allocation => PlantSimEngine.RefVector{Float64}[], + :carbon_demand => PlantSimEngine.RefVector{Float64}[]) +); +``` + +```jldoctest +julia> PlantSimEngine.vars_from_mapping(vars_mapping) +3-element Vector{Symbol}: + :A + :carbon_allocation + :carbon_demand +``` +""" +vars_from_mapping(m) = collect(Iterators.flatten(keys.(values(m)))) +vars_type_from_mapping(m) = collect(Iterators.flatten(values.(values(m)))) + +""" + create_var_ref(organ::Vector{<:AbstractString}, default::T) where {T} + create_var_ref(organ::AbstractString, default) + +Create a RefVector from a vector of organs and a default value. The RefVector will be filled with the default value. + +Create the reference to a multiscale variable. The reference is a RefVector if the organ was given as a vector, or a Ref if it is a scalar. +""" +function create_var_ref(organ::Vector{<:AbstractString}, var, default::T) where {T} + RefVector(Base.RefValue{T}[]) +end + +#! reverse parameter order , it is more logical +struct MappedVar{S<:AbstractString,T} + organ::S + var::Symbol + default::T +end + +function create_var_ref(organ::AbstractString, var, default) + MappedVar(organ, var, default) +end + +function outputs_from_other_scale!(var_outputs_from_mapping, multi_scale_outs, map_vars) + multi_scale_outs_organ = filter(x -> first(x) in keys(multi_scale_outs), map_vars) + for (var, organs) in multi_scale_outs_organ + # var, organs = multi_scale_outs_organ[1] + if isa(organs, String) + organs = [organs] + end + for org in organs + # org = organs[1] + if haskey(var_outputs_from_mapping, org) + push!(var_outputs_from_mapping[org], var => multi_scale_outs[var]) + else + var_outputs_from_mapping[org] = [var => multi_scale_outs[var]] + end + end + end +end + +# Initialisations with the mapping: +function init_simulation!(mtg, models; type_promotion=nothing, check=true) + # if check + # attr_name_sym = Set(keys(models)) + # multiscale_vars_names = collect(keys(multiscale_vars)) + # for i in multiscale_vars_names + # if isa(i, Vector{String}) + # for n in i + # push!(attr_name_sym, n) + # end + # else + # push!(attr_name_sym, i) + # end + # end + # # Check if all components have a model + # component_no_models = setdiff(MultiScaleTreeGraph.components(mtg), attr_name_sym) + # if length(component_no_models) > 0 + # @info string("No model found for component(s) ", join(component_no_models, ", ", ", and ")) maxlog = 1 + # end + # end + + # Initialise a dict that defines the multiscale variables for each organ type: + organs_mapping = Dict{String,Any}() + # Initialise a Dict that defines the variables that are outputs from a mapping, + # i.e. variables that are written by a model at another scale: + var_outputs_from_mapping = Dict{String,Vector{Pair{Symbol,Any}}}() + for organ in keys(models) + # organ = "Leaf" + map_vars = PlantSimEngine.get_mapping(models[organ]) + if length(map_vars) == 0 + continue + end + + multiscale_vars = collect(first(i) for i in map_vars) + mods = PlantSimEngine.get_models(models[organ]) + ins = merge(PlantSimEngine.inputs_.(mods)...) + outs = merge(PlantSimEngine.outputs_.(mods)...) + + # Variables in the node that are defined as multiscale: + multi_scale_ins = intersect(keys(ins), multiscale_vars) # inputs: variables that are taken from another scale + multi_scale_outs = intersect(keys(outs), multiscale_vars) # outputs: variables that are written to another scale + + multi_scale_vars = Status(PlantSimEngine.convert_vars(type_promotion, merge(ins[multi_scale_ins], outs[multi_scale_outs]))) + + # Users can provide initialisation values in a status. We get them here: + st = PlantSimEngine.get_status(models[organ]) + + # Add the values given by the user (initialisation) to the mapping, and make it a Status: + if isnothing(st) + new_st = multi_scale_vars + else + # If the user provided the multiscale variable in the status, and it is an output variable, + # we use those values for the mapping: + for i in keys(multi_scale_vars) + if i in multi_scale_outs && i in keys(st) + multi_scale_vars[i] = st[i] + end + end + # NB: we do this only for multiscale outputs, because this output cannot be + # defined from the models at the target scale, so we need to add it to this other scale + # as an output variable. + + new_st = Status(merge(NamedTuple(st), NamedTuple(multi_scale_vars))) + diff = intersect(keys(st), keys(multi_scale_vars)) + for i in diff + if isa(new_st[i], PlantSimEngine.RefVector) + new_st[i][1] = st[i] + else + new_st[i] = st[i] + end + end + end + + # Add outputs from this scale as a variable for other scales: + PlantSimEngine.outputs_from_other_scale!(var_outputs_from_mapping, NamedTuple(new_st)[(multi_scale_outs)], map_vars) + + organ_mapping = Dict{Union{String,Vector{String}},Dict{Symbol,Union{PlantSimEngine.RefVector,PlantSimEngine.MappedVar}}}() + for var_mapping in map_vars + # var_mapping = map_vars[1] + variable, organs_mapped = var_mapping + + if haskey(organ_mapping, organs_mapped) + push!(organ_mapping[organs_mapped], variable => PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + else + organ_mapping[organs_mapped] = Dict(variable => PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + end + end + + organs_mapping[organ] = Dict(k => NamedTuple(v) for (k, v) in organ_mapping) + # organs_mapping[organ] = organ_mapping + end + + # Output of the code above: + # - organs_mapping: for each organ type, the variables that are mapped to other scales, how they are mapped (RefVector or RefValue) + # and the nodes that are targeted by the mapping + # - var_outputs_from_mapping: for each organ type, the variables that are written by a model at another scale and its default value + + var_outputs_from_mapping = Dict(k => NamedTuple(v) for (k, v) in var_outputs_from_mapping) + + #! recommencer par ici. Ce qu'il faut que je fasse: + #! 1. instantier des RefVector{Type} pour chaque variable multiscale, pour chaque type de mapping, e.g. :A => ["Leaf, "Internode"] + #! 2. créer un Status pour chaque noeud de façon usuelle, sauf que: + #! - pour les variables multiscale, que l'on instancie en utilisant les RefVector vides. + #! - pour les variables multiscale qui sont des outputs d'autres échelles, on doit créer la variable avec une bonne valeur par défaut. + #! 3. traverser les MTG pour remplir les RefVector, sachant qu'ils seront automatiquement remplis partout puisque c'est des Ref (a vérifier). + #! 4. ajouter des checks, par exemple une variable output + #! 5. faire un tableau de status, qui potentiellement référence le noeud. Et faire une `sort` en fonction de l'arbre de dépendence multi-échelle + #! 6. ajouter des tests + + # Vector of statuses, pre-initialised with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: + organs_statuses = Dict{String,Status}() + #! what we need to do here: + #! 1. get the models for the node + #! 2. get the variables that are defined as multiscale (used by other scales), + #! and use them as default values for the status (do not forget to remove the default value at some point) + #! (check why we need a default value here, it was used for the output at the other scale but may not be needed anymore) + #! 3. get the variables that are written by a model from another scale (if so), and add them to the status + #! tip: utiliser `var_outputs_from_mapping` pour ajouter les variables dans le status + #! des échelles cibles. (Dict de échelle cible: variables à ajouter) + + # We make a pre-initialised status for each kind of organ: + for organ in keys(models) + # organ = "Soil" + # Parsing the models into a NamedTuple to get the process name: + node_models = PlantSimEngine.parse_models(PlantSimEngine.get_models(models[organ])) + + # Get the status if any was given by the user (this can be used as default values in the mapping): + st = PlantSimEngine.get_status(models[organ]) # User status + + if isnothing(st) + st = NamedTuple() + else + st = NamedTuple(st) + end + + # Add the variables that are defined as multiscale (coming from other scales): + if haskey(organs_mapping, organ) + st_vars_mapped = (; zip(PlantSimEngine.vars_from_mapping(organs_mapping[organ]), PlantSimEngine.vars_type_from_mapping(organs_mapping[organ]))...) + !isnothing(st_vars_mapped) && (st = merge(st, st_vars_mapped)) + end + + # Add the variable(s) written by other scales into this node scale: + haskey(var_outputs_from_mapping, organ) && (st = merge(st, var_outputs_from_mapping[organ])) + + # Then we initialise a status taking into account the status given by the user. + # This step is done to get default values for each variables: + if length(st) == 0 + st = nothing + else + st = Status(st) + end + + st = PlantSimEngine.add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) + # The status is added to the vector of statuses. + push!(organs_statuses, organ => st) + end + + # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable + # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. + for (organ, st) in organs_statuses # e.g.: organ = "Leaf"; st = organs_statuses[organ] + # If there is any MappedVar in the status: + if any(x -> isa(x, PlantSimEngine.MappedVar), values(st)) + val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) + for (k, v) in val_pointers + if isa(v, PlantSimEngine.MappedVar) + val_pointers[k] = PlantSimEngine.refvalue(organs_statuses[val.organ], val.var) + else + val_pointers[k] = PlantSimEngine.refvalue(st, k) + end + end + organs_statuses[organ] = Status(NamedTuple(val_pointers)) + end + end + + #! continue here. What we need to do: + #! 3. traverser les MTG pour remplir les RefVector, sachant qu'ils seront automatiquement remplis partout puisque c'est des Ref (a vérifier). + #! 4. ajouter des checks, par exemple une variable output + #! 5. faire un tableau de status, qui potentiellement référence le noeud. Et faire une `sort` en fonction de l'arbre de dépendence multi-échelle + #! 6. ajouter des tests + statuses = Status[] + # We traverse the MTG to initialise the mapping depending on the number of nodes and their types + traverse!(mtg) do node + # Check if the node has a model defined for its symbol: + # node = get_node(mtg, 1) + push!(statuses, organs_statuses[node.MTG.symbol]) + organs_statuses + end +end \ No newline at end of file From d72e7e1dedfba1ace5b3593d05483118027286c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 14 Sep 2023 11:12:33 +0200 Subject: [PATCH 14/97] Update PlantSimEngine.jl --- src/PlantSimEngine.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 9970a2bd..3658a136 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -34,6 +34,7 @@ include("component_models/TimeStepTable.jl") # List of models: include("component_models/ModelList.jl") +include("mtg/MultiScaleModel.jl") # Getters / setters for status: include("component_models/get_status.jl") @@ -43,6 +44,7 @@ include("dataframe.jl") # MTG compatibility: include("mtg/init_mtg_models.jl") +include("mtg/mapping.jl") # Model evaluation (statistics): include("evaluation/statistics.jl") @@ -70,7 +72,7 @@ include("run.jl") include("evaluation/fit.jl") export AbstractModel -export ModelList +export ModelList, MultiScaleModel export init_mtg_models! export RMSE, NRMSE, EF, dr export Status, TimeStepTable, status From 26fc0bf59cdec90b5c4d39e712d963f2576fefaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 18 Sep 2023 11:28:21 +0200 Subject: [PATCH 15/97] Make default status from mtg model list + check if missing init --- src/mtg/mapping.jl | 97 ++++++++++++++++++++++++- src/processes/model_initialisation.jl | 100 +++++++++++++++++++++++++- 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 9fb91c39..c4a7677e 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -1,3 +1,7 @@ +inputs_(m::MultiScaleModel) = inputs_(m.model) +outputs_(m::MultiScaleModel) = outputs_(m.model) + + """ model(m::AbstractModel) @@ -75,7 +79,7 @@ get_models(m::MultiScaleModel) = [model(m)] # Note: it is returning a vector of models, because in this case the user provided a single MultiScaleModel instead of a vector of. # Get the models of an AbstractModel: -get_models(m::AbstractModel) = model(m) +get_models(m::AbstractModel) = [model(m)] # Same, for the status (if any provided): @@ -130,8 +134,88 @@ end get_mapping(m::MultiScaleModel{T,S}) where {T,S} = mapping(m) get_mapping(m::AbstractModel) = Pair{Symbol,String}[] -# Functions to get the variables from the mapping: +""" + compute_mapping(models::Dict{String,Any}, type_promotion) + + +""" +function compute_mapping(models::Dict{String,Any}, type_promotion) + # Initialise a dict that defines the multiscale variables for each organ type: + organs_mapping = Dict{String,Any}() + # Initialise a Dict that defines the variables that are outputs from a mapping, + # i.e. variables that are written by a model at another scale: + var_outputs_from_mapping = Dict{String,Vector{Pair{Symbol,Any}}}() + for organ in keys(models) + # organ = "Leaf" + map_vars = PlantSimEngine.get_mapping(models[organ]) + if length(map_vars) == 0 + continue + end + + multiscale_vars = collect(first(i) for i in map_vars) + mods = PlantSimEngine.get_models(models[organ]) + ins = merge(PlantSimEngine.inputs_.(mods)...) + outs = merge(PlantSimEngine.outputs_.(mods)...) + + # Variables in the node that are defined as multiscale: + multi_scale_ins = intersect(keys(ins), multiscale_vars) # inputs: variables that are taken from another scale + multi_scale_outs = intersect(keys(outs), multiscale_vars) # outputs: variables that are written to another scale + + multi_scale_vars = Status(PlantSimEngine.convert_vars(type_promotion, merge(ins[multi_scale_ins], outs[multi_scale_outs]))) + + # Users can provide initialisation values in a status. We get them here: + st = PlantSimEngine.get_status(models[organ]) + + # Add the values given by the user (initialisation) to the mapping, and make it a Status: + if isnothing(st) + new_st = multi_scale_vars + else + # If the user provided the multiscale variable in the status, and it is an output variable, + # we use those values for the mapping: + for i in keys(multi_scale_vars) + if i in multi_scale_outs && i in keys(st) + multi_scale_vars[i] = st[i] + end + end + # NB: we do this only for multiscale outputs, because this output cannot be + # defined from the models at the target scale, so we need to add it to this other scale + # as an output variable. + + new_st = Status(merge(NamedTuple(st), NamedTuple(multi_scale_vars))) + diff = intersect(keys(st), keys(multi_scale_vars)) + for i in diff + if isa(new_st[i], PlantSimEngine.RefVector) + new_st[i][1] = st[i] + else + new_st[i] = st[i] + end + end + end + # Add outputs from this scale as a variable for other scales: + PlantSimEngine.outputs_from_other_scale!(var_outputs_from_mapping, NamedTuple(new_st)[(multi_scale_outs)], map_vars) + + organ_mapping = Dict{Union{String,Vector{String}},Dict{Symbol,Union{PlantSimEngine.RefVector,PlantSimEngine.MappedVar}}}() + for var_mapping in map_vars + # var_mapping = map_vars[1] + variable, organs_mapped = var_mapping + + if haskey(organ_mapping, organs_mapped) + push!(organ_mapping[organs_mapped], variable => PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + else + organ_mapping[organs_mapped] = Dict(variable => PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + end + end + + organs_mapping[organ] = Dict(k => NamedTuple(v) for (k, v) in organ_mapping) + end + + var_outputs_from_mapping = Dict(k => NamedTuple(v) for (k, v) in var_outputs_from_mapping) + + return (; organs_mapping, var_outputs_from_mapping) +end + +# Functions to get the variables from the mapping: """ vars_from_mapping(m) @@ -395,4 +479,13 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) push!(statuses, organs_statuses[node.MTG.symbol]) organs_statuses end +end + + +function map_scale(f, m, scale::String) + map_scale(f, m, [scale]) +end + +function map_scale(f, m, scales::AbstractVector{String}) + map(s -> f(m, s), scales) end \ No newline at end of file diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 04cc9ea6..8b477b78 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -106,6 +106,103 @@ function to_initialize(; verbose=true, vars...) return NamedTuple(to_init) end +# For the list of models given to an MTG: +function to_initialize(models::Dict{String,Any}, organs_statuses) + var_need_init = Dict{String,Any}() + for organ in keys(models) + # organ = "Plant" + # Get all models for the organ: + mods = PlantSimEngine.get_models(models[organ]) + map_vars = PlantSimEngine.get_mapping(models[organ]) + multiscale_vars = collect(first(i) for i in map_vars) + ins = merge(PlantSimEngine.inputs_.(mods)...) + outs = merge(PlantSimEngine.outputs_.(mods)...) + + # Variables in the node that are defined as multiscale: + multi_scale_ins = intersect(keys(ins), multiscale_vars) # inputs: variables that are taken from another scale + # multi_scale_outs = intersect(keys(outs), multiscale_vars) # outputs: variables that are written to another scale + + # Variables we need to initialise for this scale: + vars_needed_this_scale = setdiff(keys(ins), keys(outs)) + + need_initialisation = Symbol[] + need_models_from_scales = NamedTuple{(:var, :scale, :need_scales),Tuple{Symbol,String,Union{String,Vector{String}}}}[] + + for var in vars_needed_this_scale # e.g. var = :carbon_demand + # If the variable is multiscale (it is computed by anothe model), we check if there is a model at the + # other scale(s) that computes it: + if var in multi_scale_ins + # Scale(s) at which the variable is computed: + from_scales = last(map_vars[findfirst(i -> i == var, multiscale_vars)]) + # We check if there is a model at the other scale(s) that computes it: + outputs_from_scales = PlantSimEngine.map_scale(models, from_scales) do m, s + # We check that the node type exist in the model list: + haskey(m, s) || error( + "Nodes of type $organ are mapping to variable `:$var` computed from nodes of type $s, but there is no type $s in the list of models." + ) + # If it does, we get the outputs of its models: + merge(PlantSimEngine.outputs_.(PlantSimEngine.get_models(m[s]))...) + end + + outputs_from_scales = merge(outputs_from_scales...) + push!(need_models_from_scales, (var=var, scale=organ, need_scales=from_scales)) + elseif organs_statuses[organ][var] == ins[var] + push!(need_initialisation, var) + end + # Note: if the variable is an output of the model for another scale (in `multi_scale_outs`), we don't need to initialise it at this scale. + end + if length(need_initialisation) > 0 + var_need_init[organ] = (; need_initialisation, need_models_from_scales) + end + # to_initialize(ModelList(PlantSimEngine.parse_models(mods), organs_statuses[organ])) + end +end + + +function get_status(models::Dict{String,Any}, type_promotion) + organs_mapping, var_outputs_from_mapping = PlantSimEngine.compute_mapping(models, type_promotion) + # Vector of statuses, pre-initialised with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: + organs_statuses = Dict{String,Status}() + + for organ in keys(models) + # organ = "Internode" + # Parsing the models into a NamedTuple to get the process name: + node_models = PlantSimEngine.parse_models(PlantSimEngine.get_models(models[organ])) + + # Get the status if any was given by the user (this can be used as default values in the mapping): + st = PlantSimEngine.get_status(models[organ]) # User status + + if isnothing(st) + st = NamedTuple() + else + st = NamedTuple(st) + end + + # Add the variables that are defined as multiscale (coming from other scales): + if haskey(organs_mapping, organ) + st_vars_mapped = (; zip(PlantSimEngine.vars_from_mapping(organs_mapping[organ]), PlantSimEngine.vars_type_from_mapping(organs_mapping[organ]))...) + !isnothing(st_vars_mapped) && (st = merge(st, st_vars_mapped)) + end + + # Add the variable(s) written by other scales into this node scale: + haskey(var_outputs_from_mapping, organ) && (st = merge(st, var_outputs_from_mapping[organ])) + + # Then we initialise a status taking into account the status given by the user. + # This step is done to get default values for each variables: + if length(st) == 0 + st = nothing + else + st = Status(st) + end + + st = PlantSimEngine.add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) + # The status is added to the vector of statuses. + push!(organs_statuses, organ => st) + end + + return organs_statuses +end + """ init_status!(object::Dict{String,ModelList};vars...) init_status!(component::ModelList;vars...) @@ -274,7 +371,8 @@ function vars_not_init_(st::T, default_values) where {T<:Status} not_init = Symbol[] for i in keys(default_values) - if getproperty(st, i) == default_values[i] + # if the variable value is equal to the default value, or if it is an uninitialized RefVector (length == 0): + if getproperty(st, i) == default_values[i] || (isa(getproperty(st, i), RefVector) && length(getproperty(st, i)) == 0) push!(not_init, i) end end From 0112b789b38813380b6b36559d7ce9fbcbf104b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 18 Sep 2023 12:18:36 +0200 Subject: [PATCH 16/97] Simplify the code for MTG --- src/mtg/mapping.jl | 233 ++++++++++++++++++++++---- src/processes/model_initialisation.jl | 74 +++----- 2 files changed, 225 insertions(+), 82 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index c4a7677e..6c2cdd70 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -147,24 +147,24 @@ function compute_mapping(models::Dict{String,Any}, type_promotion) var_outputs_from_mapping = Dict{String,Vector{Pair{Symbol,Any}}}() for organ in keys(models) # organ = "Leaf" - map_vars = PlantSimEngine.get_mapping(models[organ]) + map_vars = get_mapping(models[organ]) if length(map_vars) == 0 continue end multiscale_vars = collect(first(i) for i in map_vars) - mods = PlantSimEngine.get_models(models[organ]) - ins = merge(PlantSimEngine.inputs_.(mods)...) - outs = merge(PlantSimEngine.outputs_.(mods)...) + mods = get_models(models[organ]) + ins = merge(inputs_.(mods)...) + outs = merge(outputs_.(mods)...) # Variables in the node that are defined as multiscale: multi_scale_ins = intersect(keys(ins), multiscale_vars) # inputs: variables that are taken from another scale multi_scale_outs = intersect(keys(outs), multiscale_vars) # outputs: variables that are written to another scale - multi_scale_vars = Status(PlantSimEngine.convert_vars(type_promotion, merge(ins[multi_scale_ins], outs[multi_scale_outs]))) + multi_scale_vars = Status(convert_vars(type_promotion, merge(ins[multi_scale_ins], outs[multi_scale_outs]))) # Users can provide initialisation values in a status. We get them here: - st = PlantSimEngine.get_status(models[organ]) + st = get_status(models[organ]) # Add the values given by the user (initialisation) to the mapping, and make it a Status: if isnothing(st) @@ -184,7 +184,7 @@ function compute_mapping(models::Dict{String,Any}, type_promotion) new_st = Status(merge(NamedTuple(st), NamedTuple(multi_scale_vars))) diff = intersect(keys(st), keys(multi_scale_vars)) for i in diff - if isa(new_st[i], PlantSimEngine.RefVector) + if isa(new_st[i], RefVector) new_st[i][1] = st[i] else new_st[i] = st[i] @@ -193,17 +193,17 @@ function compute_mapping(models::Dict{String,Any}, type_promotion) end # Add outputs from this scale as a variable for other scales: - PlantSimEngine.outputs_from_other_scale!(var_outputs_from_mapping, NamedTuple(new_st)[(multi_scale_outs)], map_vars) + outputs_from_other_scale!(var_outputs_from_mapping, NamedTuple(new_st)[(multi_scale_outs)], map_vars) - organ_mapping = Dict{Union{String,Vector{String}},Dict{Symbol,Union{PlantSimEngine.RefVector,PlantSimEngine.MappedVar}}}() + organ_mapping = Dict{Union{String,Vector{String}},Dict{Symbol,Union{RefVector,MappedVar}}}() for var_mapping in map_vars # var_mapping = map_vars[1] variable, organs_mapped = var_mapping if haskey(organ_mapping, organs_mapped) - push!(organ_mapping[organs_mapped], variable => PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + push!(organ_mapping[organs_mapped], variable => create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) else - organ_mapping[organs_mapped] = Dict(variable => PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + organ_mapping[organs_mapped] = Dict(variable => create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) end end @@ -233,15 +233,15 @@ See also `vars_type_from_mapping` to get the variables type. ```jldoctest vars_mapping = Dict( - ["Leaf"] => Dict(:A => PlantSimEngine.RefVector{Float64}[-Inf]), + ["Leaf"] => Dict(:A => RefVector{Float64}[-Inf]), ["Leaf", "Internode"] => Dict( - :carbon_allocation => PlantSimEngine.RefVector{Float64}[], - :carbon_demand => PlantSimEngine.RefVector{Float64}[]) + :carbon_allocation => RefVector{Float64}[], + :carbon_demand => RefVector{Float64}[]) ); ``` ```jldoctest -julia> PlantSimEngine.vars_from_mapping(vars_mapping) +julia> vars_from_mapping(vars_mapping) 3-element Vector{Symbol}: :A :carbon_allocation @@ -320,24 +320,24 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) var_outputs_from_mapping = Dict{String,Vector{Pair{Symbol,Any}}}() for organ in keys(models) # organ = "Leaf" - map_vars = PlantSimEngine.get_mapping(models[organ]) + map_vars = get_mapping(models[organ]) if length(map_vars) == 0 continue end multiscale_vars = collect(first(i) for i in map_vars) - mods = PlantSimEngine.get_models(models[organ]) - ins = merge(PlantSimEngine.inputs_.(mods)...) - outs = merge(PlantSimEngine.outputs_.(mods)...) + mods = get_models(models[organ]) + ins = merge(inputs_.(mods)...) + outs = merge(outputs_.(mods)...) # Variables in the node that are defined as multiscale: multi_scale_ins = intersect(keys(ins), multiscale_vars) # inputs: variables that are taken from another scale multi_scale_outs = intersect(keys(outs), multiscale_vars) # outputs: variables that are written to another scale - multi_scale_vars = Status(PlantSimEngine.convert_vars(type_promotion, merge(ins[multi_scale_ins], outs[multi_scale_outs]))) + multi_scale_vars = Status(convert_vars(type_promotion, merge(ins[multi_scale_ins], outs[multi_scale_outs]))) # Users can provide initialisation values in a status. We get them here: - st = PlantSimEngine.get_status(models[organ]) + st = get_status(models[organ]) # Add the values given by the user (initialisation) to the mapping, and make it a Status: if isnothing(st) @@ -357,7 +357,7 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) new_st = Status(merge(NamedTuple(st), NamedTuple(multi_scale_vars))) diff = intersect(keys(st), keys(multi_scale_vars)) for i in diff - if isa(new_st[i], PlantSimEngine.RefVector) + if isa(new_st[i], RefVector) new_st[i][1] = st[i] else new_st[i] = st[i] @@ -366,17 +366,17 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) end # Add outputs from this scale as a variable for other scales: - PlantSimEngine.outputs_from_other_scale!(var_outputs_from_mapping, NamedTuple(new_st)[(multi_scale_outs)], map_vars) + outputs_from_other_scale!(var_outputs_from_mapping, NamedTuple(new_st)[(multi_scale_outs)], map_vars) - organ_mapping = Dict{Union{String,Vector{String}},Dict{Symbol,Union{PlantSimEngine.RefVector,PlantSimEngine.MappedVar}}}() + organ_mapping = Dict{Union{String,Vector{String}},Dict{Symbol,Union{RefVector,MappedVar}}}() for var_mapping in map_vars # var_mapping = map_vars[1] variable, organs_mapped = var_mapping if haskey(organ_mapping, organs_mapped) - push!(organ_mapping[organs_mapped], variable => PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + push!(organ_mapping[organs_mapped], variable => create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) else - organ_mapping[organs_mapped] = Dict(variable => PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + organ_mapping[organs_mapped] = Dict(variable => create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) end end @@ -416,10 +416,10 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) for organ in keys(models) # organ = "Soil" # Parsing the models into a NamedTuple to get the process name: - node_models = PlantSimEngine.parse_models(PlantSimEngine.get_models(models[organ])) + node_models = parse_models(get_models(models[organ])) # Get the status if any was given by the user (this can be used as default values in the mapping): - st = PlantSimEngine.get_status(models[organ]) # User status + st = get_status(models[organ]) # User status if isnothing(st) st = NamedTuple() @@ -429,7 +429,7 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) # Add the variables that are defined as multiscale (coming from other scales): if haskey(organs_mapping, organ) - st_vars_mapped = (; zip(PlantSimEngine.vars_from_mapping(organs_mapping[organ]), PlantSimEngine.vars_type_from_mapping(organs_mapping[organ]))...) + st_vars_mapped = (; zip(vars_from_mapping(organs_mapping[organ]), vars_type_from_mapping(organs_mapping[organ]))...) !isnothing(st_vars_mapped) && (st = merge(st, st_vars_mapped)) end @@ -444,7 +444,7 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) st = Status(st) end - st = PlantSimEngine.add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) + st = add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) # The status is added to the vector of statuses. push!(organs_statuses, organ => st) end @@ -453,13 +453,13 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. for (organ, st) in organs_statuses # e.g.: organ = "Leaf"; st = organs_statuses[organ] # If there is any MappedVar in the status: - if any(x -> isa(x, PlantSimEngine.MappedVar), values(st)) + if any(x -> isa(x, MappedVar), values(st)) val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) for (k, v) in val_pointers - if isa(v, PlantSimEngine.MappedVar) - val_pointers[k] = PlantSimEngine.refvalue(organs_statuses[val.organ], val.var) + if isa(v, MappedVar) + val_pointers[k] = refvalue(organs_statuses[val.organ], val.var) else - val_pointers[k] = PlantSimEngine.refvalue(st, k) + val_pointers[k] = refvalue(st, k) end end organs_statuses[organ] = Status(NamedTuple(val_pointers)) @@ -488,4 +488,169 @@ end function map_scale(f, m, scales::AbstractVector{String}) map(s -> f(m, s), scales) +end + + +# Return an error if some variables are not initialized or computed by other models in the output +# from to_initialize(models, organs_statuses) +function error_mtg_init(var_need_init) + if length(var_need_init) > 0 + error_string = String[] + for need_init in var_need_init + organ_init = first(need_init) + need_initialisation = last(need_init).need_initialisation + + # A model needs initialisations: + if length(need_initialisation) > 0 + push!( + error_string, + "Nodes of type $organ_init need variables $need_initialisation, but they are not initialized or computed." + ) + end + + # The mapping is wrong: + need_models_from_scales = last(need_init).need_models_from_scales + for er in need_models_from_scales + var, scale, need_scales = er + push!( + error_string, + "Nodes of type $need_scales should provide a model to compute variable `:$var` as input for nodes of type $scale, but none is provided." + ) + end + end + + if length(error_string) > 0 + error(join(error_string, "\n")) + end + end +end + + +""" + status_template(models::Dict{String,Any}, type_promotion) + +Create a status template for a given set of models and type promotion. + +# Arguments +- `models::Dict{String,Any}`: A dictionary of models. +- `type_promotion`: The type promotion to use. + +# Returns +- A dictionary with a `status` template for each organ type. + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine, Random +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +``` + +```jldoctest mylabel +julia> models = Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ), + "Soil" => ( + ToySoilWaterModel(), + ), + ); +``` + +```jldoctest mylabel +julia> status_template(models, nothing) +Dict{String, Status} with 4 entries: + "Soil" => Status(soil_water_content = -Inf,) + "Internode" => Status(TT = -Inf, carbon_demand = -Inf, carbon_allocation = -Inf) + "Plant" => Status(A = RefVector{Float64}[], carbon_demand = RefVector{Float64}[], carbon_offer = -Inf, carbon_allocation = RefVector{Float64}[]) + "Leaf" => Status(aPPFD = 1300.0, soil_water_content = MappedVar{String, Float64}("Soil", :soil_water_content, -Inf), A = -Inf, TT = 10.0, carbon_demand = -Inf, carbon_allocation = -Inf) +``` + +Note that variables that are multiscale (*i.e.* defined in a mapping) are linked between scales, so if we write at a scale, the value will be +automatically updated at the other scale: + +```jldoctest mylabel +organs_statuses["Soil"][:soil_water_content] === organs_statuses["Leaf"][:soil_water_content] +true +``` +""" +function status_template(models::Dict{String,Any}, type_promotion) + organs_mapping, var_outputs_from_mapping = compute_mapping(models, type_promotion) + # Vector of statuses, pre-initialised with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: + organs_statuses = Dict{String,Status}() + + for organ in keys(models) + # organ = "Internode" + # Parsing the models into a NamedTuple to get the process name: + node_models = parse_models(get_models(models[organ])) + + # Get the status if any was given by the user (this can be used as default values in the mapping): + st = get_status(models[organ]) # User status + + if isnothing(st) + st = NamedTuple() + else + st = NamedTuple(st) + end + + # Add the variables that are defined as multiscale (coming from other scales): + if haskey(organs_mapping, organ) + st_vars_mapped = (; zip(vars_from_mapping(organs_mapping[organ]), vars_type_from_mapping(organs_mapping[organ]))...) + !isnothing(st_vars_mapped) && (st = merge(st, st_vars_mapped)) + end + + # Add the variable(s) written by other scales into this node scale: + haskey(var_outputs_from_mapping, organ) && (st = merge(st, var_outputs_from_mapping[organ])) + + # Then we initialise a status taking into account the status given by the user. + # This step is done to get default values for each variables: + if length(st) == 0 + st = nothing + else + st = Status(st) + end + + st = add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) + # The status is added to the vector of statuses. + push!(organs_statuses, organ => st) + end + + # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable + # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. + for (organ, st) in organs_statuses # e.g.: organ = "Leaf"; st = organs_statuses[organ] + # If there is any MappedVar in the status: + if any(x -> isa(x, MappedVar), values(st)) + val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) + for (k, v) in val_pointers + if isa(v, MappedVar) + val_pointers[k] = refvalue(organs_statuses[v.organ], v.var) + else + val_pointers[k] = refvalue(st, k) + end + end + organs_statuses[organ] = Status(NamedTuple(val_pointers)) + end + end + + return organs_statuses end \ No newline at end of file diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 8b477b78..481e68f3 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -106,8 +106,22 @@ function to_initialize(; verbose=true, vars...) return NamedTuple(to_init) end +""" + VarFromMTG(var::Symbol, scale::String) + +A strucure to hold the variables that are needed for initialisation, and that must be taken from the MTG attributes. +""" +struct VarFromMTG + var::Symbol + scale::String +end + # For the list of models given to an MTG: -function to_initialize(models::Dict{String,Any}, organs_statuses) +function to_initialize(models::Dict{String,Any}, organs_statuses, mtg) + + # Get the variables in the MTG: + vars_in_mtg = names(mtg) + var_need_init = Dict{String,Any}() for organ in keys(models) # organ = "Plant" @@ -126,6 +140,7 @@ function to_initialize(models::Dict{String,Any}, organs_statuses) vars_needed_this_scale = setdiff(keys(ins), keys(outs)) need_initialisation = Symbol[] + need_var_from_mtg = VarFromMTG[] need_models_from_scales = NamedTuple{(:var, :scale, :need_scales),Tuple{Symbol,String,Union{String,Vector{String}}}}[] for var in vars_needed_this_scale # e.g. var = :carbon_demand @@ -147,60 +162,23 @@ function to_initialize(models::Dict{String,Any}, organs_statuses) outputs_from_scales = merge(outputs_from_scales...) push!(need_models_from_scales, (var=var, scale=organ, need_scales=from_scales)) elseif organs_statuses[organ][var] == ins[var] - push!(need_initialisation, var) + # In this case the variable is an input of the model, and is not computed by other models at this scale or the others. + if var in vars_in_mtg + # If the variable can be found in the MTG, we will take it from there: + push!(need_var_from_mtg, VarFromMTG(var, organ)) + else + # Else, the user need to initialise it: + push!(need_initialisation, var) + end end # Note: if the variable is an output of the model for another scale (in `multi_scale_outs`), we don't need to initialise it at this scale. end if length(need_initialisation) > 0 - var_need_init[organ] = (; need_initialisation, need_models_from_scales) + var_need_init[organ] = (; need_initialisation, need_models_from_scales, need_var_from_mtg) end - # to_initialize(ModelList(PlantSimEngine.parse_models(mods), organs_statuses[organ])) - end -end - - -function get_status(models::Dict{String,Any}, type_promotion) - organs_mapping, var_outputs_from_mapping = PlantSimEngine.compute_mapping(models, type_promotion) - # Vector of statuses, pre-initialised with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: - organs_statuses = Dict{String,Status}() - - for organ in keys(models) - # organ = "Internode" - # Parsing the models into a NamedTuple to get the process name: - node_models = PlantSimEngine.parse_models(PlantSimEngine.get_models(models[organ])) - - # Get the status if any was given by the user (this can be used as default values in the mapping): - st = PlantSimEngine.get_status(models[organ]) # User status - - if isnothing(st) - st = NamedTuple() - else - st = NamedTuple(st) - end - - # Add the variables that are defined as multiscale (coming from other scales): - if haskey(organs_mapping, organ) - st_vars_mapped = (; zip(PlantSimEngine.vars_from_mapping(organs_mapping[organ]), PlantSimEngine.vars_type_from_mapping(organs_mapping[organ]))...) - !isnothing(st_vars_mapped) && (st = merge(st, st_vars_mapped)) - end - - # Add the variable(s) written by other scales into this node scale: - haskey(var_outputs_from_mapping, organ) && (st = merge(st, var_outputs_from_mapping[organ])) - - # Then we initialise a status taking into account the status given by the user. - # This step is done to get default values for each variables: - if length(st) == 0 - st = nothing - else - st = Status(st) - end - - st = PlantSimEngine.add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) - # The status is added to the vector of statuses. - push!(organs_statuses, organ => st) end - return organs_statuses + return var_need_init end """ From baf923c975ee771098baf3339bb7370e1ac73429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 21 Sep 2023 16:43:31 +0200 Subject: [PATCH 17/97] Add doc to compute_mapping --- src/mtg/mapping.jl | 244 +++++++++++++-------------------------------- 1 file changed, 68 insertions(+), 176 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 6c2cdd70..ca0be34d 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -137,7 +137,59 @@ get_mapping(m::AbstractModel) = Pair{Symbol,String}[] """ compute_mapping(models::Dict{String,Any}, type_promotion) +Compute the mapping of a dictionary of model mapping. +# Arguments + +- `models::Dict{String,Any}`: a dictionary of model mapping +- `type_promotion`: the type promotion to use for the variables + +# Returns + +- organs_mapping: for each organ type, the variables that are mapped to other scales, how they are mapped (RefVector or RefValue) +and the nodes that are targeted by the mapping +- var_outputs_from_mapping: for each organ type, the variables that are written by a model at another scale and its default value + + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +``` + +```jldoctest mylabel +models = Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ), +) +``` + +```jldoctest mylabel +compute_mapping(models, nothing) +``` """ function compute_mapping(models::Dict{String,Any}, type_promotion) # Initialise a dict that defines the multiscale variables for each organ type: @@ -182,8 +234,8 @@ function compute_mapping(models::Dict{String,Any}, type_promotion) # as an output variable. new_st = Status(merge(NamedTuple(st), NamedTuple(multi_scale_vars))) - diff = intersect(keys(st), keys(multi_scale_vars)) - for i in diff + diff_keys = intersect(keys(st), keys(multi_scale_vars)) + for i in diff_keys if isa(new_st[i], RefVector) new_st[i][1] = st[i] else @@ -294,183 +346,23 @@ end # Initialisations with the mapping: function init_simulation!(mtg, models; type_promotion=nothing, check=true) - # if check - # attr_name_sym = Set(keys(models)) - # multiscale_vars_names = collect(keys(multiscale_vars)) - # for i in multiscale_vars_names - # if isa(i, Vector{String}) - # for n in i - # push!(attr_name_sym, n) - # end - # else - # push!(attr_name_sym, i) - # end - # end - # # Check if all components have a model - # component_no_models = setdiff(MultiScaleTreeGraph.components(mtg), attr_name_sym) - # if length(component_no_models) > 0 - # @info string("No model found for component(s) ", join(component_no_models, ", ", ", and ")) maxlog = 1 - # end - # end - - # Initialise a dict that defines the multiscale variables for each organ type: - organs_mapping = Dict{String,Any}() - # Initialise a Dict that defines the variables that are outputs from a mapping, - # i.e. variables that are written by a model at another scale: - var_outputs_from_mapping = Dict{String,Vector{Pair{Symbol,Any}}}() - for organ in keys(models) - # organ = "Leaf" - map_vars = get_mapping(models[organ]) - if length(map_vars) == 0 - continue - end - - multiscale_vars = collect(first(i) for i in map_vars) - mods = get_models(models[organ]) - ins = merge(inputs_.(mods)...) - outs = merge(outputs_.(mods)...) - - # Variables in the node that are defined as multiscale: - multi_scale_ins = intersect(keys(ins), multiscale_vars) # inputs: variables that are taken from another scale - multi_scale_outs = intersect(keys(outs), multiscale_vars) # outputs: variables that are written to another scale - - multi_scale_vars = Status(convert_vars(type_promotion, merge(ins[multi_scale_ins], outs[multi_scale_outs]))) - - # Users can provide initialisation values in a status. We get them here: - st = get_status(models[organ]) - - # Add the values given by the user (initialisation) to the mapping, and make it a Status: - if isnothing(st) - new_st = multi_scale_vars - else - # If the user provided the multiscale variable in the status, and it is an output variable, - # we use those values for the mapping: - for i in keys(multi_scale_vars) - if i in multi_scale_outs && i in keys(st) - multi_scale_vars[i] = st[i] - end - end - # NB: we do this only for multiscale outputs, because this output cannot be - # defined from the models at the target scale, so we need to add it to this other scale - # as an output variable. - - new_st = Status(merge(NamedTuple(st), NamedTuple(multi_scale_vars))) - diff = intersect(keys(st), keys(multi_scale_vars)) - for i in diff - if isa(new_st[i], RefVector) - new_st[i][1] = st[i] - else - new_st[i] = st[i] - end - end - end - - # Add outputs from this scale as a variable for other scales: - outputs_from_other_scale!(var_outputs_from_mapping, NamedTuple(new_st)[(multi_scale_outs)], map_vars) - - organ_mapping = Dict{Union{String,Vector{String}},Dict{Symbol,Union{RefVector,MappedVar}}}() - for var_mapping in map_vars - # var_mapping = map_vars[1] - variable, organs_mapped = var_mapping - - if haskey(organ_mapping, organs_mapped) - push!(organ_mapping[organs_mapped], variable => create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) - else - organ_mapping[organs_mapped] = Dict(variable => create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) - end - end - - organs_mapping[organ] = Dict(k => NamedTuple(v) for (k, v) in organ_mapping) - # organs_mapping[organ] = organ_mapping - end - - # Output of the code above: - # - organs_mapping: for each organ type, the variables that are mapped to other scales, how they are mapped (RefVector or RefValue) - # and the nodes that are targeted by the mapping - # - var_outputs_from_mapping: for each organ type, the variables that are written by a model at another scale and its default value - - var_outputs_from_mapping = Dict(k => NamedTuple(v) for (k, v) in var_outputs_from_mapping) - - #! recommencer par ici. Ce qu'il faut que je fasse: - #! 1. instantier des RefVector{Type} pour chaque variable multiscale, pour chaque type de mapping, e.g. :A => ["Leaf, "Internode"] - #! 2. créer un Status pour chaque noeud de façon usuelle, sauf que: - #! - pour les variables multiscale, que l'on instancie en utilisant les RefVector vides. - #! - pour les variables multiscale qui sont des outputs d'autres échelles, on doit créer la variable avec une bonne valeur par défaut. - #! 3. traverser les MTG pour remplir les RefVector, sachant qu'ils seront automatiquement remplis partout puisque c'est des Ref (a vérifier). - #! 4. ajouter des checks, par exemple une variable output - #! 5. faire un tableau de status, qui potentiellement référence le noeud. Et faire une `sort` en fonction de l'arbre de dépendence multi-échelle - #! 6. ajouter des tests - - # Vector of statuses, pre-initialised with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: - organs_statuses = Dict{String,Status}() - #! what we need to do here: - #! 1. get the models for the node - #! 2. get the variables that are defined as multiscale (used by other scales), - #! and use them as default values for the status (do not forget to remove the default value at some point) - #! (check why we need a default value here, it was used for the output at the other scale but may not be needed anymore) - #! 3. get the variables that are written by a model from another scale (if so), and add them to the status - #! tip: utiliser `var_outputs_from_mapping` pour ajouter les variables dans le status - #! des échelles cibles. (Dict de échelle cible: variables à ajouter) - - # We make a pre-initialised status for each kind of organ: - for organ in keys(models) - # organ = "Soil" - # Parsing the models into a NamedTuple to get the process name: - node_models = parse_models(get_models(models[organ])) - - # Get the status if any was given by the user (this can be used as default values in the mapping): - st = get_status(models[organ]) # User status - - if isnothing(st) - st = NamedTuple() - else - st = NamedTuple(st) - end - - # Add the variables that are defined as multiscale (coming from other scales): - if haskey(organs_mapping, organ) - st_vars_mapped = (; zip(vars_from_mapping(organs_mapping[organ]), vars_type_from_mapping(organs_mapping[organ]))...) - !isnothing(st_vars_mapped) && (st = merge(st, st_vars_mapped)) - end - - # Add the variable(s) written by other scales into this node scale: - haskey(var_outputs_from_mapping, organ) && (st = merge(st, var_outputs_from_mapping[organ])) - - # Then we initialise a status taking into account the status given by the user. - # This step is done to get default values for each variables: - if length(st) == 0 - st = nothing - else - st = Status(st) - end + # We make a pre-initialised status for each kind of organ (this is a template for each node type): + organs_statuses = PlantSimEngine.status_template(models, type_promotion) - st = add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) - # The status is added to the vector of statuses. - push!(organs_statuses, organ => st) - end + # We need to know which variables are not initialized, and not computed by other models: + var_need_init = PlantSimEngine.to_initialize(models, organs_statuses) - # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable - # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. - for (organ, st) in organs_statuses # e.g.: organ = "Leaf"; st = organs_statuses[organ] - # If there is any MappedVar in the status: - if any(x -> isa(x, MappedVar), values(st)) - val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) - for (k, v) in val_pointers - if isa(v, MappedVar) - val_pointers[k] = refvalue(organs_statuses[val.organ], val.var) - else - val_pointers[k] = refvalue(st, k) - end - end - organs_statuses[organ] = Status(NamedTuple(val_pointers)) - end - end + # If we find some, we return an error: + error_mtg_init(var_need_init) #! continue here. What we need to do: - #! 3. traverser les MTG pour remplir les RefVector, sachant qu'ils seront automatiquement remplis partout puisque c'est des Ref (a vérifier). - #! 4. ajouter des checks, par exemple une variable output - #! 5. faire un tableau de status, qui potentiellement référence le noeud. Et faire une `sort` en fonction de l'arbre de dépendence multi-échelle - #! 6. ajouter des tests + #! - traverser les MTG pour initialiser un Status par organe, et mettre le vecteur de ces status dans un Dict{Organe, Status} + #! - dans le même traversal, trouver les variables qui doivent être initialisées depuis le mtg (et erreur si elles n'y sont pas) + #! - remplir les RefVector, sachant qu'ils seront automatiquement remplis partout puisque c'est des Ref (a vérifier). + #! - Ajouter la référence au noeud dans le status ? + #! - calculer le graphe de dépendence des modèles, et faire des calls en fonction + #! - ajouter des tests + #! - ajouter des checks, e.g. est-ce que tous les organes du MTG ont un modèle ou pas... statuses = Status[] # We traverse the MTG to initialise the mapping depending on the number of nodes and their types traverse!(mtg) do node @@ -504,7 +396,7 @@ function error_mtg_init(var_need_init) if length(need_initialisation) > 0 push!( error_string, - "Nodes of type $organ_init need variables $need_initialisation, but they are not initialized or computed." + "Nodes of type $organ_init need variable(s) $(join(need_initialisation, ", ")) to be initialized or computed by a model." ) end From 703d97478a53350b41753114ba0d2b44d1731707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 26 Sep 2023 18:20:23 +0200 Subject: [PATCH 18/97] Update mapping.jl Still working on the initialisation --- src/mtg/mapping.jl | 120 +++++++++++++++++++++++++++++++++------------ 1 file changed, 89 insertions(+), 31 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index ca0be34d..4b5dd2d2 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -252,14 +252,29 @@ function compute_mapping(models::Dict{String,Any}, type_promotion) # var_mapping = map_vars[1] variable, organs_mapped = var_mapping + ref_var = PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable)) if haskey(organ_mapping, organs_mapped) - push!(organ_mapping[organs_mapped], variable => create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + push!(organ_mapping[organs_mapped], variable => ref_var) else - organ_mapping[organs_mapped] = Dict(variable => create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable))) + organ_mapping[organs_mapped] = Dict(variable => ref_var) + end + + # If the mapping is one node type only and is given as a string, we add the variable of the source scale + # as a MappedVar linked to itself, so we remember to not deepcopy when we build the status for the source node. + # This is a special case for when the source scale only has one node in the MTG, and one variable is mapped. + if isa(organs_mapped, AbstractString) + if !haskey(organs_mapping, organs_mapped) + organs_mapping[organs_mapped] = Dict(organs_mapped => organ_mapping[organs_mapped]) + else + push!(organs_mapping[organs_mapped][organs_mapped], organ_mapping[organs_mapped]) + end end end + organs_mapping[organ] = organ_mapping + end - organs_mapping[organ] = Dict(k => NamedTuple(v) for (k, v) in organ_mapping) + for (k, v) in organs_mapping + organs_mapping[k] = Dict(k => NamedTuple(v) for (k, v) in v) end var_outputs_from_mapping = Dict(k => NamedTuple(v) for (k, v) in var_outputs_from_mapping) @@ -315,7 +330,6 @@ function create_var_ref(organ::Vector{<:AbstractString}, var, default::T) where RefVector(Base.RefValue{T}[]) end -#! reverse parameter order , it is more logical struct MappedVar{S<:AbstractString,T} organ::S var::Symbol @@ -350,10 +364,10 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) organs_statuses = PlantSimEngine.status_template(models, type_promotion) # We need to know which variables are not initialized, and not computed by other models: - var_need_init = PlantSimEngine.to_initialize(models, organs_statuses) + var_need_init = PlantSimEngine.to_initialize(models, organs_statuses, mtg) # If we find some, we return an error: - error_mtg_init(var_need_init) + check && PlantSimEngine.error_mtg_init(var_need_init) #! continue here. What we need to do: #! - traverser les MTG pour initialiser un Status par organe, et mettre le vecteur de ces status dans un Dict{Organe, Status} @@ -363,14 +377,75 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) #! - calculer le graphe de dépendence des modèles, et faire des calls en fonction #! - ajouter des tests #! - ajouter des checks, e.g. est-ce que tous les organes du MTG ont un modèle ou pas... - statuses = Status[] - # We traverse the MTG to initialise the mapping depending on the number of nodes and their types - traverse!(mtg) do node - # Check if the node has a model defined for its symbol: - # node = get_node(mtg, 1) - push!(statuses, organs_statuses[node.MTG.symbol]) - organs_statuses + + organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() + dict_mapped_vars = Dict{Pair,Any}() + # # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable + # # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. + for (organ, st) in organs_statuses # e.g.: organ = "Soil"; st = organs_statuses[organ] + val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) + # If there is any MappedVar in the status: + if any(x -> isa(x, PlantSimEngine.MappedVar), values(st)) + for (k, v) in val_pointers # e.g.: k = :soil_water_content; v = val_pointers[k] + if isa(v, PlantSimEngine.MappedVar) + # First time we encounter this variable as a MappedVar, we create its value into the dict_mapped_vars Dict: + if !haskey(dict_mapped_vars, v.organ => v.var) + push!(dict_mapped_vars, Pair(v.organ, v.var) => Ref(st[k].default)) + end + + # Then we replace the MappedVar by a RefValue to the actual variable: + val_pointers[k] = dict_mapped_vars[v.organ=>v.var] + else + val_pointers[k] = st[k] + end + end + end + organs_statuses_dict[organ] = val_pointers + end + + #! Continue here (bis): use organs_statuses_dict to initialise the MTG nodes statuses. The Status will be created manually + #! to control the references. + #! Standard values will be copied and a reference to this copy will be used. + #! RefValues will be used as is (i.e. the reference will be passed to the Status) + #! RefVector will be passes as is, and instantiated on the fly traversing the MTG. + + nodes_with_models = collect(keys(organs_statuses)) + # We traverse the MTG a first time to initialise the statuses linked to the nodes: + statuses = Dict(i => Status[] for i in nodes_with_models) + traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) + if node.MTG.symbol in nodes_with_models # Check if the node has a model defined for its symbol + # If there is any MappedVar in the status: + st = organs_statuses[node.MTG.symbol] + if any(x -> isa(x, PlantSimEngine.MappedVar), values(st)) + val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) + for (k, v) in val_pointers + if isa(v, PlantSimEngine.MappedVar) + val_pointers[k] = PlantSimEngine.refvalue(organs_statuses[v.organ], v.var) + else + val_pointers[k] = PlantSimEngine.refvalue(st, k) + end + end + st = Status(NamedTuple(val_pointers)) + else + st = deepcopy(st) + end + + push!(statuses[node.MTG.symbol], st) + end end + #! 1. For the soil_water_content of the soil, we need a way to know that it will be mapped, + #! so we need to reference the original status, not deepcopying it. We should use a MappedVar + #! too, referencing the "Soil" type of node in the template (i.e. itself). + #! 2. We need a way to know if a variable of a node needs to be pushed into a RefVector of another scale, + #! so this way we only traverse the MTG once and push into the RefVector of the template status. + + # Print an info if models are declared for nodes that don't exist in the MTG: + if check && any(x -> length(last(x)) == 0, statuses) + model_no_node = join(findall(x -> length(x) == 0, statuses), ", ") + @info "Models given for $model_no_node, but no node with this symbol was found in the MTG." maxlog = 1 + end + + push!(statuses[1][:carbon_allocation], PlantSimEngine.refvalue(statuses[2], :carbon_allocation)) end @@ -527,22 +602,5 @@ function status_template(models::Dict{String,Any}, type_promotion) push!(organs_statuses, organ => st) end - # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable - # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. - for (organ, st) in organs_statuses # e.g.: organ = "Leaf"; st = organs_statuses[organ] - # If there is any MappedVar in the status: - if any(x -> isa(x, MappedVar), values(st)) - val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) - for (k, v) in val_pointers - if isa(v, MappedVar) - val_pointers[k] = refvalue(organs_statuses[v.organ], v.var) - else - val_pointers[k] = refvalue(st, k) - end - end - organs_statuses[organ] = Status(NamedTuple(val_pointers)) - end - end - return organs_statuses -end \ No newline at end of file +end From 365965ea1777a69861a3af37e689810d4f54b308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 27 Sep 2023 10:50:48 +0200 Subject: [PATCH 19/97] Update mapping.jl Make a status for each node based on the template dict + references between variables --- src/mtg/mapping.jl | 92 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 19 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 4b5dd2d2..6203f552 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -414,23 +414,10 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) statuses = Dict(i => Status[] for i in nodes_with_models) traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) if node.MTG.symbol in nodes_with_models # Check if the node has a model defined for its symbol - # If there is any MappedVar in the status: - st = organs_statuses[node.MTG.symbol] - if any(x -> isa(x, PlantSimEngine.MappedVar), values(st)) - val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) - for (k, v) in val_pointers - if isa(v, PlantSimEngine.MappedVar) - val_pointers[k] = PlantSimEngine.refvalue(organs_statuses[v.organ], v.var) - else - val_pointers[k] = PlantSimEngine.refvalue(st, k) - end - end - st = Status(NamedTuple(val_pointers)) - else - st = deepcopy(st) - end - - push!(statuses[node.MTG.symbol], st) + push!( + statuses[node.MTG.symbol], + PlantSimEngine.status_from_template(organs_statuses_dict[node.MTG.symbol]) + ) end end #! 1. For the soil_water_content of the soil, we need a way to know that it will be mapped, @@ -566,8 +553,7 @@ function status_template(models::Dict{String,Any}, type_promotion) # Vector of statuses, pre-initialised with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: organs_statuses = Dict{String,Status}() - for organ in keys(models) - # organ = "Internode" + for organ in keys(models) # e.g.: organ = "Internode" # Parsing the models into a NamedTuple to get the process name: node_models = parse_models(get_models(models[organ])) @@ -604,3 +590,71 @@ function status_template(models::Dict{String,Any}, type_promotion) return organs_statuses end + +""" + status_from_template(d::Dict{Symbol,Any}) + +Create a status from a template dictionary of variables and values. If the values +are already RefValues or RefVectors, they are used as is, else they are converted to Refs. + +# Arguments + +- `d::Dict{Symbol,Any}`: A dictionary of variables and values. + +# Returns + +- A [`Status`](@ref). + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +julia> a, b = PlantSimEngine.status_from_template(Dict(:a => 1.0, :b => 2.0)); +julia> a +1.0 +julia> b +2.0 +``` +""" +function status_from_template(d::Dict{Symbol,T} where {T}) + Status(NamedTuple(first(i) => ref_var(last(i)) for i in d)) +end + +""" + ref_var(v) + +Create a reference to a variable. If the variable is already a `Base.RefValue`, +it is returned as is, else it is returned as a Ref to the copy of the value, or a +or a Ref to the `RefVector` (in case `v` is a `RefVector`). + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +julia> ref_var(1.0) +Base.RefValue{Float64}(1.0) +``` + +```jldoctest mylabel +julia> ref_var([1.0]) +Base.RefValue{Vector{Float64}}([1.0]) +``` + +```jldoctest mylabel +julia> ref_var(Base.RefValue(1.0)) +Base.RefValue{Float64}(1.0) +``` + +```jldoctest mylabel +julia> ref_var(Base.RefValue([1.0])) +Base.RefValue{Vector{Float64}}([1.0]) +``` + +```jldoctest mylabel +julia> ref_var(PlantSimEngine.RefVector([Ref(1.0), Ref(2.0), Ref(3.0)])) +Base.RefValue{PlantSimEngine.RefVector{Float64}}(RefVector{Float64}[1.0, 2.0, 3.0]) +``` +""" +ref_var(v) = Base.Ref(copy(v)) +ref_var(v::T) where {T<:Base.RefValue} = v +ref_var(v::T) where {T<:RefVector} = Base.Ref(v) \ No newline at end of file From 925ba50e91301d3d6d630d842171cb69d499a1fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 27 Sep 2023 11:29:05 +0200 Subject: [PATCH 20/97] Update mapping.jl --- src/mtg/mapping.jl | 119 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 104 insertions(+), 15 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 6203f552..a34818bd 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -377,6 +377,8 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) #! - calculer le graphe de dépendence des modèles, et faire des calls en fonction #! - ajouter des tests #! - ajouter des checks, e.g. est-ce que tous les organes du MTG ont un modèle ou pas... + #! We need a way to know if a variable of a node needs to be pushed into a RefVector of another scale, + #! so this way we only traverse the MTG once and push into the RefVector of the template status. organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() dict_mapped_vars = Dict{Pair,Any}() @@ -403,28 +405,32 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) organs_statuses_dict[organ] = val_pointers end - #! Continue here (bis): use organs_statuses_dict to initialise the MTG nodes statuses. The Status will be created manually - #! to control the references. - #! Standard values will be copied and a reference to this copy will be used. - #! RefValues will be used as is (i.e. the reference will be passed to the Status) - #! RefVector will be passes as is, and instantiated on the fly traversing the MTG. - - nodes_with_models = collect(keys(organs_statuses)) + vars_to_add_in_refvector = PlantSimEngine.reverse_mapping(models) + nodes_with_models = collect(keys(organs_statuses_dict)) # We traverse the MTG a first time to initialise the statuses linked to the nodes: statuses = Dict(i => Status[] for i in nodes_with_models) traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) if node.MTG.symbol in nodes_with_models # Check if the node has a model defined for its symbol + #! 1. if some values still need to be instantiated, look into the MTG node if we can find them, + #! and if not, throw an error. + #! 2. Add a Ref to the node into the status. + + st = PlantSimEngine.status_from_template(organs_statuses_dict[node.MTG.symbol]) push!( statuses[node.MTG.symbol], - PlantSimEngine.status_from_template(organs_statuses_dict[node.MTG.symbol]) + st ) + + # Instantiate the RefVectors on the fly for other scales that map into this scale + if haskey(vars_to_add_in_refvector, node.MTG.symbol) + for (organ, vars) in vars_to_add_in_refvector[node.MTG.symbol] + for var in vars # e.g.: var = :carbon_demand + push!(organs_statuses_dict[organ][var], PlantSimEngine.refvalue(st, var)) + end + end + end end end - #! 1. For the soil_water_content of the soil, we need a way to know that it will be mapped, - #! so we need to reference the original status, not deepcopying it. We should use a MappedVar - #! too, referencing the "Soil" type of node in the template (i.e. itself). - #! 2. We need a way to know if a variable of a node needs to be pushed into a RefVector of another scale, - #! so this way we only traverse the MTG once and push into the RefVector of the template status. # Print an info if models are declared for nodes that don't exist in the MTG: if check && any(x -> length(last(x)) == 0, statuses) @@ -432,7 +438,7 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) @info "Models given for $model_no_node, but no node with this symbol was found in the MTG." maxlog = 1 end - push!(statuses[1][:carbon_allocation], PlantSimEngine.refvalue(statuses[2], :carbon_allocation)) + return statuses end @@ -657,4 +663,87 @@ Base.RefValue{PlantSimEngine.RefVector{Float64}}(RefVector{Float64}[1.0, 2.0, 3. """ ref_var(v) = Base.Ref(copy(v)) ref_var(v::T) where {T<:Base.RefValue} = v -ref_var(v::T) where {T<:RefVector} = Base.Ref(v) \ No newline at end of file +ref_var(v::T) where {T<:RefVector} = Base.Ref(v) + + +""" + reverse_mapping(models) + +Get the reverse mapping of a dictionary of model mapping, *i.e.* the variables that are mapped to other scales. +This is used for *e.g.* knowing which scales are needed to add values to others. + +# Arguments + +- `models::Dict{String,Any}`: A dictionary of model mapping. + +# Returns + +- A dictionary of variables (keys) to a dictionary (values) of organs => vector of variables. + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +``` + +```jldoctest mylabel +julia> models = Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ), + "Soil" => ( + ToySoilWaterModel(), + ), + ); +``` + +```jldoctest mylabel +julia> reverse_mapping(models) +Dict{String, Any} with 2 entries: + "Internode" => Dict("Plant"=>[:carbon_demand, :carbon_allocation]) + "Leaf" => Dict("Plant"=>[:A, :carbon_demand, :carbon_allocation]) +``` +""" +function reverse_mapping(models) + var_to_ref = Dict{String,Any}(i => Dict{String,Vector{Symbol}}() for i in keys(models)) + for organ in keys(models) + # organ = "Plant" + map_vars = PlantSimEngine.get_mapping(models[organ]) + for i in map_vars # e.g.: i = :carbon_demand => ["Leaf", "Internode"] + mapped = last(i) # e.g.: mapped = ["Leaf", "Internode"] + if isa(mapped, Vector) + for j in mapped # e.g.: j = "Leaf" + if haskey(var_to_ref[j], organ) + push!(var_to_ref[j][organ], first(i)) + else + var_to_ref[j][organ] = [first(i)] + end + end + end + end + end + + return filter!(x -> length(last(x)) > 0, var_to_ref) +end \ No newline at end of file From 4b9ee45aaf44c1116c5515484e09debb930d531b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 27 Sep 2023 13:40:44 +0200 Subject: [PATCH 21/97] Update mapping.jl Make the status_template in one pass (with link between scales) --- src/mtg/mapping.jl | 82 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index a34818bd..e12c4553 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -373,12 +373,10 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) #! - traverser les MTG pour initialiser un Status par organe, et mettre le vecteur de ces status dans un Dict{Organe, Status} #! - dans le même traversal, trouver les variables qui doivent être initialisées depuis le mtg (et erreur si elles n'y sont pas) #! - remplir les RefVector, sachant qu'ils seront automatiquement remplis partout puisque c'est des Ref (a vérifier). - #! - Ajouter la référence au noeud dans le status ? + #! - Ajouter la référence au noeud dans le status #! - calculer le graphe de dépendence des modèles, et faire des calls en fonction #! - ajouter des tests #! - ajouter des checks, e.g. est-ce que tous les organes du MTG ont un modèle ou pas... - #! We need a way to know if a variable of a node needs to be pushed into a RefVector of another scale, - #! so this way we only traverse the MTG once and push into the RefVector of the template status. organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() dict_mapped_vars = Dict{Pair,Any}() @@ -410,23 +408,40 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) # We traverse the MTG a first time to initialise the statuses linked to the nodes: statuses = Dict(i => Status[] for i in nodes_with_models) traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) - if node.MTG.symbol in nodes_with_models # Check if the node has a model defined for its symbol - #! 1. if some values still need to be instantiated, look into the MTG node if we can find them, - #! and if not, throw an error. - #! 2. Add a Ref to the node into the status. - - st = PlantSimEngine.status_from_template(organs_statuses_dict[node.MTG.symbol]) - push!( - statuses[node.MTG.symbol], - st - ) - - # Instantiate the RefVectors on the fly for other scales that map into this scale - if haskey(vars_to_add_in_refvector, node.MTG.symbol) - for (organ, vars) in vars_to_add_in_refvector[node.MTG.symbol] - for var in vars # e.g.: var = :carbon_demand - push!(organs_statuses_dict[organ][var], PlantSimEngine.refvalue(st, var)) - end + # Check if the node has a model defined for its symbol + node.MTG.symbol ∉ nodes_with_models && return + + # We make a copy of the template status for this node: + st_template = copy(organs_statuses_dict[node.MTG.symbol]) + + # We add a reference to the node into the status, so that we can access it from the models if needed. + push!(st_template, :node => Ref(node)) + + # If some variables still need to be instantiated in the status, look into the MTG node if we can find them, + # and if so, use their value in the status: + if haskey(var_need_init, node.MTG.symbol) && length(var_need_init[node.MTG.symbol].need_var_from_mtg) > 0 + for i in var_need_init[node.MTG.symbol].need_var_from_mtg + @assert typeof(node[i.var]) == typeof(st[i.var]) string( + "Initializing variable $(i.var) using MTG node $(node.id): expected type $(typeof(st[i.var])), found $(typeof(node[i.var])). ", + "Please check the type of the variable in the MTG, and make it a $(typeof(st[i.var]))." + ) + st_template[i.var] = node[i.var] + #! NB: the variable is not a reference to the value in the MTG, but a copy of it. + #! This is because we can't reference a value in a Dict. If we need a ref, the user can use a RefValue in the MTG directly, + #! and it will be automatically passed as is. + end + end + + # Make the node status from the template: + st = PlantSimEngine.status_from_template(st_template) + + push!(statuses[node.MTG.symbol], st) + + # Instantiate the RefVectors on the fly for other scales that map into this scale + if haskey(vars_to_add_in_refvector, node.MTG.symbol) + for (organ, vars) in vars_to_add_in_refvector[node.MTG.symbol] + for var in vars # e.g.: var = :carbon_demand + push!(organs_statuses_dict[organ][var], PlantSimEngine.refvalue(st, var)) end end end @@ -594,7 +609,32 @@ function status_template(models::Dict{String,Any}, type_promotion) push!(organs_statuses, organ => st) end - return organs_statuses + organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() + dict_mapped_vars = Dict{Pair,Any}() + # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable + # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. + for (organ, st) in organs_statuses # e.g.: organ = "Soil"; st = organs_statuses[organ] + val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) + # If there is any MappedVar in the status: + if any(x -> isa(x, PlantSimEngine.MappedVar), values(st)) + for (k, v) in val_pointers # e.g.: k = :soil_water_content; v = val_pointers[k] + if isa(v, PlantSimEngine.MappedVar) + # First time we encounter this variable as a MappedVar, we create its value into the dict_mapped_vars Dict: + if !haskey(dict_mapped_vars, v.organ => v.var) + push!(dict_mapped_vars, Pair(v.organ, v.var) => Ref(st[k].default)) + end + + # Then we replace the MappedVar by a RefValue to the actual variable: + val_pointers[k] = dict_mapped_vars[v.organ=>v.var] + else + val_pointers[k] = st[k] + end + end + end + organs_statuses_dict[organ] = val_pointers + end + + return organs_statuses_dict end """ From 404d68054b976c22a9f612591c8dd3c1e2e0e698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 27 Sep 2023 14:50:28 +0200 Subject: [PATCH 22/97] Update mapping.jl Add first step of `init_simulation`: just the statuses, not the dependency graph yet --- src/mtg/mapping.jl | 166 +++++++++++++++++++++------------------------ 1 file changed, 79 insertions(+), 87 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index e12c4553..4d41c7cb 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -358,10 +358,20 @@ function outputs_from_other_scale!(var_outputs_from_mapping, multi_scale_outs, m end end -# Initialisations with the mapping: -function init_simulation!(mtg, models; type_promotion=nothing, check=true) +""" + init_simulation(mtg, models; type_promotion=nothing, check=true) + +Initialise the simulation by creating: + +- a status for each node type, considering multi-scale variables. +- the dependency graph of the models, and the order in which they should be called. +""" +function init_simulation(mtg, models; type_promotion=nothing, check=true) # We make a pre-initialised status for each kind of organ (this is a template for each node type): organs_statuses = PlantSimEngine.status_template(models, type_promotion) + # Get the reverse mapping, i.e. the variables that are mapped to other scales. This is used to initialise + # the RefVectors properly: + var_refvector = PlantSimEngine.reverse_mapping(models) # We need to know which variables are not initialized, and not computed by other models: var_need_init = PlantSimEngine.to_initialize(models, organs_statuses, mtg) @@ -378,74 +388,8 @@ function init_simulation!(mtg, models; type_promotion=nothing, check=true) #! - ajouter des tests #! - ajouter des checks, e.g. est-ce que tous les organes du MTG ont un modèle ou pas... - organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() - dict_mapped_vars = Dict{Pair,Any}() - # # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable - # # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. - for (organ, st) in organs_statuses # e.g.: organ = "Soil"; st = organs_statuses[organ] - val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) - # If there is any MappedVar in the status: - if any(x -> isa(x, PlantSimEngine.MappedVar), values(st)) - for (k, v) in val_pointers # e.g.: k = :soil_water_content; v = val_pointers[k] - if isa(v, PlantSimEngine.MappedVar) - # First time we encounter this variable as a MappedVar, we create its value into the dict_mapped_vars Dict: - if !haskey(dict_mapped_vars, v.organ => v.var) - push!(dict_mapped_vars, Pair(v.organ, v.var) => Ref(st[k].default)) - end - - # Then we replace the MappedVar by a RefValue to the actual variable: - val_pointers[k] = dict_mapped_vars[v.organ=>v.var] - else - val_pointers[k] = st[k] - end - end - end - organs_statuses_dict[organ] = val_pointers - end - - vars_to_add_in_refvector = PlantSimEngine.reverse_mapping(models) - nodes_with_models = collect(keys(organs_statuses_dict)) - # We traverse the MTG a first time to initialise the statuses linked to the nodes: - statuses = Dict(i => Status[] for i in nodes_with_models) - traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) - # Check if the node has a model defined for its symbol - node.MTG.symbol ∉ nodes_with_models && return - - # We make a copy of the template status for this node: - st_template = copy(organs_statuses_dict[node.MTG.symbol]) - - # We add a reference to the node into the status, so that we can access it from the models if needed. - push!(st_template, :node => Ref(node)) - - # If some variables still need to be instantiated in the status, look into the MTG node if we can find them, - # and if so, use their value in the status: - if haskey(var_need_init, node.MTG.symbol) && length(var_need_init[node.MTG.symbol].need_var_from_mtg) > 0 - for i in var_need_init[node.MTG.symbol].need_var_from_mtg - @assert typeof(node[i.var]) == typeof(st[i.var]) string( - "Initializing variable $(i.var) using MTG node $(node.id): expected type $(typeof(st[i.var])), found $(typeof(node[i.var])). ", - "Please check the type of the variable in the MTG, and make it a $(typeof(st[i.var]))." - ) - st_template[i.var] = node[i.var] - #! NB: the variable is not a reference to the value in the MTG, but a copy of it. - #! This is because we can't reference a value in a Dict. If we need a ref, the user can use a RefValue in the MTG directly, - #! and it will be automatically passed as is. - end - end - - # Make the node status from the template: - st = PlantSimEngine.status_from_template(st_template) - - push!(statuses[node.MTG.symbol], st) - - # Instantiate the RefVectors on the fly for other scales that map into this scale - if haskey(vars_to_add_in_refvector, node.MTG.symbol) - for (organ, vars) in vars_to_add_in_refvector[node.MTG.symbol] - for var in vars # e.g.: var = :carbon_demand - push!(organs_statuses_dict[organ][var], PlantSimEngine.refvalue(st, var)) - end - end - end - end + # Get the status of each node by node type, pre-initialised considering multi-scale variables: + statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init) # Print an info if models are declared for nodes that don't exist in the MTG: if check && any(x -> length(last(x)) == 0, statuses) @@ -511,7 +455,8 @@ Create a status template for a given set of models and type promotion. - `type_promotion`: The type promotion to use. # Returns -- A dictionary with a `status` template for each organ type. + +- A dictionary with the organ types as keys, and a dictionary of variables => default values as values. # Examples @@ -554,11 +499,11 @@ julia> models = Dict( ```jldoctest mylabel julia> status_template(models, nothing) -Dict{String, Status} with 4 entries: - "Soil" => Status(soil_water_content = -Inf,) - "Internode" => Status(TT = -Inf, carbon_demand = -Inf, carbon_allocation = -Inf) - "Plant" => Status(A = RefVector{Float64}[], carbon_demand = RefVector{Float64}[], carbon_offer = -Inf, carbon_allocation = RefVector{Float64}[]) - "Leaf" => Status(aPPFD = 1300.0, soil_water_content = MappedVar{String, Float64}("Soil", :soil_water_content, -Inf), A = -Inf, TT = 10.0, carbon_demand = -Inf, carbon_allocation = -Inf) +Dict{String, Dict{Symbol, Any}} with 4 entries: + "Soil" => Dict(:soil_water_content=>RefValue{Float64}(-Inf)) + "Internode" => Dict(:carbon_allocation=>-Inf, :TT=>-Inf, :carbon_demand=>-Inf) + "Plant" => Dict(:carbon_allocation=>RefVector{Float64}[], :A=>RefVector{Float64}[], :carbon_offer=>-Inf, :carbon_demand=>RefVector{Float64}[]) + "Leaf" => Dict(:carbon_allocation=>-Inf, :A=>-Inf, :TT=>10.0, :aPPFD=>1300.0, :soil_water_content=>RefValue{Float64}(-Inf), :carbon_demand=>-Inf) ``` Note that variables that are multiscale (*i.e.* defined in a mapping) are linked between scales, so if we write at a scale, the value will be @@ -571,8 +516,9 @@ true """ function status_template(models::Dict{String,Any}, type_promotion) organs_mapping, var_outputs_from_mapping = compute_mapping(models, type_promotion) - # Vector of statuses, pre-initialised with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: - organs_statuses = Dict{String,Status}() + # Vector of pre-initialised variables with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: + organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() + dict_mapped_vars = Dict{Pair,Any}() for organ in keys(models) # e.g.: organ = "Internode" # Parsing the models into a NamedTuple to get the process name: @@ -605,17 +551,10 @@ function status_template(models::Dict{String,Any}, type_promotion) end st = add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) - # The status is added to the vector of statuses. - push!(organs_statuses, organ => st) - end - organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() - dict_mapped_vars = Dict{Pair,Any}() - # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable - # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. - for (organ, st) in organs_statuses # e.g.: organ = "Soil"; st = organs_statuses[organ] + # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable + # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) - # If there is any MappedVar in the status: if any(x -> isa(x, PlantSimEngine.MappedVar), values(st)) for (k, v) in val_pointers # e.g.: k = :soil_water_content; v = val_pointers[k] if isa(v, PlantSimEngine.MappedVar) @@ -786,4 +725,57 @@ function reverse_mapping(models) end return filter!(x -> length(last(x)) > 0, var_to_ref) +end + +""" + init_statuses(mtg, models, status_template, var_need_init) + init_statuses(mtg, models, status_template) + +Get the status of each node in the MTG by node type, pre-initialised considering multi-scale variables +using the template given by `status_template`. +""" +function init_statuses(mtg, status_template, var_refvector, var_need_init=Dict{String,Any}()) + nodes_with_models = collect(keys(status_template)) + # We traverse the MTG a first time to initialise the statuses linked to the nodes: + statuses = Dict(i => Status[] for i in nodes_with_models) + MultiScaleTreeGraph.traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) + # Check if the node has a model defined for its symbol + node.MTG.symbol ∉ nodes_with_models && return + + # We make a copy of the template status for this node: + st_template = copy(status_template[node.MTG.symbol]) + + # We add a reference to the node into the status, so that we can access it from the models if needed. + push!(st_template, :node => Ref(node)) + + # If some variables still need to be instantiated in the status, look into the MTG node if we can find them, + # and if so, use their value in the status: + if haskey(var_need_init, node.MTG.symbol) && length(var_need_init[node.MTG.symbol].need_var_from_mtg) > 0 + for i in var_need_init[node.MTG.symbol].need_var_from_mtg + @assert typeof(node[i.var]) == typeof(st_template[i.var]) string( + "Initializing variable $(i.var) using MTG node $(node.id): expected type $(typeof(st_template[i.var])), found $(typeof(node[i.var])). ", + "Please check the type of the variable in the MTG, and make it a $(typeof(st_template[i.var]))." + ) + st_template[i.var] = node[i.var] + #! NB: the variable is not a reference to the value in the MTG, but a copy of it. + #! This is because we can't reference a value in a Dict. If we need a ref, the user can use a RefValue in the MTG directly, + #! and it will be automatically passed as is. + end + end + + # Make the node status from the template: + st = PlantSimEngine.status_from_template(st_template) + + push!(statuses[node.MTG.symbol], st) + + # Instantiate the RefVectors on the fly for other scales that map into this scale + if haskey(var_refvector, node.MTG.symbol) + for (organ, vars) in var_refvector[node.MTG.symbol] + for var in vars # e.g.: var = :carbon_demand + push!(status_template[organ][var], PlantSimEngine.refvalue(st, var)) + end + end + end + end + return statuses end \ No newline at end of file From 963bd1bf5ce9a43b3d81f908836772e52aca05d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 27 Sep 2023 14:59:40 +0200 Subject: [PATCH 23/97] Update mapping.jl Add doc for init_simulation --- src/mtg/mapping.jl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 4d41c7cb..053fbb4a 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -365,6 +365,26 @@ Initialise the simulation by creating: - a status for each node type, considering multi-scale variables. - the dependency graph of the models, and the order in which they should be called. + +# Arguments + +- `mtg`: the MTG +- `models::Dict{String,Any}`: a dictionary of model mapping +- `type_promotion`: the type promotion to use for the variables +- `check`: whether to check the mapping for errors + +# Details + +The function first computes a template of status for each organ type that has a model in the mapping. +This template is used to initialise the status of each node in the MTG, taking into account the user-defined +initialisation, and the multiscale mapping. The multiscale mapping is used to make references to the variables +that are defined at another scale, so that the values are automatically updated when the variable is changed at +the other scale. + +Note that if a variable is not computed by models or initialised from the mapping, it is searched in the MTG attributes. +The value is not a reference to the one in the attribute of the MTG, but a copy of it. This is because we can't reference +a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be +automatically passed as is. """ function init_simulation(mtg, models; type_promotion=nothing, check=true) # We make a pre-initialised status for each kind of organ (this is a template for each node type): From 0f099b7eb37e845791fc846544780ef05b549280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 27 Sep 2023 15:00:44 +0200 Subject: [PATCH 24/97] De-reference PlantSimEngine --- src/mtg/mapping.jl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 053fbb4a..2998bd92 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -252,7 +252,7 @@ function compute_mapping(models::Dict{String,Any}, type_promotion) # var_mapping = map_vars[1] variable, organs_mapped = var_mapping - ref_var = PlantSimEngine.create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable)) + ref_var = create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable)) if haskey(organ_mapping, organs_mapped) push!(organ_mapping[organs_mapped], variable => ref_var) else @@ -388,16 +388,16 @@ automatically passed as is. """ function init_simulation(mtg, models; type_promotion=nothing, check=true) # We make a pre-initialised status for each kind of organ (this is a template for each node type): - organs_statuses = PlantSimEngine.status_template(models, type_promotion) + organs_statuses = status_template(models, type_promotion) # Get the reverse mapping, i.e. the variables that are mapped to other scales. This is used to initialise # the RefVectors properly: - var_refvector = PlantSimEngine.reverse_mapping(models) + var_refvector = reverse_mapping(models) # We need to know which variables are not initialized, and not computed by other models: - var_need_init = PlantSimEngine.to_initialize(models, organs_statuses, mtg) + var_need_init = to_initialize(models, organs_statuses, mtg) # If we find some, we return an error: - check && PlantSimEngine.error_mtg_init(var_need_init) + check && error_mtg_init(var_need_init) #! continue here. What we need to do: #! - traverser les MTG pour initialiser un Status par organe, et mettre le vecteur de ces status dans un Dict{Organe, Status} @@ -409,7 +409,7 @@ function init_simulation(mtg, models; type_promotion=nothing, check=true) #! - ajouter des checks, e.g. est-ce que tous les organes du MTG ont un modèle ou pas... # Get the status of each node by node type, pre-initialised considering multi-scale variables: - statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init) + statuses = init_statuses(mtg, organs_statuses, var_refvector, var_need_init) # Print an info if models are declared for nodes that don't exist in the MTG: if check && any(x -> length(last(x)) == 0, statuses) @@ -575,9 +575,9 @@ function status_template(models::Dict{String,Any}, type_promotion) # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) - if any(x -> isa(x, PlantSimEngine.MappedVar), values(st)) + if any(x -> isa(x, MappedVar), values(st)) for (k, v) in val_pointers # e.g.: k = :soil_water_content; v = val_pointers[k] - if isa(v, PlantSimEngine.MappedVar) + if isa(v, MappedVar) # First time we encounter this variable as a MappedVar, we create its value into the dict_mapped_vars Dict: if !haskey(dict_mapped_vars, v.organ => v.var) push!(dict_mapped_vars, Pair(v.organ, v.var) => Ref(st[k].default)) @@ -614,7 +614,7 @@ are already RefValues or RefVectors, they are used as is, else they are converte ```jldoctest mylabel julia> using PlantSimEngine -julia> a, b = PlantSimEngine.status_from_template(Dict(:a => 1.0, :b => 2.0)); +julia> a, b = status_from_template(Dict(:a => 1.0, :b => 2.0)); julia> a 1.0 julia> b @@ -656,8 +656,8 @@ Base.RefValue{Vector{Float64}}([1.0]) ``` ```jldoctest mylabel -julia> ref_var(PlantSimEngine.RefVector([Ref(1.0), Ref(2.0), Ref(3.0)])) -Base.RefValue{PlantSimEngine.RefVector{Float64}}(RefVector{Float64}[1.0, 2.0, 3.0]) +julia> ref_var(RefVector([Ref(1.0), Ref(2.0), Ref(3.0)])) +Base.RefValue{RefVector{Float64}}(RefVector{Float64}[1.0, 2.0, 3.0]) ``` """ ref_var(v) = Base.Ref(copy(v)) @@ -729,7 +729,7 @@ function reverse_mapping(models) var_to_ref = Dict{String,Any}(i => Dict{String,Vector{Symbol}}() for i in keys(models)) for organ in keys(models) # organ = "Plant" - map_vars = PlantSimEngine.get_mapping(models[organ]) + map_vars = get_mapping(models[organ]) for i in map_vars # e.g.: i = :carbon_demand => ["Leaf", "Internode"] mapped = last(i) # e.g.: mapped = ["Leaf", "Internode"] if isa(mapped, Vector) @@ -784,7 +784,7 @@ function init_statuses(mtg, status_template, var_refvector, var_need_init=Dict{S end # Make the node status from the template: - st = PlantSimEngine.status_from_template(st_template) + st = status_from_template(st_template) push!(statuses[node.MTG.symbol], st) @@ -792,7 +792,7 @@ function init_statuses(mtg, status_template, var_refvector, var_need_init=Dict{S if haskey(var_refvector, node.MTG.symbol) for (organ, vars) in var_refvector[node.MTG.symbol] for var in vars # e.g.: var = :carbon_demand - push!(status_template[organ][var], PlantSimEngine.refvalue(st, var)) + push!(status_template[organ][var], refvalue(st, var)) end end end From ffc6b1715b71b505e0fb1615e98ee1c4230af53e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 29 Sep 2023 17:54:43 +0200 Subject: [PATCH 25/97] Update soft_dependencies.jl --- src/dependencies/soft_dependencies.jl | 40 ++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index 565ed0cf..f0e7bb61 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -37,7 +37,7 @@ soft_dep = soft_dependencies(hard_dep) function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, nsteps=1) # Compute the variables of each node in the hard-dependency graph: - d_vars = Dict() + d_vars = Dict{Symbol,Vector{Pair{Symbol,NamedTuple{(:inputs, :outputs),Tuple{Tuple{Vararg{Symbol}},Tuple{Vararg{Symbol}}}}}}}() for (procname, node) in d.roots var = Pair{Symbol,NamedTuple}[] traverse_dependency_graph!(node, variables, var) @@ -51,8 +51,12 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, # all_nodes = Dict(traverse_dependency_graph(d, x -> x)) # Compute the inputs and outputs of each process graph in the dependency graph - inputs_process = Dict(key => [j.first => j.second.inputs for j in val] for (key, val) in d_vars) - outputs_process = Dict(key => [j.first => j.second.outputs for j in val] for (key, val) in d_vars) + inputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Symbol}}}}}( + key => [j.first => j.second.inputs for j in val] for (key, val) in d_vars + ) + outputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Symbol}}}}}( + key => [j.first => j.second.outputs for j in val] for (key, val) in d_vars + ) soft_dep_graph = Dict( process_ => SoftDependencyNode( @@ -223,6 +227,34 @@ function search_inputs_in_output(process, inputs, outputs) return NamedTuple(inputs_as_output_of_process) end +function search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_graphs) + vars_input = PlantSimEngine.flatten_vars(inputs[process]) + + inputs_as_output_of_other_scale = Dict{String,Dict{Symbol,Vector{Symbol}}}() + for var in vars_input # e.g. var = PlantSimEngine.MappedVar{String, Nothing}("Soil", :soil_water_content, nothing) + # The variable is a multiscale variable: + if isa(var, PlantSimEngine.MappedVar) + for (proc_output, pairs_vars_output) in soft_dep_graphs[var.organ][:outputs] # e.g. proc_output = :soil_water; pairs_vars_output = [:soil_water=>(:soil_water_content,)] + process == proc_output && error("Process $process declared at two scales: $organ and $(var.organ). A process can only be simulated at one scale.") + vars_output = PlantSimEngine.flatten_vars(pairs_vars_output) + if var.var in vars_output + # The variable is found at another scale: + if haskey(inputs_as_output_of_other_scale, var.organ) + if haskey(inputs_as_output_of_other_scale[var.organ], proc_output) + push!(inputs_as_output_of_other_scale[var.organ][proc_output], var.var) + else + inputs_as_output_of_other_scale[var.organ][proc_output] = [var.var] + end + else + inputs_as_output_of_other_scale[var.organ] = Dict(proc_output => [var.var]) + end + end + end + end + end + + return inputs_as_output_of_other_scale +end """ flatten_vars(vars) @@ -249,7 +281,7 @@ Set{Symbol} with 4 elements: ``` """ function flatten_vars(vars) - vars_input = Set{Symbol}() + vars_input = Set() for (key, val) in vars for j in val push!(vars_input, j) From d68d5e21435a7712b8c1d0d33daaa289f6b1e65d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 29 Sep 2023 17:54:55 +0200 Subject: [PATCH 26/97] Update mapping.jl --- src/mtg/mapping.jl | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 2998bd92..f6a758bd 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -330,7 +330,7 @@ function create_var_ref(organ::Vector{<:AbstractString}, var, default::T) where RefVector(Base.RefValue{T}[]) end -struct MappedVar{S<:AbstractString,T} +struct MappedVar{S<:Union{A,Vector{A}} where {A<:AbstractString},T} organ::S var::Symbol default::T @@ -391,7 +391,8 @@ function init_simulation(mtg, models; type_promotion=nothing, check=true) organs_statuses = status_template(models, type_promotion) # Get the reverse mapping, i.e. the variables that are mapped to other scales. This is used to initialise # the RefVectors properly: - var_refvector = reverse_mapping(models) + var_refvector = reverse_mapping(models, all=false) + #NB: we use all=false because we only want the variables that are mapped as RefVectors. # We need to know which variables are not initialized, and not computed by other models: var_need_init = to_initialize(models, organs_statuses, mtg) @@ -666,7 +667,7 @@ ref_var(v::T) where {T<:RefVector} = Base.Ref(v) """ - reverse_mapping(models) + reverse_mapping(models; all=true) Get the reverse mapping of a dictionary of model mapping, *i.e.* the variables that are mapped to other scales. This is used for *e.g.* knowing which scales are needed to add values to others. @@ -674,6 +675,7 @@ This is used for *e.g.* knowing which scales are needed to add values to others. # Arguments - `models::Dict{String,Any}`: A dictionary of model mapping. +- `all::Bool`: Whether to get all the variables that are mapped to other scales, including the ones that are mapped as single values. # Returns @@ -725,13 +727,17 @@ Dict{String, Any} with 2 entries: "Leaf" => Dict("Plant"=>[:A, :carbon_demand, :carbon_allocation]) ``` """ -function reverse_mapping(models) +function reverse_mapping(models; all=true) var_to_ref = Dict{String,Any}(i => Dict{String,Vector{Symbol}}() for i in keys(models)) for organ in keys(models) # organ = "Plant" map_vars = get_mapping(models[organ]) for i in map_vars # e.g.: i = :carbon_demand => ["Leaf", "Internode"] mapped = last(i) # e.g.: mapped = ["Leaf", "Internode"] + + # If we want to get all the variables that are mapped to other scales, including the ones that are mapped as single values: + isa(mapped, AbstractString) && all && (mapped = [mapped]) + if isa(mapped, Vector) for j in mapped # e.g.: j = "Leaf" if haskey(var_to_ref[j], organ) @@ -798,4 +804,24 @@ function init_statuses(mtg, status_template, var_refvector, var_need_init=Dict{S end end return statuses +end + +""" + variables_multiscale(node, organ, mapping) + +Get the variables of a HardDependencyNode, taking into account the multiscale mapping, *i.e.* +defining variables as `MappedVar` if they are mapped to another scale. +""" +function variables_multiscale(node, organ, mapping) + map(variables(node)) do vars + vars_ = Vector{Union{Symbol,PlantSimEngine.MappedVar}}() + for var in vars # e.g. var = :soil_water_content + if haskey(mapping[organ], var) + push!(vars_, PlantSimEngine.MappedVar(mapping[organ][var], var, nothing)) + else + push!(vars_, var) + end + end + return (vars_...,) + end end \ No newline at end of file From 425659d726f7539ac533d5e61a573b115783da04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 29 Sep 2023 17:55:01 +0200 Subject: [PATCH 27/97] Update dependency_graph.jl --- src/dependencies/dependency_graph.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dependencies/dependency_graph.jl b/src/dependencies/dependency_graph.jl index 38f15a49..a86e7c12 100644 --- a/src/dependencies/dependency_graph.jl +++ b/src/dependencies/dependency_graph.jl @@ -12,7 +12,7 @@ end mutable struct SoftDependencyNode{T} <: AbstractDependencyNode value::T process::Symbol - hard_dependency::Union{Vector{HardDependencyNode}} + hard_dependency::Vector{HardDependencyNode} parent::Union{Nothing,Vector{SoftDependencyNode}} parent_vars::Union{Nothing,NamedTuple} children::Vector{SoftDependencyNode} From 9ae6af18ee7836fb57f8ae77f13d544ea0bc1e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 29 Sep 2023 17:55:10 +0200 Subject: [PATCH 28/97] Update RefVector.jl --- src/component_models/RefVector.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/component_models/RefVector.jl b/src/component_models/RefVector.jl index 737660d9..f16befd5 100644 --- a/src/component_models/RefVector.jl +++ b/src/component_models/RefVector.jl @@ -105,8 +105,7 @@ Base.length(rv::RefVector) = length(rv.v) Base.eltype(::Type{RefVector{T}}) where {T} = T Base.parent(v::RefVector) = v.v -# Base.push!(v::RefVector, val) = push!(parent(v), val) -Base.resize!(v::RefVector, nl::Integer) = (resize!(v.parent, nl); v) +Base.resize!(v::RefVector, nl::Integer) = (resize!(parent(v), nl); v) Base.push!(v::RefVector, x...) = (push!(parent(v), x...); v) Base.pop!(v::RefVector) = pop!(parent(v)) Base.append!(v::RefVector, items) = (append!(parent(v), items); v) From a8d88d93881ef92ff792bfc86b1115db0e2de6ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 29 Sep 2023 17:55:13 +0200 Subject: [PATCH 29/97] Create multi-scale_dependencies.jl --- src/dependencies/multi-scale_dependencies.jl | 147 +++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 src/dependencies/multi-scale_dependencies.jl diff --git a/src/dependencies/multi-scale_dependencies.jl b/src/dependencies/multi-scale_dependencies.jl new file mode 100644 index 00000000..80ff9397 --- /dev/null +++ b/src/dependencies/multi-scale_dependencies.jl @@ -0,0 +1,147 @@ +function multiscale_dep(models, verbose=true) + + mapping = Dict(first(mod) => Dict(PlantSimEngine.get_mapping(last(mod))) for mod in models) + #! continue here: we have the inputs and outputs variables for each process per scale, and if the variable can + #! be found at another scale, it is defined as a MappedVar (variables + mapped scale). + #! Now what we need to do is to compute the dependency graph for each process each scale, by searching the inputs + #! of each process in the outputs of its own scale, or the other scales. There are five cases then: + #! 1. The process has no inputs. It is completely independent. + #! 2. The process needs inputs from its own scale. We put it as a child of this other process. + #! 3. The process needs inputs from another scale. We put it as a child of this process at another scale. + #! 4. The process needs inputs from its own scale and another scale. We put it as a child of both. + #! 5. The process is a hard dependency of another process (only possible in own scale). In this case it is treated differently (uses the standard method) + #! Note that in the 5th case, we still need to check if a variable is needed from another scale. In this case, the root node of the + #! hard dependency graph is used as a child of the process at the other scale. + + #! How do we do all that? We identify the hard dependencies first. Then we link the inputs/outputs of the hard dependencies roots + #! to other scales if needed. Then we transform all these nodes into soft dependencies, that we put into a Dict of Scale => Dict(process => SoftDependencyNode). + #! Then we traverse all these and we set nodes that need outputs from other nodes as inputs as children/parents. + #! If a node has no dependency, it is set as a root node and pushed into a new Dict (independant_process_root). This Dict is the dependency graph. + + soft_dep_graphs = Dict{String,Any}(i => 0.0 for i in keys(models)) + for (organ, model) in models + # organ = "Leaf"; model = models[organ] + mods = PlantSimEngine.parse_models(PlantSimEngine.get_models(model)) + + # Move some models below others when they are manually linked (hard-dependency): + hard_deps = PlantSimEngine.hard_dependencies((; mods...), verbose=verbose) + d_vars = Dict{Symbol,Vector{Pair{Symbol,NamedTuple}}}() + for (procname, node) in hard_deps.roots + var = Pair{Symbol,NamedTuple}[] + PlantSimEngine.traverse_dependency_graph!(node, x -> PlantSimEngine.variables_multiscale(x, organ, mapping), var) + push!(d_vars, procname => var) + end + + inputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,PlantSimEngine.MappedVar}}}}}}( + key => [j.first => j.second.inputs for j in val] for (key, val) in d_vars + ) + outputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,PlantSimEngine.MappedVar}}}}}}( + key => [j.first => j.second.outputs for j in val] for (key, val) in d_vars + ) + + soft_dep_graph = Dict( + process_ => PlantSimEngine.SoftDependencyNode( + soft_dep_vars.value, + process_, # process name + PlantSimEngine.AbstractTrees.children(soft_dep_vars), # hard dependencies + nothing, + nothing, + PlantSimEngine.SoftDependencyNode[], + [0] + ) + for (process_, soft_dep_vars) in hard_deps.roots + ) + + soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process, outputs=outputs_process) + end + + independant_process_root = Dict{Symbol,PlantSimEngine.SoftDependencyNode}() + for (organ, (soft_dep_graph, ins, outs)) in soft_dep_graphs # e.g. organ = "Plant"; soft_dep_graph, inputs_process, outputs_process = soft_dep_graphs[organ] + for (proc, i) in soft_dep_graph + # proc = :photosynthesis; i = soft_dep_graph[proc] + # Search if the process has soft dependencies: + soft_deps = PlantSimEngine.search_inputs_in_output(proc, ins, outs) + + # Remove the hard dependencies from the soft dependencies: + soft_deps_not_hard = PlantSimEngine.drop_process(soft_dep_graphs, [hd.process for hd in i.hard_dependency]) + # NB: if a node is already a hard dependency of the node, it cannot be a soft dependency + + # Check if the process has soft dependencies at other scales: + soft_deps_multiscale = PlantSimEngine.search_inputs_in_multiscale_output(proc, organ, ins, soft_dep_graphs) + # Example output: "Soil" => Dict(:soil_water=>[:soil_water_content]), which means that the variable :soil_water_content + # is computed by the process :soil_water at the scale "Soil". + + if length(soft_deps_not_hard) == 0 && i.process in keys(hard_deps.roots) && length(soft_deps_multiscale) == 0 + # If the process has no soft (multiscale) dependencies, then it is independant (so it is a root) + # Note that the process is only independent if it is also a root in the hard-dependency graph + independant_process_root[proc] = i + else + + # If the process has soft dependencies at its scale, add it: + if length(soft_deps_not_hard) > 0 + # If the process has soft dependencies, then it is not independant + # and we need to add its parent(s) to the node, and the node as a child + for (parent_soft_dep, soft_dep_vars) in pairs(soft_deps_not_hard) + # parent_soft_dep = :process5; soft_dep_vars = soft_deps[parent_soft_dep] + + # preventing a cyclic dependency + if parent_soft_dep == proc + error("Cyclic model dependency detected for process $proc") + end + + # preventing a cyclic dependency: if the parent also has a dependency on the current node: + if soft_dep_graph[parent_soft_dep].parent !== nothing && any([i == p for p in soft_dep_graph[parent_soft_dep].parent]) + error( + "Cyclic dependency detected for process $proc:", + " $proc depends on $parent_soft_dep, which depends on $proc.", + " This is not allowed, but is possible via a hard dependency." + ) + end + + # preventing a cyclic dependency: if the current node has the parent node as a child: + if i.children !== nothing && any([soft_dep_graph[parent_soft_dep] == p for p in i.children]) + error( + "Cyclic dependency detected for process $proc:", + " $proc depends on $parent_soft_dep, which depends on $proc.", + " This is not allowed, but is possible via a hard dependency." + ) + end + + # Add the current node as a child of the node on which it depends + push!(soft_dep_graph[parent_soft_dep].children, i) + + # Add the node on which the current node depends as a parent + if i.parent === nothing + # If the node had no parent already, it is nothing, so we change into a vector + i.parent = [soft_dep_graph[parent_soft_dep]] + else + push!(i.parent, soft_dep_graph[parent_soft_dep]) + end + + # Add the soft dependencies (variables) of the parent to the current node + i.parent_vars = soft_deps + end + end + + # If the node has soft dependencies at other scales, add it as child of the other scale (and add its parent too): + if length(soft_deps_multiscale) > 0 + #! Continue here: add the node as a child of the other scale, and add this other node has its parent. + #! Take inspiration from the code above happening at the same scale. + #! Note that the node can have both soft dependencies at its own scale and at other scales, but it is not + #! a big deal because in the end we drop the scales and only keep the root soft-dependency nodes. + + end + #! To do: make this code work without multiscale mapping, so we have only one code base for both cases. + #! Also, put some parts into functions to make the code more readable. + end + end + end + + #! CONTINUE HERE: use the multiscale variables to compute the dependency graph. The steps would look something like: + #! 1. Get the hard-dependency graph + #! 2. Get the soft-dependency graph: do as before by computing inputs and outputs for each hard-dependency root, + #! but also look at the multiscale variables to the inputs and outputs. + #! 3. For each soft-dependency root, check if the process is independant (i.e. if it has no soft-dependencies). + #! within its own scale, but also in other scales. If it is independant, then it is a root of the soft-dependency multiscale graph. + #! If it is not, then add it as a child of the other soft-dependency node that it depends on. +end \ No newline at end of file From 9f1636e7991c1779752e69049e40e66fecba34bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 2 Oct 2023 12:28:44 +0200 Subject: [PATCH 30/97] Update multi-scale_dependencies.jl Now managing multiscale dependencies (to check) --- src/dependencies/multi-scale_dependencies.jl | 59 +++++++++++++++++--- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/src/dependencies/multi-scale_dependencies.jl b/src/dependencies/multi-scale_dependencies.jl index 80ff9397..64ea3d10 100644 --- a/src/dependencies/multi-scale_dependencies.jl +++ b/src/dependencies/multi-scale_dependencies.jl @@ -18,6 +18,9 @@ function multiscale_dep(models, verbose=true) #! Then we traverse all these and we set nodes that need outputs from other nodes as inputs as children/parents. #! If a node has no dependency, it is set as a root node and pushed into a new Dict (independant_process_root). This Dict is the dependency graph. + # First step, get the hard-dependency graph and create SoftDependencyNodes for each hard-dependency root. In other word, we want + # only the nodes that are not hard-dependency of other nodes. These nodes are taken as roots for the soft-dependency graph because they + # are independant. soft_dep_graphs = Dict{String,Any}(i => 0.0 for i in keys(models)) for (organ, model) in models # organ = "Leaf"; model = models[organ] @@ -47,7 +50,7 @@ function multiscale_dep(models, verbose=true) nothing, nothing, PlantSimEngine.SoftDependencyNode[], - [0] + [0] # Vector of zeros of length = number of time-steps ) for (process_, soft_dep_vars) in hard_deps.roots ) @@ -55,15 +58,19 @@ function multiscale_dep(models, verbose=true) soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process, outputs=outputs_process) end - independant_process_root = Dict{Symbol,PlantSimEngine.SoftDependencyNode}() - for (organ, (soft_dep_graph, ins, outs)) in soft_dep_graphs # e.g. organ = "Plant"; soft_dep_graph, inputs_process, outputs_process = soft_dep_graphs[organ] + # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the + # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the + # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children + # of the nodes that they depend on. + independant_process_root = Dict{Pair{String,Symbol},PlantSimEngine.SoftDependencyNode}() + for (organ, (soft_dep_graph, ins, outs)) in soft_dep_graphs # e.g. organ = "Plant"; soft_dep_graph, ins, outs = soft_dep_graphs[organ] for (proc, i) in soft_dep_graph - # proc = :photosynthesis; i = soft_dep_graph[proc] + # proc = :carbon_allocation; i = soft_dep_graph[proc] # Search if the process has soft dependencies: soft_deps = PlantSimEngine.search_inputs_in_output(proc, ins, outs) # Remove the hard dependencies from the soft dependencies: - soft_deps_not_hard = PlantSimEngine.drop_process(soft_dep_graphs, [hd.process for hd in i.hard_dependency]) + soft_deps_not_hard = PlantSimEngine.drop_process(soft_deps, [hd.process for hd in i.hard_dependency]) # NB: if a node is already a hard dependency of the node, it cannot be a soft dependency # Check if the process has soft dependencies at other scales: @@ -74,9 +81,8 @@ function multiscale_dep(models, verbose=true) if length(soft_deps_not_hard) == 0 && i.process in keys(hard_deps.roots) && length(soft_deps_multiscale) == 0 # If the process has no soft (multiscale) dependencies, then it is independant (so it is a root) # Note that the process is only independent if it is also a root in the hard-dependency graph - independant_process_root[proc] = i + independant_process_root[organ=>proc] = i else - # If the process has soft dependencies at its scale, add it: if length(soft_deps_not_hard) > 0 # If the process has soft dependencies, then it is not independant @@ -129,7 +135,44 @@ function multiscale_dep(models, verbose=true) #! Take inspiration from the code above happening at the same scale. #! Note that the node can have both soft dependencies at its own scale and at other scales, but it is not #! a big deal because in the end we drop the scales and only keep the root soft-dependency nodes. - + for org in keys(soft_deps_multiscale) + # org = "Leaf" + for (parent_soft_dep, soft_dep_vars) in soft_deps_multiscale[org] + # parent_soft_dep= :photosynthesis; soft_dep_vars = soft_deps_multiscale[org][parent_soft_dep] + parent_node = soft_dep_graphs[org][:soft_dep_graph][parent_soft_dep] + # preventing a cyclic dependency: if the parent also has a dependency on the current node: + if parent_node.parent !== nothing && any([i == p for p in parent_node.parent]) + error( + "Cyclic dependency detected for process $proc:", + " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one", + " This is not allowed, you may need to develop a new process that does the whole computation by itself." + ) + end + + # preventing a cyclic dependency: if the current node has the parent node as a child: + if i.children !== nothing && any([parent_node == p for p in i.children]) + error( + "Cyclic dependency detected for process $proc:", + " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one.", + " This is not allowed, you may need to develop a new process that does the whole computation by itself." + ) + end + + # Add the current node as a child of the node on which it depends: + push!(parent_node.children, i) + + # Add the node on which the current node depends as a parent + if i.parent === nothing + # If the node had no parent already, it is nothing, so we change into a vector + i.parent = [parent_node] + else + push!(i.parent, parent_node) + end + + # Add the multiscale soft dependencies variables of the parent to the current node + i.parent_vars = NamedTuple(Symbol(k) => NamedTuple(v) for (k, v) in soft_deps_multiscale) + end + end end #! To do: make this code work without multiscale mapping, so we have only one code base for both cases. #! Also, put some parts into functions to make the code more readable. From 59407dc1246250a121243aa08ab644b1fc54afcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 2 Oct 2023 12:32:45 +0200 Subject: [PATCH 31/97] Update soft_dependencies.jl Add doc to `search_inputs_in_multiscale_output` --- src/dependencies/soft_dependencies.jl | 63 ++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index f0e7bb61..53a35a85 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -227,6 +227,35 @@ function search_inputs_in_output(process, inputs, outputs) return NamedTuple(inputs_as_output_of_process) end + +""" + search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_graphs) + +# Arguments + +- `process::Symbol`: the process for which we want to find the soft dependencies at other scales. +- `organ::String`: the organ for which we want to find the soft dependencies. +- `inputs::Dict{Symbol, Vector{Pair{Symbol}, Tuple{Symbol, Vararg{Symbol}}}}`: a dict of process => [:subprocess => (:var1, :var2)]. +- `soft_dep_graphs::Dict{String, ...}`: a dict of organ => (soft_dep_graph, inputs, outputs). + +# Details + +The inputs (and similarly, outputs) give the inputs of each process, classified by the process it comes from. It can +come from itself (its own inputs), or from another process that is a hard-dependency. + +# Returns + +A dictionary with the soft dependencies variables found in outputs of other scales for each process, e.g.: + +```julia +Dict{String, Dict{Symbol, Vector{Symbol}}} with 2 entries: + "Internode" => Dict(:carbon_demand=>[:carbon_demand]) + "Leaf" => Dict(:photosynthesis=>[:A], :carbon_demand=>[:carbon_demand]) +``` + +This means that the variable `:carbon_demand` is computed by the process `:carbon_demand` at the scale "Internode", and the variable `:A` +is computed by the process `:photosynthesis` at the scale "Leaf". Those variables are used as inputs for the process that we just passed. +""" function search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_graphs) vars_input = PlantSimEngine.flatten_vars(inputs[process]) @@ -234,19 +263,31 @@ function search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_gra for var in vars_input # e.g. var = PlantSimEngine.MappedVar{String, Nothing}("Soil", :soil_water_content, nothing) # The variable is a multiscale variable: if isa(var, PlantSimEngine.MappedVar) - for (proc_output, pairs_vars_output) in soft_dep_graphs[var.organ][:outputs] # e.g. proc_output = :soil_water; pairs_vars_output = [:soil_water=>(:soil_water_content,)] - process == proc_output && error("Process $process declared at two scales: $organ and $(var.organ). A process can only be simulated at one scale.") - vars_output = PlantSimEngine.flatten_vars(pairs_vars_output) - if var.var in vars_output - # The variable is found at another scale: - if haskey(inputs_as_output_of_other_scale, var.organ) - if haskey(inputs_as_output_of_other_scale[var.organ], proc_output) - push!(inputs_as_output_of_other_scale[var.organ][proc_output], var.var) + var_organ = var.organ + + @assert var_organ != organ "$var in process $process is set to be multiscale, but points to its own scale ($organ). This is not allowed." + + if !isa(var_organ, AbstractVector) + # In case the organ is given as a singleton (e.g. "Soil" instead of ["Soil"]) + var_organ = [var_organ] + end + + for org in var_organ # e.g. org = "Soil" + # The variable is a multiscale variable: + for (proc_output, pairs_vars_output) in soft_dep_graphs[org][:outputs] # e.g. proc_output = :soil_water; pairs_vars_output = [:soil_water=>(:soil_water_content,)] + process == proc_output && error("Process $process declared at two scales: $organ and $org. A process can only be simulated at one scale.") + vars_output = PlantSimEngine.flatten_vars(pairs_vars_output) + if var.var in vars_output + # The variable is found at another scale: + if haskey(inputs_as_output_of_other_scale, org) + if haskey(inputs_as_output_of_other_scale[org], proc_output) + push!(inputs_as_output_of_other_scale[org][proc_output], var.var) + else + inputs_as_output_of_other_scale[org][proc_output] = [var.var] + end else - inputs_as_output_of_other_scale[var.organ][proc_output] = [var.var] + inputs_as_output_of_other_scale[org] = Dict(proc_output => [var.var]) end - else - inputs_as_output_of_other_scale[var.organ] = Dict(proc_output => [var.var]) end end end From 266ed236bca2aad2ff2866255f8fc304fbd36c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 2 Oct 2023 14:49:01 +0200 Subject: [PATCH 32/97] Add multi-scale_dependencies.jl --- src/PlantSimEngine.jl | 1 + src/dependencies/multi-scale_dependencies.jl | 34 +++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 3658a136..84d1ddd7 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -57,6 +57,7 @@ include("traits/parallel_traits.jl") include("dependencies/dependency_graph.jl") include("dependencies/soft_dependencies.jl") include("dependencies/hard_dependencies.jl") +include("dependencies/multi-scale_dependencies.jl") include("dependencies/dependencies.jl") # Processes: diff --git a/src/dependencies/multi-scale_dependencies.jl b/src/dependencies/multi-scale_dependencies.jl index 64ea3d10..4fdf2250 100644 --- a/src/dependencies/multi-scale_dependencies.jl +++ b/src/dependencies/multi-scale_dependencies.jl @@ -1,6 +1,6 @@ function multiscale_dep(models, verbose=true) - mapping = Dict(first(mod) => Dict(PlantSimEngine.get_mapping(last(mod))) for mod in models) + mapping = Dict(first(mod) => Dict(get_mapping(last(mod))) for mod in models) #! continue here: we have the inputs and outputs variables for each process per scale, and if the variable can #! be found at another scale, it is defined as a MappedVar (variables + mapped scale). #! Now what we need to do is to compute the dependency graph for each process each scale, by searching the inputs @@ -22,63 +22,65 @@ function multiscale_dep(models, verbose=true) # only the nodes that are not hard-dependency of other nodes. These nodes are taken as roots for the soft-dependency graph because they # are independant. soft_dep_graphs = Dict{String,Any}(i => 0.0 for i in keys(models)) + not_found = Dict{Symbol,DataType}() for (organ, model) in models # organ = "Leaf"; model = models[organ] - mods = PlantSimEngine.parse_models(PlantSimEngine.get_models(model)) + mods = parse_models(get_models(model)) # Move some models below others when they are manually linked (hard-dependency): - hard_deps = PlantSimEngine.hard_dependencies((; mods...), verbose=verbose) + hard_deps = hard_dependencies((; mods...), verbose=verbose) d_vars = Dict{Symbol,Vector{Pair{Symbol,NamedTuple}}}() for (procname, node) in hard_deps.roots var = Pair{Symbol,NamedTuple}[] - PlantSimEngine.traverse_dependency_graph!(node, x -> PlantSimEngine.variables_multiscale(x, organ, mapping), var) + traverse_dependency_graph!(node, x -> variables_multiscale(x, organ, mapping), var) push!(d_vars, procname => var) end - inputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,PlantSimEngine.MappedVar}}}}}}( + inputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( key => [j.first => j.second.inputs for j in val] for (key, val) in d_vars ) - outputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,PlantSimEngine.MappedVar}}}}}}( + outputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( key => [j.first => j.second.outputs for j in val] for (key, val) in d_vars ) soft_dep_graph = Dict( - process_ => PlantSimEngine.SoftDependencyNode( + process_ => SoftDependencyNode( soft_dep_vars.value, process_, # process name - PlantSimEngine.AbstractTrees.children(soft_dep_vars), # hard dependencies + AbstractTrees.children(soft_dep_vars), # hard dependencies nothing, nothing, - PlantSimEngine.SoftDependencyNode[], + SoftDependencyNode[], [0] # Vector of zeros of length = number of time-steps ) for (process_, soft_dep_vars) in hard_deps.roots ) soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process, outputs=outputs_process) + not_found = merge(not_found, hard_deps.not_found) end # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children # of the nodes that they depend on. - independant_process_root = Dict{Pair{String,Symbol},PlantSimEngine.SoftDependencyNode}() + independant_process_root = Dict{Pair{String,Symbol},SoftDependencyNode}() for (organ, (soft_dep_graph, ins, outs)) in soft_dep_graphs # e.g. organ = "Plant"; soft_dep_graph, ins, outs = soft_dep_graphs[organ] for (proc, i) in soft_dep_graph # proc = :carbon_allocation; i = soft_dep_graph[proc] # Search if the process has soft dependencies: - soft_deps = PlantSimEngine.search_inputs_in_output(proc, ins, outs) + soft_deps = search_inputs_in_output(proc, ins, outs) # Remove the hard dependencies from the soft dependencies: - soft_deps_not_hard = PlantSimEngine.drop_process(soft_deps, [hd.process for hd in i.hard_dependency]) + soft_deps_not_hard = drop_process(soft_deps, [hd.process for hd in i.hard_dependency]) # NB: if a node is already a hard dependency of the node, it cannot be a soft dependency # Check if the process has soft dependencies at other scales: - soft_deps_multiscale = PlantSimEngine.search_inputs_in_multiscale_output(proc, organ, ins, soft_dep_graphs) + soft_deps_multiscale = search_inputs_in_multiscale_output(proc, organ, ins, soft_dep_graphs) # Example output: "Soil" => Dict(:soil_water=>[:soil_water_content]), which means that the variable :soil_water_content # is computed by the process :soil_water at the scale "Soil". - if length(soft_deps_not_hard) == 0 && i.process in keys(hard_deps.roots) && length(soft_deps_multiscale) == 0 + if length(soft_deps_not_hard) == 0 && i.process in keys(soft_dep_graph) && length(soft_deps_multiscale) == 0 # If the process has no soft (multiscale) dependencies, then it is independant (so it is a root) # Note that the process is only independent if it is also a root in the hard-dependency graph independant_process_root[organ=>proc] = i @@ -187,4 +189,6 @@ function multiscale_dep(models, verbose=true) #! 3. For each soft-dependency root, check if the process is independant (i.e. if it has no soft-dependencies). #! within its own scale, but also in other scales. If it is independant, then it is a root of the soft-dependency multiscale graph. #! If it is not, then add it as a child of the other soft-dependency node that it depends on. -end \ No newline at end of file + + return DependencyGraph(independant_process_root, not_found) +end From db2e40191c24c3e7d2314a4a3cf07bdfb9bc85ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 2 Oct 2023 15:44:16 +0200 Subject: [PATCH 33/97] Add more doc --- src/dependencies/multi-scale_dependencies.jl | 2 +- src/mtg/mapping.jl | 92 ++++++++++++-------- 2 files changed, 57 insertions(+), 37 deletions(-) diff --git a/src/dependencies/multi-scale_dependencies.jl b/src/dependencies/multi-scale_dependencies.jl index 4fdf2250..5679c98a 100644 --- a/src/dependencies/multi-scale_dependencies.jl +++ b/src/dependencies/multi-scale_dependencies.jl @@ -1,4 +1,4 @@ -function multiscale_dep(models, verbose=true) +function multiscale_dep(models; verbose=true) mapping = Dict(first(mod) => Dict(get_mapping(last(mod))) for mod in models) #! continue here: we have the inputs and outputs variables for each process per scale, and if the variable can diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index f6a758bd..42a01767 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -319,36 +319,53 @@ vars_from_mapping(m) = collect(Iterators.flatten(keys.(values(m)))) vars_type_from_mapping(m) = collect(Iterators.flatten(values.(values(m)))) """ - create_var_ref(organ::Vector{<:AbstractString}, default::T) where {T} - create_var_ref(organ::AbstractString, default) + MappedVar(organ, var, default) -Create a RefVector from a vector of organs and a default value. The RefVector will be filled with the default value. +A variable mapped to another scale. -Create the reference to a multiscale variable. The reference is a RefVector if the organ was given as a vector, or a Ref if it is a scalar. -""" -function create_var_ref(organ::Vector{<:AbstractString}, var, default::T) where {T} - RefVector(Base.RefValue{T}[]) -end +# Arguments +- `organ`: the organ(s) that are targeted by the mapping +- `var`: the variable that is mapped +- `default`: the default value of the variable + +# Examples + +```jldoctest +julia> using PlantSimEngine +``` + +```jldoctest +julia> MappedVar("Leaf", :A, 1.0) +``` +""" struct MappedVar{S<:Union{A,Vector{A}} where {A<:AbstractString},T} organ::S var::Symbol default::T end -function create_var_ref(organ::AbstractString, var, default) - MappedVar(organ, var, default) +""" + create_var_ref(organ::Vector{<:AbstractString}, default::T) where {T} + create_var_ref(organ::AbstractString, default) + +Create a referece variable. The reference is a `RefVector` if the organ is a vector of strings, and a `MappedVar` +if it is a singleton string. This is because we want to avoid indeing into a vector of values if there is only one +value to map. +""" +function create_var_ref(organ::Vector{<:AbstractString}, var, default::T) where {T} + RefVector(Base.RefValue{T}[]) end +create_var_ref(organ::AbstractString, var, default) = MappedVar(organ, var, default) + function outputs_from_other_scale!(var_outputs_from_mapping, multi_scale_outs, map_vars) multi_scale_outs_organ = filter(x -> first(x) in keys(multi_scale_outs), map_vars) - for (var, organs) in multi_scale_outs_organ - # var, organs = multi_scale_outs_organ[1] + for (var, organs) in multi_scale_outs_organ # var, organs = multi_scale_outs_organ[1] if isa(organs, String) organs = [organs] end - for org in organs - # org = organs[1] + for org in organs # org = organs[1] if haskey(var_outputs_from_mapping, org) push!(var_outputs_from_mapping[org], var => multi_scale_outs[var]) else @@ -359,12 +376,12 @@ function outputs_from_other_scale!(var_outputs_from_mapping, multi_scale_outs, m end """ - init_simulation(mtg, models; type_promotion=nothing, check=true) + init_simulation(mtg, models; type_promotion=nothing, check=true, verbose=true) Initialise the simulation by creating: -- a status for each node type, considering multi-scale variables. -- the dependency graph of the models, and the order in which they should be called. +- a status for each node type, considering multi-scale variables +- the dependency graph of the models # Arguments @@ -372,21 +389,29 @@ Initialise the simulation by creating: - `models::Dict{String,Any}`: a dictionary of model mapping - `type_promotion`: the type promotion to use for the variables - `check`: whether to check the mapping for errors +- `verbose`: print information about errors in the mapping # Details The function first computes a template of status for each organ type that has a model in the mapping. -This template is used to initialise the status of each node in the MTG, taking into account the user-defined -initialisation, and the multiscale mapping. The multiscale mapping is used to make references to the variables +This template is used to initialise the status of each node of the MTG, taking into account the user-defined +initialisation, and the (multiscale) mapping. The mapping is used to make references to the variables that are defined at another scale, so that the values are automatically updated when the variable is changed at -the other scale. +the other scale. Two types of multiscale variables are available: `RefVector` and `MappedVar`. The first one is +used when the variable is mapped to a vector of nodes, and the second one when it is mapped to a single node. This +is given by the user through the mapping, using a string for a single node (*e.g.* `=> "Leaf"`), and a vector of strings for a vector of +nodes (*e.g.* `=> ["Leaf"]` for one type of node or `=> ["Leaf", "Internode"]` for several). + +The function also computes the dependency graph of the models, i.e. the order in which the models should be +called, considering the dependencies between them. The dependency graph is used to call the models in the right order +when the simulation is run. Note that if a variable is not computed by models or initialised from the mapping, it is searched in the MTG attributes. The value is not a reference to the one in the attribute of the MTG, but a copy of it. This is because we can't reference a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be automatically passed as is. """ -function init_simulation(mtg, models; type_promotion=nothing, check=true) +function init_simulation(mtg, models; type_promotion=nothing, check=true, verbose=true) # We make a pre-initialised status for each kind of organ (this is a template for each node type): organs_statuses = status_template(models, type_promotion) # Get the reverse mapping, i.e. the variables that are mapped to other scales. This is used to initialise @@ -400,15 +425,6 @@ function init_simulation(mtg, models; type_promotion=nothing, check=true) # If we find some, we return an error: check && error_mtg_init(var_need_init) - #! continue here. What we need to do: - #! - traverser les MTG pour initialiser un Status par organe, et mettre le vecteur de ces status dans un Dict{Organe, Status} - #! - dans le même traversal, trouver les variables qui doivent être initialisées depuis le mtg (et erreur si elles n'y sont pas) - #! - remplir les RefVector, sachant qu'ils seront automatiquement remplis partout puisque c'est des Ref (a vérifier). - #! - Ajouter la référence au noeud dans le status - #! - calculer le graphe de dépendence des modèles, et faire des calls en fonction - #! - ajouter des tests - #! - ajouter des checks, e.g. est-ce que tous les organes du MTG ont un modèle ou pas... - # Get the status of each node by node type, pre-initialised considering multi-scale variables: statuses = init_statuses(mtg, organs_statuses, var_refvector, var_need_init) @@ -418,7 +434,10 @@ function init_simulation(mtg, models; type_promotion=nothing, check=true) @info "Models given for $model_no_node, but no node with this symbol was found in the MTG." maxlog = 1 end - return statuses + # Compute the multi-scale dependency graph of the models: + mapping_dependency = multiscale_dep(models, verbose=verbose) + + return statuses, mapping_dependency end @@ -762,7 +781,7 @@ using the template given by `status_template`. """ function init_statuses(mtg, status_template, var_refvector, var_need_init=Dict{String,Any}()) nodes_with_models = collect(keys(status_template)) - # We traverse the MTG a first time to initialise the statuses linked to the nodes: + # We traverse the MTG to initialise the statuses linked to the nodes: statuses = Dict(i => Status[] for i in nodes_with_models) MultiScaleTreeGraph.traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) # Check if the node has a model defined for its symbol @@ -783,9 +802,9 @@ function init_statuses(mtg, status_template, var_refvector, var_need_init=Dict{S "Please check the type of the variable in the MTG, and make it a $(typeof(st_template[i.var]))." ) st_template[i.var] = node[i.var] - #! NB: the variable is not a reference to the value in the MTG, but a copy of it. - #! This is because we can't reference a value in a Dict. If we need a ref, the user can use a RefValue in the MTG directly, - #! and it will be automatically passed as is. + # NB: the variable is not a reference to the value in the MTG, but a copy of it. + # This is because we can't reference a value in a Dict. If we need a ref, the user can use a RefValue in the MTG directly, + # and it will be automatically passed as is. end end @@ -794,7 +813,8 @@ function init_statuses(mtg, status_template, var_refvector, var_need_init=Dict{S push!(statuses[node.MTG.symbol], st) - # Instantiate the RefVectors on the fly for other scales that map into this scale + # Instantiate the RefVectors on the fly for other scales that map into this scale, *i.e.* + # add a reference to the value of any variable that is used by another scale into its RefVector: if haskey(var_refvector, node.MTG.symbol) for (organ, vars) in var_refvector[node.MTG.symbol] for var in vars # e.g.: var = :carbon_demand From f0578ac89ce6cde2dca881d0fccfbea700158092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 2 Oct 2023 18:54:08 +0200 Subject: [PATCH 34/97] End-of-day commit (a little bit of everything) --- src/PlantSimEngine.jl | 14 +- src/dependencies/dependency_graph.jl | 1 + src/dependencies/multi-scale_dependencies.jl | 1 + src/dependencies/soft_dependencies.jl | 1 + src/mtg/mapping.jl | 39 ++++- src/run.jl | 160 ++++++++++++++----- src/traits/table_traits.jl | 7 +- 7 files changed, 167 insertions(+), 56 deletions(-) diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 84d1ddd7..9673b239 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -42,6 +42,13 @@ include("component_models/get_status.jl") # Transform into a dataframe: include("dataframe.jl") +# Model dependencies: +include("dependencies/dependency_graph.jl") +include("dependencies/soft_dependencies.jl") +include("dependencies/hard_dependencies.jl") +include("dependencies/multi-scale_dependencies.jl") +include("dependencies/dependencies.jl") + # MTG compatibility: include("mtg/init_mtg_models.jl") include("mtg/mapping.jl") @@ -53,13 +60,6 @@ include("evaluation/statistics.jl") include("traits/table_traits.jl") include("traits/parallel_traits.jl") -# Model dependencies: -include("dependencies/dependency_graph.jl") -include("dependencies/soft_dependencies.jl") -include("dependencies/hard_dependencies.jl") -include("dependencies/multi-scale_dependencies.jl") -include("dependencies/dependencies.jl") - # Processes: include("processes/model_initialisation.jl") include("processes/models_inputs_outputs.jl") diff --git a/src/dependencies/dependency_graph.jl b/src/dependencies/dependency_graph.jl index a86e7c12..1aa56ba4 100644 --- a/src/dependencies/dependency_graph.jl +++ b/src/dependencies/dependency_graph.jl @@ -12,6 +12,7 @@ end mutable struct SoftDependencyNode{T} <: AbstractDependencyNode value::T process::Symbol + scale::String hard_dependency::Vector{HardDependencyNode} parent::Union{Nothing,Vector{SoftDependencyNode}} parent_vars::Union{Nothing,NamedTuple} diff --git a/src/dependencies/multi-scale_dependencies.jl b/src/dependencies/multi-scale_dependencies.jl index 5679c98a..674880ae 100644 --- a/src/dependencies/multi-scale_dependencies.jl +++ b/src/dependencies/multi-scale_dependencies.jl @@ -47,6 +47,7 @@ function multiscale_dep(models; verbose=true) process_ => SoftDependencyNode( soft_dep_vars.value, process_, # process name + organ, # scale AbstractTrees.children(soft_dep_vars), # hard dependencies nothing, nothing, diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index 53a35a85..5b86f9e7 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -62,6 +62,7 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, process_ => SoftDependencyNode( soft_dep_vars.value, process_, # process name + "", AbstractTrees.children(soft_dep_vars), # hard dependencies nothing, nothing, diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 42a01767..4161d49e 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -378,10 +378,12 @@ end """ init_simulation(mtg, models; type_promotion=nothing, check=true, verbose=true) -Initialise the simulation by creating: +Initialise the simulation. Returns: -- a status for each node type, considering multi-scale variables +- the mtg +- a status for each node by organ type, considering multi-scale variables - the dependency graph of the models +- the models parsed as a Dict of organ type => NamedTuple of process => model mapping # Arguments @@ -435,11 +437,40 @@ function init_simulation(mtg, models; type_promotion=nothing, check=true, verbos end # Compute the multi-scale dependency graph of the models: - mapping_dependency = multiscale_dep(models, verbose=verbose) + dependency_graph = multiscale_dep(models, verbose=verbose) - return statuses, mapping_dependency + models = Dict(first(m) => PlantSimEngine.parse_models(PlantSimEngine.get_models(last(m))) for m in models) + + return mtg, statuses, dependency_graph, models +end + +""" + GraphSimulation(graph, mapping) + GraphSimulation(graph, statuses, dependency_graph) + +A type that holds all information for a simulation over a graph. + +# Arguments + +- `graph`: an graph, such as an MTG +- `mapping`: a dictionary of model mapping +- `statuses`: a structure that defines the status of each node in the graph +- `dependency_graph`: the dependency graph of the models applied to the graph +""" +struct GraphSimulation{T,S} + graph::T + statuses::S + dependency_graph::DependencyGraph + models::Dict{String,NamedTuple} +end + +function GraphSimulation(graph, mapping; type_promotion=nothing, check=true, verbose=true) + GraphSimulation(init_simulation(graph, mapping; type_promotion=type_promotion, check=check, verbose=verbose)...) end +dep(g::GraphSimulation) = g.dependency_graph +status(g::GraphSimulation) = g.statuses +get_models(g::GraphSimulation) = g.models function map_scale(f, m, scale::String) map_scale(f, m, [scale]) diff --git a/src/run.jl b/src/run.jl index 9edd2fe2..745f0d1e 100644 --- a/src/run.jl +++ b/src/run.jl @@ -312,10 +312,52 @@ function run!( end # 6- Compatibility with MTG: + +# 6.1: if we pass an MTG and a mapping, then we use them to compute a GraphSimulation object +# that we use with the first method in this file. +function run!( + object::MultiScaleTreeGraph.Node, + mapping::Dict{String,Any}, + meteo=nothing, + constants=PlantMeteo.Constants(), + extra=nothing; + check=true, + executor=ThreadedEx() +) + run!( + GraphSimulation(object, mapping, check=check), + meteo, + constants, + extra; + check, + executor + ) +end + +# 6.2: if we pass a TreeAlike object (e.g. a GraphSimulation): +function run!( + ::TreeAlike, + ::SingletonAlike, + object::GraphSimulation, + meteo, + constants=PlantMeteo.Constants(), + extra=nothing; + check=true, + executor=ThreadedEx() +) + models = get_models(object) + # Run the simulation of each soft-coupled model in the dependency graph: + # Note: hard-coupled processes handle themselves already + @floop executor for (process_key, dependency_node) in collect(dep(object).roots) + run!(object, dependency_node, 1, models, meteo, constants, extra, check, executor) + end +end + +# 6.2 bis, over several time-steps function run!( ::TreeAlike, ::TableAlike, - mtg::MultiScaleTreeGraph.Node, + object::GraphSimulation, meteo, constants=PlantMeteo.Constants(), extra=nothing; @@ -323,51 +365,34 @@ function run!( executor=ThreadedEx() ) - #! Manage parallel computations over nodes - @assert extra === nothing "The extra argument cannot be used with an MTG. It already contains the node." - - # Define the attribute name used for the models in the nodes - attr_name = Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models")) - - # Initialize the models and pre-allocate nodes attributes: - # init_mtg_models!(mtg, models, length(meteo), attr_name=attr_name) - - # # Here we make a simulation for one time-step and going to the next node. - # # This is good for models that need the result of others nodes on one time-step. - # # but not efficient for models that are completely independent. - # # Computing for each time-steps: - # for (i, meteo_i) in enumerate(meteo) - # MultiScaleTreeGraph.transform!( - # mtg, - # (node) -> run!(node[attr_name], meteo_i, constants, node, check=check, executor=executor), - # filter_fun=node -> node[attr_name] !== nothing - # ) - # end - - MultiScaleTreeGraph.transform!( - mtg, - (node) -> run!(node[attr_name], meteo, constants, node, check=check, executor=executor), - filter_fun=node -> node[attr_name] !== nothing - ) -end + dep_graph = dep(object) + if !timestep_parallelizable(dep_graph) && executor != SequentialEx() + is_ts_parallel = which_timestep_parallelizable(dep_graph) + mods_not_parallel = join([i.second.first for i in is_ts_parallel[findall(x -> x.second.second == false, is_ts_parallel)]], "; ") -# 8- Non-mutating version (make a copy before the call, and return the copy): -#! removed this method because it clashes with Base.run and it's not that usefull -# function run( -# object::O, -# meteo::T=nothing, -# constants=PlantMeteo.Constants(), -# extra=nothing; -# check=true -# ) where {O<:Union{ModelList,AbstractArray,AbstractDict},T<:Union{Nothing,PlantMeteo.AbstractAtmosphere,TimeStepTable{<:PlantMeteo.AbstractAtmosphere}}} -# object_tmp = copy(object) -# run!(object_tmp, meteo, constants, extra; check=check) -# return object_tmp -# end + check && @warn string( + "A parallel executor was provided (`executor=$(executor)`) but some models cannot be run in parallel: $mods_not_parallel. ", + "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." + ) maxlog = 1 + executor2 = SequentialEx() + else + executor2 = executor # We define executor2 to limit boxed variables: https://juliafolds.github.io/FLoops.jl/dev/howto/avoid-box/ + end + models = get_models(object) -#! Actual call to the model: + # Optionnaly in parallel over time-steps: + @floop executor2 for (i, meteo_i) in enumerate(Tables.rows(meteo)) + # In parallel over dependency roots (independant computations): + # @floop executor for (process_key, dependency_node) in collect(dep_graph.roots) + #! put this into another function so I can call an @floop again + for (process_key, dependency_node) in collect(dep_graph.roots) + run!(object, dependency_node, i, models, meteo_i, constants, extra, check, executor) + end + end +end +#! Actual calls to the model: # Running the simulation on the dependency graph (always one time-step, one object): function run!(object::T, dep_graph::DependencyGraph{Dict{Symbol,N}}, i, st, meteo, constants, extra; executor=ThreadedEx()) where { T<:ModelList, @@ -408,4 +433,55 @@ function run!( #! which is not thread-safe. run!(object, child, i, st, meteo, constants, extra) end +end + +# For a tree-alike object: +function run!( + object::T, + node::SoftDependencyNode, + i, # time-step to index into the dependency node (to know if the model has been called already) + models, + meteo, + constants, + extra, + check, + executor +) where {T<:GraphSimulation} # T is the status of each node by organ type + + # run!(status(object), dependency_node, meteo, constants, extra) + # Check if all the parents have been called before the child: + if !AbstractTrees.isroot(node) && any([p.simulation_id[1] <= node.simulation_id[1] for p in node.parent]) + # If not, this node should be called via another parent + return nothing + end + + node_statuses = status(object)[node.scale] # Get the status of the nodes at the current scale + models_at_scale = models[node.scale] + + # Check if the simulation can be parallelized over objects: + #TODO: move this check up in the call stack so we check only once per time-step + if !last(object_parallelizable(node)) && executor != SequentialEx() + check && @warn string( + "A parallel executor was provided (`executor=$(executor)`) but the model $(node.value) (or its hard dependencies) cannot be run in parallel over objects.", + "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." + ) maxlog = 1 + executor = SequentialEx() + end + + @floop executor for st in node_statuses # for each node status at the current scale (potentially in parallel over nodes) + # Actual call to the model: + run!(node.value, models_at_scale, st, meteo, constants, extra) + + #TODO: keep track of the outputs users need here. + end + + node.simulation_id[1] += 1 # increment the simulation id, to remember that the model has been called already + + # Recursively visit the children (soft dependencies only, hard dependencies are handled by the model itself): + for child in node.children + #! check if we can run this safely in a @floop loop. I would say no, + #! because we are running a parallel computation above already, modifying the node.simulation_id, + #! which is not thread-safe yet. + run!(object, child, i, models, meteo, constants, extra, check, executor) + end end \ No newline at end of file diff --git a/src/traits/table_traits.jl b/src/traits/table_traits.jl index 89a20f74..5d19cdd3 100644 --- a/src/traits/table_traits.jl +++ b/src/traits/table_traits.jl @@ -13,10 +13,11 @@ how to iterate over the data. The following data formats are supported: `TimeStepTable`. The data is iterated over by rows using the `Tables.jl` interface. - `SingletonAlike`: The data is a singleton-like object, e.g. a `NamedTuple` or a `TimeStepRow`. The data is iterated over by columns. +- `TreeAlike`: The data is a tree-like object, e.g. a `Node`. The default implementation returns `TableAlike` for `AbstractDataFrame`, -`TimeStepTable`, `AbstractVector` and `Dict`, `TreeAlike` for `Node`, `SingletonAlike` -for `Status`, `ModelList`, `NamedTuple` and `TimeStepRow`. +`TimeStepTable`, `AbstractVector` and `Dict`, `TreeAlike` for `GraphSimulation`, +`SingletonAlike` for `Status`, `ModelList`, `NamedTuple` and `TimeStepRow`. The default implementation for `Any` throws an error. Users that want to use another input should define this trait for the new data format, e.g.: @@ -57,7 +58,7 @@ DataFormat(::Type{<:NamedTuple}) = SingletonAlike() DataFormat(::Type{<:Status}) = SingletonAlike() DataFormat(::Type{<:ModelList{Mo,S} where {Mo,S<:Status}}) = SingletonAlike() DataFormat(::Type{<:ModelList{Mo,S}}) where {Mo,S} = TableAlike() -DataFormat(::Type{<:MultiScaleTreeGraph.Node}) = TreeAlike() +DataFormat(::Type{<:GraphSimulation}) = TreeAlike() DataFormat(::Type{<:PlantMeteo.AbstractAtmosphere}) = SingletonAlike() DataFormat(::Type{<:PlantMeteo.TimeStepRow}) = SingletonAlike() From 3c9ccf618cf5378d8b3fc76b728126abc21c2bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 3 Oct 2023 14:51:11 +0200 Subject: [PATCH 35/97] Make the simulation work on an mtg --- src/mtg/mapping.jl | 22 +++++++++++++++------- src/processes/model_initialisation.jl | 2 +- src/run.jl | 7 +++++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 4161d49e..84183414 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -191,7 +191,7 @@ models = Dict( compute_mapping(models, nothing) ``` """ -function compute_mapping(models::Dict{String,Any}, type_promotion) +function compute_mapping(models::Dict{String,T}, type_promotion) where {T} # Initialise a dict that defines the multiscale variables for each organ type: organs_mapping = Dict{String,Any}() # Initialise a Dict that defines the variables that are outputs from a mapping, @@ -353,7 +353,7 @@ Create a referece variable. The reference is a `RefVector` if the organ is a vec if it is a singleton string. This is because we want to avoid indeing into a vector of values if there is only one value to map. """ -function create_var_ref(organ::Vector{<:AbstractString}, var, default::T) where {T} +function create_var_ref(organ::Vector{<:AbstractString}, var, default::AbstractVector{T}) where {T} RefVector(Base.RefValue{T}[]) end @@ -362,14 +362,22 @@ create_var_ref(organ::AbstractString, var, default) = MappedVar(organ, var, defa function outputs_from_other_scale!(var_outputs_from_mapping, multi_scale_outs, map_vars) multi_scale_outs_organ = filter(x -> first(x) in keys(multi_scale_outs), map_vars) for (var, organs) in multi_scale_outs_organ # var, organs = multi_scale_outs_organ[1] + if isa(multi_scale_outs[var], AbstractVector) + var_default_value = multi_scale_outs[var][1] + else + error( + "The variable $var is an output variable mapped to nodes of type $organs, but its default value is not a vector. " * + "Make sure the model that computes this variable has a vector of values as outputs." + ) + end if isa(organs, String) organs = [organs] end for org in organs # org = organs[1] if haskey(var_outputs_from_mapping, org) - push!(var_outputs_from_mapping[org], var => multi_scale_outs[var]) + push!(var_outputs_from_mapping[org], var => var_default_value) else - var_outputs_from_mapping[org] = [var => multi_scale_outs[var]] + var_outputs_from_mapping[org] = [var => var_default_value] end end end @@ -457,11 +465,11 @@ A type that holds all information for a simulation over a graph. - `statuses`: a structure that defines the status of each node in the graph - `dependency_graph`: the dependency graph of the models applied to the graph """ -struct GraphSimulation{T,S} +struct GraphSimulation{T,S,U} graph::T statuses::S dependency_graph::DependencyGraph - models::Dict{String,NamedTuple} + models::Dict{String,U} end function GraphSimulation(graph, mapping; type_promotion=nothing, check=true, verbose=true) @@ -585,7 +593,7 @@ organs_statuses["Soil"][:soil_water_content] === organs_statuses["Leaf"][:soil_w true ``` """ -function status_template(models::Dict{String,Any}, type_promotion) +function status_template(models::Dict{String,T}, type_promotion) where {T} organs_mapping, var_outputs_from_mapping = compute_mapping(models, type_promotion) # Vector of pre-initialised variables with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 481e68f3..575d2d9b 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -117,7 +117,7 @@ struct VarFromMTG end # For the list of models given to an MTG: -function to_initialize(models::Dict{String,Any}, organs_statuses, mtg) +function to_initialize(models::Dict{String,T}, organs_statuses, mtg) where {T} # Get the variables in the MTG: vars_in_mtg = names(mtg) diff --git a/src/run.jl b/src/run.jl index 745f0d1e..e7112a16 100644 --- a/src/run.jl +++ b/src/run.jl @@ -317,21 +317,24 @@ end # that we use with the first method in this file. function run!( object::MultiScaleTreeGraph.Node, - mapping::Dict{String,Any}, + mapping::Dict{String,T} where {T}, meteo=nothing, constants=PlantMeteo.Constants(), extra=nothing; check=true, executor=ThreadedEx() ) + sim = GraphSimulation(object, mapping, check=check) run!( - GraphSimulation(object, mapping, check=check), + sim, meteo, constants, extra; check, executor ) + + return sim end # 6.2: if we pass a TreeAlike object (e.g. a GraphSimulation): From 15bcae047ea2b14937bb68abd39f12db10045e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 3 Oct 2023 14:51:48 +0200 Subject: [PATCH 36/97] Update example models that use MTG (now simpler implementation) --- examples/ToyAssimGrowthModel.jl | 2 +- examples/ToyAssimModel.jl | 2 +- examples/ToyCAllocationModel.jl | 35 +++++++++++---------------------- examples/ToyDegreeDays.jl | 4 +--- examples/ToyLAIModel.jl | 10 +++++----- examples/ToyRUEGrowthModel.jl | 2 +- examples/ToySoilModel.jl | 8 +++----- 7 files changed, 23 insertions(+), 40 deletions(-) diff --git a/examples/ToyAssimGrowthModel.jl b/examples/ToyAssimGrowthModel.jl index 07f8eca4..6ccad8cb 100644 --- a/examples/ToyAssimGrowthModel.jl +++ b/examples/ToyAssimGrowthModel.jl @@ -73,7 +73,7 @@ function PlantSimEngine.run!(::ToyAssimGrowthModel, models, status, meteo, const status.biomass_increment = NPP - status.Rg # The biomass is the biomass from the previous time-step plus the biomass increment: - status.biomass = PlantMeteo.prev_value(status, :biomass; default=0.0) + status.biomass_increment + status.biomass += status.biomass_increment end # And optionally, we can tell PlantSimEngine that we can safely parallelize our model over space (objects): diff --git a/examples/ToyAssimModel.jl b/examples/ToyAssimModel.jl index 3ea57ee3..8f03643f 100644 --- a/examples/ToyAssimModel.jl +++ b/examples/ToyAssimModel.jl @@ -48,7 +48,7 @@ Base.eltype(::ToyAssimModel{T}) where {T} = T # Implement the growth model: function PlantSimEngine.run!(::ToyAssimModel, models, status, meteo, constants, extra) # The assimilation is simply the absorbed photosynthetic photon flux density (aPPFD) times the light use efficiency (LUE): - status.A = status.aPPFD * models.growth.LUE * status.soil_water_content + status.A = status.aPPFD * models.photosynthesis.LUE * status.soil_water_content end # And optionally, we can tell PlantSimEngine that we can safely parallelize our model over space (objects): diff --git a/examples/ToyCAllocationModel.jl b/examples/ToyCAllocationModel.jl index 1b821081..0451a44e 100644 --- a/examples/ToyCAllocationModel.jl +++ b/examples/ToyCAllocationModel.jl @@ -23,49 +23,36 @@ struct ToyCAllocationModel <: AbstractCarbon_AllocationModel end # Define inputs: function PlantSimEngine.inputs_(::ToyCAllocationModel) - (A=-Inf, carbon_demand=-Inf,) + (A=[-Inf], carbon_demand=[-Inf],) end # Define outputs: function PlantSimEngine.outputs_(::ToyCAllocationModel) - (carbon_offer=-Inf, carbon_allocation=-Inf) + (carbon_offer=-Inf, carbon_allocation=[-Inf]) end function PlantSimEngine.run!(::ToyCAllocationModel, models, status, meteo, constants, mtg) - carbon_demand_organs = Vector{eltype(status.carbon_demand)}() - MultiScaleTreeGraph.traverse!(mtg, symbol=["Leaf", "Internode"]) do node - push!(carbon_demand_organs, node[:models].status[:carbon_demand]) - end - - carbon_demand = sum(carbon_demand_organs) - status.carbon_offer = 0.0 - MultiScaleTreeGraph.traverse!(mtg, symbol="Leaf") do node - status.carbon_offer += node[:models].status[:A] - end + carbon_demand_tot = sum(status.carbon_demand) + status.carbon_offer = sum(status.A) + #Note: this model is multiscale, so status.carbon_demand, status.carbon_allocation, and status.A are vectors. # If the total demand is positive, we try allocating carbon: - if carbon_demand > 0.0 + if carbon_demand_tot > 0.0 # Proportion of the demand of each leaf compared to the total leaf demand: - proportion_carbon_demand = carbon_demand_organs ./ carbon_demand + proportion_carbon_demand = status.carbon_demand ./ carbon_demand_tot - if carbon_demand <= status.carbon_offer + if carbon_demand_tot <= status.carbon_offer # If the carbon demand is lower than the offer we allocate the offer: - carbon_allocation_organs = carbon_demand + carbon_allocation_organs = carbon_demand_tot else # Here we don't have enough carbon offer carbon_allocation_organs = status.carbon_offer end - carbon_allocation_organ = carbon_allocation_organs .* proportion_carbon_demand + status.carbon_allocation .= carbon_allocation_organs .* proportion_carbon_demand else # If the carbon demand is 0.0, we allocate nothing: - carbon_allocation_organs = 0.0 - carbon_allocation_organ = zeros(typeof(carbon_demand_organs[1]), length(carbon_demand_organs)) - end - - # We allocate the carbon to the organs: - MultiScaleTreeGraph.traverse!(mtg, symbol=["Leaf", "Internode"]) do organ - organ[:models].status[:carbon_allocation] = popfirst!(carbon_allocation_organ) + status.carbon_allocation .= 0.0 end end diff --git a/examples/ToyDegreeDays.jl b/examples/ToyDegreeDays.jl index bf795109..049f6d11 100644 --- a/examples/ToyDegreeDays.jl +++ b/examples/ToyDegreeDays.jl @@ -16,9 +16,7 @@ PlantSimEngine.outputs_(::ToyDegreeDaysCumulModel) = (degree_days_cu=-Inf,) # Implementing the actual algorithm by adding a method to the run! function for our model: function PlantSimEngine.run!(m::ToyDegreeDaysCumulModel, models, status, meteo, constants=nothing, extra=nothing) - status.degree_days_cu = - PlantMeteo.prev_value(status, :degree_days_cu, default=m.init_degreedays) + status.degree_days - println("step = ", status.step) + status.degree_days_cu += status.degree_days end # The computation of ToyDegreeDaysCumulModel dependents on previous values, but it is independent of other objects. diff --git a/examples/ToyLAIModel.jl b/examples/ToyLAIModel.jl index 295b67f8..97cee390 100644 --- a/examples/ToyLAIModel.jl +++ b/examples/ToyLAIModel.jl @@ -23,13 +23,13 @@ PlantSimEngine.outputs_(::ToyLAIModel) = (LAI=-Inf,) function PlantSimEngine.run!(::ToyLAIModel, models, status, meteo, constants=nothing, extra=nothing) status.LAI = models.LAI_Dynamic.max_lai * - (1 / - (1 + exp((models.LAI_Dynamic.dd_incslope - status.degree_days_cu) / models.LAI_Dynamic.inc_slope)) - - 1 / (1 + exp((models.LAI_Dynamic.dd_decslope - status.degree_days_cu) / models.LAI_Dynamic.dec_slope)) + (1.0 / + (1.0 + exp((models.LAI_Dynamic.dd_incslope - status.degree_days_cu) / models.LAI_Dynamic.inc_slope)) - + 1.0 / (1.0 + exp((models.LAI_Dynamic.dd_decslope - status.degree_days_cu) / models.LAI_Dynamic.dec_slope)) ) - if status.LAI < 0 - status.LAI = 0 + if status.LAI < 0.0 + status.LAI = 0.0 end end diff --git a/examples/ToyRUEGrowthModel.jl b/examples/ToyRUEGrowthModel.jl index c5d5aff5..2b9d89a1 100644 --- a/examples/ToyRUEGrowthModel.jl +++ b/examples/ToyRUEGrowthModel.jl @@ -44,7 +44,7 @@ Base.eltype(x::ToyRUEGrowthModel{T}) where {T} = T # Implement the growth model: function PlantSimEngine.run!(::ToyRUEGrowthModel, models, status, meteo, constants, extra) status.biomass_increment = status.aPPFD * models.growth.efficiency - status.biomass = PlantMeteo.prev_value(status, :biomass; default=0.0) + status.biomass_increment + status.biomass += status.biomass_increment end # And optionally, we can tell PlantSimEngine that we can safely parallelize our model over space (objects): diff --git a/examples/ToySoilModel.jl b/examples/ToySoilModel.jl index a744a9cb..b124f7fa 100644 --- a/examples/ToySoilModel.jl +++ b/examples/ToySoilModel.jl @@ -1,4 +1,3 @@ -using Random # Declaring the process of LAI dynamic: @process "soil_water" verbose = false @@ -9,7 +8,7 @@ using Random ToySoilWaterModel(values,rng) A toy model to compute the soil water content. The model simply take a random value in -the `values` range using the `rng` random number generator. +the `values` range using `rand`. # Outputs @@ -17,11 +16,10 @@ the `values` range using the `rng` random number generator. """ struct ToySoilWaterModel <: AbstractSoil_WaterModel values::AbstractRange{Float64} - rng::AbstractRNG end # Defining a method with keyword arguments and default values: -ToySoilWaterModel(; values=0.1:0.1:1.0, rng=MersenneTwister(1234)) = ToySoilWaterModel(values, rng) +ToySoilWaterModel(; values=0.1:0.1:1.0) = ToySoilWaterModel(values) # Defining the inputs and outputs of the model: PlantSimEngine.inputs_(::ToySoilWaterModel) = NamedTuple() @@ -29,7 +27,7 @@ PlantSimEngine.outputs_(::ToySoilWaterModel) = (soil_water_content=-Inf,) # Implementing the actual algorithm by adding a method to the run! function for our model: function PlantSimEngine.run!(m::ToySoilWaterModel, models, status, meteo, constants=nothing, extra=nothing) - soil_water_content = rand(m.values) + status.soil_water_content = rand(m.values) end # The computation of ToySoilWaterModel is independant of previous values and other objects. We can add this information as From 86c56ef89e2d713ae36211b51dfbfed09046ac9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 3 Oct 2023 14:52:22 +0200 Subject: [PATCH 37/97] Update Project.toml remove Random pkg (removed the dependency in the example toy soil model) --- test/Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Project.toml b/test/Project.toml index 48535d85..6ae7f9b2 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -5,7 +5,6 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" PlantMeteo = "4630fe09-e0fb-4da5-a846-781cb73437b6" -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From 35a78bfa2306f0ab59174abf5cf897c89d2700cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 3 Oct 2023 17:09:32 +0200 Subject: [PATCH 38/97] Update mapping.jl Use type promotion on the status given by the user --- src/mtg/mapping.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 84183414..15c8bfac 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -190,6 +190,10 @@ models = Dict( ```jldoctest mylabel compute_mapping(models, nothing) ``` + +```jldoctest mylabel +compute_mapping(models, Dict(Float64 => Float32)) +``` """ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} # Initialise a dict that defines the multiscale variables for each organ type: @@ -224,7 +228,7 @@ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} else # If the user provided the multiscale variable in the status, and it is an output variable, # we use those values for the mapping: - for i in keys(multi_scale_vars) + for i in keys(multi_scale_vars) # e.g. i = keys(multi_scale_vars)[1] if i in multi_scale_outs && i in keys(st) multi_scale_vars[i] = st[i] end @@ -233,7 +237,7 @@ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} # defined from the models at the target scale, so we need to add it to this other scale # as an output variable. - new_st = Status(merge(NamedTuple(st), NamedTuple(multi_scale_vars))) + new_st = Status(merge(convert_vars(type_promotion, st), NamedTuple(multi_scale_vars))) diff_keys = intersect(keys(st), keys(multi_scale_vars)) for i in diff_keys if isa(new_st[i], RefVector) @@ -252,11 +256,11 @@ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} # var_mapping = map_vars[1] variable, organs_mapped = var_mapping - ref_var = create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable)) + ref_var_ = create_var_ref(organs_mapped, variable, getproperty(multi_scale_vars, variable)) if haskey(organ_mapping, organs_mapped) - push!(organ_mapping[organs_mapped], variable => ref_var) + push!(organ_mapping[organs_mapped], variable => ref_var_) else - organ_mapping[organs_mapped] = Dict(variable => ref_var) + organ_mapping[organs_mapped] = Dict(variable => ref_var_) end # If the mapping is one node type only and is given as a string, we add the variable of the source scale From cf90b2314ab9a6b056110bc609707268222d3566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 3 Oct 2023 19:55:26 +0200 Subject: [PATCH 39/97] Update mapping.jl --- src/mtg/mapping.jl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 15c8bfac..5bf744eb 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -153,15 +153,15 @@ and the nodes that are targeted by the mapping # Examples -```jldoctest mylabel -julia> using PlantSimEngine -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +```julia +using PlantSimEngine +include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); ``` -```jldoctest mylabel +```julia models = Dict( "Plant" => MultiScaleModel( @@ -187,12 +187,12 @@ models = Dict( ) ``` -```jldoctest mylabel -compute_mapping(models, nothing) +```julia +organs_mapping, var_outputs_from_mapping = compute_mapping(models, nothing); ``` -```jldoctest mylabel -compute_mapping(models, Dict(Float64 => Float32)) +```julia +compute_mapping(models, Dict(Float64 => Float32, Vector{Float64} => Vector{Float32})) ``` """ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} @@ -237,7 +237,7 @@ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} # defined from the models at the target scale, so we need to add it to this other scale # as an output variable. - new_st = Status(merge(convert_vars(type_promotion, st), NamedTuple(multi_scale_vars))) + new_st = Status(merge(NamedTuple(convert_vars(type_promotion, st)), NamedTuple(multi_scale_vars))) diff_keys = intersect(keys(st), keys(multi_scale_vars)) for i in diff_keys if isa(new_st[i], RefVector) From cd5ddcffd319ac38975b19df379f24323246538a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 3 Oct 2023 19:55:48 +0200 Subject: [PATCH 40/97] Add test-mtg-multiscale.jl --- test/runtests.jl | 8 +- test/test-mtg-multiscale.jl | 284 ++++++++++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+), 2 deletions(-) create mode 100644 test/test-mtg-multiscale.jl diff --git a/test/runtests.jl b/test/runtests.jl index aee95e0c..0ef46868 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,7 +4,6 @@ using Tables, DataFrames, CSV using MultiScaleTreeGraph using PlantMeteo, Statistics using Documenter # for doctests -using Random # for ToySoilModel # Include the example dummy processes: include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) @@ -59,7 +58,12 @@ include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")) include("test-mtg.jl") end - if VERSION == v"1.8" + @testset "MTG" begin + include("test-mtg-multiscale.jl") + end + + + if VERSION >= v"1.8" # Error formating changed in Julia 1.8 (or was it 1.7?), so the doctest # that returns an error in PlantSimEngine.check_dimensions(models, w) # fails in Julia 1.6. So we test the doctests only in Julia 1.8 and later. diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl new file mode 100644 index 00000000..19298e18 --- /dev/null +++ b/test/test-mtg-multiscale.jl @@ -0,0 +1,284 @@ + +# Example meteo: +meteo = Weather( + [ + Atmosphere(T=20.0, Wind=1.0, Rh=0.65), + Atmosphere(T=25.0, Wind=0.5, Rh=0.8) +] +) + + +# Example MTG: + +mtg = begin + scene = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) + soil = Node(scene, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + internode1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + leaf1 = Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + internode2 = Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) + leaf2 = Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + scene +end + +# Testing with a simple mapping (just the soil model, no multiscale mapping): + +@testset "run! on MTG: simple mapping" begin + out = @test_nowarn run!(mtg, Dict("Soil" => (ToySoilWaterModel(),)), meteo) + @test out.statuses["Soil"][1].node == soil + @test out.models == Dict("Soil" => (soil_water=ToySoilWaterModel(0.1:0.1:1.0),)) + @test length(out.dependency_graph.roots) == 1 + @test collect(keys(out.dependency_graph.roots))[1] == Pair("Soil", :soil_water) + @test out.graph == mtg + + leaf_mapping = Dict("Leaf" => (ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), Status(TT=10.0))) + out = run!(mtg, leaf_mapping, meteo) + @test collect(keys(out.statuses)) == ["Leaf"] + @test length(out.statuses["Leaf"]) == 2 + @test out.statuses["Leaf"][1].TT == 10.0 # As initialized in the mapping + @test out.statuses["Leaf"][1].carbon_demand == 0.5 + + @test out.statuses["Leaf"][1].node == leaf1 + @test out.statuses["Leaf"][2].node == leaf2 +end + +# A mapping with all different types of mapping (single, multi-scale, model as is, or tuple of): +@testset "run! on MTG with complete mapping (missing init)" begin + mapping_all = Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ), + "Soil" => ( + ToySoilWaterModel(), + ), + ) + # The mapping above should throw an error because TT is not initialized for the Internode: + @test_throws "Nodes of type Internode need variable(s) TT to be initialized or computed by a model." run!(mtg, mapping_all, meteo) + # It should work if we don't check the mapping though: + out = @test_nowarn run!(mtg, mapping_all, meteo, check=false) + # Note that the outputs are garbage because the TT is not initialized. + + @test out.models == Dict{String,NamedTuple}( + "Soil" => (soil_water=ToySoilWaterModel(0.1:0.1:1.0),), + "Internode" => (carbon_demand=ToyCDemandModel{Float64}(10.0, 200.0),), + "Plant" => (carbon_allocation=ToyCAllocationModel(),), + "Leaf" => (photosynthesis=ToyAssimModel{Float64}(0.2), carbon_demand=ToyCDemandModel{Float64}(10.0, 200.0)) + ) + + @test length(out.dependency_graph.roots) == 3 # 3 because the plant is not a root (its model has dependencies) + @test out.statuses["Internode"][1].TT === -Inf + @test out.statuses["Internode"][1].carbon_demand === -Inf + + @test out.statuses["Leaf"][1].TT == 10.0 + @test out.statuses["Leaf"][1].carbon_demand == 0.5 + @test out.statuses["Leaf"][1].A == 260.0 + @test out.statuses["Leaf"][1].carbon_allocation == 0.0 +end + +# A mapping that actually works (same as before but with the init for TT): +mapping = Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(TT=10.0) + ), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ), + "Soil" => ( + ToySoilWaterModel(), + ), +) + +@testset "run! on MTG with complete mapping (with init)" begin + out = @test_nowarn run!(mtg, mapping, meteo, executor=ThreadedEx()) + + @test typeof(out.statuses) == Dict{String,Vector{Status}} + @test length(out.statuses["Plant"]) == 1 + @test length(out.statuses["Leaf"]) == 2 + @test length(out.statuses["Internode"]) == 2 + @test length(out.statuses["Soil"]) == 1 + @test out.statuses["Soil"][1].node == soil + @test out.statuses["Soil"][1].soil_water_content !== -Inf + + # Testing if the value in the status of the leaves is the same as the one in the status of the soil: + @test out.statuses["Soil"][1].soil_water_content === out.statuses["Leaf"][1].soil_water_content + @test out.statuses["Soil"][1].soil_water_content === out.statuses["Leaf"][2].soil_water_content + + leaf1_status = out.statuses["Leaf"][1] + + # This is the model that computes the assimilation (testing manually that we get the right result here): + @test leaf1_status.A == leaf1_status.aPPFD * out.models["Leaf"].photosynthesis.LUE * leaf1_status.soil_water_content + + @test out.statuses["Plant"][1].carbon_demand[[1, 3]] == [i.carbon_demand for i in out.statuses["Internode"]] + @test out.statuses["Plant"][1].carbon_demand[[2, 4]] == [i.carbon_demand for i in out.statuses["Leaf"]] + + # Testing the reference directly: + ref_values_cdemand = getfield(out.statuses["Plant"][1].carbon_demand, :v) + + for (j, i) in enumerate([1, 3]) + @test ref_values_cdemand[i] === PlantSimEngine.refvalue(out.statuses["Internode"][j], :carbon_demand) + end + + for (j, i) in enumerate([2, 4]) + @test ref_values_cdemand[i] === PlantSimEngine.refvalue(out.statuses["Leaf"][j], :carbon_demand) + end + + # Testing that carbon allocation in Leaf and Internode was added as a variable from the model at the Plant scale: + + @test hasproperty(out.statuses["Internode"][1], :carbon_allocation) + @test hasproperty(out.statuses["Leaf"][1], :carbon_allocation) + + @test out.statuses["Internode"][1].carbon_allocation == 0.5 + @test out.statuses["Leaf"][1].carbon_allocation == 0.5 + + # Testing the reference directly: + ref_values_callocation = getfield(out.statuses["Plant"][1].carbon_allocation, :v) + + for (j, i) in enumerate([1, 3]) + @test ref_values_callocation[i] === PlantSimEngine.refvalue(out.statuses["Internode"][j], :carbon_allocation) + end + + for (j, i) in enumerate([2, 4]) + @test ref_values_callocation[i] === PlantSimEngine.refvalue(out.statuses["Leaf"][j], :carbon_allocation) + end +end + + +@testset "status_template" begin + organs_statuses = PlantSimEngine.status_template(mapping, nothing) + @test collect(keys(organs_statuses)) == ["Soil", "Internode", "Plant", "Leaf"] + # Check that the soil_water_content is linked between the soil and the leaves: + @test organs_statuses["Soil"][:soil_water_content][] === -Inf + @test organs_statuses["Leaf"][:soil_water_content][] === -Inf + + @test organs_statuses["Soil"][:soil_water_content][] === organs_statuses["Leaf"][:soil_water_content][] + + organs_statuses["Soil"][:soil_water_content][] = 1.0 + @test organs_statuses["Leaf"][:soil_water_content][] == 1.0 + + @test organs_statuses["Plant"][:A] == PlantSimEngine.RefVector{Float64}[] + @test organs_statuses["Plant"][:carbon_allocation] == PlantSimEngine.RefVector{Float64}[] + @test organs_statuses["Internode"][:carbon_allocation] == -Inf + @test organs_statuses["Leaf"][:carbon_demand] == -Inf + + # Testing with a different type: + organs_statuses = PlantSimEngine.status_template(mapping, Dict(Float64 => Float32, Vector{Float64} => Vector{Float32})) + + @test isa(organs_statuses["Plant"][:A], PlantSimEngine.RefVector{Float32}) + @test isa(organs_statuses["Plant"][:carbon_allocation], PlantSimEngine.RefVector{Float32}) + @test isa(organs_statuses["Internode"][:carbon_allocation], Float32) + @test isa(organs_statuses["Leaf"][:carbon_demand], Float32) + @test isa(organs_statuses["Soil"][:soil_water_content], Base.RefValue{Float32}) +end + +# Here we initialise var1 to a constant value: +@testset "MTG initialisation" begin + var1 = 1.0 + mapping = Dict( + "Leaf" => ( + Process1Model(1.0), + Process2Model(), + Process3Model(), + Status(var1=var1,) + ) + ) + + # Need init for var2, so it returns an error: + @test_throws "Nodes of type Leaf need variable(s) var2 to be initialized or computed by a model." PlantSimEngine.init_simulation(mtg, mapping) + + mapping = Dict( + "Leaf" => ( + Process1Model(1.0), + Process2Model(), + Process3Model(), + Status(var1=var1, var2=1.0) + ) + ) + + out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo) + + @test out.statuses["Leaf"][1].var1 === var1 + @test out.statuses["Leaf"][1].var2 === 1.0 + @test out.statuses["Leaf"][1].var3 === 2.0 + @test out.statuses["Leaf"][1].var6 === 40.4 +end + +@testset "MTG with complex mapping" begin + mapping = + Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(TT=10.0) + ), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Process1Model(1.0), + Process2Model(), + Process3Model(), + Process4Model(), + Process5Model(), + Process6Model(), + Status(aPPFD=1300.0, TT=10.0, var0=1.0, var9=1.0), + ), + "Soil" => ( + ToySoilWaterModel(), + ), + ) + out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo) + + @test length(out.dependency_graph.roots) == 4 + @test out.statuses["Leaf"][1].var1 === 1.01 + @test out.statuses["Leaf"][1].var2 === 1.03 + @test out.statuses["Leaf"][1].var8 ≈ 1015.47786908 atol = 1e-6 +end \ No newline at end of file From 4c52eda14170a259f6fd981fd9eb4f6dccc063ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 4 Oct 2023 10:30:56 +0200 Subject: [PATCH 41/97] Update model_initialisation.jl Remove the need for organs' statuses with mapping --- src/processes/model_initialisation.jl | 76 ++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 575d2d9b..8c179b07 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -1,12 +1,23 @@ """ - to_initialize(v::T, vars...) where T <: Union{Missing,AbstractModel} + to_initialize(; verbose=true, vars...) to_initialize(m::T) where T <: ModelList to_initialize(m::DependencyGraph) + to_initialize(mapping::Dict{String,T}, graph=nothing) Return the variables that must be initialized providing a set of models and processes. The function takes into account model coupling and only returns the variables that are needed considering that some variables that are outputs of some models are used as inputs of others. +# Arguments + +- `verbose`: if `true`, print information messages. +- `vars...`: the models and processes to consider. +- `m::T`: a [`ModelList`](@ref). +- `m::DependencyGraph`: a [`DependencyGraph`](@ref). +- `mapping::Dict{String,T}`: a mapping that associates models to organs. +- `graph`: a graph representing a plant or a scene, *e.g.* a multiscale tree graph. The graph + is used to check if variables that are not initialized can be found in the graph nodes attributes. + # Examples ```@example @@ -28,6 +39,30 @@ m = ModelList( ), Status(var1 = 5.0, var2 = -Inf, var3 = -Inf, var4 = -Inf, var5 = -Inf) ) + +to_initialize(m) +``` + +Or with a mapping: + +```@example +using PlantSimEngine + +# Including an example script that implements dummy processes and models: +include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) + +mapping = Dict( + "Leaf" => ModelList( + process1=Process1Model(1.0), + process2=Process2Model(), + process3=Process3Model() + ), + "Internode" => ModelList( + process1=Process1Model(1.0), + ) +) + +to_initialize(mapping) ``` """ function to_initialize(m::ModelList; verbose::Bool=true) @@ -116,28 +151,41 @@ struct VarFromMTG scale::String end -# For the list of models given to an MTG: -function to_initialize(models::Dict{String,T}, organs_statuses, mtg) where {T} +# For the list of mapping given to an MTG: +function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} # Get the variables in the MTG: - vars_in_mtg = names(mtg) + if isnothing(graph) + vars_in_mtg = names(graph) + else + vars_in_mtg = Symbol[] + end var_need_init = Dict{String,Any}() - for organ in keys(models) + for organ in keys(mapping) # organ = "Plant" - # Get all models for the organ: - mods = PlantSimEngine.get_models(models[organ]) - map_vars = PlantSimEngine.get_mapping(models[organ]) + # Get all mapping for the organ: + mods = PlantSimEngine.get_models(mapping[organ]) + map_vars = PlantSimEngine.get_mapping(mapping[organ]) + user_st = PlantSimEngine.get_status(mapping[organ]) # User status + + if isnothing(user_st) + user_st = NamedTuple() + else + user_st = NamedTuple(user_st) + end + multiscale_vars = collect(first(i) for i in map_vars) ins = merge(PlantSimEngine.inputs_.(mods)...) outs = merge(PlantSimEngine.outputs_.(mods)...) # Variables in the node that are defined as multiscale: multi_scale_ins = intersect(keys(ins), multiscale_vars) # inputs: variables that are taken from another scale - # multi_scale_outs = intersect(keys(outs), multiscale_vars) # outputs: variables that are written to another scale # Variables we need to initialise for this scale: vars_needed_this_scale = setdiff(keys(ins), keys(outs)) + # And that are not provided by the user: + setdiff!(vars_needed_this_scale, keys(user_st)) need_initialisation = Symbol[] need_var_from_mtg = VarFromMTG[] @@ -150,19 +198,19 @@ function to_initialize(models::Dict{String,T}, organs_statuses, mtg) where {T} # Scale(s) at which the variable is computed: from_scales = last(map_vars[findfirst(i -> i == var, multiscale_vars)]) # We check if there is a model at the other scale(s) that computes it: - outputs_from_scales = PlantSimEngine.map_scale(models, from_scales) do m, s + outputs_from_scales = PlantSimEngine.map_scale(mapping, from_scales) do m, s # We check that the node type exist in the model list: haskey(m, s) || error( - "Nodes of type $organ are mapping to variable `:$var` computed from nodes of type $s, but there is no type $s in the list of models." + "Nodes of type $organ are mapping to variable `:$var` computed from nodes of type $s, but there is no type $s in the list of mapping." ) - # If it does, we get the outputs of its models: + # If it does, we get the outputs of its mapping: merge(PlantSimEngine.outputs_.(PlantSimEngine.get_models(m[s]))...) end outputs_from_scales = merge(outputs_from_scales...) push!(need_models_from_scales, (var=var, scale=organ, need_scales=from_scales)) - elseif organs_statuses[organ][var] == ins[var] - # In this case the variable is an input of the model, and is not computed by other models at this scale or the others. + else + # In this case the variable is an input of the model, and is not computed by other mapping at this scale or the others. if var in vars_in_mtg # If the variable can be found in the MTG, we will take it from there: push!(need_var_from_mtg, VarFromMTG(var, organ)) From 9bb339be6c73f75c53b36f4b20854dc3386f6125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 4 Oct 2023 10:31:10 +0200 Subject: [PATCH 42/97] Update mapping.jl Update call accordingly --- src/mtg/mapping.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 5bf744eb..9f7d24d1 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -434,7 +434,7 @@ function init_simulation(mtg, models; type_promotion=nothing, check=true, verbos #NB: we use all=false because we only want the variables that are mapped as RefVectors. # We need to know which variables are not initialized, and not computed by other models: - var_need_init = to_initialize(models, organs_statuses, mtg) + var_need_init = to_initialize(models, mtg) # If we find some, we return an error: check && error_mtg_init(var_need_init) From 0784aba9343eda5a1c9554053abf2dea861104be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 4 Oct 2023 10:45:52 +0200 Subject: [PATCH 43/97] Update model_initialisation.jl fix error --- src/processes/model_initialisation.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 8c179b07..7779eb2a 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -156,9 +156,9 @@ function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} # Get the variables in the MTG: if isnothing(graph) - vars_in_mtg = names(graph) - else vars_in_mtg = Symbol[] + else + vars_in_mtg = names(graph) end var_need_init = Dict{String,Any}() From b628f655f0623d17cff97d827f2284a928d1c4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 4 Oct 2023 11:09:11 +0200 Subject: [PATCH 44/97] model() and mapping() are internals, using suffix `_` in their names --- src/mtg/MultiScaleModel.jl | 8 ++++---- src/mtg/mapping.jl | 30 +++++++++++++++--------------- test/test-simulation.jl | 1 + 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/mtg/MultiScaleModel.jl b/src/mtg/MultiScaleModel.jl index ed96e494..5b73487c 100644 --- a/src/mtg/MultiScaleModel.jl +++ b/src/mtg/MultiScaleModel.jl @@ -56,13 +56,13 @@ MultiScaleModel{ToyCAllocationModel, String}(ToyCAllocationModel(), ["carbon_all We can access the mapping and the model: ```jldoctest mylabel -julia> PlantSimEngine.mapping(multiscale_model) +julia> PlantSimEngine.mapping_(multiscale_model) 1-element Vector{Pair{Symbol, Vector{String}}}: :carbon_allocation => ["Leaf", "Internode"] ``` ```jldoctest mylabel -julia> PlantSimEngine.model(multiscale_model) +julia> PlantSimEngine.model_(multiscale_model) ToyCAllocationModel() ``` """ @@ -80,5 +80,5 @@ end MultiScaleModel(; model, mapping) = MultiScaleModel(model, mapping) -mapping(m::MultiScaleModel) = m.mapping -model(m::MultiScaleModel) = m.model \ No newline at end of file +mapping_(m::MultiScaleModel) = m.mapping +model_(m::MultiScaleModel) = m.model \ No newline at end of file diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 9f7d24d1..934c1bdc 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -3,11 +3,11 @@ outputs_(m::MultiScaleModel) = outputs_(m.model) """ - model(m::AbstractModel) + model_(m::AbstractModel) Get the model of an AbstractModel (it is the model itself if it is not a MultiScaleModel). """ -model(m::AbstractModel) = m +model_(m::AbstractModel) = m # Functions to get the models from the dictionary that defines the mapping: @@ -72,14 +72,14 @@ julia> get_models(models2) ToyCDemandModel{Float64}(10.0, 200.0) ``` """ -get_models(m) = [model(i) for i in m if !isa(i, Status)] +get_models(m) = [model_(i) for i in m if !isa(i, Status)] # Get the models of a MultiScaleModel: -get_models(m::MultiScaleModel) = [model(m)] +get_models(m::MultiScaleModel) = [model_(m)] # Note: it is returning a vector of models, because in this case the user provided a single MultiScaleModel instead of a vector of. # Get the models of an AbstractModel: -get_models(m::AbstractModel) = [model(m)] +get_models(m::AbstractModel) = [model_(m)] # Same, for the status (if any provided): @@ -124,14 +124,14 @@ Returns a vector of pairs of symbols and strings or vectors of strings See [`get_models`](@ref) for examples. """ function get_mapping(m) - mod_mapping = [mapping(i) for i in m if isa(i, MultiScaleModel)] + mod_mapping = [mapping_(i) for i in m if isa(i, MultiScaleModel)] if length(mod_mapping) == 0 return Pair{Symbol,String}[] end return reduce(vcat, mod_mapping) end -get_mapping(m::MultiScaleModel{T,S}) where {T,S} = mapping(m) +get_mapping(m::MultiScaleModel{T,S}) where {T,S} = mapping_(m) get_mapping(m::AbstractModel) = Pair{Symbol,String}[] """ @@ -388,7 +388,7 @@ function outputs_from_other_scale!(var_outputs_from_mapping, multi_scale_outs, m end """ - init_simulation(mtg, models; type_promotion=nothing, check=true, verbose=true) + init_simulation(mtg, mapping; type_promotion=nothing, check=true, verbose=true) Initialise the simulation. Returns: @@ -400,7 +400,7 @@ Initialise the simulation. Returns: # Arguments - `mtg`: the MTG -- `models::Dict{String,Any}`: a dictionary of model mapping +- `mapping::Dict{String,Any}`: a dictionary of model mapping - `type_promotion`: the type promotion to use for the variables - `check`: whether to check the mapping for errors - `verbose`: print information about errors in the mapping @@ -425,16 +425,16 @@ The value is not a reference to the one in the attribute of the MTG, but a copy a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be automatically passed as is. """ -function init_simulation(mtg, models; type_promotion=nothing, check=true, verbose=true) +function init_simulation(mtg, mapping; type_promotion=nothing, check=true, verbose=true) # We make a pre-initialised status for each kind of organ (this is a template for each node type): - organs_statuses = status_template(models, type_promotion) + organs_statuses = status_template(mapping, type_promotion) # Get the reverse mapping, i.e. the variables that are mapped to other scales. This is used to initialise # the RefVectors properly: - var_refvector = reverse_mapping(models, all=false) + var_refvector = reverse_mapping(mapping, all=false) #NB: we use all=false because we only want the variables that are mapped as RefVectors. # We need to know which variables are not initialized, and not computed by other models: - var_need_init = to_initialize(models, mtg) + var_need_init = to_initialize(mapping, mtg) # If we find some, we return an error: check && error_mtg_init(var_need_init) @@ -449,9 +449,9 @@ function init_simulation(mtg, models; type_promotion=nothing, check=true, verbos end # Compute the multi-scale dependency graph of the models: - dependency_graph = multiscale_dep(models, verbose=verbose) + dependency_graph = multiscale_dep(mapping, verbose=verbose) - models = Dict(first(m) => PlantSimEngine.parse_models(PlantSimEngine.get_models(last(m))) for m in models) + models = Dict(first(m) => PlantSimEngine.parse_models(PlantSimEngine.get_models(last(m))) for m in mapping) return mtg, statuses, dependency_graph, models end diff --git a/test/test-simulation.jl b/test/test-simulation.jl index adde632e..da82acab 100644 --- a/test/test-simulation.jl +++ b/test/test-simulation.jl @@ -187,6 +187,7 @@ end; process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model() + Process1Model(1.0), ) ) From a88f88d75357d0749bbe3a6cf2b34fb85a1b0a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 4 Oct 2023 11:09:28 +0200 Subject: [PATCH 45/97] Update model_initialisation.jl fix issue when variable is needed only in the mtg --- src/processes/model_initialisation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 7779eb2a..04b958a7 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -221,7 +221,7 @@ function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} end # Note: if the variable is an output of the model for another scale (in `multi_scale_outs`), we don't need to initialise it at this scale. end - if length(need_initialisation) > 0 + if length(need_initialisation) > 0 || length(need_var_from_mtg) > 0 || length(need_models_from_scales) > 0 var_need_init[organ] = (; need_initialisation, need_models_from_scales, need_var_from_mtg) end end From bb35c06a94b4b41c33278d273cc6f486172535a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 4 Oct 2023 15:30:03 +0200 Subject: [PATCH 46/97] Add vars_not_propagated in ModelList --- src/component_models/ModelList.jl | 25 +++++++++++++++++++++++-- src/dataframe.jl | 6 +++--- src/traits/table_traits.jl | 4 ++-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/component_models/ModelList.jl b/src/component_models/ModelList.jl index aa2f63ec..76282259 100644 --- a/src/component_models/ModelList.jl +++ b/src/component_models/ModelList.jl @@ -177,9 +177,14 @@ julia> status(m) Note that computations will be slower using DataFrame, so if performance is an issue, use TimeStepTable instead (or a NamedTuple as shown in the example). """ -struct ModelList{M<:NamedTuple,S} +struct ModelList{M<:NamedTuple,S,V<:Tuple{Vararg{Symbol}}} models::M status::S + vars_not_propagated::V +end + +function ModelList(models::M, status::S) where {M<:NamedTuple{names,T} where {names,T<:NTuple{N,<:AbstractModel} where {N}},S} + ModelList(models, status, ()) end # General interface: @@ -215,12 +220,19 @@ function ModelList( # Make a vector of NamedTuples from the input (please implement yours if you need it) ts_kwargs = homogeneous_ts_kwargs(status, nsteps) + # Variables for which a value was given for each time-step by the user: + vars_not_propagated = get_vars_not_propagated(status) + # Note: that the length was checked in homogeneous_ts_kwargs, so we don't need to check it again here. + # Note 2: we need to know these variables because they will not be propagated between time-steps, but set at + # the given value instead. + # Add the missing variables required by the models (set to default value): ts_kwargs = add_model_vars(ts_kwargs, mods, type_promotion; init_fun=init_fun, nsteps=nsteps) model_list = ModelList( mods, - ts_kwargs + ts_kwargs, + vars_not_propagated ) variables_check && !is_initialized(model_list) @@ -341,6 +353,15 @@ function homogeneous_ts_kwargs(kwargs::NamedTuple{N,T}, nsteps) where {N,T} return vars_array end + +""" + get_vars_not_propagated(status) + +Returns all variables that are given for several time-steps in the status. +""" +get_vars_not_propagated(status) = (findall(x -> length(x) > 1, status)...,) +get_vars_not_propagated(::Type{Nothing}) = () + """ Base.copy(l::ModelList) Base.copy(l::ModelList, status) diff --git a/src/dataframe.jl b/src/dataframe.jl index b7cf684d..0904fda8 100644 --- a/src/dataframe.jl +++ b/src/dataframe.jl @@ -68,15 +68,15 @@ end Implementation of `DataFrame` for a `ModelList` model with several time steps. """ -function DataFrames.DataFrame(components::ModelList{T,S}) where {T,S<:TimeStepTable} +function DataFrames.DataFrame(components::ModelList{T,S,V}) where {T,S<:TimeStepTable,V} DataFrames.DataFrame([(NamedTuple(j)..., timestep=i) for (i, j) in enumerate(status(components))]) end """ - DataFrame(components::ModelList{T,S}) where {T,S<:AbstractDict} + DataFrame(components::ModelList{T,S,V}) where {T,S<:Status,V} Implementation of `DataFrame` for a `ModelList` model with one time step. """ -function DataFrames.DataFrame(components::ModelList{T,S}) where {T,S<:Status} +function DataFrames.DataFrame(components::ModelList{T,S,V}) where {T,S<:Status,V} DataFrames.DataFrame([NamedTuple(status(components)[1])]) end diff --git a/src/traits/table_traits.jl b/src/traits/table_traits.jl index 5d19cdd3..39c1c98e 100644 --- a/src/traits/table_traits.jl +++ b/src/traits/table_traits.jl @@ -56,8 +56,8 @@ DataFormat(::Type{<:Dict}) = TableAlike() DataFormat(::Type{<:NamedTuple}) = SingletonAlike() DataFormat(::Type{<:Status}) = SingletonAlike() -DataFormat(::Type{<:ModelList{Mo,S} where {Mo,S<:Status}}) = SingletonAlike() -DataFormat(::Type{<:ModelList{Mo,S}}) where {Mo,S} = TableAlike() +DataFormat(::Type{<:ModelList{Mo,S,V} where {Mo,S<:Status,V}}) = SingletonAlike() +DataFormat(::Type{<:ModelList{Mo,S,V}}) where {Mo,S,V} = TableAlike() DataFormat(::Type{<:GraphSimulation}) = TreeAlike() DataFormat(::Type{<:PlantMeteo.AbstractAtmosphere}) = SingletonAlike() From c61829436b1e6af76ef6223619574022509df5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 4 Oct 2023 18:06:57 +0200 Subject: [PATCH 47/97] Update dummy.jl Make the cal to run! explicit that is from PlantSimEngine --- examples/dummy.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/dummy.jl b/examples/dummy.jl index 12a180df..4230da4f 100644 --- a/examples/dummy.jl +++ b/examples/dummy.jl @@ -37,7 +37,7 @@ PlantSimEngine.outputs_(::Process2Model) = (var4=-Inf, var5=-Inf) PlantSimEngine.dep(::Process2Model) = (process1=AbstractProcess1Model,) function PlantSimEngine.run!(::Process2Model, models, status, meteo, constants=nothing, extra=nothing) # computing var3 using process1: - run!(models.process1, models, status, meteo, constants) + PlantSimEngine.run!(models.process1, models, status, meteo, constants) # computing var4 and var5: status.var4 = status.var3 * 2.0 status.var5 = status.var4 + 1.0 * meteo.T + 2.0 * meteo.Wind + 3.0 * meteo.Rh @@ -61,7 +61,7 @@ PlantSimEngine.outputs_(::Process3Model) = (var4=-Inf, var6=-Inf) PlantSimEngine.dep(::Process3Model) = (process2=Process2Model,) function PlantSimEngine.run!(::Process3Model, models, status, meteo, constants=nothing, extra=nothing) # computing var3 using process1: - run!(models.process2, models, status, meteo, constants, extra) + PlantSimEngine.run!(models.process2, models, status, meteo, constants, extra) # re-computing var4: status.var4 = status.var4 * 2.0 status.var6 = status.var5 + status.var4 From b2732d2ab2f71ae62e93374b77637620aaedd94b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 4 Oct 2023 18:08:40 +0200 Subject: [PATCH 48/97] Better handle initializations over several time-steps for ModelList + force computations to be sequential over time-steps for MTG --- src/component_models/ModelList.jl | 8 +- src/component_models/Status.jl | 33 +++++++- src/processes/model_initialisation.jl | 8 +- src/run.jl | 112 +++++++++++--------------- 4 files changed, 92 insertions(+), 69 deletions(-) diff --git a/src/component_models/ModelList.jl b/src/component_models/ModelList.jl index 76282259..d6ce6ec0 100644 --- a/src/component_models/ModelList.jl +++ b/src/component_models/ModelList.jl @@ -360,7 +360,7 @@ end Returns all variables that are given for several time-steps in the status. """ get_vars_not_propagated(status) = (findall(x -> length(x) > 1, status)...,) -get_vars_not_propagated(::Type{Nothing}) = () +get_vars_not_propagated(::Nothing) = () """ Base.copy(l::ModelList) @@ -394,14 +394,16 @@ ml3 = copy(models, TimeStepTable([Status(var1=20.0, var2=0.5))]) function Base.copy(m::T) where {T<:ModelList} ModelList( m.models, - deepcopy(m.status) + deepcopy(m.status), + m.vars_not_propagated ) end function Base.copy(m::T, status) where {T<:ModelList} ModelList( m.models, - status + status, + m.vars_not_propagated ) end diff --git a/src/component_models/Status.jl b/src/component_models/Status.jl index ebb315cb..2c5ddf10 100644 --- a/src/component_models/Status.jl +++ b/src/component_models/Status.jl @@ -124,4 +124,35 @@ Base.lastindex(mnt::Status) = lastindex(NamedTuple(mnt)) function Base.indexed_iterate(mnt::Status, i::Int, state=1) Base.indexed_iterate(NamedTuple(mnt), i, state) -end \ No newline at end of file +end + + +""" + propagate_values!(status1::Dict, status2::Dict, vars_not_propagated::Set) + +Propagates the values of all variables in `status1` to `status2`, except for vars in `vars_not_propagated`. + +# Arguments + +- `status1::Dict`: A dictionary containing the current values of variables. +- `status2::Dict`: A dictionary to which the values of variables will be propagated. +- `vars_not_propagated::Set`: A set of variables whose values should not be propagated. + +# Examples + +```jldoctest st1 +julia> status1 = Status(var1 = 15.0, var2 = 0.3); +julia> status2 = Status(var1 = 16.0, var2 = -Inf); +julia> vars_not_propagated = (:var1,); +julia> propagate_values!(status1, status2, vars_not_propagated); +julia> status2.var2 == status1.var2 +true +julia> status2.var1 == status1.var1 +false +``` +""" +function propagate_values!(status1, status2, vars_not_propagated) + for var in setdiff(keys(status1), vars_not_propagated) + status2[var] = status1[var] + end +end diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 04b958a7..e3bfd1ef 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -208,7 +208,13 @@ function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} end outputs_from_scales = merge(outputs_from_scales...) - push!(need_models_from_scales, (var=var, scale=organ, need_scales=from_scales)) + if var in keys(outputs_from_scales) + # If the variable is computed by a model at the other scale, we don't need to initialise it: + continue + else + # Else, we need to initialise it: + push!(need_models_from_scales, (var=var, scale=organ, need_scales=from_scales)) + end else # In this case the variable is an input of the model, and is not computed by other mapping at this scale or the others. if var in vars_in_mtg diff --git a/src/run.jl b/src/run.jl index e7112a16..9761a3aa 100644 --- a/src/run.jl +++ b/src/run.jl @@ -143,27 +143,6 @@ function run!( @floop executor_obj for obj in collect(values(object)) run!(obj, meteo, constants, extra, check=check, executor=executor) - # # Check if the simulation can be parallelized over time-steps: - # if !timestep_parallelizable(dep_graphs[obj_index]) && executor != SequentialEx() - # executor_time = SequentialEx() - # else - # executor_time = executor - # end - - # @floop executor_time for (i, meteo_i) in enumerate(meteo_rows) - # if check - # # Check if the meteo data and the status have the same length (or length 1) - # check_dimensions(obj, meteo) - - # if length(dep_graphs[obj_index].not_found) > 0 - # error( - # "The following processes are missing in the ModelList: ", - # dep_graphs[obj_index].not_found - # ) - # end - # end - # run!(obj, dep_graphs[obj_index], i, obj[i], meteo_i, constants, extra) - # end end end @@ -201,19 +180,27 @@ function run!( ) end - if !timestep_parallelizable(dep_graph) && executor != SequentialEx() - is_ts_parallel = which_timestep_parallelizable(dep_graph) - mods_not_parallel = join([i.second.first for i in is_ts_parallel[findall(x -> x.second.second == false, is_ts_parallel)]], "; ") + if !timestep_parallelizable(dep_graph) + if executor != SequentialEx() + is_ts_parallel = which_timestep_parallelizable(dep_graph) + mods_not_parallel = join([i.second.first for i in is_ts_parallel[findall(x -> x.second.second == false, is_ts_parallel)]], "; ") - check && @warn string( - "A parallel executor was provided (`executor=$(executor)`) but some models cannot be run in parallel: $mods_not_parallel. ", - "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." - ) maxlog = 1 - executor = SequentialEx() - end - - @floop executor for (i, row) in enumerate(sim_rows) - run!(object, dep_graph, i, row, meteo, constants, extra) + check && @warn string( + "A parallel executor was provided (`executor=$(executor)`) but some models cannot be run in parallel: $mods_not_parallel. ", + "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." + ) maxlog = 1 + end + # Not parallelizable over time-steps, it means some values depend on the previous value. + # In this case we propagate the values of the variables from one time-step to the other, except for + # the variables the user provided for all time-steps. + for (i, row) in enumerate(sim_rows) + i > 1 && propagate_values!(sim_rows[i-1], row, object.vars_not_propagated) + run!(object, dep_graph, i, row, meteo, constants, extra) + end + else + @floop executor for (i, row) in enumerate(sim_rows) + run!(object, dep_graph, i, row, meteo, constants, extra) + end end end @@ -243,20 +230,29 @@ function run!( end end - if !timestep_parallelizable(dep_graph) && executor != SequentialEx() - is_ts_parallel = which_timestep_parallelizable(dep_graph) - mods_not_parallel = join([i.second.first for i in is_ts_parallel[findall(x -> x.second.second == false, is_ts_parallel)]], "; ") + if !timestep_parallelizable(dep_graph) + if executor != SequentialEx() + is_ts_parallel = which_timestep_parallelizable(dep_graph) + mods_not_parallel = join([i.second.first for i in is_ts_parallel[findall(x -> x.second.second == false, is_ts_parallel)]], "; ") - check && @warn string( - "A parallel executor was provided (`executor=$(executor)`) but some models cannot be run in parallel: $mods_not_parallel. ", - "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." - ) maxlog = 1 - executor = SequentialEx() - end + check && @warn string( + "A parallel executor was provided (`executor=$(executor)`) but some models cannot be run in parallel: $mods_not_parallel. ", + "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." + ) maxlog = 1 + end - # Computing for each time-step: - @floop executor for (i, meteo_i) in enumerate(meteo_rows) - run!(object, dep_graph, i, object[i], meteo_i, constants, extra) + # Not parallelizable over time-steps, it means some values depend on the previous value. + # In this case we propagate the values of the variables from one time-step to the other, except for + # the variables the user provided for all time-steps. + for (i, meteo_i) in enumerate(meteo_rows) + i > 1 && propagate_values!(object[i-1], object[i], object.vars_not_propagated) + run!(object, dep_graph, i, object[i], meteo_i, constants, extra) + end + else + # Computing time-steps in parallel: + @floop executor for (i, meteo_i) in enumerate(meteo_rows) + run!(object, dep_graph, i, object[i], meteo_i, constants, extra) + end end end @@ -369,27 +365,15 @@ function run!( ) dep_graph = dep(object) - if !timestep_parallelizable(dep_graph) && executor != SequentialEx() - is_ts_parallel = which_timestep_parallelizable(dep_graph) - mods_not_parallel = join([i.second.first for i in is_ts_parallel[findall(x -> x.second.second == false, is_ts_parallel)]], "; ") - - check && @warn string( - "A parallel executor was provided (`executor=$(executor)`) but some models cannot be run in parallel: $mods_not_parallel. ", - "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." - ) maxlog = 1 - executor2 = SequentialEx() - else - executor2 = executor # We define executor2 to limit boxed variables: https://juliafolds.github.io/FLoops.jl/dev/howto/avoid-box/ - end - models = get_models(object) - # Optionnaly in parallel over time-steps: - @floop executor2 for (i, meteo_i) in enumerate(Tables.rows(meteo)) - # In parallel over dependency roots (independant computations): - # @floop executor for (process_key, dependency_node) in collect(dep_graph.roots) - #! put this into another function so I can call an @floop again - for (process_key, dependency_node) in collect(dep_graph.roots) + # Note: The object is not thread safe here, because we write all meteo time-steps into the same Status (for each node) + # This is because the number of nodes is usually higher than the number of cores anyway, so we don't gain much by parallelizing over + # meteo time-steps in addition. This way we also reduce the memory footprint that can grow very large if we have many time-steps. + for (i, meteo_i) in enumerate(Tables.rows(meteo)) + # In parallel over dependency root, i.e. for independant computations: + @floop executor for (process_key, dependency_node) in collect(dep_graph.roots) + # Note: parallelization over objects is handled by the run! method below run!(object, dependency_node, i, models, meteo_i, constants, extra, check, executor) end end From 8bee01141f27c8130b0c293899ed49d6c23c0409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 4 Oct 2023 18:09:07 +0200 Subject: [PATCH 49/97] Update tests accordingly --- test/runtests.jl | 2 +- test/test-mtg-multiscale.jl | 133 ++++++++++++++++++++++++++++++++++-- test/test-simulation.jl | 62 +++-------------- test/test-toy_models.jl | 2 +- 4 files changed, 140 insertions(+), 59 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 0ef46868..54887e4f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -58,7 +58,7 @@ include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")) include("test-mtg.jl") end - @testset "MTG" begin + @testset "MTG with multiscale mapping" begin include("test-mtg-multiscale.jl") end diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index 19298e18..ca3d703b 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -21,8 +21,124 @@ mtg = begin scene end -# Testing with a simple mapping (just the soil model, no multiscale mapping): +# Testing the mappings: +@testset "Mapping: missing initialisation" begin + mapping = Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ), + "Soil" => ( + ToySoilWaterModel(), + ), + ) + + to_init = @test_nowarn to_initialize(mapping) + + @test to_init["Internode"].need_initialisation == [:TT] + @test to_init["Internode"].need_models_from_scales == [] + @test to_init["Internode"].need_var_from_mtg == [] +end + +@testset "Mapping: missing organ in mapping (Soil)" begin + mapping = Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ) + ) + + @test_throws "Nodes of type Leaf are mapping to variable `:soil_water_content` computed from nodes of type Soil, but there is no type Soil in the list of mapping." to_initialize(mapping) +end + +@testset "Mapping: missing model at other scale (soil_water_content) + missing init + var1 from MTG" begin + mtg_var = let + scene = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) + soil = Node(scene, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + internode1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + leaf1 = Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + scene + end + + mapping = Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ), + "Soil" => ( + Process1Model(1.0), + ), + ) + + soil_node = mtg_var[1] + soil_node[:var1] = 1.0 + to_init = to_initialize(mapping, mtg_var) + @test to_init["Soil"].need_initialisation == [:var2]# var1 would be here if not present in the MTG + @test to_init["Soil"].need_models_from_scales == [] + @test to_init["Soil"].need_var_from_mtg == [PlantSimEngine.VarFromMTG(:var1, "Soil")] + + @test to_init["Leaf"].need_initialisation == [] + @test to_init["Leaf"].need_models_from_scales == [(var=:soil_water_content, scale="Leaf", need_scales="Soil")] + @test to_init["Leaf"].need_var_from_mtg == [] + @test to_init["Internode"].need_initialisation == [:TT] + @test to_init["Internode"].need_models_from_scales == [] + @test to_init["Internode"].need_var_from_mtg == [] +end + +# Testing with a simple mapping (just the soil model, no multiscale mapping): @testset "run! on MTG: simple mapping" begin out = @test_nowarn run!(mtg, Dict("Soil" => (ToySoilWaterModel(),)), meteo) @test out.statuses["Soil"][1].node == soil @@ -87,10 +203,12 @@ end @test out.statuses["Internode"][1].TT === -Inf @test out.statuses["Internode"][1].carbon_demand === -Inf - @test out.statuses["Leaf"][1].TT == 10.0 - @test out.statuses["Leaf"][1].carbon_demand == 0.5 - @test out.statuses["Leaf"][1].A == 260.0 - @test out.statuses["Leaf"][1].carbon_allocation == 0.0 + st_leaf1 = out.statuses["Leaf"][1] + @test st_leaf1.TT == 10.0 + @test st_leaf1.carbon_demand == 0.5 + # This one depends on the soil, which is random, so we test using the computation directly: + @test st_leaf1.A == st_leaf1.aPPFD * out.models["Leaf"].photosynthesis.LUE * st_leaf1.soil_water_content + @test st_leaf1.carbon_allocation == 0.0 end # A mapping that actually works (same as before but with the init for TT): @@ -275,10 +393,13 @@ end ToySoilWaterModel(), ), ) + out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo) @test length(out.dependency_graph.roots) == 4 @test out.statuses["Leaf"][1].var1 === 1.01 @test out.statuses["Leaf"][1].var2 === 1.03 - @test out.statuses["Leaf"][1].var8 ≈ 1015.47786908 atol = 1e-6 + @test out.statuses["Leaf"][1].var4 ≈ 8.1612000000000013 atol = 1e-6 + @test out.statuses["Leaf"][1].var5 == 32.4806 + @test out.statuses["Leaf"][1].var8 ≈ 1321.0700490800002 atol = 1e-6 end \ No newline at end of file diff --git a/test/test-simulation.jl b/test/test-simulation.jl index da82acab..d4b5e0a8 100644 --- a/test/test-simulation.jl +++ b/test/test-simulation.jl @@ -182,12 +182,11 @@ end; leaf[:var1] = [15.0, 16.0] leaf[:var2] = 0.3 - models = Dict( - "Leaf" => ModelList( - process1=Process1Model(1.0), - process2=Process2Model(), - process3=Process3Model() + mapping = Dict( + "Leaf" => ( Process1Model(1.0), + Process2Model(), + Process3Model() ) ) @@ -198,54 +197,15 @@ end; ] ) - init_mtg_models!(mtg, models, length(meteo)) - run!(mtg, meteo) - df_leaf = DataFrame(leaf) - vars = (:var4, :var6, :var5, :var1, :var2, :var3) - @test [df_leaf[1, i] for i in vars] == [ - [22.0, 23.2], - [56.95, 63.2], - [34.95, 40.0], - [15.0, 16.0], - [0.3, 0.3], - [5.5, 5.8], - ] -end; - - -@testset "Simulation: 2 time-step, 2 Atmospheres, MTG" begin - mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) - internode = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) - leaf = Node(mtg, MultiScaleTreeGraph.NodeMTG("<", "Leaf", 1, 2)) - leaf[:var1] = [15.0, 16.0] - leaf[:var2] = 0.3 + # var1 is taken from the MTG attributes but is a vector instead of a scalar, expecting an error: + @test_throws AssertionError run!(mtg, mapping, meteo) - models = Dict( - "Leaf" => ModelList( - process1=Process1Model(1.0), - process2=Process2Model(), - process3=Process3Model() - ) - ) - - meteo = Weather( - [ - Atmosphere(T=20.0, Wind=1.0, Rh=0.65), - Atmosphere(T=25.0, Wind=0.5, Rh=0.8) - ] - ) + leaf[:var1] = 15.0 - init_mtg_models!(mtg, models, length(meteo)) - run!(mtg, meteo) + out = @test_nowarn run!(mtg, mapping, meteo) - df_leaf = DataFrame(leaf) vars = (:var4, :var6, :var5, :var1, :var2, :var3) - @test [df_leaf[1, i] for i in vars] == [ - [22.0, 23.2], - [56.95, 63.2], - [34.95, 40.0], - [15.0, 16.0], - [0.3, 0.3], - [5.5, 5.8], + @test [out.statuses["Leaf"][1][i] for i in vars] == [ + 22.0, 61.4, 39.4, 15.0, 0.3, 5.5 ] -end; \ No newline at end of file +end; diff --git a/test/test-toy_models.jl b/test/test-toy_models.jl index 8a48841d..8d3e5fc4 100644 --- a/test/test-toy_models.jl +++ b/test/test-toy_models.jl @@ -109,5 +109,5 @@ end @test mean(models.status[:aPPFD]) ≈ 9.511021781482347 @test mean(models.status[:LAI]) ≈ 1.098492557536525 - @test models.status[:biomass][end] ≈ 1041.4568850723167 + @test models.status[:biomass][end] ≈ 1041.4687939085675 rtol = 1e-4 end \ No newline at end of file From 9a4dddaee062f2e516990483ecae2d37ff5443c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 5 Oct 2023 11:15:22 +0200 Subject: [PATCH 50/97] Fix doctests --- src/component_models/ModelList.jl | 5 +- src/component_models/RefVector.jl | 2 +- src/component_models/Status.jl | 16 +- src/mtg/MultiScaleModel.jl | 12 +- src/mtg/mapping.jl | 239 ++++++++++++++++++------------ src/run.jl | 2 +- test/test-mtg-multiscale.jl | 4 +- 7 files changed, 176 insertions(+), 104 deletions(-) diff --git a/src/component_models/ModelList.jl b/src/component_models/ModelList.jl index d6ce6ec0..0546b5e8 100644 --- a/src/component_models/ModelList.jl +++ b/src/component_models/ModelList.jl @@ -71,7 +71,7 @@ julia> models = ModelList(process1=Process1Model(1.0), process2=Process2Model(), ```jldoctest 1 julia> typeof(models) -ModelList{NamedTuple{(:process1, :process2, :process3), Tuple{Process1Model, Process2Model, Process3Model}}, TimeStepTable{Status{(:var4, :var5, :var6, :var1, :var3, :var2), NTuple{6, Base.RefValue{Float64}}}}} +ModelList{NamedTuple{(:process1, :process2, :process3), Tuple{Process1Model, Process2Model, Process3Model}}, TimeStepTable{Status{(:var4, :var5, :var6, :var1, :var3, :var2), NTuple{6, Base.RefValue{Float64}}}}, Tuple{}} ``` No variables were given as keyword arguments, that means that the status of the ModelList is not @@ -281,7 +281,7 @@ function add_model_vars(x, models, type_promotion; init_fun=init_fun_default, ns if Tables.istable(x) # Making a vars for each ith value in the user vars: - x_full = [merge(ref_vars, NamedTuple(x[1]))] + x_full = [merge(ref_vars, NamedTuple(Tables.rows(x)[1]))] for r in Tables.rows(x)[2:end] push!(x_full, merge(ref_vars, NamedTuple(r))) end @@ -360,6 +360,7 @@ end Returns all variables that are given for several time-steps in the status. """ get_vars_not_propagated(status) = (findall(x -> length(x) > 1, status)...,) +get_vars_not_propagated(df::DataFrames.DataFrame) = (propertynames(df)...,) get_vars_not_propagated(::Nothing) = () """ diff --git a/src/component_models/RefVector.jl b/src/component_models/RefVector.jl index f16befd5..5c9e96e6 100644 --- a/src/component_models/RefVector.jl +++ b/src/component_models/RefVector.jl @@ -51,7 +51,6 @@ We can access the values of the RefVector: ```jldoctest mylabel julia> rv[1] 1.0 -1.0 ``` Updating the value in the RefVector will update the value in the original struct: @@ -86,6 +85,7 @@ julia> rv = PlantSimEngine.RefVector(vec) ```jldoctest mylabel julia> rv[1] +1.0 ``` """ struct RefVector{T} <: AbstractVector{T} diff --git a/src/component_models/Status.jl b/src/component_models/Status.jl index 2c5ddf10..ee3af99f 100644 --- a/src/component_models/Status.jl +++ b/src/component_models/Status.jl @@ -142,11 +142,25 @@ Propagates the values of all variables in `status1` to `status2`, except for var ```jldoctest st1 julia> status1 = Status(var1 = 15.0, var2 = 0.3); +``` + +```jldoctest st1 julia> status2 = Status(var1 = 16.0, var2 = -Inf); +``` + +```jldoctest st1 julia> vars_not_propagated = (:var1,); -julia> propagate_values!(status1, status2, vars_not_propagated); + +```jldoctest st1 +julia> PlantSimEngine.propagate_values!(status1, status2, vars_not_propagated); +``` + +```jldoctest st1 julia> status2.var2 == status1.var2 true +``` + +```jldoctest st1 julia> status2.var1 == status1.var1 false ``` diff --git a/src/mtg/MultiScaleModel.jl b/src/mtg/MultiScaleModel.jl index 5b73487c..30884df7 100644 --- a/src/mtg/MultiScaleModel.jl +++ b/src/mtg/MultiScaleModel.jl @@ -23,7 +23,11 @@ of one node, they will be updated in the other nodes. # Examples ```jldoctest mylabel -julia> using PlantSimEngine +julia> using PlantSimEngine; +``` + +```jldoctest mylabel +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); ``` Let's take a model: @@ -49,15 +53,15 @@ between the `carbon_allocation` variable and the `Leaf` and `Internode` nodes. We can now make the model multi-scale by passing the model and the mapping to the `MultiScaleModel` constructor : ```jldoctest mylabel -julia> multiscale_model = MultiScaleModel(model, mapping) -MultiScaleModel{ToyCAllocationModel, String}(ToyCAllocationModel(), ["carbon_allocation" => ["Leaf", "Internode"]]) +julia> multiscale_model = PlantSimEngine.MultiScaleModel(model, mapping) +MultiScaleModel{ToyCAllocationModel, String}(ToyCAllocationModel(), Pair{Symbol, Union{String, Vector{String}}}[:carbon_allocation => ["Leaf", "Internode"]]) ``` We can access the mapping and the model: ```jldoctest mylabel julia> PlantSimEngine.mapping_(multiscale_model) -1-element Vector{Pair{Symbol, Vector{String}}}: +1-element Vector{Pair{Symbol, Union{String, Vector{String}}}}: :carbon_allocation => ["Leaf", "Internode"] ``` diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 934c1bdc..2319e7d3 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -25,28 +25,40 @@ Returns a vector of models # Examples ```jldoctest mylabel -julia> using PlantSimEngine +julia> using PlantSimEngine; +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); ``` If we just give a MultiScaleModel, we get its model as a one-element vector: ```jldoctest mylabel -julia> models = MultiScaleModel( - model=ToyCAllocationModel(), - mapping=[ - :A => ["Leaf"], - :carbon_demand => ["Leaf", "Internode"], - :carbon_allocation => ["Leaf", "Internode"] - ], +julia> models = MultiScaleModel( \ + model=ToyCAllocationModel(), \ + mapping=[ \ + :A => ["Leaf"], \ + :carbon_demand => ["Leaf", "Internode"], \ + :carbon_allocation => ["Leaf", "Internode"] \ + ], \ ); ``` ```jldoctest mylabel -julia> get_models(models) +julia> PlantSimEngine.get_models(models) 1-element Vector{ToyCAllocationModel}: ToyCAllocationModel() ``` @@ -54,19 +66,20 @@ julia> get_models(models) If we give a tuple of models, we get each model in a vector: ```jldoctest mylabel -julia> models2 = ( - MultiScaleModel( - model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], - # Notice we provide "Soil", not ["Soil"], so a single value is expected here - ), - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - Status(aPPFD=1300.0, TT=10.0), +julia> models2 = ( \ + MultiScaleModel( \ + model=ToyAssimModel(), \ + mapping=[:soil_water_content => "Soil",], \ + ), \ + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + Status(aPPFD=1300.0, TT=10.0), \ ); ``` +Notice that we provide "Soil", not ["Soil"] in the mapping because a single value is expected for the mapping here. + ```jldoctest mylabel -julia> get_models(models2) +julia> PlantSimEngine.get_models(models2) 2-element Vector{AbstractModel}: ToyAssimModel{Float64}(0.2) ToyCDemandModel{Float64}(10.0, 200.0) @@ -302,17 +315,21 @@ See also `vars_type_from_mapping` to get the variables type. # Examples -```jldoctest -vars_mapping = Dict( - ["Leaf"] => Dict(:A => RefVector{Float64}[-Inf]), - ["Leaf", "Internode"] => Dict( - :carbon_allocation => RefVector{Float64}[], - :carbon_demand => RefVector{Float64}[]) -); +```jldoctest test1 +julia> vars_mapping = Dict( \ + ["Leaf"] => Dict(:A => PlantSimEngine.RefVector{Float64}[]), \ + ["Leaf", "Internode"] => Dict( \ + :carbon_allocation => PlantSimEngine.RefVector{Float64}[], \ + :carbon_demand => PlantSimEngine.RefVector{Float64}[] \ + ) \ +) +Dict{Vector{String}, Dict{Symbol, Vector{PlantSimEngine.RefVector{Float64}}}} with 2 entries: + ["Leaf"] => Dict(:A=>[]) + ["Leaf", "Internode"] => Dict(:carbon_allocation=>[], :carbon_demand=>[]) ``` -```jldoctest -julia> vars_from_mapping(vars_mapping) +```jldoctest test1 +julia> PlantSimEngine.vars_from_mapping(vars_mapping) 3-element Vector{Symbol}: :A :carbon_allocation @@ -340,7 +357,8 @@ julia> using PlantSimEngine ``` ```jldoctest -julia> MappedVar("Leaf", :A, 1.0) +julia> PlantSimEngine.MappedVar("Leaf", :A, 1.0) +PlantSimEngine.MappedVar{String, Float64}("Leaf", :A, 1.0) ``` """ struct MappedVar{S<:Union{A,Vector{A}} where {A<:AbstractString},T} @@ -544,56 +562,65 @@ Create a status template for a given set of models and type promotion. # Examples ```jldoctest mylabel -julia> using PlantSimEngine, Random +julia> using PlantSimEngine; +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); ``` ```jldoctest mylabel -julia> models = Dict( - "Plant" => - MultiScaleModel( - model=ToyCAllocationModel(), - mapping=[ - # inputs - :A => ["Leaf"], - :carbon_demand => ["Leaf", "Internode"], - # outputs - :carbon_allocation => ["Leaf", "Internode"] - ], - ), - "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - "Leaf" => ( - MultiScaleModel( - model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], - # Notice we provide "Soil", not ["Soil"], so a single value is expected here - ), - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - Status(aPPFD=1300.0, TT=10.0), - ), - "Soil" => ( - ToySoilWaterModel(), - ), +julia> models = Dict( \ + "Plant" => \ + MultiScaleModel( \ + model=ToyCAllocationModel(), \ + mapping=[ \ + :A => ["Leaf"], \ + :carbon_demand => ["Leaf", "Internode"], \ + :carbon_allocation => ["Leaf", "Internode"] \ + ], \ + ), \ + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + "Leaf" => ( \ + MultiScaleModel( \ + model=ToyAssimModel(), \ + mapping=[:soil_water_content => "Soil",], \ + ), \ + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + Status(aPPFD=1300.0, TT=10.0), \ + ), \ + "Soil" => ( \ + ToySoilWaterModel(), \ + ), \ ); ``` ```jldoctest mylabel -julia> status_template(models, nothing) +julia> organs_statuses = PlantSimEngine.status_template(models, nothing) Dict{String, Dict{Symbol, Any}} with 4 entries: "Soil" => Dict(:soil_water_content=>RefValue{Float64}(-Inf)) "Internode" => Dict(:carbon_allocation=>-Inf, :TT=>-Inf, :carbon_demand=>-Inf) - "Plant" => Dict(:carbon_allocation=>RefVector{Float64}[], :A=>RefVector{Float64}[], :carbon_offer=>-Inf, :carbon_demand=>RefVector{Float64}[]) - "Leaf" => Dict(:carbon_allocation=>-Inf, :A=>-Inf, :TT=>10.0, :aPPFD=>1300.0, :soil_water_content=>RefValue{Float64}(-Inf), :carbon_demand=>-Inf) + "Plant" => Dict(:carbon_allocation=>RefVector{Float64}[], :A=>RefVector{F… + "Leaf" => Dict(:carbon_allocation=>-Inf, :A=>-Inf, :TT=>10.0, :aPPFD=>13… ``` Note that variables that are multiscale (*i.e.* defined in a mapping) are linked between scales, so if we write at a scale, the value will be automatically updated at the other scale: ```jldoctest mylabel -organs_statuses["Soil"][:soil_water_content] === organs_statuses["Leaf"][:soil_water_content] +julia> organs_statuses["Soil"][:soil_water_content] === organs_statuses["Leaf"][:soil_water_content] true ``` """ @@ -677,9 +704,18 @@ are already RefValues or RefVectors, they are used as is, else they are converte ```jldoctest mylabel julia> using PlantSimEngine -julia> a, b = status_from_template(Dict(:a => 1.0, :b => 2.0)); +``` + +```jldoctest mylabel +julia> a, b = PlantSimEngine.status_from_template(Dict(:a => 1.0, :b => 2.0)); +``` + +```jldoctest mylabel julia> a 1.0 +``` + +```jldoctest mylabel julia> b 2.0 ``` @@ -698,29 +734,32 @@ or a Ref to the `RefVector` (in case `v` is a `RefVector`). # Examples ```jldoctest mylabel -julia> using PlantSimEngine -julia> ref_var(1.0) +julia> using PlantSimEngine; +``` + +```jldoctest mylabel +julia> PlantSimEngine.ref_var(1.0) Base.RefValue{Float64}(1.0) ``` ```jldoctest mylabel -julia> ref_var([1.0]) +julia> PlantSimEngine.ref_var([1.0]) Base.RefValue{Vector{Float64}}([1.0]) ``` ```jldoctest mylabel -julia> ref_var(Base.RefValue(1.0)) +julia> PlantSimEngine.ref_var(Base.RefValue(1.0)) Base.RefValue{Float64}(1.0) ``` ```jldoctest mylabel -julia> ref_var(Base.RefValue([1.0])) +julia> PlantSimEngine.ref_var(Base.RefValue([1.0])) Base.RefValue{Vector{Float64}}([1.0]) ``` ```jldoctest mylabel -julia> ref_var(RefVector([Ref(1.0), Ref(2.0), Ref(3.0)])) -Base.RefValue{RefVector{Float64}}(RefVector{Float64}[1.0, 2.0, 3.0]) +julia> PlantSimEngine.ref_var(PlantSimEngine.RefVector([Ref(1.0), Ref(2.0), Ref(3.0)])) +Base.RefValue{PlantSimEngine.RefVector{Float64}}(RefVector{Float64}[1.0, 2.0, 3.0]) ``` """ ref_var(v) = Base.Ref(copy(v)) @@ -747,44 +786,58 @@ This is used for *e.g.* knowing which scales are needed to add values to others. ```jldoctest mylabel julia> using PlantSimEngine +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +``` + +```jldoctest mylabel julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); ``` ```jldoctest mylabel -julia> models = Dict( - "Plant" => - MultiScaleModel( - model=ToyCAllocationModel(), - mapping=[ - # inputs - :A => ["Leaf"], - :carbon_demand => ["Leaf", "Internode"], - # outputs - :carbon_allocation => ["Leaf", "Internode"] - ], - ), - "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - "Leaf" => ( - MultiScaleModel( - model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], - # Notice we provide "Soil", not ["Soil"], so a single value is expected here - ), - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - Status(aPPFD=1300.0, TT=10.0), - ), - "Soil" => ( - ToySoilWaterModel(), - ), +julia> models = Dict( \ + "Plant" => \ + MultiScaleModel( \ + model=ToyCAllocationModel(), \ + mapping=[ \ + :A => ["Leaf"], \ + :carbon_demand => ["Leaf", "Internode"], \ + :carbon_allocation => ["Leaf", "Internode"] \ + ], \ + ), \ + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + "Leaf" => ( \ + MultiScaleModel( \ + model=ToyAssimModel(), \ + mapping=[:soil_water_content => "Soil",], \ + ), \ + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + Status(aPPFD=1300.0, TT=10.0), \ + ), \ + "Soil" => ( \ + ToySoilWaterModel(), \ + ), \ ); ``` +Notice we provide "Soil", not ["Soil"] in the mapping of the `ToyAssimModel` for the `Leaf`. This is because +we expect a single value for the `soil_water_content` to be mapped here (there is only one soil). This allows +to get the value as a singleton instead of a vector of values. + ```jldoctest mylabel -julia> reverse_mapping(models) -Dict{String, Any} with 2 entries: +julia> PlantSimEngine.reverse_mapping(models) +Dict{String, Any} with 3 entries: + "Soil" => Dict("Leaf"=>[:soil_water_content]) "Internode" => Dict("Plant"=>[:carbon_demand, :carbon_allocation]) "Leaf" => Dict("Plant"=>[:A, :carbon_demand, :carbon_allocation]) ``` diff --git a/src/run.jl b/src/run.jl index 9761a3aa..da8da0ff 100644 --- a/src/run.jl +++ b/src/run.jl @@ -450,7 +450,7 @@ function run!( if !last(object_parallelizable(node)) && executor != SequentialEx() check && @warn string( "A parallel executor was provided (`executor=$(executor)`) but the model $(node.value) (or its hard dependencies) cannot be run in parallel over objects.", - "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." + " The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." ) maxlog = 1 executor = SequentialEx() end diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index ca3d703b..7094eb88 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -243,7 +243,7 @@ mapping = Dict( ) @testset "run! on MTG with complete mapping (with init)" begin - out = @test_nowarn run!(mtg, mapping, meteo, executor=ThreadedEx()) + out = @test_nowarn run!(mtg, mapping, meteo, executor=SequentialEx()) @test typeof(out.statuses) == Dict{String,Vector{Status}} @test length(out.statuses["Plant"]) == 1 @@ -394,7 +394,7 @@ end ), ) - out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo) + out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, executor=SequentialEx()) @test length(out.dependency_graph.roots) == 4 @test out.statuses["Leaf"][1].var1 === 1.01 From 64be9ebec10cdf5a2cf2fb736ffec16e41c101e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 5 Oct 2023 11:21:31 +0200 Subject: [PATCH 51/97] Remove `PlantSimEngine.` in the code --- src/dependencies/soft_dependencies.jl | 8 ++++---- src/mtg/mapping.jl | 6 +++--- src/processes/model_initialisation.jl | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index 5b86f9e7..514ed8d3 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -258,12 +258,12 @@ This means that the variable `:carbon_demand` is computed by the process `:carbo is computed by the process `:photosynthesis` at the scale "Leaf". Those variables are used as inputs for the process that we just passed. """ function search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_graphs) - vars_input = PlantSimEngine.flatten_vars(inputs[process]) + vars_input = flatten_vars(inputs[process]) inputs_as_output_of_other_scale = Dict{String,Dict{Symbol,Vector{Symbol}}}() - for var in vars_input # e.g. var = PlantSimEngine.MappedVar{String, Nothing}("Soil", :soil_water_content, nothing) + for var in vars_input # e.g. var = MappedVar{String, Nothing}("Soil", :soil_water_content, nothing) # The variable is a multiscale variable: - if isa(var, PlantSimEngine.MappedVar) + if isa(var, MappedVar) var_organ = var.organ @assert var_organ != organ "$var in process $process is set to be multiscale, but points to its own scale ($organ). This is not allowed." @@ -277,7 +277,7 @@ function search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_gra # The variable is a multiscale variable: for (proc_output, pairs_vars_output) in soft_dep_graphs[org][:outputs] # e.g. proc_output = :soil_water; pairs_vars_output = [:soil_water=>(:soil_water_content,)] process == proc_output && error("Process $process declared at two scales: $organ and $org. A process can only be simulated at one scale.") - vars_output = PlantSimEngine.flatten_vars(pairs_vars_output) + vars_output = flatten_vars(pairs_vars_output) if var.var in vars_output # The variable is found at another scale: if haskey(inputs_as_output_of_other_scale, org) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 2319e7d3..8126be5e 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -469,7 +469,7 @@ function init_simulation(mtg, mapping; type_promotion=nothing, check=true, verbo # Compute the multi-scale dependency graph of the models: dependency_graph = multiscale_dep(mapping, verbose=verbose) - models = Dict(first(m) => PlantSimEngine.parse_models(PlantSimEngine.get_models(last(m))) for m in mapping) + models = Dict(first(m) => parse_models(get_models(last(m))) for m in mapping) return mtg, statuses, dependency_graph, models end @@ -930,10 +930,10 @@ defining variables as `MappedVar` if they are mapped to another scale. """ function variables_multiscale(node, organ, mapping) map(variables(node)) do vars - vars_ = Vector{Union{Symbol,PlantSimEngine.MappedVar}}() + vars_ = Vector{Union{Symbol,MappedVar}}() for var in vars # e.g. var = :soil_water_content if haskey(mapping[organ], var) - push!(vars_, PlantSimEngine.MappedVar(mapping[organ][var], var, nothing)) + push!(vars_, MappedVar(mapping[organ][var], var, nothing)) else push!(vars_, var) end diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index e3bfd1ef..f577caf2 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -165,9 +165,9 @@ function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} for organ in keys(mapping) # organ = "Plant" # Get all mapping for the organ: - mods = PlantSimEngine.get_models(mapping[organ]) - map_vars = PlantSimEngine.get_mapping(mapping[organ]) - user_st = PlantSimEngine.get_status(mapping[organ]) # User status + mods = get_models(mapping[organ]) + map_vars = get_mapping(mapping[organ]) + user_st = get_status(mapping[organ]) # User status if isnothing(user_st) user_st = NamedTuple() @@ -176,8 +176,8 @@ function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} end multiscale_vars = collect(first(i) for i in map_vars) - ins = merge(PlantSimEngine.inputs_.(mods)...) - outs = merge(PlantSimEngine.outputs_.(mods)...) + ins = merge(inputs_.(mods)...) + outs = merge(outputs_.(mods)...) # Variables in the node that are defined as multiscale: multi_scale_ins = intersect(keys(ins), multiscale_vars) # inputs: variables that are taken from another scale @@ -198,13 +198,13 @@ function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} # Scale(s) at which the variable is computed: from_scales = last(map_vars[findfirst(i -> i == var, multiscale_vars)]) # We check if there is a model at the other scale(s) that computes it: - outputs_from_scales = PlantSimEngine.map_scale(mapping, from_scales) do m, s + outputs_from_scales = map_scale(mapping, from_scales) do m, s # We check that the node type exist in the model list: haskey(m, s) || error( "Nodes of type $organ are mapping to variable `:$var` computed from nodes of type $s, but there is no type $s in the list of mapping." ) # If it does, we get the outputs of its mapping: - merge(PlantSimEngine.outputs_.(PlantSimEngine.get_models(m[s]))...) + merge(outputs_.(get_models(m[s]))...) end outputs_from_scales = merge(outputs_from_scales...) From db6dce6159f5e420b2d82f63bee2e566a498c545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 5 Oct 2023 15:37:27 +0200 Subject: [PATCH 52/97] Update CI.yml Remove nightly and 1.8 and replace by 1 (==release) --- .github/workflows/CI.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 07c31c32..82418afe 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,8 +19,7 @@ jobs: matrix: version: - "1.7" - - "1.8" - - "nightly" + - "1" os: - ubuntu-latest arch: From 20ec2d0f550a4c3bf4b56578ff8b7735fd27c657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 5 Oct 2023 15:37:54 +0200 Subject: [PATCH 53/97] Adapt tests to julia V<1.8 (error format changed) --- test/test-mtg-multiscale.jl | 70 +++++++++++++++++++++++++++++++++++-- test/test-mtg.jl | 7 ++-- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index 7094eb88..fe90f389 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -82,7 +82,12 @@ end ) ) - @test_throws "Nodes of type Leaf are mapping to variable `:soil_water_content` computed from nodes of type Soil, but there is no type Soil in the list of mapping." to_initialize(mapping) + + if VERSION < v"1.8" # We test differently depending on the julia version because the format of the error message changed + @test_throws ErrorException to_initialize(mapping) + else + @test_throws "Nodes of type Leaf are mapping to variable `:soil_water_content` computed from nodes of type Soil, but there is no type Soil in the list of mapping." to_initialize(mapping) + end end @testset "Mapping: missing model at other scale (soil_water_content) + missing init + var1 from MTG" begin @@ -187,7 +192,12 @@ end ), ) # The mapping above should throw an error because TT is not initialized for the Internode: - @test_throws "Nodes of type Internode need variable(s) TT to be initialized or computed by a model." run!(mtg, mapping_all, meteo) + if VERSION < v"1.8" # We test differently depending on the julia version because the format of the error message changed + @test_throws ErrorException run!(mtg, mapping_all, meteo) + else + @test_throws "Nodes of type Internode need variable(s) TT to be initialized or computed by a model." run!(mtg, mapping_all, meteo) + end + # It should work if we don't check the mapping though: out = @test_nowarn run!(mtg, mapping_all, meteo, check=false) # Note that the outputs are garbage because the TT is not initialized. @@ -337,7 +347,11 @@ end ) # Need init for var2, so it returns an error: - @test_throws "Nodes of type Leaf need variable(s) var2 to be initialized or computed by a model." PlantSimEngine.init_simulation(mtg, mapping) + if VERSION < v"1.8" # We test differently depending on the julia version because the format of the error message changed + @test_throws ErrorException PlantSimEngine.init_simulation(mtg, mapping) + else + @test_throws "Nodes of type Leaf need variable(s) var2 to be initialized or computed by a model." PlantSimEngine.init_simulation(mtg, mapping) + end mapping = Dict( "Leaf" => ( @@ -396,6 +410,56 @@ end out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, executor=SequentialEx()) + @test length(out.dependency_graph.roots) == 4 + @test out.statuses["Leaf"][1].var1 === 1.01 + @test out.statuses["Leaf"][1].var2 === 1.03 + @test out.statuses["Leaf"][1].var4 ≈ 8.1612000000000013 atol = 1e-6 + @test out.statuses["Leaf"][1].var5 == 32.4806 + @test out.statuses["Leaf"][1].var8 ≈ 1321.0700490800002 atol = 1e-6 +end + + + +@testset "MTG with dynamic output variables" begin + mapping = + Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(TT=10.0) + ), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Process1Model(1.0), + Process2Model(), + Process3Model(), + Process4Model(), + Process5Model(), + Process6Model(), + Status(aPPFD=1300.0, TT=10.0, var0=1.0, var9=1.0), + ), + "Soil" => ( + ToySoilWaterModel(), + ), + ) + + out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, executor=SequentialEx()) + @test length(out.dependency_graph.roots) == 4 @test out.statuses["Leaf"][1].var1 === 1.01 @test out.statuses["Leaf"][1].var2 === 1.03 diff --git a/test/test-mtg.jl b/test/test-mtg.jl index e684b95b..3f3e7a34 100644 --- a/test/test-mtg.jl +++ b/test/test-mtg.jl @@ -30,8 +30,11 @@ meteo = Weather( @test NamedTuple(get_node(mtg, 3)[:models][1]) == (var4=-Inf, var5=-Inf, var6=-Inf, var1=var1, var3=-Inf, var2=var2) # The following shouldn't work because var2 has only one value: - @test_throws ["The attribute", "in node 3"] init_mtg_models!(mtg, models, 10) - + if VERSION < v"1.8" # We test differently depending on the julia version because the format of the error message changed + @test_throws ErrorException init_mtg_models!(mtg, models, 10) + else + @test_throws ["The attribute", "in node 3"] init_mtg_models!(mtg, models, 10) + end # Same with two time-steps: mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) internode = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) From 66db07eaaaa3b93c0f40353208389fd1c4e6d94f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 11:16:32 +0200 Subject: [PATCH 54/97] Add get_nsteps --- Project.toml | 1 + src/checks/dimensions.jl | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/Project.toml b/Project.toml index 66753680..f6cb3d7f 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.8.2" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" FLoops = "cc61a311-1640-44b5-9fba-1b764f453329" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" diff --git a/src/checks/dimensions.jl b/src/checks/dimensions.jl index 0c046e0f..9804ebac 100644 --- a/src/checks/dimensions.jl +++ b/src/checks/dimensions.jl @@ -80,4 +80,22 @@ end function check_dimensions(::SingletonAlike, ::SingletonAlike, st, weather) return nothing +end + + +""" + get_nsteps(t) + +Get the number of steps in the object. +""" +function get_nsteps(t) + get_nsteps(DataFormat(t), t) +end + +function get_nsteps(::SingletonAlike, t) + 1 +end + +function get_nsteps(::TableAlike, t) + DataAPI.nrow(t) end \ No newline at end of file From 1a7046aab3a18ff3f91abd7037a61df9fa29cbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 11:16:45 +0200 Subject: [PATCH 55/97] Update PlantSimEngine.jl --- src/PlantSimEngine.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 9673b239..d16b34a3 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -3,6 +3,7 @@ module PlantSimEngine # FOr data formatting: import DataFrames import Tables +import DataAPI import CSV # For reading csv files with variables() From dc568f0c57d5f9352880c7494415952d71accb98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 11:17:09 +0200 Subject: [PATCH 56/97] Now we can get any variable in dynamic for the MTG --- src/mtg/mapping.jl | 137 +++++++++++++++++++++++++++++++++++++++++++-- src/run.jl | 8 ++- 2 files changed, 138 insertions(+), 7 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 8126be5e..977ba6c7 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -443,7 +443,7 @@ The value is not a reference to the one in the attribute of the MTG, but a copy a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be automatically passed as is. """ -function init_simulation(mtg, mapping; type_promotion=nothing, check=true, verbose=true) +function init_simulation(mtg, mapping; nsteps=1, outputs=Dict{String,Any}(), type_promotion=nothing, check=true, verbose=true) # We make a pre-initialised status for each kind of organ (this is a template for each node type): organs_statuses = status_template(mapping, type_promotion) # Get the reverse mapping, i.e. the variables that are mapped to other scales. This is used to initialise @@ -471,12 +471,14 @@ function init_simulation(mtg, mapping; type_promotion=nothing, check=true, verbo models = Dict(first(m) => parse_models(get_models(last(m))) for m in mapping) - return mtg, statuses, dependency_graph, models + outputs = pre_allocate_outputs(statuses, outputs, nsteps) + + return (; mtg, statuses, dependency_graph, models, outputs) end """ GraphSimulation(graph, mapping) - GraphSimulation(graph, statuses, dependency_graph) + GraphSimulation(graph, statuses, dependency_graph, models, outputs) A type that holds all information for a simulation over a graph. @@ -486,21 +488,25 @@ A type that holds all information for a simulation over a graph. - `mapping`: a dictionary of model mapping - `statuses`: a structure that defines the status of each node in the graph - `dependency_graph`: the dependency graph of the models applied to the graph +- `models`: a dictionary of models +- `outputs`: a dictionary of outputs """ -struct GraphSimulation{T,S,U} +struct GraphSimulation{T,S,U,O} graph::T statuses::S dependency_graph::DependencyGraph models::Dict{String,U} + outputs::Dict{String,O} end -function GraphSimulation(graph, mapping; type_promotion=nothing, check=true, verbose=true) - GraphSimulation(init_simulation(graph, mapping; type_promotion=type_promotion, check=check, verbose=verbose)...) +function GraphSimulation(graph, mapping; nsteps=1, outputs=Dict{String,Vararg{Symbol}}(), type_promotion=nothing, check=true, verbose=true) + GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose)...) end dep(g::GraphSimulation) = g.dependency_graph status(g::GraphSimulation) = g.statuses get_models(g::GraphSimulation) = g.models +outputs(g::GraphSimulation) = g.outputs function map_scale(f, m, scale::String) map_scale(f, m, [scale]) @@ -940,4 +946,123 @@ function variables_multiscale(node, organ, mapping) end return (vars_...,) end +end + + +""" + pre_allocate_outputs(statuses, outputs) + +Pre-allocate the outputs of needed variable for each node type in vectors of vectors. +The first level vectors have length nsteps, and the second level vectors have length n_nodes of this type. + +# Arguments + +- `statuses`: a dictionary of status by node type +- `outputs`: a dictionary of outputs by node type + +# Returns + +- A dictionary of pre-allocated output of vector of time-step and vector of node of that type. + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +``` + +```jldoctest mylabel +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +``` + +```jldoctest mylabel +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +``` + +```jldoctest mylabel +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +``` + +```jldoctest mylabel +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +``` + +```jldoctest mylabel +julia> models = Dict( \ + "Plant" => \ + MultiScaleModel( \ + model=ToyCAllocationModel(), \ + mapping=[ \ + :A => ["Leaf"], \ + :carbon_demand => ["Leaf", "Internode"], \ + :carbon_allocation => ["Leaf", "Internode"] \ + ], \ + ), \ + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + "Leaf" => ( \ + MultiScaleModel( \ + model=ToyAssimModel(), \ + mapping=[:soil_water_content => "Soil",], \ + ), \ + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + Status(aPPFD=1300.0, TT=10.0), \ + ), \ + "Soil" => ( \ + ToySoilWaterModel(), \ + ), \ + ); +``` + +```jldoctest mylabel +mtg = let \ + scene = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) \ + soil = Node(scene, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)) \ + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) \ + internode1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) \ + leaf1 = Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) \ + internode2 = Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) \ + leaf2 = Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) \ + scene \ +end +``` + +```jldoctest mylabel +julia> organs_statuses = PlantSimEngine.status_template(models, nothing); +``` + +```jldoctest mylabel +julia> var_refvector = PlantSimEngine.reverse_mapping(mapping, all=false) +``` + +```jldoctest mylabel +julia> var_need_init = PlantSimEngine.to_initialize(mapping, mtg) +``` + +```jldoctest mylabel +julia> statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init) +``` + +```jldoctest mylabel +julia> outputs = Dict("Leaf" => (:A, :carbon_demand), "Soil" => (:soil_water_content,)); +``` + +```jldoctest mylabel +julia> PlantSimEngine.pre_allocate_outputs(statuses, outputs, 2) +Dict{String, Dict{Symbol, Vector{Vector{Float64}}}} with 2 entries: + "Soil" => Dict(:soil_water_content=>[[], []]) + "Leaf" => Dict(:A=>[[], []], :carbon_demand=>[[], []]) +``` +""" +function pre_allocate_outputs(statuses, outputs, nsteps) + Dict(organ => Dict(var => [typeof(statuses[organ][1][var])[] for n in 1:nsteps] for var in vars) for (organ, vars) in outputs) +end + +function save_results!(object::GraphSimulation, i) + outs = outputs(object) + statuses = status(object) + + for (organ, vars) in outs + for (var, values) in vars + values[i] = [status[var] for status in statuses[organ]] + end + end end \ No newline at end of file diff --git a/src/run.jl b/src/run.jl index da8da0ff..5a8ac13e 100644 --- a/src/run.jl +++ b/src/run.jl @@ -317,10 +317,14 @@ function run!( meteo=nothing, constants=PlantMeteo.Constants(), extra=nothing; + nsteps=nothing, + outputs::Dict{String,Tuple{Symbol,Vararg{Symbol}}}=Dict{String,Tuple{Symbol,Vararg{Symbol}}}(), check=true, executor=ThreadedEx() ) - sim = GraphSimulation(object, mapping, check=check) + isnothing(nsteps) && (nsteps = get_nsteps(meteo)) + + sim = GraphSimulation(object, mapping, nsteps=nsteps, check=check, outputs=outputs) run!( sim, meteo, @@ -376,6 +380,8 @@ function run!( # Note: parallelization over objects is handled by the run! method below run!(object, dependency_node, i, models, meteo_i, constants, extra, check, executor) end + # At the end of the time-step, we save the results of the simulation in the object: + save_results!(object, i) end end From 38b05be63b6fc6058fc6feb376118dffa192ecc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 16:43:51 +0200 Subject: [PATCH 57/97] Update mapping.jl Add checks on outputs variables requested by the user --- src/mtg/mapping.jl | 67 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 977ba6c7..0c43e7ad 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -471,7 +471,7 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=Dict{String,Any}(), typ models = Dict(first(m) => parse_models(get_models(last(m))) for m in mapping) - outputs = pre_allocate_outputs(statuses, outputs, nsteps) + outputs = pre_allocate_outputs(statuses, outputs, nsteps, check=check) return (; mtg, statuses, dependency_graph, models, outputs) end @@ -950,15 +950,21 @@ end """ - pre_allocate_outputs(statuses, outputs) + pre_allocate_outputs(statuses, outs, nsteps; check=true) Pre-allocate the outputs of needed variable for each node type in vectors of vectors. The first level vectors have length nsteps, and the second level vectors have length n_nodes of this type. +Note that we pre-allocate the vectors for the time-steps, but not for each organ, because we don't +know how many nodes will be in each organ in the future (organs can appear or disapear). + # Arguments - `statuses`: a dictionary of status by node type -- `outputs`: a dictionary of outputs by node type +- `outs`: a dictionary of outputs by node type +- `nsteps`: the number of time-steps +- `check`: whether to check the mapping for errors. Default (`true`) returns an error if some variables do not exist. +If false and some variables are missing, return an info, remove the unknown variables and continue. # Returns @@ -1052,8 +1058,59 @@ Dict{String, Dict{Symbol, Vector{Vector{Float64}}}} with 2 entries: "Leaf" => Dict(:A=>[[], []], :carbon_demand=>[[], []]) ``` """ -function pre_allocate_outputs(statuses, outputs, nsteps) - Dict(organ => Dict(var => [typeof(statuses[organ][1][var])[] for n in 1:nsteps] for var in vars) for (organ, vars) in outputs) +function pre_allocate_outputs(statuses, outs, nsteps; check=true) + + outs_ = copy(outs) + # Checking that organs in outputs exist in the mtg (in the statuses): + if !all(i in keys(statuses) for i in keys(outs_)) + not_in_statuses = setdiff(keys(outs_), keys(statuses)) + e = string( + "You requested outputs for organs ", + join(keys(outs_), ", "), + ", but organs ", + join(not_in_statuses, ", "), + " have no models." + ) + + if check + error(e) + else + @info e + [delete!(outs_, i) for i in not_in_statuses] + end + end + + # Checking that variables in outputs exist in the statuses: + for (organ, vars) in outs_ + if !all(i in collect(keys(statuses[organ][1])) for i in vars) + not_in_statuses = (setdiff(vars, keys(statuses[organ][1]))...,) + e = string( + "You requested outputs for variables ", + join(vars, ", "), + ", but variables ", + join(not_in_statuses, ", "), + " have no models." + ) + if check + error(e) + else + @info e + existing_vars_requested = setdiff(outs_[organ], not_in_statuses) + if length(existing_vars_requested) == 0 + # None of the variables requested by the user exist at this scale for this set of models + delete!(outs_, organ) + else + # Some still exist, we onl use the ones that do: + outs_[organ] = (existing_vars_requested...,) + end + end + end + end + + # Making the pre-allocated outputs: + Dict(organ => Dict(var => [typeof(statuses[organ][1][var])[] for n in 1:nsteps] for var in vars) for (organ, vars) in outs_) + # Note: we use the type of the variable from the first status for each organ to pre-allocate the outputs, because they are + # all the same type for others. end function save_results!(object::GraphSimulation, i) From 00110e4956fc4f209753f64fdf16f58dbab34db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 16:43:55 +0200 Subject: [PATCH 58/97] Update run.jl --- src/run.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/run.jl b/src/run.jl index 5a8ac13e..274cf8c0 100644 --- a/src/run.jl +++ b/src/run.jl @@ -318,7 +318,7 @@ function run!( constants=PlantMeteo.Constants(), extra=nothing; nsteps=nothing, - outputs::Dict{String,Tuple{Symbol,Vararg{Symbol}}}=Dict{String,Tuple{Symbol,Vararg{Symbol}}}(), + outputs=Dict{String,Tuple{Symbol,Vararg{Symbol}}}(), check=true, executor=ThreadedEx() ) From 0469a000e38b80a7c28080d7dc12c88d88b6e67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 16:44:02 +0200 Subject: [PATCH 59/97] Update test-mtg-multiscale.jl Add more tests --- test/test-mtg-multiscale.jl | 184 ++++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 61 deletions(-) diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index fe90f389..0e19f2b6 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -7,9 +7,38 @@ meteo = Weather( ] ) +# A mapping that actually works (same as before but with the init for TT): +mapping_1 = Dict( + "Plant" => + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + # inputs + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + "Internode" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(TT=10.0) + ), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0), + ), + "Soil" => ( + ToySoilWaterModel(), + ), +) # Example MTG: - mtg = begin scene = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) soil = Node(scene, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)) @@ -21,6 +50,98 @@ mtg = begin scene end + +@testset "status_template" begin + organs_statuses = PlantSimEngine.status_template(mapping, nothing) + @test collect(keys(organs_statuses)) == ["Soil", "Internode", "Plant", "Leaf"] + # Check that the soil_water_content is linked between the soil and the leaves: + @test organs_statuses["Soil"][:soil_water_content][] === -Inf + @test organs_statuses["Leaf"][:soil_water_content][] === -Inf + + @test organs_statuses["Soil"][:soil_water_content][] === organs_statuses["Leaf"][:soil_water_content][] + + organs_statuses["Soil"][:soil_water_content][] = 1.0 + @test organs_statuses["Leaf"][:soil_water_content][] == 1.0 + + @test organs_statuses["Plant"][:A] == PlantSimEngine.RefVector{Float64}[] + @test organs_statuses["Plant"][:carbon_allocation] == PlantSimEngine.RefVector{Float64}[] + @test organs_statuses["Internode"][:carbon_allocation] == -Inf + @test organs_statuses["Leaf"][:carbon_demand] == -Inf + + # Testing with a different type: + organs_statuses = PlantSimEngine.status_template(mapping, Dict(Float64 => Float32, Vector{Float64} => Vector{Float32})) + + @test isa(organs_statuses["Plant"][:A], PlantSimEngine.RefVector{Float32}) + @test isa(organs_statuses["Plant"][:carbon_allocation], PlantSimEngine.RefVector{Float32}) + @test isa(organs_statuses["Internode"][:carbon_allocation], Float32) + @test isa(organs_statuses["Leaf"][:carbon_demand], Float32) + @test isa(organs_statuses["Soil"][:soil_water_content], Base.RefValue{Float32}) +end + + +@testset "Multiscale initialisations and outputs" begin + outs = Dict( + "Flowers" => (:A, :carbon_demand), # There are no flowers in this MTG + "Leaf" => (:A, :carbon_demand, :non_existing_variable), # :non_existing_variable is not computed by any model + "Soil" => (:soil_water_content,), + ) + + type_promotion = nothing + nsteps = 2 + organs_statuses = PlantSimEngine.status_template(mapping_1, type_promotion) + + @test collect(keys(organs_statuses)) == ["Soil", "Internode", "Plant", "Leaf"] + @test collect(keys(organs_statuses["Soil"])) == [:soil_water_content] + @test collect(keys(organs_statuses["Leaf"])) == [:carbon_allocation, :A, :TT, :aPPFD, :soil_water_content, :carbon_demand] + @test collect(keys(organs_statuses["Plant"])) == [:carbon_allocation, :A, :carbon_offer, :carbon_demand] + @test organs_statuses["Soil"][:soil_water_content][] === -Inf + @test organs_statuses["Leaf"][:carbon_allocation] === -Inf + @test organs_statuses["Leaf"][:TT] === 10.0 + @test typeof(organs_statuses["Plant"][:carbon_allocation]) === PlantSimEngine.RefVector{Float64} + + + + @test PlantSimEngine.reverse_mapping(mapping_1, all=true) == Dict{String,Any}( + "Soil" => Dict("Leaf" => [:soil_water_content]), + "Internode" => Dict("Plant" => [:carbon_demand, :carbon_allocation]), + "Leaf" => Dict("Plant" => [:A, :carbon_demand, :carbon_allocation]) + ) + + var_refvector_1 = PlantSimEngine.reverse_mapping(mapping_1, all=false) + @test var_refvector == Dict{String,Any}( + "Internode" => Dict("Plant" => [:carbon_demand, :carbon_allocation]), + "Leaf" => Dict("Plant" => [:A, :carbon_demand, :carbon_allocation]) + ) + + @test PlantSimEngine.reverse_mapping(filter(x -> x.first == "Soil", mapping_1)) == Dict{String,Any}() + + var_need_init = PlantSimEngine.to_initialize(mapping_1, mtg) + @test var_need_init == Dict{String,Any}() + + statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init) + @test collect(keys(statuses)) == ["Soil", "Internode", "Plant", "Leaf"] + + @test length(statuses["Internode"]) == length(statuses["Leaf"]) == 2 + @test length(statuses["Soil"]) == length(statuses["Plant"]) == 1 + + e_1 = "You requested outputs for organs Soil, Flowers, Leaf, but organs Flowers have no models." + e_2 = "You requested outputs for variables A, carbon_demand, non_existing_variable, but variables non_existing_variable have no models." + + # If check is true, this should return an error (some outputs are not computed): + if VERSION < v"1.8" # We test differently depending on the julia version because the format of the error message changed + @test_throws ErrorException PlantSimEngine.pre_allocate_outputs(statuses, outs, nsteps) + else + @test_throws e_1 PlantSimEngine.pre_allocate_outputs(statuses, outs, nsteps) + end + + outs_ = @test_logs (:info, "$e_1") (:info, "$e_2") PlantSimEngine.pre_allocate_outputs(statuses, outs, nsteps, check=false) + + @test outs_ == Dict( + "Soil" => Dict(:soil_water_content => [[], []]), + "Leaf" => Dict(:A => [[], []], :carbon_demand => [[], []]) + ) +end + # Testing the mappings: @testset "Mapping: missing initialisation" begin mapping = Dict( @@ -221,39 +342,8 @@ end @test st_leaf1.carbon_allocation == 0.0 end -# A mapping that actually works (same as before but with the init for TT): -mapping = Dict( - "Plant" => - MultiScaleModel( - model=ToyCAllocationModel(), - mapping=[ - # inputs - :A => ["Leaf"], - :carbon_demand => ["Leaf", "Internode"], - # outputs - :carbon_allocation => ["Leaf", "Internode"] - ], - ), - "Internode" => ( - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - Status(TT=10.0) - ), - "Leaf" => ( - MultiScaleModel( - model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], - # Notice we provide "Soil", not ["Soil"], so a single value is expected here - ), - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - Status(aPPFD=1300.0, TT=10.0), - ), - "Soil" => ( - ToySoilWaterModel(), - ), -) - @testset "run! on MTG with complete mapping (with init)" begin - out = @test_nowarn run!(mtg, mapping, meteo, executor=SequentialEx()) + out = @test_nowarn run!(mtg, mapping_1, meteo, executor=SequentialEx()) @test typeof(out.statuses) == Dict{String,Vector{Status}} @test length(out.statuses["Plant"]) == 1 @@ -306,34 +396,6 @@ mapping = Dict( end end - -@testset "status_template" begin - organs_statuses = PlantSimEngine.status_template(mapping, nothing) - @test collect(keys(organs_statuses)) == ["Soil", "Internode", "Plant", "Leaf"] - # Check that the soil_water_content is linked between the soil and the leaves: - @test organs_statuses["Soil"][:soil_water_content][] === -Inf - @test organs_statuses["Leaf"][:soil_water_content][] === -Inf - - @test organs_statuses["Soil"][:soil_water_content][] === organs_statuses["Leaf"][:soil_water_content][] - - organs_statuses["Soil"][:soil_water_content][] = 1.0 - @test organs_statuses["Leaf"][:soil_water_content][] == 1.0 - - @test organs_statuses["Plant"][:A] == PlantSimEngine.RefVector{Float64}[] - @test organs_statuses["Plant"][:carbon_allocation] == PlantSimEngine.RefVector{Float64}[] - @test organs_statuses["Internode"][:carbon_allocation] == -Inf - @test organs_statuses["Leaf"][:carbon_demand] == -Inf - - # Testing with a different type: - organs_statuses = PlantSimEngine.status_template(mapping, Dict(Float64 => Float32, Vector{Float64} => Vector{Float32})) - - @test isa(organs_statuses["Plant"][:A], PlantSimEngine.RefVector{Float32}) - @test isa(organs_statuses["Plant"][:carbon_allocation], PlantSimEngine.RefVector{Float32}) - @test isa(organs_statuses["Internode"][:carbon_allocation], Float32) - @test isa(organs_statuses["Leaf"][:carbon_demand], Float32) - @test isa(organs_statuses["Soil"][:soil_water_content], Base.RefValue{Float32}) -end - # Here we initialise var1 to a constant value: @testset "MTG initialisation" begin var1 = 1.0 From be2ec5d4f5b2841cbba853c7f46753ef01b7a0e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 17:02:25 +0200 Subject: [PATCH 60/97] outputs is now `nothing` by default --- src/mtg/mapping.jl | 6 ++++-- src/run.jl | 2 +- test/test-mtg-multiscale.jl | 6 ++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 0c43e7ad..b744301f 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -443,7 +443,7 @@ The value is not a reference to the one in the attribute of the MTG, but a copy a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be automatically passed as is. """ -function init_simulation(mtg, mapping; nsteps=1, outputs=Dict{String,Any}(), type_promotion=nothing, check=true, verbose=true) +function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) # We make a pre-initialised status for each kind of organ (this is a template for each node type): organs_statuses = status_template(mapping, type_promotion) # Get the reverse mapping, i.e. the variables that are mapped to other scales. This is used to initialise @@ -499,7 +499,7 @@ struct GraphSimulation{T,S,U,O} outputs::Dict{String,O} end -function GraphSimulation(graph, mapping; nsteps=1, outputs=Dict{String,Vararg{Symbol}}(), type_promotion=nothing, check=true, verbose=true) +function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose)...) end @@ -1113,6 +1113,8 @@ function pre_allocate_outputs(statuses, outs, nsteps; check=true) # all the same type for others. end +pre_allocate_outputs(statuses, ::Nothing, nsteps; check=true) = Dict{String,Tuple{Symbol,Vararg{Symbol}}}() + function save_results!(object::GraphSimulation, i) outs = outputs(object) statuses = status(object) diff --git a/src/run.jl b/src/run.jl index 274cf8c0..d838c7c7 100644 --- a/src/run.jl +++ b/src/run.jl @@ -318,7 +318,7 @@ function run!( constants=PlantMeteo.Constants(), extra=nothing; nsteps=nothing, - outputs=Dict{String,Tuple{Symbol,Vararg{Symbol}}}(), + outputs=nothing, check=true, executor=ThreadedEx() ) diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index 0e19f2b6..2616312c 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -52,7 +52,7 @@ end @testset "status_template" begin - organs_statuses = PlantSimEngine.status_template(mapping, nothing) + organs_statuses = PlantSimEngine.status_template(mapping_1, nothing) @test collect(keys(organs_statuses)) == ["Soil", "Internode", "Plant", "Leaf"] # Check that the soil_water_content is linked between the soil and the leaves: @test organs_statuses["Soil"][:soil_water_content][] === -Inf @@ -69,7 +69,7 @@ end @test organs_statuses["Leaf"][:carbon_demand] == -Inf # Testing with a different type: - organs_statuses = PlantSimEngine.status_template(mapping, Dict(Float64 => Float32, Vector{Float64} => Vector{Float32})) + organs_statuses = PlantSimEngine.status_template(mapping_1, Dict(Float64 => Float32, Vector{Float64} => Vector{Float32})) @test isa(organs_statuses["Plant"][:A], PlantSimEngine.RefVector{Float32}) @test isa(organs_statuses["Plant"][:carbon_allocation], PlantSimEngine.RefVector{Float32}) @@ -480,8 +480,6 @@ end @test out.statuses["Leaf"][1].var8 ≈ 1321.0700490800002 atol = 1e-6 end - - @testset "MTG with dynamic output variables" begin mapping = Dict( From 5d72770ff87e6c56e49aa0f318d5357014f77dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 17:27:20 +0200 Subject: [PATCH 61/97] Update Project.toml add compat for DataAPI --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index f6cb3d7f..abca1c12 100644 --- a/Project.toml +++ b/Project.toml @@ -19,6 +19,7 @@ Term = "22787eb5-b846-44ae-b979-8e399b8463ab" [compat] AbstractTrees = "0.4" CSV = "0.10" +DataAPI = "1.15" DataFrames = "1" FLoops = "0.2" MultiScaleTreeGraph = "0.12" From ff07372f4c4b07c9fd107f7eb57b73b1b9b426b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 17:27:29 +0200 Subject: [PATCH 62/97] Update test-mtg-multiscale.jl fix issue in test --- test/test-mtg-multiscale.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index 2616312c..d0d631a0 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -99,8 +99,6 @@ end @test organs_statuses["Leaf"][:TT] === 10.0 @test typeof(organs_statuses["Plant"][:carbon_allocation]) === PlantSimEngine.RefVector{Float64} - - @test PlantSimEngine.reverse_mapping(mapping_1, all=true) == Dict{String,Any}( "Soil" => Dict("Leaf" => [:soil_water_content]), "Internode" => Dict("Plant" => [:carbon_demand, :carbon_allocation]), @@ -108,7 +106,7 @@ end ) var_refvector_1 = PlantSimEngine.reverse_mapping(mapping_1, all=false) - @test var_refvector == Dict{String,Any}( + @test var_refvector_1 == Dict{String,Any}( "Internode" => Dict("Plant" => [:carbon_demand, :carbon_allocation]), "Leaf" => Dict("Plant" => [:A, :carbon_demand, :carbon_allocation]) ) @@ -118,7 +116,7 @@ end var_need_init = PlantSimEngine.to_initialize(mapping_1, mtg) @test var_need_init == Dict{String,Any}() - statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init) + statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector_1, var_need_init) @test collect(keys(statuses)) == ["Soil", "Internode", "Plant", "Leaf"] @test length(statuses["Internode"]) == length(statuses["Leaf"]) == 2 From 8a7fe24e5fba65ef1ded05adb825c304dc4fd2c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 17:42:10 +0200 Subject: [PATCH 63/97] Fix issue in doc for `pre_allocate_outputs` --- src/mtg/mapping.jl | 47 ++++++++++++++++++++++++------------- test/test-mtg-multiscale.jl | 6 +++++ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index b744301f..d9ed3d94 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -973,7 +973,7 @@ If false and some variables are missing, return an info, remove the unknown vari # Examples ```jldoctest mylabel -julia> using PlantSimEngine +julia> using PlantSimEngine, MultiScaleTreeGraph ``` ```jldoctest mylabel @@ -993,7 +993,7 @@ julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); ``` ```jldoctest mylabel -julia> models = Dict( \ +julia> mapping = Dict( \ "Plant" => \ MultiScaleModel( \ model=ToyCAllocationModel(), \ @@ -1019,32 +1019,47 @@ julia> models = Dict( \ ``` ```jldoctest mylabel -mtg = let \ - scene = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) \ - soil = Node(scene, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)) \ - plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) \ - internode1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) \ - leaf1 = Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) \ - internode2 = Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) \ - leaf2 = Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) \ - scene \ -end +julia> mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)); +``` + +```jldoctest mylabel +julia> soil = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)); +``` + +```jldoctest mylabel +julia> plant = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)); +``` + +```jldoctest mylabel +julia> internode1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)); +``` + +```jldoctest mylabel +julia> leaf1 = Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); +``` + +```jldoctest mylabel +julia> internode2 = Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)); +``` + +```jldoctest mylabel +julia> leaf2 = Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); ``` ```jldoctest mylabel -julia> organs_statuses = PlantSimEngine.status_template(models, nothing); +julia> organs_statuses = PlantSimEngine.status_template(mapping, nothing); ``` ```jldoctest mylabel -julia> var_refvector = PlantSimEngine.reverse_mapping(mapping, all=false) +julia> var_refvector = PlantSimEngine.reverse_mapping(mapping, all=false); ``` ```jldoctest mylabel -julia> var_need_init = PlantSimEngine.to_initialize(mapping, mtg) +julia> var_need_init = PlantSimEngine.to_initialize(mapping, mtg); ``` ```jldoctest mylabel -julia> statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init) +julia> statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init); ``` ```jldoctest mylabel diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index d0d631a0..e3c96ad7 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -516,6 +516,12 @@ end ), ) + out_vars = Dict( + "Leaf" => (:A, :carbon_demand, :soil_water_content), + "Internode" => (:carbon_allocation,), + "Plant" => (:carbon_allocation,), + "Soil" => (:soil_water_content,), + ) out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, executor=SequentialEx()) @test length(out.dependency_graph.roots) == 4 From bc54f236615df55939eb30c0c3921ec543c34664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 17:46:45 +0200 Subject: [PATCH 64/97] Update run.jl Use extra to pass statuses --- src/run.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/run.jl b/src/run.jl index d838c7c7..f77cfe85 100644 --- a/src/run.jl +++ b/src/run.jl @@ -349,10 +349,12 @@ function run!( executor=ThreadedEx() ) models = get_models(object) + st = status(object) + # Run the simulation of each soft-coupled model in the dependency graph: # Note: hard-coupled processes handle themselves already @floop executor for (process_key, dependency_node) in collect(dep(object).roots) - run!(object, dependency_node, 1, models, meteo, constants, extra, check, executor) + run!(object, dependency_node, 1, models, meteo, constants, st, check, executor) end end @@ -370,6 +372,9 @@ function run!( dep_graph = dep(object) models = get_models(object) + st = status(object) + + !isnothing(extra) && error("Extra parameters are not allowed for the simulation of an MTG (already used for statuses).") # Note: The object is not thread safe here, because we write all meteo time-steps into the same Status (for each node) # This is because the number of nodes is usually higher than the number of cores anyway, so we don't gain much by parallelizing over @@ -378,7 +383,7 @@ function run!( # In parallel over dependency root, i.e. for independant computations: @floop executor for (process_key, dependency_node) in collect(dep_graph.roots) # Note: parallelization over objects is handled by the run! method below - run!(object, dependency_node, i, models, meteo_i, constants, extra, check, executor) + run!(object, dependency_node, i, models, meteo_i, constants, st, check, executor) end # At the end of the time-step, we save the results of the simulation in the object: save_results!(object, i) From 38416af801303944d09443dcf17ccbbd2d5cf71e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 6 Oct 2023 17:57:38 +0200 Subject: [PATCH 65/97] Add more tests on multiscale with several time-steps --- test/test-mtg-multiscale.jl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index e3c96ad7..d37d2d92 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -517,12 +517,12 @@ end ) out_vars = Dict( - "Leaf" => (:A, :carbon_demand, :soil_water_content), + "Leaf" => (:A, :carbon_demand, :soil_water_content, :carbon_allocation), "Internode" => (:carbon_allocation,), "Plant" => (:carbon_allocation,), "Soil" => (:soil_water_content,), ) - out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, executor=SequentialEx()) + out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, outputs=out_vars, executor=SequentialEx()) @test length(out.dependency_graph.roots) == 4 @test out.statuses["Leaf"][1].var1 === 1.01 @@ -530,4 +530,11 @@ end @test out.statuses["Leaf"][1].var4 ≈ 8.1612000000000013 atol = 1e-6 @test out.statuses["Leaf"][1].var5 == 32.4806 @test out.statuses["Leaf"][1].var8 ≈ 1321.0700490800002 atol = 1e-6 + + @test out.outputs["Leaf"][:carbon_demand] == [[0.5, 0.5], [0.5, 0.5]] + @test out.outputs["Leaf"][:soil_water_content][1] == fill(out.outputs["Soil"][:soil_water_content][1][1], 2) + @test out.outputs["Leaf"][:soil_water_content][2] == fill(out.outputs["Soil"][:soil_water_content][2][1], 2) + + @test out.outputs["Leaf"][:carbon_allocation] == out.outputs["Internode"][:carbon_allocation] + @test out.outputs["Plant"][:carbon_allocation][1][1][1] === out.outputs["Internode"][:carbon_allocation][1][1] end \ No newline at end of file From 6765f500cab6f61b3483c75aaa853d036e072542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Sat, 7 Oct 2023 16:06:59 +0200 Subject: [PATCH 66/97] `multiscale_dep` becomes `dep`, as for the other cases --- src/PlantSimEngine.jl | 1 - src/dependencies/dependencies.jl | 215 ++++++++++++++++++- src/dependencies/multi-scale_dependencies.jl | 195 ----------------- src/mtg/mapping.jl | 12 +- src/run.jl | 8 +- 5 files changed, 222 insertions(+), 209 deletions(-) delete mode 100644 src/dependencies/multi-scale_dependencies.jl diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index d16b34a3..9729452b 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -47,7 +47,6 @@ include("dataframe.jl") include("dependencies/dependency_graph.jl") include("dependencies/soft_dependencies.jl") include("dependencies/hard_dependencies.jl") -include("dependencies/multi-scale_dependencies.jl") include("dependencies/dependencies.jl") # MTG compatibility: diff --git a/src/dependencies/dependencies.jl b/src/dependencies/dependencies.jl index 2cf86925..603142b5 100644 --- a/src/dependencies/dependencies.jl +++ b/src/dependencies/dependencies.jl @@ -2,12 +2,39 @@ dep(::T, nsteps=1) where {T<:AbstractModel} = NamedTuple() """ dep(m::ModelList, nsteps=1; verbose::Bool=true) + dep(mapping::Dict{String,T}; verbose=true) -Get the model dependency graph given a ModelList. If one graph is returned, then all models are -coupled. If several graphs are returned, then only the models inside each graph are coupled, and +Get the model dependency graph given a ModelList or a multiscale model mapping. If one graph is returned, +then all models are coupled. If several graphs are returned, then only the models inside each graph are coupled, and the models in different graphs are not coupled. `nsteps` is the number of steps the dependency graph will be used over. It is used to determine -the length of the `simulation_id` argument for each soft dependencies in the graph. +the length of the `simulation_id` argument for each soft dependencies in the graph. It is set to `1` in the case of a +multiscale mapping. + +# Details + +The dependency graph is computed by searching the inputs of each process in the outputs of its own scale, or the other scales. There are five cases +for every model (one model simulates one process): + +1. The process has no inputs. It is completely independent, and is placed as one of the roots of the dependency graph. +2. The process needs inputs from models at its own scale. We put it as a child of this other process. +3. The process needs inputs from another scale. We put it as a child of this process at another scale. +4. The process needs inputs from its own scale and another scale. We put it as a child of both. +5. The process is a hard dependency of another process (only possible at the same scale). In this case, the process is set as a hard-dependency of the +other process, and its simulation is handled directly from this process. + +For the 4th case, the process have two parent processes. This is OK because the process will only be computed once during simulation as we check if both +parents were run before running the process. + +Note that in the 5th case, we still need to check if a variable is needed from another scale. In this case, the parent node is +used as a child of the process at the other scale. Note there can be several levels of hard dependency graph, so this is done recursively. + +How do we do all that? We identify the hard dependencies first. Then we link the inputs/outputs of the hard dependencies roots +to other scales if needed. Then we transform all these nodes into soft dependencies, that we put into a Dict of Scale => Dict(process => SoftDependencyNode). +Then we traverse all these and we set nodes that need outputs from other nodes as inputs as children/parents. +If a node has no dependency, it is set as a root node and pushed into a new Dict (independant_process_root). This Dict is the returned dependency graph. And +it presents root nodes as independent starting points for the sub-graphs, which are the models that are coupled together. We can then traverse each of +these graphs independently to retrieve the models that are coupled together, in the right order of execution. # Examples @@ -55,4 +82,184 @@ end function dep(m::NamedTuple, nsteps=1; verbose::Bool=true) dep(nsteps; verbose=verbose, m...) -end \ No newline at end of file +end + +function dep(mapping::Dict{String,T}; verbose=true) where {T} + + full_mapping = Dict(first(mod) => Dict(get_mapping(last(mod))) for mod in mapping) + + # First step, get the hard-dependency graph and create SoftDependencyNodes for each hard-dependency root. In other word, we want + # only the nodes that are not hard-dependency of other nodes. These nodes are taken as roots for the soft-dependency graph because they + # are independant. + soft_dep_graphs = Dict{String,Any}(i => 0.0 for i in keys(mapping)) + not_found = Dict{Symbol,DataType}() + for (organ, model) in mapping + # organ = "Leaf"; model = mapping[organ] + mods = parse_models(get_models(model)) + + # Move some models below others when they are manually linked (hard-dependency): + hard_deps = hard_dependencies((; mods...), verbose=verbose) + d_vars = Dict{Symbol,Vector{Pair{Symbol,NamedTuple}}}() + for (procname, node) in hard_deps.roots + var = Pair{Symbol,NamedTuple}[] + traverse_dependency_graph!(node, x -> variables_multiscale(x, organ, full_mapping), var) + push!(d_vars, procname => var) + end + + inputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( + key => [j.first => j.second.inputs for j in val] for (key, val) in d_vars + ) + outputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( + key => [j.first => j.second.outputs for j in val] for (key, val) in d_vars + ) + + soft_dep_graph = Dict( + process_ => SoftDependencyNode( + soft_dep_vars.value, + process_, # process name + organ, # scale + AbstractTrees.children(soft_dep_vars), # hard dependencies + nothing, + nothing, + SoftDependencyNode[], + [0] # Vector of zeros of length = number of time-steps + ) + for (process_, soft_dep_vars) in hard_deps.roots + ) + + soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process, outputs=outputs_process) + not_found = merge(not_found, hard_deps.not_found) + end + + # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the + # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the + # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children + # of the nodes that they depend on. + independant_process_root = Dict{Pair{String,Symbol},SoftDependencyNode}() + for (organ, (soft_dep_graph, ins, outs)) in soft_dep_graphs # e.g. organ = "Plant"; soft_dep_graph, ins, outs = soft_dep_graphs[organ] + for (proc, i) in soft_dep_graph + # proc = :carbon_allocation; i = soft_dep_graph[proc] + # Search if the process has soft dependencies: + soft_deps = search_inputs_in_output(proc, ins, outs) + + # Remove the hard dependencies from the soft dependencies: + soft_deps_not_hard = drop_process(soft_deps, [hd.process for hd in i.hard_dependency]) + # NB: if a node is already a hard dependency of the node, it cannot be a soft dependency + + # Check if the process has soft dependencies at other scales: + soft_deps_multiscale = search_inputs_in_multiscale_output(proc, organ, ins, soft_dep_graphs) + # Example output: "Soil" => Dict(:soil_water=>[:soil_water_content]), which means that the variable :soil_water_content + # is computed by the process :soil_water at the scale "Soil". + + if length(soft_deps_not_hard) == 0 && i.process in keys(soft_dep_graph) && length(soft_deps_multiscale) == 0 + # If the process has no soft (multiscale) dependencies, then it is independant (so it is a root) + # Note that the process is only independent if it is also a root in the hard-dependency graph + independant_process_root[organ=>proc] = i + else + # If the process has soft dependencies at its scale, add it: + if length(soft_deps_not_hard) > 0 + # If the process has soft dependencies, then it is not independant + # and we need to add its parent(s) to the node, and the node as a child + for (parent_soft_dep, soft_dep_vars) in pairs(soft_deps_not_hard) + # parent_soft_dep = :process5; soft_dep_vars = soft_deps[parent_soft_dep] + + # preventing a cyclic dependency + if parent_soft_dep == proc + error("Cyclic model dependency detected for process $proc") + end + + # preventing a cyclic dependency: if the parent also has a dependency on the current node: + if soft_dep_graph[parent_soft_dep].parent !== nothing && any([i == p for p in soft_dep_graph[parent_soft_dep].parent]) + error( + "Cyclic dependency detected for process $proc:", + " $proc depends on $parent_soft_dep, which depends on $proc.", + " This is not allowed, but is possible via a hard dependency." + ) + end + + # preventing a cyclic dependency: if the current node has the parent node as a child: + if i.children !== nothing && any([soft_dep_graph[parent_soft_dep] == p for p in i.children]) + error( + "Cyclic dependency detected for process $proc:", + " $proc depends on $parent_soft_dep, which depends on $proc.", + " This is not allowed, but is possible via a hard dependency." + ) + end + + # Add the current node as a child of the node on which it depends + push!(soft_dep_graph[parent_soft_dep].children, i) + + # Add the node on which the current node depends as a parent + if i.parent === nothing + # If the node had no parent already, it is nothing, so we change into a vector + i.parent = [soft_dep_graph[parent_soft_dep]] + else + push!(i.parent, soft_dep_graph[parent_soft_dep]) + end + + # Add the soft dependencies (variables) of the parent to the current node + i.parent_vars = soft_deps + end + end + + # If the node has soft dependencies at other scales, add it as child of the other scale (and add its parent too): + if length(soft_deps_multiscale) > 0 + #! Continue here: add the node as a child of the other scale, and add this other node has its parent. + #! Take inspiration from the code above happening at the same scale. + #! Note that the node can have both soft dependencies at its own scale and at other scales, but it is not + #! a big deal because in the end we drop the scales and only keep the root soft-dependency nodes. + for org in keys(soft_deps_multiscale) + # org = "Leaf" + for (parent_soft_dep, soft_dep_vars) in soft_deps_multiscale[org] + # parent_soft_dep= :photosynthesis; soft_dep_vars = soft_deps_multiscale[org][parent_soft_dep] + parent_node = soft_dep_graphs[org][:soft_dep_graph][parent_soft_dep] + # preventing a cyclic dependency: if the parent also has a dependency on the current node: + if parent_node.parent !== nothing && any([i == p for p in parent_node.parent]) + error( + "Cyclic dependency detected for process $proc:", + " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one", + " This is not allowed, you may need to develop a new process that does the whole computation by itself." + ) + end + + # preventing a cyclic dependency: if the current node has the parent node as a child: + if i.children !== nothing && any([parent_node == p for p in i.children]) + error( + "Cyclic dependency detected for process $proc:", + " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one.", + " This is not allowed, you may need to develop a new process that does the whole computation by itself." + ) + end + + # Add the current node as a child of the node on which it depends: + push!(parent_node.children, i) + + # Add the node on which the current node depends as a parent + if i.parent === nothing + # If the node had no parent already, it is nothing, so we change into a vector + i.parent = [parent_node] + else + push!(i.parent, parent_node) + end + + # Add the multiscale soft dependencies variables of the parent to the current node + i.parent_vars = NamedTuple(Symbol(k) => NamedTuple(v) for (k, v) in soft_deps_multiscale) + end + end + end + #! To do: make this code work without multiscale mapping, so we have only one code base for both cases. + #! Also, put some parts into functions to make the code more readable. + end + end + end + + #! CONTINUE HERE: use the multiscale variables to compute the dependency graph. The steps would look something like: + #! 1. Get the hard-dependency graph + #! 2. Get the soft-dependency graph: do as before by computing inputs and outputs for each hard-dependency root, + #! but also look at the multiscale variables to the inputs and outputs. + #! 3. For each soft-dependency root, check if the process is independant (i.e. if it has no soft-dependencies). + #! within its own scale, but also in other scales. If it is independant, then it is a root of the soft-dependency multiscale graph. + #! If it is not, then add it as a child of the other soft-dependency node that it depends on. + + return DependencyGraph(independant_process_root, not_found) +end diff --git a/src/dependencies/multi-scale_dependencies.jl b/src/dependencies/multi-scale_dependencies.jl deleted file mode 100644 index 674880ae..00000000 --- a/src/dependencies/multi-scale_dependencies.jl +++ /dev/null @@ -1,195 +0,0 @@ -function multiscale_dep(models; verbose=true) - - mapping = Dict(first(mod) => Dict(get_mapping(last(mod))) for mod in models) - #! continue here: we have the inputs and outputs variables for each process per scale, and if the variable can - #! be found at another scale, it is defined as a MappedVar (variables + mapped scale). - #! Now what we need to do is to compute the dependency graph for each process each scale, by searching the inputs - #! of each process in the outputs of its own scale, or the other scales. There are five cases then: - #! 1. The process has no inputs. It is completely independent. - #! 2. The process needs inputs from its own scale. We put it as a child of this other process. - #! 3. The process needs inputs from another scale. We put it as a child of this process at another scale. - #! 4. The process needs inputs from its own scale and another scale. We put it as a child of both. - #! 5. The process is a hard dependency of another process (only possible in own scale). In this case it is treated differently (uses the standard method) - #! Note that in the 5th case, we still need to check if a variable is needed from another scale. In this case, the root node of the - #! hard dependency graph is used as a child of the process at the other scale. - - #! How do we do all that? We identify the hard dependencies first. Then we link the inputs/outputs of the hard dependencies roots - #! to other scales if needed. Then we transform all these nodes into soft dependencies, that we put into a Dict of Scale => Dict(process => SoftDependencyNode). - #! Then we traverse all these and we set nodes that need outputs from other nodes as inputs as children/parents. - #! If a node has no dependency, it is set as a root node and pushed into a new Dict (independant_process_root). This Dict is the dependency graph. - - # First step, get the hard-dependency graph and create SoftDependencyNodes for each hard-dependency root. In other word, we want - # only the nodes that are not hard-dependency of other nodes. These nodes are taken as roots for the soft-dependency graph because they - # are independant. - soft_dep_graphs = Dict{String,Any}(i => 0.0 for i in keys(models)) - not_found = Dict{Symbol,DataType}() - for (organ, model) in models - # organ = "Leaf"; model = models[organ] - mods = parse_models(get_models(model)) - - # Move some models below others when they are manually linked (hard-dependency): - hard_deps = hard_dependencies((; mods...), verbose=verbose) - d_vars = Dict{Symbol,Vector{Pair{Symbol,NamedTuple}}}() - for (procname, node) in hard_deps.roots - var = Pair{Symbol,NamedTuple}[] - traverse_dependency_graph!(node, x -> variables_multiscale(x, organ, mapping), var) - push!(d_vars, procname => var) - end - - inputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( - key => [j.first => j.second.inputs for j in val] for (key, val) in d_vars - ) - outputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( - key => [j.first => j.second.outputs for j in val] for (key, val) in d_vars - ) - - soft_dep_graph = Dict( - process_ => SoftDependencyNode( - soft_dep_vars.value, - process_, # process name - organ, # scale - AbstractTrees.children(soft_dep_vars), # hard dependencies - nothing, - nothing, - SoftDependencyNode[], - [0] # Vector of zeros of length = number of time-steps - ) - for (process_, soft_dep_vars) in hard_deps.roots - ) - - soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process, outputs=outputs_process) - not_found = merge(not_found, hard_deps.not_found) - end - - # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the - # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the - # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children - # of the nodes that they depend on. - independant_process_root = Dict{Pair{String,Symbol},SoftDependencyNode}() - for (organ, (soft_dep_graph, ins, outs)) in soft_dep_graphs # e.g. organ = "Plant"; soft_dep_graph, ins, outs = soft_dep_graphs[organ] - for (proc, i) in soft_dep_graph - # proc = :carbon_allocation; i = soft_dep_graph[proc] - # Search if the process has soft dependencies: - soft_deps = search_inputs_in_output(proc, ins, outs) - - # Remove the hard dependencies from the soft dependencies: - soft_deps_not_hard = drop_process(soft_deps, [hd.process for hd in i.hard_dependency]) - # NB: if a node is already a hard dependency of the node, it cannot be a soft dependency - - # Check if the process has soft dependencies at other scales: - soft_deps_multiscale = search_inputs_in_multiscale_output(proc, organ, ins, soft_dep_graphs) - # Example output: "Soil" => Dict(:soil_water=>[:soil_water_content]), which means that the variable :soil_water_content - # is computed by the process :soil_water at the scale "Soil". - - if length(soft_deps_not_hard) == 0 && i.process in keys(soft_dep_graph) && length(soft_deps_multiscale) == 0 - # If the process has no soft (multiscale) dependencies, then it is independant (so it is a root) - # Note that the process is only independent if it is also a root in the hard-dependency graph - independant_process_root[organ=>proc] = i - else - # If the process has soft dependencies at its scale, add it: - if length(soft_deps_not_hard) > 0 - # If the process has soft dependencies, then it is not independant - # and we need to add its parent(s) to the node, and the node as a child - for (parent_soft_dep, soft_dep_vars) in pairs(soft_deps_not_hard) - # parent_soft_dep = :process5; soft_dep_vars = soft_deps[parent_soft_dep] - - # preventing a cyclic dependency - if parent_soft_dep == proc - error("Cyclic model dependency detected for process $proc") - end - - # preventing a cyclic dependency: if the parent also has a dependency on the current node: - if soft_dep_graph[parent_soft_dep].parent !== nothing && any([i == p for p in soft_dep_graph[parent_soft_dep].parent]) - error( - "Cyclic dependency detected for process $proc:", - " $proc depends on $parent_soft_dep, which depends on $proc.", - " This is not allowed, but is possible via a hard dependency." - ) - end - - # preventing a cyclic dependency: if the current node has the parent node as a child: - if i.children !== nothing && any([soft_dep_graph[parent_soft_dep] == p for p in i.children]) - error( - "Cyclic dependency detected for process $proc:", - " $proc depends on $parent_soft_dep, which depends on $proc.", - " This is not allowed, but is possible via a hard dependency." - ) - end - - # Add the current node as a child of the node on which it depends - push!(soft_dep_graph[parent_soft_dep].children, i) - - # Add the node on which the current node depends as a parent - if i.parent === nothing - # If the node had no parent already, it is nothing, so we change into a vector - i.parent = [soft_dep_graph[parent_soft_dep]] - else - push!(i.parent, soft_dep_graph[parent_soft_dep]) - end - - # Add the soft dependencies (variables) of the parent to the current node - i.parent_vars = soft_deps - end - end - - # If the node has soft dependencies at other scales, add it as child of the other scale (and add its parent too): - if length(soft_deps_multiscale) > 0 - #! Continue here: add the node as a child of the other scale, and add this other node has its parent. - #! Take inspiration from the code above happening at the same scale. - #! Note that the node can have both soft dependencies at its own scale and at other scales, but it is not - #! a big deal because in the end we drop the scales and only keep the root soft-dependency nodes. - for org in keys(soft_deps_multiscale) - # org = "Leaf" - for (parent_soft_dep, soft_dep_vars) in soft_deps_multiscale[org] - # parent_soft_dep= :photosynthesis; soft_dep_vars = soft_deps_multiscale[org][parent_soft_dep] - parent_node = soft_dep_graphs[org][:soft_dep_graph][parent_soft_dep] - # preventing a cyclic dependency: if the parent also has a dependency on the current node: - if parent_node.parent !== nothing && any([i == p for p in parent_node.parent]) - error( - "Cyclic dependency detected for process $proc:", - " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one", - " This is not allowed, you may need to develop a new process that does the whole computation by itself." - ) - end - - # preventing a cyclic dependency: if the current node has the parent node as a child: - if i.children !== nothing && any([parent_node == p for p in i.children]) - error( - "Cyclic dependency detected for process $proc:", - " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one.", - " This is not allowed, you may need to develop a new process that does the whole computation by itself." - ) - end - - # Add the current node as a child of the node on which it depends: - push!(parent_node.children, i) - - # Add the node on which the current node depends as a parent - if i.parent === nothing - # If the node had no parent already, it is nothing, so we change into a vector - i.parent = [parent_node] - else - push!(i.parent, parent_node) - end - - # Add the multiscale soft dependencies variables of the parent to the current node - i.parent_vars = NamedTuple(Symbol(k) => NamedTuple(v) for (k, v) in soft_deps_multiscale) - end - end - end - #! To do: make this code work without multiscale mapping, so we have only one code base for both cases. - #! Also, put some parts into functions to make the code more readable. - end - end - end - - #! CONTINUE HERE: use the multiscale variables to compute the dependency graph. The steps would look something like: - #! 1. Get the hard-dependency graph - #! 2. Get the soft-dependency graph: do as before by computing inputs and outputs for each hard-dependency root, - #! but also look at the multiscale variables to the inputs and outputs. - #! 3. For each soft-dependency root, check if the process is independant (i.e. if it has no soft-dependencies). - #! within its own scale, but also in other scales. If it is independant, then it is a root of the soft-dependency multiscale graph. - #! If it is not, then add it as a child of the other soft-dependency node that it depends on. - - return DependencyGraph(independant_process_root, not_found) -end diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index d9ed3d94..03618b55 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -467,7 +467,7 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion end # Compute the multi-scale dependency graph of the models: - dependency_graph = multiscale_dep(mapping, verbose=verbose) + dependency_graph = dep(mapping, verbose=verbose) models = Dict(first(m) => parse_models(get_models(last(m))) for m in mapping) @@ -630,18 +630,18 @@ julia> organs_statuses["Soil"][:soil_water_content] === organs_statuses["Leaf"][ true ``` """ -function status_template(models::Dict{String,T}, type_promotion) where {T} - organs_mapping, var_outputs_from_mapping = compute_mapping(models, type_promotion) +function status_template(mapping::Dict{String,T}, type_promotion) where {T} + organs_mapping, var_outputs_from_mapping = compute_mapping(mapping, type_promotion) # Vector of pre-initialised variables with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() dict_mapped_vars = Dict{Pair,Any}() - for organ in keys(models) # e.g.: organ = "Internode" + for organ in keys(mapping) # e.g.: organ = "Internode" # Parsing the models into a NamedTuple to get the process name: - node_models = parse_models(get_models(models[organ])) + node_models = parse_models(get_models(mapping[organ])) # Get the status if any was given by the user (this can be used as default values in the mapping): - st = get_status(models[organ]) # User status + st = get_status(mapping[organ]) # User status if isnothing(st) st = NamedTuple() diff --git a/src/run.jl b/src/run.jl index f77cfe85..7fff4a27 100644 --- a/src/run.jl +++ b/src/run.jl @@ -1,5 +1,6 @@ """ run!(object, meteo, constants, extra=nothing; check=true, executor=Floops.ThreadedEx()) + run!(object, mapping, meteo, constants, extra; nsteps, outputs, check, executor) Run the simulation for each model in the model list in the correct order, *i.e.* respecting the dependency graph. @@ -8,7 +9,7 @@ If several time-steps are given, the models are run sequentially for each time-s # Arguments -- `object`: a [`ModelList`](@ref), an array or dict of `ModelList`, or an MTG. +- `object`: a [`ModelList`](@ref), an array or dict of `ModelList`, or a plant graph (MTG). - `meteo`: a [`PlantMeteo.TimeStepTable`](https://palmstudio.github.io/PlantMeteo.jl/stable/API/#PlantMeteo.TimeStepTable) of [`PlantMeteo.Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/API/#PlantMeteo.Atmosphere) or a single `PlantMeteo.Atmosphere`. - `constants`: a [`PlantMeteo.Constants`](https://palmstudio.github.io/PlantMeteo.jl/stable/API/#PlantMeteo.Constants) object, or a `NamedTuple` of constant keys and values. @@ -16,6 +17,9 @@ If several time-steps are given, the models are run sequentially for each time-s - `check`: if `true`, check the validity of the model list before running the simulation (takes a little bit of time), and return more information while running. - `executor`: the [`Floops`](https://juliafolds.github.io/FLoops.jl/stable/) executor used to run the simulation either in sequential (`executor=SequentialEx()`), in a multi-threaded way (`executor=ThreadedEx()`, the default), or in a distributed way (`executor=DistributedEx()`). +- `mapping`: a mapping between the MTG and the model list. +- `nsteps`: the number of time-steps to run, only needed if no meteo is given (else it is infered from it). +- `outputs`: the outputs to get in dynamic for each node type of the MTG. # Returns @@ -469,8 +473,6 @@ function run!( @floop executor for st in node_statuses # for each node status at the current scale (potentially in parallel over nodes) # Actual call to the model: run!(node.value, models_at_scale, st, meteo, constants, extra) - - #TODO: keep track of the outputs users need here. end node.simulation_id[1] += 1 # increment the simulation id, to remember that the model has been called already From f7dd52b8cc7d43e4100ed074eefe5b0db92db1ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Sat, 7 Oct 2023 16:47:22 +0200 Subject: [PATCH 67/97] Divide more the dep method for multiscale model mapping --- src/dependencies/dependencies.jl | 181 ++------------------------ src/dependencies/hard_dependencies.jl | 53 ++++++++ src/dependencies/soft_dependencies.jl | 124 ++++++++++++++++++ 3 files changed, 185 insertions(+), 173 deletions(-) diff --git a/src/dependencies/dependencies.jl b/src/dependencies/dependencies.jl index 603142b5..7de48a62 100644 --- a/src/dependencies/dependencies.jl +++ b/src/dependencies/dependencies.jl @@ -54,7 +54,7 @@ models = ModelList( dep(models) # or directly with the processes: -vars = ( +models = ( process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), @@ -64,11 +64,11 @@ vars = ( process7=Process7Model(), ) -dep(;vars...) +dep(;models...) ``` """ -function dep(nsteps=1; verbose::Bool=true, vars...) - hard_dep = hard_dependencies((; vars...), verbose=verbose) +function dep(nsteps::Int=1; verbose::Bool=true, models...) + hard_dep = hard_dependencies((; models...), verbose=verbose) deps = soft_dependencies(hard_dep, nsteps) # Return the dependency graph @@ -84,182 +84,17 @@ function dep(m::NamedTuple, nsteps=1; verbose::Bool=true) dep(nsteps; verbose=verbose, m...) end -function dep(mapping::Dict{String,T}; verbose=true) where {T} - - full_mapping = Dict(first(mod) => Dict(get_mapping(last(mod))) for mod in mapping) - +function dep(mapping::Dict{String,T}; verbose::Bool=true) where {T} # First step, get the hard-dependency graph and create SoftDependencyNodes for each hard-dependency root. In other word, we want # only the nodes that are not hard-dependency of other nodes. These nodes are taken as roots for the soft-dependency graph because they # are independant. - soft_dep_graphs = Dict{String,Any}(i => 0.0 for i in keys(mapping)) - not_found = Dict{Symbol,DataType}() - for (organ, model) in mapping - # organ = "Leaf"; model = mapping[organ] - mods = parse_models(get_models(model)) - - # Move some models below others when they are manually linked (hard-dependency): - hard_deps = hard_dependencies((; mods...), verbose=verbose) - d_vars = Dict{Symbol,Vector{Pair{Symbol,NamedTuple}}}() - for (procname, node) in hard_deps.roots - var = Pair{Symbol,NamedTuple}[] - traverse_dependency_graph!(node, x -> variables_multiscale(x, organ, full_mapping), var) - push!(d_vars, procname => var) - end - - inputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( - key => [j.first => j.second.inputs for j in val] for (key, val) in d_vars - ) - outputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( - key => [j.first => j.second.outputs for j in val] for (key, val) in d_vars - ) - - soft_dep_graph = Dict( - process_ => SoftDependencyNode( - soft_dep_vars.value, - process_, # process name - organ, # scale - AbstractTrees.children(soft_dep_vars), # hard dependencies - nothing, - nothing, - SoftDependencyNode[], - [0] # Vector of zeros of length = number of time-steps - ) - for (process_, soft_dep_vars) in hard_deps.roots - ) - - soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process, outputs=outputs_process) - not_found = merge(not_found, hard_deps.not_found) - end + soft_dep_graphs_roots = hard_dependencies(mapping; verbose=verbose) # Second step, compute the soft-dependency graph between SoftDependencyNodes computed in the first step. To do so, we search the # inputs of each process into the outputs of the other processes, at the same scale, but also between scales. Then we keep only the # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children # of the nodes that they depend on. - independant_process_root = Dict{Pair{String,Symbol},SoftDependencyNode}() - for (organ, (soft_dep_graph, ins, outs)) in soft_dep_graphs # e.g. organ = "Plant"; soft_dep_graph, ins, outs = soft_dep_graphs[organ] - for (proc, i) in soft_dep_graph - # proc = :carbon_allocation; i = soft_dep_graph[proc] - # Search if the process has soft dependencies: - soft_deps = search_inputs_in_output(proc, ins, outs) - - # Remove the hard dependencies from the soft dependencies: - soft_deps_not_hard = drop_process(soft_deps, [hd.process for hd in i.hard_dependency]) - # NB: if a node is already a hard dependency of the node, it cannot be a soft dependency - - # Check if the process has soft dependencies at other scales: - soft_deps_multiscale = search_inputs_in_multiscale_output(proc, organ, ins, soft_dep_graphs) - # Example output: "Soil" => Dict(:soil_water=>[:soil_water_content]), which means that the variable :soil_water_content - # is computed by the process :soil_water at the scale "Soil". - - if length(soft_deps_not_hard) == 0 && i.process in keys(soft_dep_graph) && length(soft_deps_multiscale) == 0 - # If the process has no soft (multiscale) dependencies, then it is independant (so it is a root) - # Note that the process is only independent if it is also a root in the hard-dependency graph - independant_process_root[organ=>proc] = i - else - # If the process has soft dependencies at its scale, add it: - if length(soft_deps_not_hard) > 0 - # If the process has soft dependencies, then it is not independant - # and we need to add its parent(s) to the node, and the node as a child - for (parent_soft_dep, soft_dep_vars) in pairs(soft_deps_not_hard) - # parent_soft_dep = :process5; soft_dep_vars = soft_deps[parent_soft_dep] - - # preventing a cyclic dependency - if parent_soft_dep == proc - error("Cyclic model dependency detected for process $proc") - end - - # preventing a cyclic dependency: if the parent also has a dependency on the current node: - if soft_dep_graph[parent_soft_dep].parent !== nothing && any([i == p for p in soft_dep_graph[parent_soft_dep].parent]) - error( - "Cyclic dependency detected for process $proc:", - " $proc depends on $parent_soft_dep, which depends on $proc.", - " This is not allowed, but is possible via a hard dependency." - ) - end - - # preventing a cyclic dependency: if the current node has the parent node as a child: - if i.children !== nothing && any([soft_dep_graph[parent_soft_dep] == p for p in i.children]) - error( - "Cyclic dependency detected for process $proc:", - " $proc depends on $parent_soft_dep, which depends on $proc.", - " This is not allowed, but is possible via a hard dependency." - ) - end - - # Add the current node as a child of the node on which it depends - push!(soft_dep_graph[parent_soft_dep].children, i) - - # Add the node on which the current node depends as a parent - if i.parent === nothing - # If the node had no parent already, it is nothing, so we change into a vector - i.parent = [soft_dep_graph[parent_soft_dep]] - else - push!(i.parent, soft_dep_graph[parent_soft_dep]) - end - - # Add the soft dependencies (variables) of the parent to the current node - i.parent_vars = soft_deps - end - end - - # If the node has soft dependencies at other scales, add it as child of the other scale (and add its parent too): - if length(soft_deps_multiscale) > 0 - #! Continue here: add the node as a child of the other scale, and add this other node has its parent. - #! Take inspiration from the code above happening at the same scale. - #! Note that the node can have both soft dependencies at its own scale and at other scales, but it is not - #! a big deal because in the end we drop the scales and only keep the root soft-dependency nodes. - for org in keys(soft_deps_multiscale) - # org = "Leaf" - for (parent_soft_dep, soft_dep_vars) in soft_deps_multiscale[org] - # parent_soft_dep= :photosynthesis; soft_dep_vars = soft_deps_multiscale[org][parent_soft_dep] - parent_node = soft_dep_graphs[org][:soft_dep_graph][parent_soft_dep] - # preventing a cyclic dependency: if the parent also has a dependency on the current node: - if parent_node.parent !== nothing && any([i == p for p in parent_node.parent]) - error( - "Cyclic dependency detected for process $proc:", - " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one", - " This is not allowed, you may need to develop a new process that does the whole computation by itself." - ) - end - - # preventing a cyclic dependency: if the current node has the parent node as a child: - if i.children !== nothing && any([parent_node == p for p in i.children]) - error( - "Cyclic dependency detected for process $proc:", - " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one.", - " This is not allowed, you may need to develop a new process that does the whole computation by itself." - ) - end - - # Add the current node as a child of the node on which it depends: - push!(parent_node.children, i) - - # Add the node on which the current node depends as a parent - if i.parent === nothing - # If the node had no parent already, it is nothing, so we change into a vector - i.parent = [parent_node] - else - push!(i.parent, parent_node) - end - - # Add the multiscale soft dependencies variables of the parent to the current node - i.parent_vars = NamedTuple(Symbol(k) => NamedTuple(v) for (k, v) in soft_deps_multiscale) - end - end - end - #! To do: make this code work without multiscale mapping, so we have only one code base for both cases. - #! Also, put some parts into functions to make the code more readable. - end - end - end - - #! CONTINUE HERE: use the multiscale variables to compute the dependency graph. The steps would look something like: - #! 1. Get the hard-dependency graph - #! 2. Get the soft-dependency graph: do as before by computing inputs and outputs for each hard-dependency root, - #! but also look at the multiscale variables to the inputs and outputs. - #! 3. For each soft-dependency root, check if the process is independant (i.e. if it has no soft-dependencies). - #! within its own scale, but also in other scales. If it is independant, then it is a root of the soft-dependency multiscale graph. - #! If it is not, then add it as a child of the other soft-dependency node that it depends on. + dep_graph = soft_dependencies_multiscale(soft_dep_graphs_roots) - return DependencyGraph(independant_process_root, not_found) + return dep_graph end diff --git a/src/dependencies/hard_dependencies.jl b/src/dependencies/hard_dependencies.jl index 3d91d23c..47a195ed 100644 --- a/src/dependencies/hard_dependencies.jl +++ b/src/dependencies/hard_dependencies.jl @@ -1,3 +1,9 @@ +""" + hard_dependencies(models; verbose::Bool=true) + hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) + +Compute the hard dependencies between models. +""" function hard_dependencies(models; verbose::Bool=true) dep_graph = Dict( p => HardDependencyNode( @@ -69,4 +75,51 @@ function hard_dependencies(models; verbose::Bool=true) end return DependencyGraph(unique_roots, dep_not_found) +end + +# When we use a mapping (multiscale), we return the set of soft-dependencies (we put the hard-dependencies as their children): +function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) where {T} + full_mapping = Dict(first(mod) => Dict(get_mapping(last(mod))) for mod in mapping) + + soft_dep_graphs = Dict{String,Any}(i => 0.0 for i in keys(mapping)) + not_found = Dict{Symbol,DataType}() + for (organ, model) in mapping + # organ = "Leaf"; model = mapping[organ] + mods = parse_models(get_models(model)) + + # Move some models below others when they are manually linked (hard-dependency): + hard_deps = hard_dependencies((; mods...), verbose=verbose) + d_vars = Dict{Symbol,Vector{Pair{Symbol,NamedTuple}}}() + for (procname, node) in hard_deps.roots + var = Pair{Symbol,NamedTuple}[] + traverse_dependency_graph!(node, x -> variables_multiscale(x, organ, full_mapping), var) + push!(d_vars, procname => var) + end + + inputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( + key => [j.first => j.second.inputs for j in val] for (key, val) in d_vars + ) + outputs_process = Dict{Symbol,Vector{Pair{Symbol,Tuple{Vararg{Union{Symbol,MappedVar}}}}}}( + key => [j.first => j.second.outputs for j in val] for (key, val) in d_vars + ) + + soft_dep_graph = Dict( + process_ => SoftDependencyNode( + soft_dep_vars.value, + process_, # process name + organ, # scale + AbstractTrees.children(soft_dep_vars), # hard dependencies + nothing, + nothing, + SoftDependencyNode[], + [0] # Vector of zeros of length = number of time-steps + ) + for (process_, soft_dep_vars) in hard_deps.roots + ) + + soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process, outputs=outputs_process) + not_found = merge(not_found, hard_deps.not_found) + end + + return DependencyGraph(soft_dep_graphs, not_found) end \ No newline at end of file diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index 514ed8d3..a6be6271 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -135,6 +135,130 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, return DependencyGraph(independant_process_root, d.not_found) end +# For multiscale mapping: +function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dict{String,Any}}) + independant_process_root = Dict{Pair{String,Symbol},SoftDependencyNode}() + for (organ, (soft_dep_graph, ins, outs)) in soft_dep_graphs_roots.roots # e.g. organ = "Plant"; soft_dep_graph, ins, outs = soft_dep_graphs[organ] + for (proc, i) in soft_dep_graph + # proc = :carbon_allocation; i = soft_dep_graph[proc] + # Search if the process has soft dependencies: + soft_deps = search_inputs_in_output(proc, ins, outs) + + # Remove the hard dependencies from the soft dependencies: + soft_deps_not_hard = drop_process(soft_deps, [hd.process for hd in i.hard_dependency]) + # NB: if a node is already a hard dependency of the node, it cannot be a soft dependency + + # Check if the process has soft dependencies at other scales: + soft_deps_multiscale = search_inputs_in_multiscale_output(proc, organ, ins, soft_dep_graphs_roots.roots) + # Example output: "Soil" => Dict(:soil_water=>[:soil_water_content]), which means that the variable :soil_water_content + # is computed by the process :soil_water at the scale "Soil". + + if length(soft_deps_not_hard) == 0 && i.process in keys(soft_dep_graph) && length(soft_deps_multiscale) == 0 + # If the process has no soft (multiscale) dependencies, then it is independant (so it is a root) + # Note that the process is only independent if it is also a root in the hard-dependency graph + independant_process_root[organ=>proc] = i + else + # If the process has soft dependencies at its scale, add it: + if length(soft_deps_not_hard) > 0 + # If the process has soft dependencies, then it is not independant + # and we need to add its parent(s) to the node, and the node as a child + for (parent_soft_dep, soft_dep_vars) in pairs(soft_deps_not_hard) + # parent_soft_dep = :process5; soft_dep_vars = soft_deps[parent_soft_dep] + + # preventing a cyclic dependency + if parent_soft_dep == proc + error("Cyclic model dependency detected for process $proc") + end + + # preventing a cyclic dependency: if the parent also has a dependency on the current node: + if soft_dep_graph[parent_soft_dep].parent !== nothing && any([i == p for p in soft_dep_graph[parent_soft_dep].parent]) + error( + "Cyclic dependency detected for process $proc:", + " $proc depends on $parent_soft_dep, which depends on $proc.", + " This is not allowed, but is possible via a hard dependency." + ) + end + + # preventing a cyclic dependency: if the current node has the parent node as a child: + if i.children !== nothing && any([soft_dep_graph[parent_soft_dep] == p for p in i.children]) + error( + "Cyclic dependency detected for process $proc:", + " $proc depends on $parent_soft_dep, which depends on $proc.", + " This is not allowed, but is possible via a hard dependency." + ) + end + + # Add the current node as a child of the node on which it depends + push!(soft_dep_graph[parent_soft_dep].children, i) + + # Add the node on which the current node depends as a parent + if i.parent === nothing + # If the node had no parent already, it is nothing, so we change into a vector + i.parent = [soft_dep_graph[parent_soft_dep]] + else + push!(i.parent, soft_dep_graph[parent_soft_dep]) + end + + # Add the soft dependencies (variables) of the parent to the current node + i.parent_vars = soft_deps + end + end + + # If the node has soft dependencies at other scales, add it as child of the other scale (and add its parent too): + if length(soft_deps_multiscale) > 0 + #! Continue here: add the node as a child of the other scale, and add this other node has its parent. + #! Take inspiration from the code above happening at the same scale. + #! Note that the node can have both soft dependencies at its own scale and at other scales, but it is not + #! a big deal because in the end we drop the scales and only keep the root soft-dependency nodes. + for org in keys(soft_deps_multiscale) + # org = "Leaf" + for (parent_soft_dep, soft_dep_vars) in soft_deps_multiscale[org] + # parent_soft_dep= :photosynthesis; soft_dep_vars = soft_deps_multiscale[org][parent_soft_dep] + parent_node = soft_dep_graphs_roots.roots[org][:soft_dep_graph][parent_soft_dep] + # preventing a cyclic dependency: if the parent also has a dependency on the current node: + if parent_node.parent !== nothing && any([i == p for p in parent_node.parent]) + error( + "Cyclic dependency detected for process $proc:", + " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one", + " This is not allowed, you may need to develop a new process that does the whole computation by itself." + ) + end + + # preventing a cyclic dependency: if the current node has the parent node as a child: + if i.children !== nothing && any([parent_node == p for p in i.children]) + error( + "Cyclic dependency detected for process $proc:", + " $proc for organ $organ depends on $parent_soft_dep from organ $org, which depends on the first one.", + " This is not allowed, you may need to develop a new process that does the whole computation by itself." + ) + end + + # Add the current node as a child of the node on which it depends: + push!(parent_node.children, i) + + # Add the node on which the current node depends as a parent + if i.parent === nothing + # If the node had no parent already, it is nothing, so we change into a vector + i.parent = [parent_node] + else + push!(i.parent, parent_node) + end + + # Add the multiscale soft dependencies variables of the parent to the current node + i.parent_vars = NamedTuple(Symbol(k) => NamedTuple(v) for (k, v) in soft_deps_multiscale) + end + end + end + #! To do: make this code work without multiscale mapping, so we have only one code base for both cases. + #! Also, put some parts into functions to make the code more readable. + end + end + end + + return DependencyGraph(independant_process_root, soft_dep_graphs_roots.not_found) +end + + """ drop_process(proc_vars, process) From de672c6e0ce97f1b12f428ae927e12a6e5bcfcc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Sat, 7 Oct 2023 17:32:59 +0200 Subject: [PATCH 68/97] Update dependencies.jl --- src/dependencies/dependencies.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dependencies/dependencies.jl b/src/dependencies/dependencies.jl index 7de48a62..5a714d39 100644 --- a/src/dependencies/dependencies.jl +++ b/src/dependencies/dependencies.jl @@ -67,8 +67,8 @@ models = ( dep(;models...) ``` """ -function dep(nsteps::Int=1; verbose::Bool=true, models...) - hard_dep = hard_dependencies((; models...), verbose=verbose) +function dep(nsteps=1; verbose::Bool=true, vars...) + hard_dep = hard_dependencies((; vars...), verbose=verbose) deps = soft_dependencies(hard_dep, nsteps) # Return the dependency graph From d1036920a138203fa663439e6ae240025c71245c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 9 Oct 2023 09:16:45 +0200 Subject: [PATCH 69/97] Add methods for mapping I/O --- src/processes/models_inputs_outputs.jl | 47 +++++++++++++++++++++++++- test/test-mtg-multiscale.jl | 20 +++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/processes/models_inputs_outputs.jl b/src/processes/models_inputs_outputs.jl index 974e3f77..923887ff 100644 --- a/src/processes/models_inputs_outputs.jl +++ b/src/processes/models_inputs_outputs.jl @@ -36,6 +36,21 @@ function inputs_(model::Missing) NamedTuple() end +""" + inputs(mapping::Dict{String,T}) + +Get the inputs of the models in a mapping, for each process and organ type. +""" +function inputs(mapping::Dict{String,T}) where {T} + vars = Dict{String,NamedTuple}() + for organ in keys(mapping) + mods = pairs(parse_models(get_models(mapping[organ]))) + push!(vars, organ => (; (i.first => (inputs(i.second)...,) for i in mods)...)) + end + return vars +end + + """ outputs(model::AbstractModel) outputs(...) @@ -66,6 +81,21 @@ function outputs(v::T, vars...) where {T<:AbstractModel} length((vars...,)) > 0 ? union(outputs(v), outputs(vars...)) : outputs(v) end +""" + outputs(mapping::Dict{String,T}) + +Get the outputs of the models in a mapping, for each process and organ type. +""" +function outputs(mapping::Dict{String,T}) where {T} + vars = Dict{String,NamedTuple}() + for organ in keys(mapping) + mods = pairs(parse_models(get_models(mapping[organ]))) + push!(vars, organ => (; (i.first => (outputs(i.second)...,) for i in mods)...)) + end + return vars +end + + function outputs_(model::AbstractModel) NamedTuple() end @@ -157,6 +187,21 @@ function variables(pkg::Module) sort!(CSV.read(joinpath(dirname(dirname(pathof(pkg))), "data", "variables.csv"), DataFrames.DataFrame)) end +""" + variables(mapping::Dict{String,T}) + +Get the variables (inputs and outputs) of the models in a mapping, for each +process and organ type. +""" +function variables(mapping::Dict{String,T}) where {T} + vars = Dict{String,NamedTuple}() + for organ in keys(mapping) + mods = pairs(parse_models(get_models(mapping[organ]))) + push!(vars, organ => (; (i.first => (variables(i.second)...,) for i in mods)...)) + end + return vars +end + """ variables_typed(model) variables_typed(model, models...) @@ -246,4 +291,4 @@ function diff_vars(x, y) end end return vars_different_types -end +end \ No newline at end of file diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index d37d2d92..4278a60d 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -51,6 +51,26 @@ mtg = begin end +@testset "inputs and outputs of a mapping" begin + ins = inputs(mapping_1) + + @test collect(keys(ins)) == collect(keys(mapping_1)) + @test ins["Soil"] == (soil_water=(),) + @test ins["Leaf"] == (photosynthesis=(:aPPFD, :soil_water_content), carbon_demand=(:TT,)) + + outs = outputs(mapping_1) + @test collect(keys(outs)) == collect(keys(mapping_1)) + @test outs["Soil"] == (soil_water=(:soil_water_content,),) + @test outs["Leaf"] == (photosynthesis=(:A,), carbon_demand=(:carbon_demand,)) + @test outs["Plant"] == (carbon_allocation=(:carbon_offer, :carbon_allocation),) + + vars = variables(mapping_1) + @test collect(keys(vars)) == collect(keys(mapping_1)) + @test vars["Soil"] == outs["Soil"] + @test vars["Plant"] == (carbon_allocation=(:A, :carbon_demand, :carbon_offer, :carbon_allocation),) + @test vars["Leaf"] == (photosynthesis=(:aPPFD, :soil_water_content, :A), carbon_demand=(:TT, :carbon_demand)) +end + @testset "status_template" begin organs_statuses = PlantSimEngine.status_template(mapping_1, nothing) @test collect(keys(organs_statuses)) == ["Soil", "Internode", "Plant", "Leaf"] From 49cf25e103138e401bb4d611ac27402ba8913092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 10 Oct 2023 11:22:58 +0200 Subject: [PATCH 70/97] Fix doc --- docs/Project.toml | 1 + docs/make.jl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index e4dfd4be..41aedce0 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,5 +3,6 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" PlantMeteo = "4630fe09-e0fb-4da5-a846-781cb73437b6" PlantSimEngine = "9a576370-710b-4269-adf9-4f603a9c6423" diff --git a/docs/make.jl b/docs/make.jl index eaa53ad0..28c34f40 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -39,6 +39,6 @@ makedocs(; ) deploydocs(; - repo="github.com/VirtualPlantLab/PlantSimEngine.jl", + repo="github.com/VirtualPlantLab/PlantSimEngine.jl.git", devbranch="main" ) From c9eade89934cb5655b7c82bde036579122f12a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 13 Oct 2023 09:50:30 +0200 Subject: [PATCH 71/97] Update TimeStepTable.jl Remove schema (waiting for better way from PlantMeteo) --- src/component_models/TimeStepTable.jl | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/component_models/TimeStepTable.jl b/src/component_models/TimeStepTable.jl index 48ad495f..d0225353 100644 --- a/src/component_models/TimeStepTable.jl +++ b/src/component_models/TimeStepTable.jl @@ -48,12 +48,15 @@ function PlantMeteo.TimeStepTable{Status}(df::DataFrames.DataFrame, metadata=Nam PlantMeteo.TimeStepTable((propertynames(df)...,), metadata, [Status(NamedTuple(ts)) for ts in Tables.rows(df)]) end -""" - Tables.schema(m::TimeStepTable{Status}) - -Create a schema for a `TimeStepTable{Status}`. -""" -function Tables.schema(m::PlantMeteo.TimeStepTable{T}) where {T<:Status} - # This one is complicated because the types of the variables are hidden in the Status as RefValues: - Tables.Schema(names(m), DataType[i.types[1] for i in T.parameters[2].parameters]) -end \ No newline at end of file +# """ +# Tables.schema(m::TimeStepTable{Status}) + +# Create a schema for a `TimeStepTable{Status}`. +# """ +# function Tables.schema(m::PlantMeteo.TimeStepTable{T}) where {T<:Status} +# # This one is complicated because the types of the variables are hidden in the Status as RefValues: +# # col_types = fieldtypes(getfield(m, :ts)[1]) + +# # # Tables.Schema(names(m), DataType[i.types[1] for i in T.parameters[2].parameters]) +# # Tables.Schema(names(m), col_types) +# end \ No newline at end of file From 8c84145a0f4bb01d78b1094b6729d983dcf7a747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 13 Oct 2023 09:50:59 +0200 Subject: [PATCH 72/97] Clarify status/outputs from GraphSimulation + clean-up the code --- src/PlantSimEngine.jl | 5 + src/doc_templates/mtg-related.jl | 78 +++++++++++ src/mtg/GraphSimulation.jl | 125 +++++++++++++++++ src/mtg/mapping.jl | 231 +------------------------------ src/mtg/save_results.jl | 205 +++++++++++++++++++++++++++ test/test-mtg-multiscale.jl | 16 +++ 6 files changed, 433 insertions(+), 227 deletions(-) create mode 100644 src/doc_templates/mtg-related.jl create mode 100644 src/mtg/GraphSimulation.jl create mode 100644 src/mtg/save_results.jl diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 9729452b..e18b0a39 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -23,6 +23,9 @@ import Statistics using PlantMeteo +# Docs templates: +include("doc_templates/mtg-related.jl") + # Models: include("Abstract_model_structs.jl") @@ -50,8 +53,10 @@ include("dependencies/hard_dependencies.jl") include("dependencies/dependencies.jl") # MTG compatibility: +include("mtg/GraphSimulation.jl") include("mtg/init_mtg_models.jl") include("mtg/mapping.jl") +include("mtg/save_results.jl") # Model evaluation (statistics): include("evaluation/statistics.jl") diff --git a/src/doc_templates/mtg-related.jl b/src/doc_templates/mtg-related.jl new file mode 100644 index 00000000..0e00027b --- /dev/null +++ b/src/doc_templates/mtg-related.jl @@ -0,0 +1,78 @@ +const MTG_EXAMPLE = """ +```@example +mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)); +``` + +```@example +soil = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)); +``` + +```@example +plant = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)); +``` + +```@example +internode1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)); +``` + +```@example +leaf1 = Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); +``` + +```@example +internode2 = Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)); +``` + +```@example +leaf2 = Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); +``` +""" + +const MAPPING_EXAMPLE = """ +```@example +mapping = Dict( \ + "Plant" => \ + MultiScaleModel( \ + model=ToyCAllocationModel(), \ + mapping=[ \ + :A => ["Leaf"], \ + :carbon_demand => ["Leaf", "Internode"], \ + :carbon_allocation => ["Leaf", "Internode"] \ + ], \ + ), \ + "Internode" => ( \ + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + Status(TT=10.0) \ + ), \ + "Leaf" => ( \ + MultiScaleModel( \ + model=ToyAssimModel(), \ + mapping=[:soil_water_content => "Soil",], \ + ), \ + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + Status(aPPFD=1300.0, TT=10.0), \ + ), \ + "Soil" => ( \ + ToySoilWaterModel(), \ + ), \ +) +``` +""" + +const EXAMPLE_MTG_MODELS = """ +```@example +include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +``` + +```@example +include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +``` + +```@example +include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +``` + +```@example +include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +``` +""" \ No newline at end of file diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl new file mode 100644 index 00000000..4c6c0ce7 --- /dev/null +++ b/src/mtg/GraphSimulation.jl @@ -0,0 +1,125 @@ +""" + GraphSimulation(graph, mapping) + GraphSimulation(graph, statuses, dependency_graph, models, outputs) + +A type that holds all information for a simulation over a graph. + +# Arguments + +- `graph`: an graph, such as an MTG +- `mapping`: a dictionary of model mapping +- `statuses`: a structure that defines the status of each node in the graph +- `dependency_graph`: the dependency graph of the models applied to the graph +- `models`: a dictionary of models +- `outputs`: a dictionary of outputs +""" +struct GraphSimulation{T,S,U,O} + graph::T + statuses::S + dependency_graph::DependencyGraph + models::Dict{String,U} + outputs::Dict{String,O} +end + +function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) + GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose)...) +end + +dep(g::GraphSimulation) = g.dependency_graph +status(g::GraphSimulation) = g.statuses +get_models(g::GraphSimulation) = g.models +outputs(g::GraphSimulation) = g.outputs + +""" + outputs(sim::GraphSimulation, sink) + +Get the outputs from a simulation made on a plant graph. + +# Details + +The first method returns a vector of `NamedTuple`, the second formats it +sing the sink function, for exemple a `DataFrame`. + +# Arguments + +- `sim::GraphSimulation`: the simulation object, typically returned by `run!`. +- `sink`: a sink compatible with the Tables.jl interface (*e.g.* a `DataFrame`) + +# Examples + +```@example +using PlantSimEngine, MultiScaleTreeGraph, DataFrames +``` + +$EXAMPLE_MTG_MODELS + +$MAPPING_EXAMPLE + +$MTG_EXAMPLE + +```@example +sim = run!(mtg, mapping, meteo, outputs = Dict( + "Leaf" => (:A, :carbon_demand, :soil_water_content, :carbon_allocation), + "Internode" => (:carbon_allocation,), + "Plant" => (:carbon_allocation,), + "Soil" => (:soil_water_content,), +)); +``` + +```@example +outputs(sim, DataFrames) +``` +""" +function outputs(sim::GraphSimulation, sink) + @assert Tables.istable(sink) "The sink argument must be compatible with the Tables.jl interface (`Tables.istable(sink)` must return `true`, *e.g.* `DataFrame`)" + + outs = outputs(sim) + + variables_names_types = Iterators.flatten(collect(i.first => eltype(i.second[1]) for i in filter(x -> x.first != :node, vars)) for (organs, vars) in outs) |> collect + variables_names_types_dict = Dict{Symbol,Any}() + + for (k, v) in variables_names_types + if !haskey(variables_names_types_dict, k) + variables_names_types_dict[k] = Union{Nothing,v} + else + variables_names_types_dict[k] = Union{variables_names_types_dict[k],v} + end + end + + + variables_names_types = (timestep=Int, organ=String, node=Int, NamedTuple(variables_names_types_dict)...) + var_names_all = keys(variables_names_types) + t = NamedTuple{var_names_all,Tuple{values(variables_names_types)...}}[] + + for (organ, vars) in outs # organ = "Leaf"; vars = outs[organ] + var_names = setdiff(collect(keys(vars)), [:node]) + if length(var_names) == 0 + continue + end + steps_iterable = axes(vars[var_names[1]], 1) + for timestep in steps_iterable # timestep = 1 + node_iterable = axes(vars[var_names[1]][timestep], 1) + for node in node_iterable # node = 1 + vars_values = (; timestep=timestep, organ=organ, node=vars[:node][timestep][node].id, zip(var_names, [vars[v][timestep][node] for v in var_names])...) + vars_no_values = setdiff(var_names_all, keys(vars_values)) + if length(vars_no_values) > 0 + vars_values = (; vars_values..., zip(vars_no_values, [nothing for v in vars_no_values])...) + end + push!( + t, + NamedTuple{var_names_all}(vars_values) + ) + end + end + end + + return sink(t) +end + +function outputs(sim::GraphSimulation, key::Symbol) + Tables.columns(outputs(sim, Vector{NamedTuple}))[key] +end + +function outputs(sim::GraphSimulation, i::T) where {T<:Integer} + Tables.columns(outputs(sim, Vector{NamedTuple}))[i] +end \ No newline at end of file diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 03618b55..09752544 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -406,7 +406,7 @@ function outputs_from_other_scale!(var_outputs_from_mapping, multi_scale_outs, m end """ - init_simulation(mtg, mapping; type_promotion=nothing, check=true, verbose=true) + init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) Initialise the simulation. Returns: @@ -414,11 +414,14 @@ Initialise the simulation. Returns: - a status for each node by organ type, considering multi-scale variables - the dependency graph of the models - the models parsed as a Dict of organ type => NamedTuple of process => model mapping +- the pre-allocated outputs # Arguments - `mtg`: the MTG - `mapping::Dict{String,Any}`: a dictionary of model mapping +- `nsteps`: the number of steps of the simulation +- `outputs`: the dynamic outputs needed for the simulation - `type_promotion`: the type promotion to use for the variables - `check`: whether to check the mapping for errors - `verbose`: print information about errors in the mapping @@ -476,38 +479,6 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion return (; mtg, statuses, dependency_graph, models, outputs) end -""" - GraphSimulation(graph, mapping) - GraphSimulation(graph, statuses, dependency_graph, models, outputs) - -A type that holds all information for a simulation over a graph. - -# Arguments - -- `graph`: an graph, such as an MTG -- `mapping`: a dictionary of model mapping -- `statuses`: a structure that defines the status of each node in the graph -- `dependency_graph`: the dependency graph of the models applied to the graph -- `models`: a dictionary of models -- `outputs`: a dictionary of outputs -""" -struct GraphSimulation{T,S,U,O} - graph::T - statuses::S - dependency_graph::DependencyGraph - models::Dict{String,U} - outputs::Dict{String,O} -end - -function GraphSimulation(graph, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) - GraphSimulation(init_simulation(graph, mapping; nsteps=nsteps, outputs=outputs, type_promotion=type_promotion, check=check, verbose=verbose)...) -end - -dep(g::GraphSimulation) = g.dependency_graph -status(g::GraphSimulation) = g.statuses -get_models(g::GraphSimulation) = g.models -outputs(g::GraphSimulation) = g.outputs - function map_scale(f, m, scale::String) map_scale(f, m, [scale]) end @@ -516,7 +487,6 @@ function map_scale(f, m, scales::AbstractVector{String}) map(s -> f(m, s), scales) end - # Return an error if some variables are not initialized or computed by other models in the output # from to_initialize(models, organs_statuses) function error_mtg_init(var_need_init) @@ -946,197 +916,4 @@ function variables_multiscale(node, organ, mapping) end return (vars_...,) end -end - - -""" - pre_allocate_outputs(statuses, outs, nsteps; check=true) - -Pre-allocate the outputs of needed variable for each node type in vectors of vectors. -The first level vectors have length nsteps, and the second level vectors have length n_nodes of this type. - -Note that we pre-allocate the vectors for the time-steps, but not for each organ, because we don't -know how many nodes will be in each organ in the future (organs can appear or disapear). - -# Arguments - -- `statuses`: a dictionary of status by node type -- `outs`: a dictionary of outputs by node type -- `nsteps`: the number of time-steps -- `check`: whether to check the mapping for errors. Default (`true`) returns an error if some variables do not exist. -If false and some variables are missing, return an info, remove the unknown variables and continue. - -# Returns - -- A dictionary of pre-allocated output of vector of time-step and vector of node of that type. - -# Examples - -```jldoctest mylabel -julia> using PlantSimEngine, MultiScaleTreeGraph -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); -``` - -```jldoctest mylabel -julia> mapping = Dict( \ - "Plant" => \ - MultiScaleModel( \ - model=ToyCAllocationModel(), \ - mapping=[ \ - :A => ["Leaf"], \ - :carbon_demand => ["Leaf", "Internode"], \ - :carbon_allocation => ["Leaf", "Internode"] \ - ], \ - ), \ - "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ - "Leaf" => ( \ - MultiScaleModel( \ - model=ToyAssimModel(), \ - mapping=[:soil_water_content => "Soil",], \ - ), \ - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ - Status(aPPFD=1300.0, TT=10.0), \ - ), \ - "Soil" => ( \ - ToySoilWaterModel(), \ - ), \ - ); -``` - -```jldoctest mylabel -julia> mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)); -``` - -```jldoctest mylabel -julia> soil = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)); -``` - -```jldoctest mylabel -julia> plant = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)); -``` - -```jldoctest mylabel -julia> internode1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)); -``` - -```jldoctest mylabel -julia> leaf1 = Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); -``` - -```jldoctest mylabel -julia> internode2 = Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)); -``` - -```jldoctest mylabel -julia> leaf2 = Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); -``` - -```jldoctest mylabel -julia> organs_statuses = PlantSimEngine.status_template(mapping, nothing); -``` - -```jldoctest mylabel -julia> var_refvector = PlantSimEngine.reverse_mapping(mapping, all=false); -``` - -```jldoctest mylabel -julia> var_need_init = PlantSimEngine.to_initialize(mapping, mtg); -``` - -```jldoctest mylabel -julia> statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init); -``` - -```jldoctest mylabel -julia> outputs = Dict("Leaf" => (:A, :carbon_demand), "Soil" => (:soil_water_content,)); -``` - -```jldoctest mylabel -julia> PlantSimEngine.pre_allocate_outputs(statuses, outputs, 2) -Dict{String, Dict{Symbol, Vector{Vector{Float64}}}} with 2 entries: - "Soil" => Dict(:soil_water_content=>[[], []]) - "Leaf" => Dict(:A=>[[], []], :carbon_demand=>[[], []]) -``` -""" -function pre_allocate_outputs(statuses, outs, nsteps; check=true) - - outs_ = copy(outs) - # Checking that organs in outputs exist in the mtg (in the statuses): - if !all(i in keys(statuses) for i in keys(outs_)) - not_in_statuses = setdiff(keys(outs_), keys(statuses)) - e = string( - "You requested outputs for organs ", - join(keys(outs_), ", "), - ", but organs ", - join(not_in_statuses, ", "), - " have no models." - ) - - if check - error(e) - else - @info e - [delete!(outs_, i) for i in not_in_statuses] - end - end - - # Checking that variables in outputs exist in the statuses: - for (organ, vars) in outs_ - if !all(i in collect(keys(statuses[organ][1])) for i in vars) - not_in_statuses = (setdiff(vars, keys(statuses[organ][1]))...,) - e = string( - "You requested outputs for variables ", - join(vars, ", "), - ", but variables ", - join(not_in_statuses, ", "), - " have no models." - ) - if check - error(e) - else - @info e - existing_vars_requested = setdiff(outs_[organ], not_in_statuses) - if length(existing_vars_requested) == 0 - # None of the variables requested by the user exist at this scale for this set of models - delete!(outs_, organ) - else - # Some still exist, we onl use the ones that do: - outs_[organ] = (existing_vars_requested...,) - end - end - end - end - - # Making the pre-allocated outputs: - Dict(organ => Dict(var => [typeof(statuses[organ][1][var])[] for n in 1:nsteps] for var in vars) for (organ, vars) in outs_) - # Note: we use the type of the variable from the first status for each organ to pre-allocate the outputs, because they are - # all the same type for others. -end - -pre_allocate_outputs(statuses, ::Nothing, nsteps; check=true) = Dict{String,Tuple{Symbol,Vararg{Symbol}}}() - -function save_results!(object::GraphSimulation, i) - outs = outputs(object) - statuses = status(object) - - for (organ, vars) in outs - for (var, values) in vars - values[i] = [status[var] for status in statuses[organ]] - end - end end \ No newline at end of file diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl new file mode 100644 index 00000000..7bea6091 --- /dev/null +++ b/src/mtg/save_results.jl @@ -0,0 +1,205 @@ +""" + pre_allocate_outputs(statuses, outs, nsteps; check=true) + +Pre-allocate the outputs of needed variable for each node type in vectors of vectors. +The first level vectors have length nsteps, and the second level vectors have length n_nodes of this type. + +Note that we pre-allocate the vectors for the time-steps, but not for each organ, because we don't +know how many nodes will be in each organ in the future (organs can appear or disapear). + +# Arguments + +- `statuses`: a dictionary of status by node type +- `outs`: a dictionary of outputs by node type +- `nsteps`: the number of time-steps +- `check`: whether to check the mapping for errors. Default (`true`) returns an error if some variables do not exist. +If false and some variables are missing, return an info, remove the unknown variables and continue. + +# Returns + +- A dictionary of pre-allocated output of vector of time-step and vector of node of that type. + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine, MultiScaleTreeGraph +``` + +```jldoctest mylabel +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); +``` + +```jldoctest mylabel +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +``` + +```jldoctest mylabel +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +``` + +```jldoctest mylabel +julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +``` + +```jldoctest mylabel +julia> mapping = Dict( \ + "Plant" => \ + MultiScaleModel( \ + model=ToyCAllocationModel(), \ + mapping=[ \ + :A => ["Leaf"], \ + :carbon_demand => ["Leaf", "Internode"], \ + :carbon_allocation => ["Leaf", "Internode"] \ + ], \ + ), \ + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + "Leaf" => ( \ + MultiScaleModel( \ + model=ToyAssimModel(), \ + mapping=[:soil_water_content => "Soil",], \ + ), \ + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + Status(aPPFD=1300.0, TT=10.0), \ + ), \ + "Soil" => ( \ + ToySoilWaterModel(), \ + ), \ + ); +``` + +```jldoctest mylabel +julia> mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)); +``` + +```jldoctest mylabel +julia> soil = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)); +``` + +```jldoctest mylabel +julia> plant = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)); +``` + +```jldoctest mylabel +julia> internode1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)); +``` + +```jldoctest mylabel +julia> leaf1 = Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); +``` + +```jldoctest mylabel +julia> internode2 = Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)); +``` + +```jldoctest mylabel +julia> leaf2 = Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); +``` + +```jldoctest mylabel +julia> organs_statuses = PlantSimEngine.status_template(mapping, nothing); +``` + +```jldoctest mylabel +julia> var_refvector = PlantSimEngine.reverse_mapping(mapping, all=false); +``` + +```jldoctest mylabel +julia> var_need_init = PlantSimEngine.to_initialize(mapping, mtg); +``` + +```jldoctest mylabel +julia> statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init); +``` + +```jldoctest mylabel +julia> outputs = Dict("Leaf" => (:A, :carbon_demand), "Soil" => (:soil_water_content,)); +``` + +```jldoctest mylabel +julia> PlantSimEngine.pre_allocate_outputs(statuses, outputs, 2) +Dict{String, Dict{Symbol, Vector{Vector{Float64}}}} with 2 entries: + "Soil" => Dict(:soil_water_content=>[[], []]) + "Leaf" => Dict(:A=>[[], []], :carbon_demand=>[[], []]) +``` +""" +function pre_allocate_outputs(statuses, outs, nsteps; check=true) + outs_ = copy(outs) + + # Add the :node variable to the outputs: + for (organ, vars) in outs_ + if :node ∉ outs_[organ] + outs_[organ] = (vars..., :node) + end + end + + # Checking that organs in outputs exist in the mtg (in the statuses): + if !all(i in keys(statuses) for i in keys(outs_)) + not_in_statuses = setdiff(keys(outs_), keys(statuses)) + e = string( + "You requested outputs for organs ", + join(keys(outs_), ", "), + ", but organs ", + join(not_in_statuses, ", "), + " have no models." + ) + + if check + error(e) + else + @info e + [delete!(outs_, i) for i in not_in_statuses] + end + end + + # Checking that variables in outputs exist in the statuses: + for (organ, vars) in outs_ + if !all(i in collect(keys(statuses[organ][1])) for i in vars) + not_in_statuses = (setdiff(vars, keys(statuses[organ][1]))...,) + e = string( + "You requested outputs for variables ", + join(vars, ", "), + ", but variables ", + join(not_in_statuses, ", "), + " have no models." + ) + if check + error(e) + else + @info e + existing_vars_requested = setdiff(outs_[organ], not_in_statuses) + if length(existing_vars_requested) == 0 + # None of the variables requested by the user exist at this scale for this set of models + delete!(outs_, organ) + else + # Some still exist, we only use the ones that do: + outs_[organ] = (existing_vars_requested...,) + end + end + end + end + + # Making the pre-allocated outputs: + Dict(organ => Dict(var => [typeof(statuses[organ][1][var])[] for n in 1:nsteps] for var in vars) for (organ, vars) in outs_) + # Note: we use the type of the variable from the first status for each organ to pre-allocate the outputs, because they are + # all the same type for others. +end + +pre_allocate_outputs(statuses, ::Nothing, nsteps; check=true) = Dict{String,Tuple{Symbol,Vararg{Symbol}}}() + +""" + save_results!(object::GraphSimulation, i) + +Save the results of the simulation for time-step `i` into the +object. For a `GraphSimulation` object, this will save the results +from the `status(object)` in the `outputs(object)`. +""" +function save_results!(object::GraphSimulation, i) + outs = outputs(object) + statuses = status(object) + + for (organ, vars) in outs + for (var, values) in vars + values[i] = [status[var] for status in statuses[organ]] + end + end +end \ No newline at end of file diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index 4278a60d..1a68ab96 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -557,4 +557,20 @@ end @test out.outputs["Leaf"][:carbon_allocation] == out.outputs["Internode"][:carbon_allocation] @test out.outputs["Plant"][:carbon_allocation][1][1][1] === out.outputs["Internode"][:carbon_allocation][1][1] + + # Testing the outputs if transformed into a DataFrame: + outs = outputs(out, DataFrame) + + @test isa(outs, DataFrame) + @test size(outs) == (12, 7) + + @test unique(outs.timestep) == [1, 2] + @test sort(unique(outs.organ)) == sort(collect(keys(out_vars))) + @test length(filter(x -> x !== nothing, outs.A)) == length(filter(x -> x, traverse(mtg, node -> node.MTG.scale == 2))) + # a = status(out, TimeStepTable{Status}) + A = outputs(out, :A) + @test A == outs.A + + A2 = outputs(out, 5) + @test A == A2 end \ No newline at end of file From e4aa5a8f2b0023ceb65bdb6daf60a89aca07e0e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 20 Oct 2023 14:37:35 +0200 Subject: [PATCH 73/97] Make tests pass --- src/mtg/save_results.jl | 40 ++++++++++++++++++++++++++----------- test/test-mtg-multiscale.jl | 6 +++--- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl index 7bea6091..aec7f941 100644 --- a/src/mtg/save_results.jl +++ b/src/mtg/save_results.jl @@ -115,23 +115,35 @@ julia> statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvect julia> outputs = Dict("Leaf" => (:A, :carbon_demand), "Soil" => (:soil_water_content,)); ``` +Pre-allocate the outputs as a dictionary: + +```jldoctest mylabel +julia> outs = PlantSimEngine.pre_allocate_outputs(statuses, outputs, 2); +``` + +The dictionary has a key for each organ from which we want outputs: + +```jldoctest mylabel +julia> collect(keys(outs)) +2-element Vector{String}: + "Soil" + "Leaf" +``` + +Each organ has a dictionary of variables for which we want outputs from, +with the pre-allocated empty vectors (one per time-step that will be filled with one value per node): + ```jldoctest mylabel -julia> PlantSimEngine.pre_allocate_outputs(statuses, outputs, 2) -Dict{String, Dict{Symbol, Vector{Vector{Float64}}}} with 2 entries: - "Soil" => Dict(:soil_water_content=>[[], []]) - "Leaf" => Dict(:A=>[[], []], :carbon_demand=>[[], []]) +julia> collect(keys(outs["Leaf"])) +3-element Vector{Symbol}: + :A + :node + :carbon_demand ``` """ function pre_allocate_outputs(statuses, outs, nsteps; check=true) outs_ = copy(outs) - # Add the :node variable to the outputs: - for (organ, vars) in outs_ - if :node ∉ outs_[organ] - outs_[organ] = (vars..., :node) - end - end - # Checking that organs in outputs exist in the mtg (in the statuses): if !all(i in keys(statuses) for i in keys(outs_)) not_in_statuses = setdiff(keys(outs_), keys(statuses)) @@ -151,7 +163,7 @@ function pre_allocate_outputs(statuses, outs, nsteps; check=true) end end - # Checking that variables in outputs exist in the statuses: + # Checking that variables in outputs exist in the statuses, and adding the :node variable: for (organ, vars) in outs_ if !all(i in collect(keys(statuses[organ][1])) for i in vars) not_in_statuses = (setdiff(vars, keys(statuses[organ][1]))...,) @@ -176,6 +188,10 @@ function pre_allocate_outputs(statuses, outs, nsteps; check=true) end end end + + if :node ∉ outs_[organ] + outs_[organ] = (outs_[organ]..., :node) + end end # Making the pre-allocated outputs: diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index 1a68ab96..1fef9a5e 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -152,11 +152,11 @@ end @test_throws e_1 PlantSimEngine.pre_allocate_outputs(statuses, outs, nsteps) end - outs_ = @test_logs (:info, "$e_1") (:info, "$e_2") PlantSimEngine.pre_allocate_outputs(statuses, outs, nsteps, check=false) + outs_ = @test_logs (:info, "You requested outputs for organs Soil, Flowers, Leaf, but organs Flowers have no models.") (:info, "You requested outputs for variables A, carbon_demand, non_existing_variable, but variables non_existing_variable have no models.") PlantSimEngine.pre_allocate_outputs(statuses, outs, nsteps, check=false) @test outs_ == Dict( - "Soil" => Dict(:soil_water_content => [[], []]), - "Leaf" => Dict(:A => [[], []], :carbon_demand => [[], []]) + "Soil" => Dict(:node => [[], []], :soil_water_content => [[], []]), + "Leaf" => Dict(:A => [[], []], :node => [[], []], :carbon_demand => [[], []]) ) end From 3307fd10b63748c43587d40691ace6758f42110d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 24 Oct 2023 11:44:30 +0200 Subject: [PATCH 74/97] Add a function to import all example models for MTG --- src/PlantSimEngine.jl | 3 ++ src/doc_templates/mtg-related.jl | 18 ----------- src/examples_import.jl | 26 ++++++++++++++++ src/mtg/GraphSimulation.jl | 6 +++- src/mtg/mapping.jl | 52 ++++++++------------------------ src/mtg/save_results.jl | 14 ++------- 6 files changed, 49 insertions(+), 70 deletions(-) create mode 100644 src/examples_import.jl diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index e18b0a39..d9663ed3 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -77,6 +77,9 @@ include("run.jl") # Fitting include("evaluation/fit.jl") +# Examples +include("examples_import.jl") + export AbstractModel export ModelList, MultiScaleModel export init_mtg_models! diff --git a/src/doc_templates/mtg-related.jl b/src/doc_templates/mtg-related.jl index 0e00027b..d0e9ce21 100644 --- a/src/doc_templates/mtg-related.jl +++ b/src/doc_templates/mtg-related.jl @@ -57,22 +57,4 @@ mapping = Dict( \ ), \ ) ``` -""" - -const EXAMPLE_MTG_MODELS = """ -```@example -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); -``` - -```@example -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); -``` - -```@example -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); -``` - -```@example -include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); -``` """ \ No newline at end of file diff --git a/src/examples_import.jl b/src/examples_import.jl new file mode 100644 index 00000000..fd4d7926 --- /dev/null +++ b/src/examples_import.jl @@ -0,0 +1,26 @@ +""" + import_multiscale_example() + +Import the examples used in the documentation for a set of multiscale models. +The models can be found in the `examples` folder of the package, and are stored +in the following files: + +- `ToyAssimModel.jl` +- `ToyCDemandModel.jl` +- `ToyCAllocationModel.jl` +- `ToySoilModel.jl` + +# Examples + +```jl +using PlantSimEngine + +import_multiscale_example() +``` +""" +function import_multiscale_example() + include(joinpath(@__DIR__, "../examples/ToyAssimModel.jl")) + include(joinpath(@__DIR__, "../examples/ToyCDemandModel.jl")) + include(joinpath(@__DIR__, "../examples/ToyCAllocationModel.jl")) + include(joinpath(@__DIR__, "../examples/ToySoilModel.jl")) +end \ No newline at end of file diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index 4c6c0ce7..d0ceba78 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -51,7 +51,11 @@ sing the sink function, for exemple a `DataFrame`. using PlantSimEngine, MultiScaleTreeGraph, DataFrames ``` -$EXAMPLE_MTG_MODELS +Import example models (can be found in the `examples` folder of the package): + +```@example +import_multiscale_example(); +``` $MAPPING_EXAMPLE diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 09752544..91907247 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -28,20 +28,10 @@ Returns a vector of models julia> using PlantSimEngine; ``` -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); -``` +Import example models (can be found in the `examples` folder of the package): ```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +import_multiscale_example(); ``` If we just give a MultiScaleModel, we get its model as a one-element vector: @@ -168,10 +158,12 @@ and the nodes that are targeted by the mapping ```julia using PlantSimEngine -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); -include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +``` + +Import example models (can be found in the `examples` folder of the package): + +```julia +import_multiscale_example(); ``` ```julia @@ -541,20 +533,10 @@ Create a status template for a given set of models and type promotion. julia> using PlantSimEngine; ``` -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); -``` +Import example models (can be found in the `examples` folder of the package): ```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +import_multiscale_example(); ``` ```jldoctest mylabel @@ -764,20 +746,10 @@ This is used for *e.g.* knowing which scales are needed to add values to others. julia> using PlantSimEngine ``` -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); -``` +Import example models (can be found in the `examples` folder of the package): ```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); +import_multiscale_example(); ``` ```jldoctest mylabel diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl index aec7f941..78411a52 100644 --- a/src/mtg/save_results.jl +++ b/src/mtg/save_results.jl @@ -25,21 +25,13 @@ If false and some variables are missing, return an info, remove the unknown vari julia> using PlantSimEngine, MultiScaleTreeGraph ``` -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")); -``` +Import example models (can be found in the `examples` folder of the package): ```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")); +import_multiscale_example(); ``` -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); -``` - -```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")); -``` +Define the models mapping: ```jldoctest mylabel julia> mapping = Dict( \ From 0f776a1509c833be40f572d2fd5457d879b5f3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 6 Nov 2023 15:42:08 +0100 Subject: [PATCH 75/97] Update examples_import.jl Add example mtg --- src/examples_import.jl | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/examples_import.jl b/src/examples_import.jl index fd4d7926..4c219ff7 100644 --- a/src/examples_import.jl +++ b/src/examples_import.jl @@ -23,4 +23,40 @@ function import_multiscale_example() include(joinpath(@__DIR__, "../examples/ToyCDemandModel.jl")) include(joinpath(@__DIR__, "../examples/ToyCAllocationModel.jl")) include(joinpath(@__DIR__, "../examples/ToySoilModel.jl")) +end + + + +""" + import_mtg_example() + +Returns an example multiscale tree graph (MTG) with a scene, a soil, and a plant with two internodes and two leaves. + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +``` + +```jldoctest +julia> PlantSimEngine.import_mtg_example() +/ 1: Scene +├─ / 2: Soil +└─ + 3: Plant + └─ / 4: Internode + ├─ + 5: Leaf + └─ < 6: Internode + └─ + 7: Leaf +``` +""" +function import_mtg_example() + mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) + MultiScaleTreeGraph.Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)) + plant = MultiScaleTreeGraph.Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + internode1 = MultiScaleTreeGraph.Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + internode2 = MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + return mtg end \ No newline at end of file From 55c2699b4d4d3f593d0ba4d493526553fa6a20f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 7 Nov 2023 00:31:31 +0100 Subject: [PATCH 76/97] Update mapping.jl Fix issues when a variable has different default values between scales (uses the upper-stream model now) and fix an issue for mapping to variables with one node only --- src/mtg/mapping.jl | 67 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index 91907247..eb3eff46 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -28,10 +28,10 @@ Returns a vector of models julia> using PlantSimEngine; ``` -Import example models (can be found in the `examples` folder of the package): +Import example models (can be found in the `examples` folder of the package, or in the `Examples` sub-modules): ```jldoctest mylabel -import_multiscale_example(); +julia> using PlantSimEngine.Examples; ``` If we just give a MultiScaleModel, we get its model as a one-element vector: @@ -160,10 +160,10 @@ and the nodes that are targeted by the mapping using PlantSimEngine ``` -Import example models (can be found in the `examples` folder of the package): +Import example models (can be found in the `examples` folder of the package, or in the `Examples` sub-modules): -```julia -import_multiscale_example(); +```jldoctest mylabel +julia> using PlantSimEngine.Examples; ``` ```julia @@ -207,7 +207,7 @@ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} # i.e. variables that are written by a model at another scale: var_outputs_from_mapping = Dict{String,Vector{Pair{Symbol,Any}}}() for organ in keys(models) - # organ = "Leaf" + # organ = "Plant" map_vars = get_mapping(models[organ]) if length(map_vars) == 0 continue @@ -217,12 +217,43 @@ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} mods = get_models(models[organ]) ins = merge(inputs_.(mods)...) outs = merge(outputs_.(mods)...) - - # Variables in the node that are defined as multiscale: - multi_scale_ins = intersect(keys(ins), multiscale_vars) # inputs: variables that are taken from another scale multi_scale_outs = intersect(keys(outs), multiscale_vars) # outputs: variables that are written to another scale - multi_scale_vars = Status(convert_vars(type_promotion, merge(ins[multi_scale_ins], outs[multi_scale_outs]))) + multi_scale_vars_vec = Pair{Symbol,Any}[] + for (var, scales) in map_vars # e.g. var = :A; scales = ["Leaf"] + isa(scales, AbstractString) && (scales = [scales]) + + # The variable default value is always taken from the upper-stream model: + if var in keys(ins) + # The variable is taken as an input from another scale. We take its default value from the model at the other scale: + mapped_out_var = [] + for s in scales + @assert haskey(models, s) "Scale $s required as a mapping for scale $organ, but not found in the mapping." + mapped_out = merge(PlantSimEngine.outputs_.(PlantSimEngine.get_models(models[s]))...) + @assert hasproperty(mapped_out, var) "No model computes variable $var at scale $s, need one for scale $organ" + push!(mapped_out_var, mapped_out[var]) + end + mapped_out = unique(mapped_out_var) + if length(mapped_out) > 1 + @info "Found different default values for variable $var in models at scales $scales: $mapped_out. Taking the first one." + end + mapped_out = mapped_out[1] + + # If the variable is given as a vector as default value, it means it will be taken from several organs. + # In this case, we keep the vector format: + if isa(ins[var], AbstractVector) + mapped_out = fill(mapped_out, length(ins[var])) + end + push!(multi_scale_vars_vec, var => mapped_out) + elseif var in keys(outs) + # The variable is an output of this scale for another scale. We take its default value from this scale: + push!(multi_scale_vars_vec, var => outs[var]) + else + error("Variable $var required to be mapped from scale(s) $scales to scale $organ was not found in any model from the scale(s) $scales.") + end + end + + multi_scale_vars = Status(PlantSimEngine.convert_vars(type_promotion, NamedTuple(multi_scale_vars_vec))) # Users can provide initialisation values in a status. We get them here: st = get_status(models[organ]) @@ -274,8 +305,12 @@ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} if isa(organs_mapped, AbstractString) if !haskey(organs_mapping, organs_mapped) organs_mapping[organs_mapped] = Dict(organs_mapped => organ_mapping[organs_mapped]) + elseif !haskey(organs_mapping[organs_mapped], organs_mapped) + push!(organs_mapping[organs_mapped], organs_mapped => organ_mapping[organs_mapped]) + elseif !haskey(organs_mapping[organs_mapped][organs_mapped], variable) + push!(organs_mapping[organs_mapped][organs_mapped], variable => organ_mapping[organs_mapped][variable]) else - push!(organs_mapping[organs_mapped][organs_mapped], organ_mapping[organs_mapped]) + @info "Variable $variable already mapped from scale $organs_mapped to scale $organs_mapped. Skipping." end end end @@ -364,7 +399,7 @@ end create_var_ref(organ::AbstractString, default) Create a referece variable. The reference is a `RefVector` if the organ is a vector of strings, and a `MappedVar` -if it is a singleton string. This is because we want to avoid indeing into a vector of values if there is only one +if it is a singleton string. This is because we want to avoid indexing into a vector of values if there is only one value to map. """ function create_var_ref(organ::Vector{<:AbstractString}, var, default::AbstractVector{T}) where {T} @@ -533,10 +568,10 @@ Create a status template for a given set of models and type promotion. julia> using PlantSimEngine; ``` -Import example models (can be found in the `examples` folder of the package): +Import example models (can be found in the `examples` folder of the package, or in the `Examples` sub-modules): ```jldoctest mylabel -import_multiscale_example(); +julia> using PlantSimEngine.Examples; ``` ```jldoctest mylabel @@ -746,10 +781,10 @@ This is used for *e.g.* knowing which scales are needed to add values to others. julia> using PlantSimEngine ``` -Import example models (can be found in the `examples` folder of the package): +Import example models (can be found in the `examples` folder of the package, or in the `Examples` sub-modules): ```jldoctest mylabel -import_multiscale_example(); +julia> using PlantSimEngine.Examples; ``` ```jldoctest mylabel From dbdcc3ecf940affb1afccea4677845266180e37c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 7 Nov 2023 00:32:00 +0100 Subject: [PATCH 77/97] Fix all example scripts --- examples/Beer.jl | 4 ++-- examples/ToyAssimModel.jl | 2 +- examples/ToyCAllocationModel.jl | 2 +- examples/ToyCDemandModel.jl | 2 +- examples/ToyDegreeDays.jl | 21 ++++++++++++++++----- examples/ToyLAIModel.jl | 8 ++++---- examples/ToySoilModel.jl | 2 +- examples/benchmark.jl | 4 ++-- examples/meteo_day.csv | 2 +- 9 files changed, 29 insertions(+), 18 deletions(-) diff --git a/examples/Beer.jl b/examples/Beer.jl index 9bab267d..f9ff9702 100644 --- a/examples/Beer.jl +++ b/examples/Beer.jl @@ -4,7 +4,7 @@ # allows us to not have a (cyclic) dependency on PlantBiophysics.jl in the docs. # Generate the light interception process methods: -@process "light_interception" verbose = false +PlantSimEngine.@process "light_interception" verbose = false """ Beer(k) @@ -57,7 +57,7 @@ m[:aPPFD] function PlantSimEngine.run!(::Beer, models, status, meteo, constants, extra=nothing) status.aPPFD = meteo.Ri_PAR_f * - (1 - exp(-models.light_interception.k * status.LAI)) * + (1.0 - exp(-models.light_interception.k * status.LAI)) * constants.J_to_umol end diff --git a/examples/ToyAssimModel.jl b/examples/ToyAssimModel.jl index 8f03643f..6189ce94 100644 --- a/examples/ToyAssimModel.jl +++ b/examples/ToyAssimModel.jl @@ -1,7 +1,7 @@ # using PlantSimEngine, PlantMeteo # Import the necessary packages, PlantMeteo is used for the meteorology # Defining the process: -@process "photosynthesis" verbose = false +PlantSimEngine.@process "photosynthesis" verbose = false # Make the struct to hold the parameters, with its documentation: """ diff --git a/examples/ToyCAllocationModel.jl b/examples/ToyCAllocationModel.jl index 0451a44e..21b035ec 100644 --- a/examples/ToyCAllocationModel.jl +++ b/examples/ToyCAllocationModel.jl @@ -1,7 +1,7 @@ # using PlantSimEngine, PlantMeteo # Import the necessary packages, PlantMeteo is used for the meteorology # Defining the process: -@process "carbon_allocation" verbose = false +PlantSimEngine.@process "carbon_allocation" verbose = false # Make the struct to hold the parameters, with its documentation: """ diff --git a/examples/ToyCDemandModel.jl b/examples/ToyCDemandModel.jl index 318409b9..b376aa83 100644 --- a/examples/ToyCDemandModel.jl +++ b/examples/ToyCDemandModel.jl @@ -1,7 +1,7 @@ # using PlantSimEngine, PlantMeteo # Import the necessary packages, PlantMeteo is used for the meteorology # Defining the process: -@process "carbon_demand" verbose = false +PlantSimEngine.@process "carbon_demand" verbose = false # Make the struct to hold the parameters, with its documentation: """ diff --git a/examples/ToyDegreeDays.jl b/examples/ToyDegreeDays.jl index 049f6d11..72d77b70 100644 --- a/examples/ToyDegreeDays.jl +++ b/examples/ToyDegreeDays.jl @@ -1,22 +1,33 @@ # Declaring the process of LAI dynamic: -@process "Degreedays" verbose = false +PlantSimEngine.@process "Degreedays" verbose = false # Declaring the model of LAI dynamic with its parameter values: + +""" + ToyDegreeDaysCumulModel(;init_TT=0.0, T_base=10.0, T_max=43.0) + +Computes the thermal time in degree days and cumulated degree-days based on the average daily temperature (`T`), +the initial cumulated degree days, the base temperature below which there is no growth, and the maximum +temperature for growh. +""" struct ToyDegreeDaysCumulModel <: AbstractDegreedaysModel - init_degreedays::Float64 + init_TT::Float64 + T_base::Float64 + T_max::Float64 end # Defining default values: -ToyDegreeDaysCumulModel(init_degreedays=0.0) = ToyDegreeDaysCumulModel(init_degreedays) +ToyDegreeDaysCumulModel(; init_TT=0.0, T_base=10.0, T_max=43.0) = ToyDegreeDaysCumulModel(init_TT, T_base, T_max) # Defining the inputs and outputs of the model: PlantSimEngine.inputs_(::ToyDegreeDaysCumulModel) = NamedTuple() -PlantSimEngine.outputs_(::ToyDegreeDaysCumulModel) = (degree_days_cu=-Inf,) +PlantSimEngine.outputs_(m::ToyDegreeDaysCumulModel) = (TT=-Inf, TT_cu=0.0,) # Implementing the actual algorithm by adding a method to the run! function for our model: function PlantSimEngine.run!(m::ToyDegreeDaysCumulModel, models, status, meteo, constants=nothing, extra=nothing) - status.degree_days_cu += status.degree_days + status.TT = max(0.0, min(meteo.T, m.T_max) - m.T_base) + status.TT_cu += status.TT end # The computation of ToyDegreeDaysCumulModel dependents on previous values, but it is independent of other objects. diff --git a/examples/ToyLAIModel.jl b/examples/ToyLAIModel.jl index 97cee390..d50f3e59 100644 --- a/examples/ToyLAIModel.jl +++ b/examples/ToyLAIModel.jl @@ -1,6 +1,6 @@ # Declaring the process of LAI dynamic: -@process "LAI_Dynamic" verbose = false +PlantSimEngine.@process "LAI_Dynamic" verbose = false # Declaring the model of LAI dynamic with its parameter values: @@ -16,7 +16,7 @@ end ToyLAIModel(; max_lai=8.0, dd_incslope=800, inc_slope=110, dd_decslope=1500, dec_slope=20) = ToyLAIModel(max_lai, dd_incslope, inc_slope, dd_decslope, dec_slope) # Defining the inputs and outputs of the model: -PlantSimEngine.inputs_(::ToyLAIModel) = (degree_days_cu=-Inf,) +PlantSimEngine.inputs_(::ToyLAIModel) = (TT_cu=-Inf,) PlantSimEngine.outputs_(::ToyLAIModel) = (LAI=-Inf,) # Implementing the actual algorithm by adding a method to the run! function for our model: @@ -24,8 +24,8 @@ function PlantSimEngine.run!(::ToyLAIModel, models, status, meteo, constants=not status.LAI = models.LAI_Dynamic.max_lai * (1.0 / - (1.0 + exp((models.LAI_Dynamic.dd_incslope - status.degree_days_cu) / models.LAI_Dynamic.inc_slope)) - - 1.0 / (1.0 + exp((models.LAI_Dynamic.dd_decslope - status.degree_days_cu) / models.LAI_Dynamic.dec_slope)) + (1.0 + exp((models.LAI_Dynamic.dd_incslope - status.TT_cu) / models.LAI_Dynamic.inc_slope)) - + 1.0 / (1.0 + exp((models.LAI_Dynamic.dd_decslope - status.TT_cu) / models.LAI_Dynamic.dec_slope)) ) if status.LAI < 0.0 diff --git a/examples/ToySoilModel.jl b/examples/ToySoilModel.jl index b124f7fa..ca598bad 100644 --- a/examples/ToySoilModel.jl +++ b/examples/ToySoilModel.jl @@ -1,5 +1,5 @@ # Declaring the process of LAI dynamic: -@process "soil_water" verbose = false +PlantSimEngine.@process "soil_water" verbose = false """ diff --git a/examples/benchmark.jl b/examples/benchmark.jl index 1ff615ae..3d12b1c9 100644 --- a/examples/benchmark.jl +++ b/examples/benchmark.jl @@ -10,7 +10,7 @@ using PlantSimEngine, PlantMeteo, DataFrames, CSV, Dates, Statistics meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) models = ModelList( ToyLAIModel(), - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) # Match the warning on the executor, the default is ThreadedEx() but ToyRUEGrowthModel can't be run in parallel: @@ -26,7 +26,7 @@ median_time_seq_ns = median(time_run_seq.times) / nrow(meteo_day) models_coupled = ModelList( ToyLAIModel(), Beer(0.5), - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) # Match the warning on the executor, the default is ThreadedEx() but ToyRUEGrowthModel can't be run in parallel: diff --git a/examples/meteo_day.csv b/examples/meteo_day.csv index 65fa3a85..bd29bcca 100644 --- a/examples/meteo_day.csv +++ b/examples/meteo_day.csv @@ -15,7 +15,7 @@ #'timezone: "UTC" #'elevation: 45.0 #' -year,dayofyear,date,duration,Tmin,Tmax,T,Precipitations,Wind,P,Rh,Ca,Ri_SW_f,Ri_PAR_f,Ri_NIR_f,degree_days +year,dayofyear,date,duration,Tmin,Tmax,T,Precipitations,Wind,P,Rh,Ca,Ri_SW_f,Ri_PAR_f,Ri_NIR_f,TT 2021,1,2021-01-01,1,3.9,9.2,5.949999999999999,0.5,4.290416666666667,100.15041666666666,0.7545833333333335,400.0,6.5304,3.134592,3.395808,0.0 2021,2,2021-01-02,1,3.5,6.0,4.420833333333333,1.0,6.740833333333332,100.17666666666668,0.6220833333333333,400.0,3.5748,1.715904,1.858896,0.0 2021,3,2021-01-03,1,3.3,6.4,4.329166666666667,0.0,6.1570833333333335,100.51333333333332,0.5975000000000001,400.0,5.180400000000001,2.4865920000000004,2.693808,0.0 From 5b1ed3cf6de8a8f11dcc531f65e1da86907f49b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 7 Nov 2023 00:32:41 +0100 Subject: [PATCH 78/97] Update degree_days into TT --- docs/src/FAQ/translate_a_model.md | 24 ++++++++++++------------ docs/src/index.md | 14 +++++++------- docs/src/model_switching.md | 8 ++++---- test/test-toy_models.jl | 12 ++++++------ 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/src/FAQ/translate_a_model.md b/docs/src/FAQ/translate_a_model.md index daaf8550..9999cffc 100644 --- a/docs/src/FAQ/translate_a_model.md +++ b/docs/src/FAQ/translate_a_model.md @@ -6,8 +6,8 @@ using CairoMakie using CSV, DataFrames include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) -function lai_toymodel(degree_days_cu; max_lai=8.0, dd_incslope=500, inc_slope=70, dd_decslope=1000, dec_slope=20) - LAI = max_lai * (1 / (1 + exp((dd_incslope - degree_days_cu) / inc_slope)) - 1 / (1 + exp((dd_decslope - degree_days_cu) / dec_slope))) +function lai_toymodel(TT_cu; max_lai=8.0, dd_incslope=500, inc_slope=70, dd_decslope=1000, dec_slope=20) + LAI = max_lai * (1 / (1 + exp((dd_incslope - TT_cu) / inc_slope)) - 1 / (1 + exp((dd_decslope - TT_cu) / dec_slope))) if LAI < 0 LAI = 0 end @@ -33,15 +33,15 @@ Simulate leaf area index (LAI, m² m⁻²) for a crop based on the amount of deg # Arguments -- `degree_days_cu`: degree-days since sowing +- `TT_cu`: degree-days since sowing - `max_lai=8`: Maximum value for LAI - `dd_incslope=500`: degree-days at which we get the maximal increase in LAI - `inc_slope=5`: slope of the increasing part of the LAI curve - `dd_decslope=1000`: degree-days at which we get the maximal decrease in LAI - `dec_slope=2`: slope of the decreasing part of the LAI curve """ -function lai_toymodel(degree_days_cu; max_lai=8.0, dd_incslope=500, inc_slope=70, dd_decslope=1000, dec_slope=20) - LAI = max_lai * (1 / (1 + exp((dd_incslope - degree_days_cu) / inc_slope)) - 1 / (1 + exp((dd_decslope - degree_days_cu) / dec_slope))) +function lai_toymodel(TT_cu; max_lai=8.0, dd_incslope=500, inc_slope=70, dd_decslope=1000, dec_slope=20) + LAI = max_lai * (1 / (1 + exp((dd_incslope - TT_cu) / inc_slope)) - 1 / (1 + exp((dd_decslope - TT_cu) / dec_slope))) if LAI < 0 LAI = 0 end @@ -96,7 +96,7 @@ This way users can create a model with default parameters just by calling `ToyLA Then we can define the inputs and outputs of the model, and the default value at initialization: ```julia -PlantSimEngine.inputs_(::ToyLAIModel) = (degree_days_cu=-Inf,) +PlantSimEngine.inputs_(::ToyLAIModel) = (TT_cu=-Inf,) PlantSimEngine.outputs_(::ToyLAIModel) = (LAI=-Inf,) ``` @@ -109,7 +109,7 @@ Finally, we can define the model function that will be called at each time step: ```julia function PlantSimEngine.run!(::ToyLAIModel, models, status, meteo, constants=nothing, extra=nothing) - status.LAI = models.LAI_Dynamic.max_lai * (1 / (1 + exp((models.LAI_Dynamic.dd_incslope - status.degree_days_cu) / model.LAI_Dynamic.inc_slope)) - 1 / (1 + exp((models.LAI_Dynamic.dd_decslope - status.degree_days_cu) / models.LAI_Dynamic.dec_slope))) + status.LAI = models.LAI_Dynamic.max_lai * (1 / (1 + exp((models.LAI_Dynamic.dd_incslope - status.TT_cu) / model.LAI_Dynamic.inc_slope)) - 1 / (1 + exp((models.LAI_Dynamic.dd_decslope - status.TT_cu) / models.LAI_Dynamic.dec_slope))) if status.LAI < 0 status.LAI = 0 @@ -138,21 +138,21 @@ period = [Dates.Date("2021-01-01"), Dates.Date("2021-12-31")] meteo = get_weather(43.649777, 3.869889, period, sink = DataFrame) # Compute the degree-days with a base temperature of 10°C: -meteo.degree_days = max.(meteo.T .- 10.0, 0.0) +meteo.TT = max.(meteo.T .- 10.0, 0.0) # Aggregate the weather data to daily values: -meteo_day = to_daily(meteo, :degree_days => (x -> sum(x) / 24) => :degree_days) +meteo_day = to_daily(meteo, :TT => (x -> sum(x) / 24) => :TT) ``` -Then we can define our list of models, passing the values for `degree_days_cu` in the status at initialization: +Then we can define our list of models, passing the values for `TT_cu` in the status at initialization: ```@example mymodel m = ModelList( ToyLAIModel(), - status = (degree_days_cu = cumsum(meteo_day.degree_days),), + status = (TT_cu = cumsum(meteo_day.TT),), ) run!(m) -lines(m[:degree_days_cu], m[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Days since sowing")) +lines(m[:TT_cu], m[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Days since sowing")) ``` diff --git a/docs/src/index.md b/docs/src/index.md index a64d3dbf..cfcd5dfa 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -15,7 +15,7 @@ meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), # Define the model: model = ModelList( ToyLAIModel(), - status=(degree_days_cu=1.0:2000.0,), # Pass the cumulated degree-days as input to the model + status=(TT_cu=1.0:2000.0,), # Pass the cumulated degree-days as input to the model ) run!(model) @@ -24,7 +24,7 @@ run!(model) model2 = ModelList( ToyLAIModel(), Beer(0.6), - status=(degree_days_cu=cumsum(meteo_day[:, :degree_days]),), # Pass the cumulated degree-days as input to `ToyLAIModel`, this could also be done using another model + status=(TT_cu=cumsum(meteo_day[:, :TT]),), # Pass the cumulated degree-days as input to `ToyLAIModel`, this could also be done using another model ) run!(model2, meteo_day) @@ -89,7 +89,7 @@ include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) # Define the model: model = ModelList( ToyLAIModel(), - status=(degree_days_cu=1.0:2000.0,), # Pass the cumulated degree-days as input to the model + status=(TT_cu=1.0:2000.0,), # Pass the cumulated degree-days as input to the model ) run!(model) # run the model @@ -106,7 +106,7 @@ Of course you can plot the outputs quite easily: # ] add CairoMakie using CairoMakie -lines(model[:degree_days_cu], model[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Cumulated growing degree days since sowing (°C)")) +lines(model[:TT_cu], model[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Cumulated growing degree days since sowing (°C)")) ``` ### Model coupling @@ -128,7 +128,7 @@ meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), model2 = ModelList( ToyLAIModel(), Beer(0.6), - status=(degree_days_cu=cumsum(meteo_day[:, :degree_days]),), # Pass the cumulated degree-days as input to `ToyLAIModel`, this could also be done using another model + status=(TT_cu=cumsum(meteo_day[:, :TT]),), # Pass the cumulated degree-days as input to `ToyLAIModel`, this could also be done using another model ) # Run the simulation: @@ -163,10 +163,10 @@ using CairoMakie fig = Figure(resolution=(800, 600)) ax = Axis(fig[1, 1], ylabel="LAI (m² m⁻²)") -lines!(ax, model2[:degree_days_cu], model2[:LAI], color=:mediumseagreen) +lines!(ax, model2[:TT_cu], model2[:LAI], color=:mediumseagreen) ax2 = Axis(fig[2, 1], xlabel="Cumulated growing degree days since sowing (°C)", ylabel="aPPFD (mol m⁻² d⁻¹)") -lines!(ax2, model2[:degree_days_cu], model2[:aPPFD], color=:firebrick1) +lines!(ax2, model2[:TT_cu], model2[:aPPFD], color=:firebrick1) fig ``` diff --git a/docs/src/model_switching.md b/docs/src/model_switching.md index b66e4419..ecfb19cb 100644 --- a/docs/src/model_switching.md +++ b/docs/src/model_switching.md @@ -13,14 +13,14 @@ models = ModelList( ToyLAIModel(), Beer(0.5), ToyRUEGrowthModel(0.2), - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) run!(models, meteo_day) models2 = ModelList( ToyLAIModel(), Beer(0.5), ToyAssimGrowthModel(), - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) run!(models2, meteo_day) ``` @@ -52,7 +52,7 @@ models = ModelList( ToyLAIModel(), Beer(0.5), ToyRUEGrowthModel(0.2), - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) nothing # hide @@ -94,7 +94,7 @@ models2 = ModelList( ToyLAIModel(), Beer(0.5), ToyAssimGrowthModel(), # This was `ToyRUEGrowthModel(0.2)` before - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) nothing # hide diff --git a/test/test-toy_models.jl b/test/test-toy_models.jl index 8d3e5fc4..4d2c65d6 100644 --- a/test/test-toy_models.jl +++ b/test/test-toy_models.jl @@ -7,20 +7,20 @@ meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), @testset "ToyLAIModel" begin @test_nowarn ModelList(ToyLAIModel()) - @test_nowarn ModelList(ToyLAIModel(), status=(degree_days_cu=10,)) + @test_nowarn ModelList(ToyLAIModel(), status=(TT_cu=10,)) @test_nowarn ModelList( ToyLAIModel(), - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) m = ModelList( ToyLAIModel(), - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) @test_nowarn run!(m) - @test m[:degree_days_cu] == cumsum(meteo_day.degree_days) + @test m[:TT_cu] == cumsum(meteo_day.TT) @test m[:LAI][begin] ≈ 0.00554987593080316 @test m[:LAI][end] ≈ 0.0 end @@ -29,7 +29,7 @@ end models = ModelList( ToyLAIModel(), Beer(0.5), - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) run!(models, meteo_day) @@ -98,7 +98,7 @@ end ToyLAIModel(), Beer(0.5), ToyRUEGrowthModel(rue), - status=(degree_days_cu=cumsum(meteo_day.degree_days),), + status=(TT_cu=cumsum(meteo_day.TT),), ) # Match the warning on the executor, the default is ThreadedEx() but ToyRUEGrowthModel can't be run in parallel: From e27c5d93fbe830613af8dbd4c4d311a2d938502d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 7 Nov 2023 00:33:03 +0100 Subject: [PATCH 79/97] Update examples_import.jl Now uses a sub-module for the examples --- src/examples_import.jl | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/examples_import.jl b/src/examples_import.jl index 4c219ff7..c08e7fe6 100644 --- a/src/examples_import.jl +++ b/src/examples_import.jl @@ -1,7 +1,7 @@ """ - import_multiscale_example() +A sub-module with example models. -Import the examples used in the documentation for a set of multiscale models. +Examples used in the documentation for a set of multiscale models. The models can be found in the `examples` folder of the package, and are stored in the following files: @@ -14,18 +14,20 @@ in the following files: ```jl using PlantSimEngine - -import_multiscale_example() +using PlantSimEngine.Example +ToyAssimModel() ``` """ -function import_multiscale_example() - include(joinpath(@__DIR__, "../examples/ToyAssimModel.jl")) - include(joinpath(@__DIR__, "../examples/ToyCDemandModel.jl")) - include(joinpath(@__DIR__, "../examples/ToyCAllocationModel.jl")) - include(joinpath(@__DIR__, "../examples/ToySoilModel.jl")) -end - +module Examples +using PlantSimEngine, MultiScaleTreeGraph +include(joinpath(@__DIR__, "../examples/ToyDegreeDays.jl")) +include(joinpath(@__DIR__, "../examples/Beer.jl")) +include(joinpath(@__DIR__, "../examples/ToyLAIModel.jl")) +include(joinpath(@__DIR__, "../examples/ToyAssimModel.jl")) +include(joinpath(@__DIR__, "../examples/ToyCDemandModel.jl")) +include(joinpath(@__DIR__, "../examples/ToyCAllocationModel.jl")) +include(joinpath(@__DIR__, "../examples/ToySoilModel.jl")) """ import_mtg_example() @@ -35,11 +37,11 @@ Returns an example multiscale tree graph (MTG) with a scene, a soil, and a plant # Examples ```jldoctest mylabel -julia> using PlantSimEngine +julia> using PlantSimEngine.Examples ``` -```jldoctest -julia> PlantSimEngine.import_mtg_example() +```jldoctest mylabel +julia> import_mtg_example() / 1: Scene ├─ / 2: Soil └─ + 3: Plant @@ -59,4 +61,9 @@ function import_mtg_example() MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) return mtg +end + +export Beer, ToyLAIModel, ToyDegreeDaysCumulModel +export ToyAssimModel, ToyCAllocationModel, ToyCDemandModel, ToySoilWaterModel +export import_mtg_example end \ No newline at end of file From af0d3f1b19398786b9cccf5f27e00fb953270e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 7 Nov 2023 00:33:25 +0100 Subject: [PATCH 80/97] Update doc accordingly --- docs/src/API.md | 18 +++++++++++++++++- src/mtg/GraphSimulation.jl | 12 +++++++----- src/mtg/save_results.jl | 32 +++++--------------------------- 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/docs/src/API.md b/docs/src/API.md index 6cc4f732..6f1fcc95 100644 --- a/docs/src/API.md +++ b/docs/src/API.md @@ -22,4 +22,20 @@ Private functions, types or constants from `PlantSimEngine`. These are not expor Modules = [PlantSimEngine] Public = false Private = true -``` \ No newline at end of file +``` + +## Example models + +PlantSimEngine provides example processes and models to users. They are available from a sub-module called `Examples`. To get access to these models, you can simply use this sub-module: + +```julia +using PlantSimEngine.Examples +``` + +The models are detailed below. + +```@autodocs +Modules = [PlantSimEngine.Examples] +Public = true +Private = true +``` diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index d0ceba78..270b2de2 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -48,18 +48,20 @@ sing the sink function, for exemple a `DataFrame`. # Examples ```@example -using PlantSimEngine, MultiScaleTreeGraph, DataFrames +using PlantSimEngine, MultiScaleTreeGraph, DataFrames, PlantSimEngine.Examples ``` -Import example models (can be found in the `examples` folder of the package): +Import example models (can be found in the `examples` folder of the package, or in the `Examples` sub-modules): -```@example -import_multiscale_example(); +```jldoctest mylabel +julia> using PlantSimEngine.Examples; ``` $MAPPING_EXAMPLE -$MTG_EXAMPLE +```@example +import_mtg_example(); +``` ```@example sim = run!(mtg, mapping, meteo, outputs = Dict( diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl index 78411a52..03b05706 100644 --- a/src/mtg/save_results.jl +++ b/src/mtg/save_results.jl @@ -22,13 +22,13 @@ If false and some variables are missing, return an info, remove the unknown vari # Examples ```jldoctest mylabel -julia> using PlantSimEngine, MultiScaleTreeGraph +julia> using PlantSimEngine, MultiScaleTreeGraph, PlantSimEngine.Examples ``` -Import example models (can be found in the `examples` folder of the package): +Import example models (can be found in the `examples` folder of the package, or in the `Examples` sub-modules): ```jldoctest mylabel -import_multiscale_example(); +julia> using PlantSimEngine.Examples; ``` Define the models mapping: @@ -59,32 +59,10 @@ julia> mapping = Dict( \ ); ``` -```jldoctest mylabel -julia> mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)); -``` - -```jldoctest mylabel -julia> soil = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)); -``` - -```jldoctest mylabel -julia> plant = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)); -``` - -```jldoctest mylabel -julia> internode1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)); -``` - -```jldoctest mylabel -julia> leaf1 = Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); -``` - -```jldoctest mylabel -julia> internode2 = Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)); -``` +Importing an example MTG provided by the package: ```jldoctest mylabel -julia> leaf2 = Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)); +julia> mtg = import_mtg_example(); ``` ```jldoctest mylabel From 322b589deea982874d11438ae7865cc953640bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 7 Nov 2023 00:33:33 +0100 Subject: [PATCH 81/97] forgot readme --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4e96e612..2d4e1b3e 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) # Define the model: model = ModelList( ToyLAIModel(), - status=(degree_days_cu=1.0:2000.0,), # Pass the cumulated degree-days as input to the model + status=(TT_cu=1.0:2000.0,), # Pass the cumulated degree-days as input to the model ) run!(model) # run the model @@ -71,9 +71,9 @@ status(model) # extract the status, i.e. the output of the model Which gives: ``` -TimeStepTable{Status{(:degree_days_cu, :LAI...}(1300 x 2): +TimeStepTable{Status{(:TT_cu, :LAI...}(1300 x 2): ╭─────┬────────────────┬────────────╮ -│ Row │ degree_days_cu │ LAI │ +│ Row │ TT_cu │ LAI │ │ │ Float64 │ Float64 │ ├─────┼────────────────┼────────────┤ │ 1 │ 1.0 │ 0.00560052 │ @@ -95,7 +95,7 @@ Of course you can plot the outputs quite easily: # ] add CairoMakie using CairoMakie -lines(model[:degree_days_cu], model[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Cumulated growing degree days since sowing (°C)")) +lines(model[:TT_cu], model[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Cumulated growing degree days since sowing (°C)")) ``` ![LAI Growth](examples/LAI_growth.png) @@ -119,7 +119,7 @@ meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), model = ModelList( ToyLAIModel(), Beer(0.6), - status=(degree_days_cu=cumsum(meteo_day[:, :degree_days]),), # Pass the cumulated degree-days as input to `ToyLAIModel`, this could also be done using another model + status=(TT_cu=cumsum(meteo_day[:, :TT]),), # Pass the cumulated degree-days as input to `ToyLAIModel`, this could also be done using another model ) ``` @@ -152,9 +152,9 @@ status(model) Which returns: ``` -TimeStepTable{Status{(:degree_days_cu, :LAI...}(365 x 3): +TimeStepTable{Status{(:TT_cu, :LAI...}(365 x 3): ╭─────┬────────────────┬────────────┬───────────╮ -│ Row │ degree_days_cu │ LAI │ aPPFD │ +│ Row │ TT_cu │ LAI │ aPPFD │ │ │ Float64 │ Float64 │ Float64 │ ├─────┼────────────────┼────────────┼───────────┤ │ 1 │ 0.0 │ 0.00554988 │ 0.0476221 │ @@ -173,10 +173,10 @@ using CairoMakie fig = Figure(resolution=(800, 600)) ax = Axis(fig[1, 1], ylabel="LAI (m² m⁻²)") -lines!(ax, model[:degree_days_cu], model[:LAI], color=:mediumseagreen) +lines!(ax, model[:TT_cu], model[:LAI], color=:mediumseagreen) ax2 = Axis(fig[2, 1], xlabel="Cumulated growing degree days since sowing (°C)", ylabel="aPPFD (mol m⁻² d⁻¹)") -lines!(ax2, model[:degree_days_cu], model[:aPPFD], color=:firebrick1) +lines!(ax2, model[:TT_cu], model[:aPPFD], color=:firebrick1) fig ``` From c70161289d5dbd9067da363b6c52f4f15369ed5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 7 Nov 2023 00:33:50 +0100 Subject: [PATCH 82/97] Add doc for multi-scale simulation --- docs/make.jl | 5 +- docs/src/model_coupling/multiscale.md | 241 ++++++++++++++++++++++++++ 2 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 docs/src/model_coupling/multiscale.md diff --git a/docs/make.jl b/docs/make.jl index 28c34f40..5303faa6 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -30,7 +30,10 @@ makedocs(; "Input types" => "./extending/inputs.md", ], "Coupling" => [ - "Users" => "./model_coupling/model_coupling_user.md", + "Users" => [ + "Simple case" => "./model_coupling/model_coupling_user.md", + "Multi-scale modelling" => "./model_coupling/multiscale.md", + ], "Modelers" => "./model_coupling/model_coupling_modeler.md", ], "FAQ" => ["./FAQ/translate_a_model.md"], diff --git a/docs/src/model_coupling/multiscale.md b/docs/src/model_coupling/multiscale.md new file mode 100644 index 00000000..8e4d13ff --- /dev/null +++ b/docs/src/model_coupling/multiscale.md @@ -0,0 +1,241 @@ +# Multi-scale modeling + +## What is multi-scale modeling? + +Multi-scale modeling is the process of simulating a system at multiple levels of detail simultaneously. For example, some models can run at the organ scale while others run at the plot scale. Each model can access variables at its scale and other scales if needed, allowing for a more comprehensive system representation. It can also help identify emergent properties that are not apparent at a single level of detail. + +For example, a model of photosynthesis at the leaf scale can be combined with a model of carbon allocation at the plant scale to simulate the growth and development of the plant. Another example is a combination of models to simulate the energy balance of a forest. To simulate it, you need a model for each organ type of the plant, another for the soil, and finally, one at the plot scale, integrating all others. + +PlantSimEngine provides a framework for multi-scale modeling to seamlessly integrate models at different scales, keeping all nice functionalities provided at one scale. A nice feature is that models do not need to be aware of the scale at which they are running, nor about the scales at which their inputs are computed, or outputs will be given, which means the model can be reused at different scales or no scale. + +PlantSimEngine automatically computes the dependency graph between mono and multi-scale models, considering every combination of models at any scale, to determine the order of model execution. This means that the user does not need to worry about the order of model execution and can focus on the model definition and the mapping between models and scales. + +Using PlantSimEngine for multi-scale modeling is relatively easy and follows the same rules as mono-scale models. Let's dive into the details with a short tutorial. + +## Simple mapping between models and scales + +To get started, we have to define a mapping between models and scales. + +Let's import the `PlantSimEngine` package and example models we will use in this tutorial: + +```@example usepkg +using PlantSimEngine +using PlantSimEngine.Examples # Import some example models +``` + +!!! note + The `Examples` submodule exports a few simple models we will use in this tutorial. The models are also found in the `examples` folder of the package. + +We now have access to models for the simulation of different processes. We can associate each model with a scale by defining a mapping between models and scales. The mapping is a dictionary with the name of the scale as the key and the model as the value. For example, we can define a mapping to simulate the assimilation process at the leaf scale with `ToyAssimModel` as follows: + +```@example usepkg +mapping = Dict("Leaf" => ToyAssimModel()) +``` + +In this example, the dictionary's key is the name of the scale (`"Leaf"`), and the value is the model. The model is an example model provided by `PlantSimEngine`, so we must prefix it with the module name. + +We can check if the mapping is valid by calling `to_initialize`: + +```@example usepkg +to_initialize(mapping) +``` + +The `to_initialize` function checks if models from any scale need further initialization before simulation. This is the case when some input variables of the model are not computed by another model. In this example, the `ToyAssimModel` needs `:aPPFD` and `:soil_water_content` as inputs. To run a simulation, we must provide a value for the variables or a model that simulates them. + +The initialization values for the variables can be provided using the `Status` type along with the model, *e.g.*: + +```@example usepkg +mapping = Dict( + "Leaf" => ( + ToyAssimModel(), + Status(aPPFD=1300.0, soil_water_content=0.5), + ), +) +``` + +!!! note + The model and the `Status` are provided as a `Tuple` to the `"Leaf"` scale. + +If we re-execute `to_initialize`, we get an empty dictionary, meaning the mapping is valid, and we can start the simulation: + +```@example usepkg +to_initialize(mapping) +``` + +## Multiscale mapping between models and scales + +In our previous example, we provided the value for the `soil_water_content` variable. However, we could also provide a model that simulates it at the soil scale. The only difference now is that we have to tell PlantSimEngine that our +`ToyAssimModel` is now multiscale and takes the `soil_water_content` variable from the `"Soil"` scale. We can do that by wrapping the `ToyAssimModel` in a `MultiScaleModel`: + +```@example usepkg +mapping = Dict( + "Soil" => ToySoilWaterModel(), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil",], + ), + Status(aPPFD=1300.0), + ), +) +``` + +The `MultiScaleModel` takes two arguments: the model and the mapping between the model and the scales. The mapping is a vector of pairs mapping the variable's name with the name of the scale its value comes from. In this example, we map the `soil_water_content` variable to the `"Soil"` scale. + +!!! note + The variable `aPPFD` is still provided in the `Status` type as a constant value. + +We can check again if the mapping is valid by calling `to_initialize`: + +```@example usepkg +to_initialize(mapping) +``` + +`to_initialize` returns an empty dictionary, meaning the mapping is valid. + +## More on MultiScaleModel + +`MultiScaleModel` is a wrapper around a model that allows it to take inputs or give outputs from other scales. It takes two arguments: the model and the mapping between the model and the scales. The mapping is a vector of pairs mapping the variable's name with the name of the scale its value comes from. + +The variable can map a single value if there is only one node to map to or a vector of values if there are several. It can also map to several types of nodes at the same time. + +Let's take a look at a more complex example of a mapping: + +```@example usepkg +mapping = Dict( + "Scene" => ToyDegreeDaysCumulModel(), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapping=[ + :TT_cu => "Scene", + ], + ), + Beer(0.6), + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + ), + "Internode" => ( + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapping=[:TT => "Scene",], + ) + ), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil", :aPPFD => "Plant"], + ), + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapping=[:TT => "Scene",], + ), + ), + "Soil" => ( + ToySoilWaterModel(), + ), +); +``` + +In this example, we expect to make a simulation at five different scales: `"Scene"`, `"Plant"`, `"Internode"`, `"Leaf"`, and `"Soil"`. The `"Scene"` scale represents the whole scene, where one or several plants can live. The `"Plant"` scale is, well, the whole plant scale, `"Internode"` and `"Leaf"` are organ scales, and `"Soil"` is the soil scale. This mapping is used to compute the carbon allocation (`ToyCAllocationModel`) to the different organs of the plant (`"Leaf"` and `"Internode"`) from the assimilation at the `"Leaf"` scale (*i.e.* the offer) and their carbon demand (`ToyCDemandModel`). The `"Soil"` scale is used to compute the soil water content (`ToySoilWaterModel`), which is needed to calculate the assimilation at the `"Leaf"` scale (`ToyAssimModel`). + +We see that all scales are interconnected, with computations at the organ scale that may depend on the soil scale and at the plant scale that depends on the organ scale and scene scale. + +Something important to note here is that we have different ways to define the mapping for the `MultiScaleModel`. For example, we have `:A => ["Leaf"]` at the plant scale for `ToyCAllocationModel`. This mapping means that the variable `A` is mapped to the `"Leaf"` scale. However, we could also have `:A => "Leaf"`, which is equivalent. + +!!! note + Note the difference between `:A => ["Leaf"]` and `:A => "Leaf"` is that "Leaf" is given as a vector in the first definition, and as a scalar in the second one. + +The difference is that the first maps to a vector of values, while the second maps to a single value. The first one is useful when we don't know how many nodes there will be in the plant of type `"Leaf"`. In this case, the values are available as a vector in the `A` variable of the `status` inside the model. The second one should only be used if we are sure that there will be only one node at this scale, and in this case, the one and single value is given as a scalar in the `A` variable of the `status` inside the model. + +A third form for the mapping would be `:A => ["Leaf", "Internode"]`. This form is useful when we need values for a variable from several scales simultaneously. In this case, the values are available as a vector in the `A` variable of the `status` inside the model, sorted in the same order as nodes are traversed in the graph. + +## Running a simulation + +Now that we have a valid mapping, we can run a simulation. Running a multiscale simulation requires two more things compared to what we saw previously: a plant graph and the definition of the output variables we want dynamically for each scale. + +### Plant graph + +We can import an example multi-scale tree graph like so: + +```@example usepkg +mtg = import_mtg_example() +``` + +!!! note + You can use `import_mtg_example` only if you previously imported the `Examples` sub-module of PlantSimEngine, *i.e.* `using PlantSimEngine.Examples`. + +This graph has a root node that defines a scene, then a soil, and a plant with two internodes and two leaves. + +### Output variables + +Models can access only one time step at a time, so the output at the end of a simulation is only the last time step. However, we can define a list of variables we want to get dynamically for each time step and each scale. This list is given as a dictionary with the name of the scale as the key and a vector of variables as the value. For example, we can define a list of variables we want to get at each time step for different scales as follows: + +```@example usepkg +outs = Dict( + "Scene" => (:TT, :TT_cu, :node), + "Plant" => (:aPPFD, :LAI), + "Leaf" => (:A, :carbon_demand, :carbon_allocation, :TT), + "Internode" => (:carbon_allocation,), + "Soil" => (:soil_water_content,), +) +``` + +These variables will be available in the `outputs` field of the simulation object, with a value for each time step. + +### Meteorological data + +As for mono-scale models, we need to provide meteorological data to run a simulation. We can use the `PlantMeteo` package to generate some dummy data for two time steps: + +```@example usepkg +meteo = Weather( + [ + Atmosphere(T=20.0, Wind=1.0, Rh=0.65, Ri_PAR_f = 200.0), + Atmosphere(T=25.0, Wind=0.5, Rh=0.8, Ri_PAR_f = 180.0) +] +) +``` + +### Simulation + +Let's make a simulation using the graph and outputs we just defined: + +```@example usepkg +sim = run!(mtg, mapping, meteo, outputs = outs); +nothing # hide +``` + +And that's it! + +We can now access the outputs for each scale as a dictionary of vectors of values per variable and scale like this: + +```@example usepkg +outputs(sim) +``` + +Or as a `DataFrame` using the `DataFrames` package: + +```@example usepkg +using DataFrames +outputs(sim, DataFrame) +``` + +The values for the last time-step of the simulation are also available from the statuses: + +```@example usepkg +status(sim) +``` + +This is a dictionary with the scale as the key and a vector of `Status` as values, one per node of that scale. So, in this example, the `"Leaf"` scale has two nodes, so the value is a vector of two `Status` objects, and the `"Soil"` scale has only one node, so the value is a vector of one `Status` object. + +### Wrapping up + +In this section, we saw how to define a mapping between models and scales, run a simulation, and access the outputs. + +This is just a simple example, but PlantSimEngine can be used to define and combine much more complex models at multiple scales of detail. With its modular architecture and intuitive API, PlantSimEngine is a powerful tool for multi-scale plant growth and development modeling. \ No newline at end of file From 1a41dcfe77c0446cda296675cb9f9468302d1909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 13 Nov 2023 10:12:54 +0100 Subject: [PATCH 83/97] Fix depreciation warning Wrapping `Vararg` directly in UnionAll is deprecated (wrap the tuple instead). See: https://github.com/JuliaLang/julia/issues/49553 --- src/component_models/Status.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/component_models/Status.jl b/src/component_models/Status.jl index ee3af99f..debca9d2 100644 --- a/src/component_models/Status.jl +++ b/src/component_models/Status.jl @@ -54,7 +54,7 @@ julia> st[1] = 22.0 22.0 ``` """ -struct Status{N,T<:Tuple{Vararg{<:Ref}}} +struct Status{N,T<:Tuple{Vararg{Ref}}} vars::NamedTuple{N,T} end From 4b30713906f486eec176d9f06d1b6517aaea7083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 13 Nov 2023 14:30:15 +0100 Subject: [PATCH 84/97] Replace include([examples]) by using PlantSimEngine.Examples --- README.md | 7 +++--- docs/src/FAQ/translate_a_model.md | 3 ++- docs/src/design.md | 10 ++++---- docs/src/extending/implement_a_model.md | 7 +++--- docs/src/fitting.md | 6 +++-- docs/src/index.md | 14 +++++------ .../model_coupling/model_coupling_modeler.md | 4 +++- .../src/model_coupling/model_coupling_user.md | 15 +++++++----- docs/src/model_switching.md | 12 ++++------ docs/src/reducing_dof.md | 6 +++-- examples/Beer.jl | 10 +++++++- examples/ToyAssimGrowthModel.jl | 4 +--- examples/ToyRUEGrowthModel.jl | 4 ++-- examples/benchmark.jl | 5 +--- examples/dummy.jl | 14 +++++------ src/checks/dimensions.jl | 2 +- src/component_models/ModelList.jl | 12 +++++----- src/component_models/get_status.jl | 4 ++-- src/dependencies/dependencies.jl | 4 ++-- src/dependencies/dependency_graph.jl | 5 ++-- src/dependencies/soft_dependencies.jl | 2 +- src/evaluation/fit.jl | 4 +++- src/examples_import.jl | 12 ++++++++-- src/mtg/MultiScaleModel.jl | 4 +++- src/mtg/init_mtg_models.jl | 4 ++-- src/processes/model_initialisation.jl | 24 +++++++++---------- src/processes/models_inputs_outputs.jl | 16 ++++++------- src/run.jl | 4 ++-- test/runtests.jl | 16 +------------ test/test-ModelList.jl | 2 -- test/test-fitting.jl | 2 -- test/test-simulation.jl | 2 -- test/test-toy_models.jl | 5 ---- 33 files changed, 120 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 2d4e1b3e..8b63f943 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ Here's a simple example of a model that simulates the growth of a plant, using a # ] add PlantSimEngine using PlantSimEngine -# Include the model definition from the examples folder: -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) +# Include the model definition from the examples sub-module: +using PlantSimEngine.Examples # Define the model: model = ModelList( @@ -109,8 +109,7 @@ Model coupling is done automatically by the package, and is based on the depende using PlantSimEngine, PlantMeteo, DataFrames, CSV # Include the model definition from the examples folder: -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +using PlantSimEngine.Examples # Import the example meteorological data: meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) diff --git a/docs/src/FAQ/translate_a_model.md b/docs/src/FAQ/translate_a_model.md index 9999cffc..ebea01d0 100644 --- a/docs/src/FAQ/translate_a_model.md +++ b/docs/src/FAQ/translate_a_model.md @@ -4,7 +4,8 @@ using PlantSimEngine using CairoMakie using CSV, DataFrames -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) +# Import the example models defined in the `Examples` sub-module: +using PlantSimEngine.Examples function lai_toymodel(TT_cu; max_lai=8.0, dd_incslope=500, inc_slope=70, dd_decslope=1000, dec_slope=20) LAI = max_lai * (1 / (1 + exp((dd_incslope - TT_cu) / inc_slope)) - 1 / (1 + exp((dd_decslope - TT_cu) / dec_slope))) diff --git a/docs/src/design.md b/docs/src/design.md index f43532cd..d6037c03 100644 --- a/docs/src/design.md +++ b/docs/src/design.md @@ -4,7 +4,7 @@ ```@setup usepkg using PlantSimEngine, PlantMeteo -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +using PlantSimEngine.Examples meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) leaf = ModelList(Beer(0.5), status = (LAI = 2.0,)) run!(leaf, meteo) @@ -56,10 +56,10 @@ Importing the package: using PlantSimEngine ``` -Including the script that defines `light_interception` and `Beer`: +Import the examples defined in the `Examples` sub-module (`light_interception` and `Beer`): ```julia -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +using PlantSimEngine.Examples ``` And then making a [`ModelList`](@ref) with the `Beer` model: @@ -187,8 +187,8 @@ For example we can simulate the `light_interception` of a leaf like so: ```@example usepkg using PlantSimEngine, PlantMeteo -# Including the script defining light_interception and Beer: -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +# Import the examples defined in the `Examples` sub-module +using PlantSimEngine.Examples meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) diff --git a/docs/src/extending/implement_a_model.md b/docs/src/extending/implement_a_model.md index fa5adf79..218116bf 100644 --- a/docs/src/extending/implement_a_model.md +++ b/docs/src/extending/implement_a_model.md @@ -33,12 +33,13 @@ In those files, you'll see that in order to implement a new model you'll need to If you create your own process, the function will print a short tutorial on how to do all that, adapted to the process you just created (see [Implement a new process](@ref)). -In this page, we'll just implement a model for a process that already exists: the light interception. This process is defined in `PlantBiophysics.jl`, but also in an example script in `PlantSimEngine.jl` here: [`examples/Beer.jl`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/Beer.jl). +In this page, we'll just implement a model for a process that already exists: the light interception. This process is defined in `PlantBiophysics.jl`, and also made available as an example model from the `Examples` sub-module. You can access the script from here: [`examples/Beer.jl`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/Beer.jl). -We can include this file like so: +We can import the model like so: ```julia -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +# Import the example models defined in the `Examples` sub-module: +using PlantSimEngine.Examples ``` But instead of just using it, we will review the script line by line. diff --git a/docs/src/fitting.md b/docs/src/fitting.md index 16237460..f946ae09 100644 --- a/docs/src/fitting.md +++ b/docs/src/fitting.md @@ -2,7 +2,8 @@ ```@setup usepkg using PlantSimEngine, PlantMeteo, DataFrames, Statistics -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +using PlantSimEngine.Examples + meteo = Atmosphere(T=20.0, Wind=1.0, P=101.3, Rh=0.65, Ri_PAR_f=300.0) m = ModelList(Beer(0.6), status=(LAI=2.0,)) run!(m, meteo) @@ -42,7 +43,8 @@ Importing the script first: ```julia using PlantSimEngine, PlantMeteo, DataFrames, Statistics -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +# Import the examples defined in the `Examples` sub-module: +using PlantSimEngine.Examples ``` Defining the meteo data: diff --git a/docs/src/index.md b/docs/src/index.md index cfcd5dfa..8e478e30 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -5,9 +5,8 @@ CurrentModule = PlantSimEngine ```@setup readme using PlantSimEngine, PlantMeteo, DataFrames, CSV -# Include the model definition from the examples folder: -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +# Import the examples defined in the `Examples` sub-module: +using PlantSimEngine.Examples # Import the example meteorological data: meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) @@ -83,8 +82,8 @@ Here's a simple example of a model that simulates the growth of a plant, using a # ] add PlantSimEngine using PlantSimEngine -# Include the model definition from the examples folder: -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) +# Import the examples defined in the `Examples` sub-module +using PlantSimEngine.Examples # Define the model: model = ModelList( @@ -117,9 +116,8 @@ Model coupling is done automatically by the package, and is based on the depende # ] add PlantSimEngine, DataFrames, CSV using PlantSimEngine, PlantMeteo, DataFrames, CSV -# Include the model definition from the examples folder: -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +# Import the examples defined in the `Examples` sub-module +using PlantSimEngine.Examples # Import the example meteorological data: meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) diff --git a/docs/src/model_coupling/model_coupling_modeler.md b/docs/src/model_coupling/model_coupling_modeler.md index 4e63cb60..8a5f424c 100644 --- a/docs/src/model_coupling/model_coupling_modeler.md +++ b/docs/src/model_coupling/model_coupling_modeler.md @@ -2,7 +2,9 @@ ```@setup usepkg using PlantSimEngine, PlantMeteo -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Import the example models defined in the `Examples` sub-module: +using PlantSimEngine.Examples + m = ModelList( Process1Model(2.0), Process2Model(), diff --git a/docs/src/model_coupling/model_coupling_user.md b/docs/src/model_coupling/model_coupling_user.md index 035a3ec4..cb2d57dd 100644 --- a/docs/src/model_coupling/model_coupling_user.md +++ b/docs/src/model_coupling/model_coupling_user.md @@ -2,7 +2,9 @@ ```@setup usepkg using PlantSimEngine, PlantMeteo -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Import the example models defined in the `Examples` sub-module: +using PlantSimEngine.Examples + m = ModelList( Process1Model(2.0), Process2Model(), @@ -29,16 +31,17 @@ The other models for the other processes are called `Process4Model`, `Process5Mo Back to our example, using `Process3Model` requires a "process2" model, and in our case the only model available is `Process2Model`. The latter also requires a "process1" model, and again we only have one model implementation for this process, which is `Process1Model`. -Let's include this script so we can play around: +Let's use the `Examples` sub-module so we can play around: ```julia -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Import the example models defined in the `Examples` sub-module: +using PlantSimEngine.Examples ``` !!! tip Use subtype(x) to know which models are available for a process, e.g. for "process1" you can do `subtypes(AbstractProcess1Model)`. -Here is how we can make the models coupling: +Here is how we can make the model coupling: ```@example usepkg m = ModelList(Process1Model(2.0), Process2Model(), Process3Model()) @@ -47,11 +50,11 @@ nothing # hide We can see that only the first model has a parameter. You can usually know that by looking at the help of the structure (*e.g.* `?Process1Model`), else, you can still look at the field names of the structure like so `fieldnames(Process1Model)`. -Note that the user only declares the models, not the way the models are coupled, because `PlantSimEngine.jl` deals with that automatically. +Note that the user only declares the models, not the way the models are coupled because `PlantSimEngine.jl` deals with that automatically. Now the example above returns some warnings saying we need to initialize some variables: `var1` and `var2`. `PlantSimEngine.jl` automatically computes which variables should be initialized based on the inputs and outputs of all models, considering their hard or soft-coupling. -For example `Process1Model` requires the following variables as inputs: +For example, `Process1Model` requires the following variables as inputs: ```@example usepkg inputs(Process1Model(2.0)) diff --git a/docs/src/model_switching.md b/docs/src/model_switching.md index ecfb19cb..765c736c 100644 --- a/docs/src/model_switching.md +++ b/docs/src/model_switching.md @@ -2,10 +2,8 @@ ```@setup usepkg using PlantSimEngine, PlantMeteo, CSV, DataFrames -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimGrowthModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyRUEGrowthModel.jl")) +# Import the examples defined in the `Examples` sub-module +using PlantSimEngine.Examples meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) @@ -39,10 +37,8 @@ Importing the models from the scripts: ```julia using PlantSimEngine -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimGrowthModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyRUEGrowthModel.jl")) +# Import the examples defined in the `Examples` sub-module: +using PlantSimEngine.Examples ``` Coupling the models in a `ModelList`: diff --git a/docs/src/reducing_dof.md b/docs/src/reducing_dof.md index b3cc1800..8f6b53aa 100644 --- a/docs/src/reducing_dof.md +++ b/docs/src/reducing_dof.md @@ -2,7 +2,8 @@ ```@setup usepkg using PlantSimEngine, PlantMeteo -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Import the examples defined in the `Examples` sub-module: +using PlantSimEngine.Examples meteo = Atmosphere(T = 20.0, Wind = 1.0, P = 101.3, Rh = 0.65) struct ForceProcess1Model <: AbstractProcess1Model end @@ -44,7 +45,8 @@ Let's define a model list as usual with the seven processes from `examples/dummy ```@example usepkg using PlantSimEngine, PlantMeteo -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Import the examples defined in the `Examples` sub-module: +using PlantSimEngine.Examples meteo = Atmosphere(T = 20.0, Wind = 1.0, P = 101.3, Rh = 0.65) m = ModelList( diff --git a/examples/Beer.jl b/examples/Beer.jl index f9ff9702..f7d1893b 100644 --- a/examples/Beer.jl +++ b/examples/Beer.jl @@ -85,8 +85,16 @@ Compute the `k` parameter of the Beer-Lambert law from measurements. # Examples +Import the example models defined in the `Examples` sub-module: + +```julia +using PlantSimEngine +using PlantSimEngine.Examples +``` + +Create a model list with a Beer model, and fit it to the data: + ```julia -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) m = ModelList(Beer(0.6), status=(LAI=2.0,)) meteo = Atmosphere(T=20.0, Wind=1.0, P=101.3, Rh=0.65, Ri_PAR_f=300.0) run!(m, meteo) diff --git a/examples/ToyAssimGrowthModel.jl b/examples/ToyAssimGrowthModel.jl index 6ccad8cb..95b200b4 100644 --- a/examples/ToyAssimGrowthModel.jl +++ b/examples/ToyAssimGrowthModel.jl @@ -1,7 +1,5 @@ -# using PlantSimEngine, PlantMeteo # Import the necessary packages, PlantMeteo is used for the meteorology - # Defining the process: -@process "growth" verbose = false +PlantSimEngine.@process "growth" verbose = false # Make the struct to hold the parameters, with its documentation: """ diff --git a/examples/ToyRUEGrowthModel.jl b/examples/ToyRUEGrowthModel.jl index 2b9d89a1..997337e9 100644 --- a/examples/ToyRUEGrowthModel.jl +++ b/examples/ToyRUEGrowthModel.jl @@ -1,7 +1,7 @@ # using PlantSimEngine, PlantMeteo # Import the necessary packages, PlantMeteo is used for the meteorology -# Defining the process: -@process "growth" verbose = false +# The process is defined in ToyAssimGrowthModel.jl: +# PlantSimEngine.@process "growth" verbose = false # Make the struct to hold the parameters, with its documentation: """ diff --git a/examples/benchmark.jl b/examples/benchmark.jl index 3d12b1c9..0ecfdbfc 100644 --- a/examples/benchmark.jl +++ b/examples/benchmark.jl @@ -2,10 +2,7 @@ using BenchmarkTools using PlantSimEngine, PlantMeteo, DataFrames, CSV, Dates, Statistics -# include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) -# include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) -# include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimGrowthModel.jl")) -# include(joinpath(pkgdir(PlantSimEngine), "examples/ToyRUEGrowthModel.jl")) +# using PlantSimEngine.Examples meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) models = ModelList( diff --git a/examples/dummy.jl b/examples/dummy.jl index 4230da4f..d27f994e 100644 --- a/examples/dummy.jl +++ b/examples/dummy.jl @@ -3,7 +3,7 @@ # Defining a process called "process1" and a model # that implements an algorithm (Process1Model): -@process "process1" verbose = false +PlantSimEngine.@process "process1" verbose = false """ Process1Model(a) @@ -24,7 +24,7 @@ PlantSimEngine.ObjectDependencyTrait(::Type{<:Process1Model}) = PlantSimEngine.I # Defining a 2nd process called "process2", and a model # that implements an algorithm, and that depends on the first one: -@process "process2" verbose = false +PlantSimEngine.@process "process2" verbose = false """ Process2Model() @@ -48,7 +48,7 @@ PlantSimEngine.ObjectDependencyTrait(::Type{<:Process2Model}) = PlantSimEngine.I # Defining a 3d process called "process3", and a model # that implements an algorithm, and that depends on the second one (and # by extension on the first one): -@process "process3" verbose = false +PlantSimEngine.@process "process3" verbose = false """ Process3Model() @@ -72,7 +72,7 @@ PlantSimEngine.ObjectDependencyTrait(::Type{<:Process3Model}) = PlantSimEngine.I # Defining a 4th process called "process4", and a model # that implements an algorithm, and that computes the # inputs of the root of the previous ones (process3): -@process "process4" verbose = false +PlantSimEngine.@process "process4" verbose = false """ Process4Model() @@ -95,7 +95,7 @@ PlantSimEngine.ObjectDependencyTrait(::Type{<:Process4Model}) = PlantSimEngine.I # Defining a 5th process called "process5", and a model # that implements an algorithm, and that computes other # variables from outputs of process 1-2-3 (soft coupling): -@process "process5" verbose = false +PlantSimEngine.@process "process5" verbose = false """ Process5Model() @@ -116,7 +116,7 @@ PlantSimEngine.ObjectDependencyTrait(::Type{<:Process5Model}) = PlantSimEngine.I # Defining a 6th process called "process6", and a model # that implements an algorithm, and that computes other # variables from outputs of process 5 (soft-coupling): -@process "process6" verbose = false +PlantSimEngine.@process "process6" verbose = false """ Process6Model() @@ -138,7 +138,7 @@ PlantSimEngine.ObjectDependencyTrait(::Type{<:Process6Model}) = PlantSimEngine.I # that depends on nothing but var0 so it is independant. # But Process6Model depends on its output, so it is a soft-coupling: # variables from outputs of process 5 (soft-coupling):(var0=-Inf,) -@process "process7" verbose = false +PlantSimEngine.@process "process7" verbose = false """ Process7Model() diff --git a/src/checks/dimensions.jl b/src/checks/dimensions.jl index 9804ebac..ee6f6373 100644 --- a/src/checks/dimensions.jl +++ b/src/checks/dimensions.jl @@ -10,7 +10,7 @@ recycled (length 1 for one of them). using PlantSimEngine, PlantMeteo # Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +using PlantSimEngine.Examples # Creating a dummy weather: w = Atmosphere(T = 20.0, Rh = 0.5, Wind = 1.0) diff --git a/src/component_models/ModelList.jl b/src/component_models/ModelList.jl index 0546b5e8..b731168c 100644 --- a/src/component_models/ModelList.jl +++ b/src/component_models/ModelList.jl @@ -58,10 +58,10 @@ one model implementation each: `Process1Model`, `Process2Model` and `Process3Mod julia> using PlantSimEngine; ``` -Including an example script that implements dummy processes and models: +Including example processes and models: ```jldoctest 1 -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")); +julia> using PlantSimEngine.Examples; ``` ```jldoctest 1 @@ -374,8 +374,8 @@ Copy a [`ModelList`](@ref), eventually with new values for the status. ```@example using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Including example processes and models: +using PlantSimEngine.Examples; # Create a model list: models = ModelList( @@ -440,8 +440,8 @@ If we want all the variables that are Reals to be Float32, we can use: ```julia using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Including example processes and models: +using PlantSimEngine.Examples; ref_vars = init_variables( process1=Process1Model(1.0), diff --git a/src/component_models/get_status.jl b/src/component_models/get_status.jl index f21dfe69..5a17e482 100644 --- a/src/component_models/get_status.jl +++ b/src/component_models/get_status.jl @@ -12,8 +12,8 @@ See also [`is_initialized`](@ref) and [`to_initialize`](@ref) ```jldoctest using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")); +# Including example models and processes: +using PlantSimEngine.Examples; # Create a ModelList models = ModelList( diff --git a/src/dependencies/dependencies.jl b/src/dependencies/dependencies.jl index 5a714d39..f7e62010 100644 --- a/src/dependencies/dependencies.jl +++ b/src/dependencies/dependencies.jl @@ -41,8 +41,8 @@ these graphs independently to retrieve the models that are coupled together, in ```@example using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Including example processes and models: +using PlantSimEngine.Examples; models = ModelList( process1=Process1Model(1.0), diff --git a/src/dependencies/dependency_graph.jl b/src/dependencies/dependency_graph.jl index 1aa56ba4..9a62af1b 100644 --- a/src/dependencies/dependency_graph.jl +++ b/src/dependencies/dependency_graph.jl @@ -78,12 +78,13 @@ Return a vector of pairs of the node and the result of the function `f`. ```julia using PlantSimEngine +# Including example processes and models: +using PlantSimEngine.Examples; + function f(node) node.value end -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) - vars = ( process1=Process1Model(1.0), process2=Process2Model(), diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index a6be6271..6e172dc1 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -15,7 +15,7 @@ can be inferred from the inputs and outputs of the processes. using PlantSimEngine # Load the dummy models given as example in the package: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +using PlantSimEngine.Examples # Create a model list: models = ModelList( diff --git a/src/evaluation/fit.jl b/src/evaluation/fit.jl index 51d72758..e477f425 100644 --- a/src/evaluation/fit.jl +++ b/src/evaluation/fit.jl @@ -26,7 +26,9 @@ Here is an example usage with the `Beer` model, where we fit the `k` parameter f and `Ri_PAR_f`. ```julia -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) +# Including example processes and models: +using PlantSimEngine.Examples; + m = ModelList(Beer(0.6), status=(LAI=2.0,)) meteo = Atmosphere(T=20.0, Wind=1.0, P=101.3, Rh=0.65, Ri_PAR_f=300.0) run!(m, meteo) diff --git a/src/examples_import.jl b/src/examples_import.jl index c08e7fe6..bc15d11f 100644 --- a/src/examples_import.jl +++ b/src/examples_import.jl @@ -14,18 +14,22 @@ in the following files: ```jl using PlantSimEngine -using PlantSimEngine.Example +using PlantSimEngine.Examples ToyAssimModel() ``` """ module Examples -using PlantSimEngine, MultiScaleTreeGraph +using PlantSimEngine, MultiScaleTreeGraph, PlantMeteo, Statistics + +include(joinpath(@__DIR__, "../examples/dummy.jl")) include(joinpath(@__DIR__, "../examples/ToyDegreeDays.jl")) include(joinpath(@__DIR__, "../examples/Beer.jl")) include(joinpath(@__DIR__, "../examples/ToyLAIModel.jl")) include(joinpath(@__DIR__, "../examples/ToyAssimModel.jl")) include(joinpath(@__DIR__, "../examples/ToyCDemandModel.jl")) +include(joinpath(@__DIR__, "../examples/ToyAssimGrowthModel.jl")) +include(joinpath(@__DIR__, "../examples/ToyRUEGrowthModel.jl")) include(joinpath(@__DIR__, "../examples/ToyCAllocationModel.jl")) include(joinpath(@__DIR__, "../examples/ToySoilModel.jl")) @@ -65,5 +69,9 @@ end export Beer, ToyLAIModel, ToyDegreeDaysCumulModel export ToyAssimModel, ToyCAllocationModel, ToyCDemandModel, ToySoilWaterModel +export ToyAssimGrowthModel, ToyRUEGrowthModel +export Process1Model, Process2Model, Process3Model, Process4Model, Process5Model +export Process6Model, Process7Model + export import_mtg_example end \ No newline at end of file diff --git a/src/mtg/MultiScaleModel.jl b/src/mtg/MultiScaleModel.jl index 30884df7..178196ca 100644 --- a/src/mtg/MultiScaleModel.jl +++ b/src/mtg/MultiScaleModel.jl @@ -26,8 +26,10 @@ of one node, they will be updated in the other nodes. julia> using PlantSimEngine; ``` +Including example processes and models: + ```jldoctest mylabel -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")); +julia> using PlantSimEngine.Examples; ``` Let's take a model: diff --git a/src/mtg/init_mtg_models.jl b/src/mtg/init_mtg_models.jl index 5f48479e..b3775aed 100644 --- a/src/mtg/init_mtg_models.jl +++ b/src/mtg/init_mtg_models.jl @@ -27,8 +27,8 @@ Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models")) ```@example using PlantSimEngine, MultiScaleTreeGraph -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Including example processes and models: +using PlantSimEngine.Examples; # Make an MTG: mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index f577caf2..64f6e516 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -23,8 +23,8 @@ considering that some variables that are outputs of some models are used as inpu ```@example using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples to_initialize(process1=Process1Model(1.0), process2=Process2Model()) @@ -48,8 +48,8 @@ Or with a mapping: ```@example using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples mapping = Dict( "Leaf" => ModelList( @@ -246,8 +246,8 @@ Initialise model variables for components with user input. ```@example using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples models = Dict( "Leaf" => ModelList( @@ -300,8 +300,8 @@ inputs and outputs of the models. ```@example using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples init_variables(Process1Model(2.0)) init_variables(process1=Process1Model(2.0), process2=Process2Model()) @@ -360,8 +360,8 @@ for other models. ```@example using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples models = ModelList( process1=Process1Model(1.0), @@ -435,8 +435,8 @@ Return an initialisation of the model variables with given values. ```@example using PlantSimEngine -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples models = ModelList( process1=Process1Model(1.0), diff --git a/src/processes/models_inputs_outputs.jl b/src/processes/models_inputs_outputs.jl index 923887ff..5ca7bb00 100644 --- a/src/processes/models_inputs_outputs.jl +++ b/src/processes/models_inputs_outputs.jl @@ -11,8 +11,8 @@ Returns an empty tuple by default for `AbstractModel`s (no inputs) or `Missing` ```jldoctest using PlantSimEngine; -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")); +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples; inputs(Process1Model(1.0)) @@ -64,8 +64,8 @@ Returns an empty tuple by default for `AbstractModel`s (no outputs) or `Missing` ```jldoctest using PlantSimEngine; -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")); +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples; outputs(Process1Model(1.0)) @@ -120,8 +120,8 @@ Each model can (and should) have a method for this function. using PlantSimEngine; -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")); +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples; variables(Process1Model(1.0)) @@ -214,8 +214,8 @@ union of those for several models. ```jldoctest using PlantSimEngine; -# Including an example script that implements dummy processes and models: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")); +# Load the dummy models given as example in the package: +using PlantSimEngine.Examples; PlantSimEngine.variables_typed(Process1Model(1.0)) (var1 = Float64, var2 = Float64, var3 = Float64) diff --git a/src/run.jl b/src/run.jl index 7fff4a27..5554d0fc 100644 --- a/src/run.jl +++ b/src/run.jl @@ -53,10 +53,10 @@ Import the packages: julia> using PlantSimEngine, PlantMeteo; ``` -Load the dummy models given as example in the package: +Load the dummy models given as example in the `Examples` sub-module: ```jldoctest run -julia> include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")); +julia> using PlantSimEngine.Examples; ``` Create a model list: diff --git a/test/runtests.jl b/test/runtests.jl index 54887e4f..dedb79f9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,17 +6,7 @@ using PlantMeteo, Statistics using Documenter # for doctests # Include the example dummy processes: -include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimGrowthModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyRUEGrowthModel.jl")) - -# For the multiscale models: -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCDemandModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToyCAllocationModel.jl")) -include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")) +using PlantSimEngine.Examples @testset "Testing PlantSimEngine" begin Aqua.test_all(PlantSimEngine, ambiguities=false) @@ -54,10 +44,6 @@ include(joinpath(pkgdir(PlantSimEngine), "examples/ToySoilModel.jl")) include("test-toy_models.jl") end - @testset "MTG" begin - include("test-mtg.jl") - end - @testset "MTG with multiscale mapping" begin include("test-mtg-multiscale.jl") end diff --git a/test/test-ModelList.jl b/test/test-ModelList.jl index f8c12aaa..97a5536d 100644 --- a/test/test-ModelList.jl +++ b/test/test-ModelList.jl @@ -1,5 +1,3 @@ -# include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) - # Tests: # Defining a list of models without status: @testset "ModelList with no status" begin diff --git a/test/test-fitting.jl b/test/test-fitting.jl index 8d1624d5..d5898b49 100644 --- a/test/test-fitting.jl +++ b/test/test-fitting.jl @@ -1,5 +1,3 @@ -# include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) - # Tests: # Defining a list of models without status: @testset "Fitting Beer" begin diff --git a/test/test-simulation.jl b/test/test-simulation.jl index d4b5e0a8..aec68fd5 100644 --- a/test/test-simulation.jl +++ b/test/test-simulation.jl @@ -1,5 +1,3 @@ -# include(joinpath(pkgdir(PlantSimEngine), "examples/dummy.jl")) - @testset "Check missing model" begin # No problem here: @test_nowarn ModelList( diff --git a/test/test-toy_models.jl b/test/test-toy_models.jl index 4d2c65d6..13d9057b 100644 --- a/test/test-toy_models.jl +++ b/test/test-toy_models.jl @@ -1,8 +1,3 @@ -# include(joinpath(pkgdir(PlantSimEngine), "examples/ToyLAIModel.jl")) -# include(joinpath(pkgdir(PlantSimEngine), "examples/Beer.jl")) -# include(joinpath(pkgdir(PlantSimEngine), "examples/ToyAssimGrowthModel.jl")) -# include(joinpath(pkgdir(PlantSimEngine), "examples/ToyRUEGrowthModel.jl")) - meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) @testset "ToyLAIModel" begin From fbd286baa9f252005e457e27b690fa750973b02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 13 Nov 2023 14:30:52 +0100 Subject: [PATCH 85/97] Update mtg tests Now that we changed how we make simulations over MTGs, we remove the old tests --- test/test-mtg-multiscale.jl | 25 +++++++ test/test-mtg.jl | 138 ------------------------------------ 2 files changed, 25 insertions(+), 138 deletions(-) delete mode 100644 test/test-mtg.jl diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index 1fef9a5e..cc13acfd 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -7,6 +7,31 @@ meteo = Weather( ] ) +@testset "MTG initialisation" begin + mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) + internode = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + leaf = Node(mtg, MultiScaleTreeGraph.NodeMTG("<", "Leaf", 1, 2)) + var1 = 15.0 + var2 = 0.3 + leaf[:var2] = var2 + + models = Dict( + "Leaf" => ( + Process1Model(1.0), + Process2Model(), + Process3Model(), + Status(var1=var1,) + ) + ) + + @test descendants(mtg, :var1) == [nothing, nothing] + @test descendants(mtg, :var2) == [nothing, var2] + + to_init = to_initialize(models) + @test to_init["Leaf"].need_initialisation == Symbol[:var2] + @test get_node(mtg, 3)[:var2] == var2 +end + # A mapping that actually works (same as before but with the init for TT): mapping_1 = Dict( "Plant" => diff --git a/test/test-mtg.jl b/test/test-mtg.jl deleted file mode 100644 index 3f3e7a34..00000000 --- a/test/test-mtg.jl +++ /dev/null @@ -1,138 +0,0 @@ -meteo = Weather( - [ - Atmosphere(T=20.0, Wind=1.0, Rh=0.65), - Atmosphere(T=25.0, Wind=0.5, Rh=0.8) -] -) -# Here we initialise var1 to a constant value: -@testset "MTG initialisation" begin - mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) - internode = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) - leaf = Node(mtg, MultiScaleTreeGraph.NodeMTG("<", "Leaf", 1, 2)) - var1 = 15.0 - var2 = 0.3 - leaf[:var2] = var2 - - models = Dict( - "Leaf" => ModelList( - process1=Process1Model(1.0), - process2=Process2Model(), - process3=Process3Model(), - status=(var1=var1,) - ) - ) - - @test descendants(mtg, :var1) == [nothing, nothing] - @test descendants(mtg, :var2) == [nothing, var2] - - to_init = init_mtg_models!(mtg, models, length(meteo), attr_name=:models) - @test to_init == Dict{String,Set{Symbol}}("Leaf" => Set(Symbol[:var2])) - @test NamedTuple(get_node(mtg, 3)[:models][1]) == (var4=-Inf, var5=-Inf, var6=-Inf, var1=var1, var3=-Inf, var2=var2) - - # The following shouldn't work because var2 has only one value: - if VERSION < v"1.8" # We test differently depending on the julia version because the format of the error message changed - @test_throws ErrorException init_mtg_models!(mtg, models, 10) - else - @test_throws ["The attribute", "in node 3"] init_mtg_models!(mtg, models, 10) - end - # Same with two time-steps: - mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) - internode = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) - leaf = Node(internode, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) - var1 = [15.0, 16.0] - var2 = [0.3, 0.4] - leaf[:var2] = var2 - - models = Dict( - "Leaf" => ModelList( - process1=Process1Model(1.0), - process2=Process2Model(), - process3=Process3Model(), - status=(var1=var1,) - ) - ) - to_init = init_mtg_models!(mtg, models, length(meteo), attr_name=:models) - @test NamedTuple(status(get_node(mtg, 3)[:models])[2]) == (var4=-Inf, var5=-Inf, var6=-Inf, var1=16.0, var3=-Inf, var2=0.4) -end - -@testset "MTG status update" begin - # After initialization, the output variables for computation are pre-allocated: - mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) - internode = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) - leaf = Node(mtg, MultiScaleTreeGraph.NodeMTG("<", "Leaf", 1, 2)) - var1 = [15.0, 16.0] - var2 = 0.3 - leaf[:var1] = var1 - leaf[:var2] = var2 - - models = Dict( - "Leaf" => ModelList( - process1=Process1Model(1.0), - process2=Process2Model(), - process3=Process3Model() - ) - ) - - to_init = init_mtg_models!(mtg, models, length(meteo), attr_name=:models) - - nsteps = length(meteo) - - @test NamedTuple(status(get_node(mtg, 3)[:models])[1]) == (var4=-Inf, var5=-Inf, var6=-Inf, var1=15.0, var3=-Inf, var2=0.3) - - # If we change the value of var1 in the status...: - status(get_node(mtg, 3)[:models])[1][:var1] = 16.0 - # ... the value in the attributes are too (because they are the same object): - @test get_node(mtg, 3)[:var1][1] == 16.0 -end - - -@testset "MTG simulation: Dict attributes" begin - mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) - internode = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) - leaf = Node(mtg, MultiScaleTreeGraph.NodeMTG("<", "Leaf", 1, 2)) - var1 = [15.0, 16.0] - var2 = 0.3 - leaf[:var1] = var1 - leaf[:var2] = var2 - - models = Dict( - "Leaf" => ModelList( - process1=Process1Model(1.0), - process2=Process2Model(), - process3=Process3Model() - ) - ) - - to_init = init_mtg_models!(mtg, models, length(meteo), attr_name=:models) - - attr_before_sim = deepcopy(leaf.attributes) - - @test attr_before_sim[:var1] == var1 - # :var2 was repeated for each time-step at init, so it should be a vector now: - @test attr_before_sim[:var2] == [var2, var2] - # :var3 was not initialized, so it should be a vector of -Inf: - @test attr_before_sim[:var3] == [-Inf, -Inf] - @test attr_before_sim[:var4] == [-Inf, -Inf] - @test attr_before_sim[:var5] == [-Inf, -Inf] - @test attr_before_sim[:var6] == [-Inf, -Inf] - - # Making the simulation: - constants = PlantMeteo.Constants() - MultiScaleTreeGraph.transform!( - mtg, - (node) -> run!(node[:models], meteo, constants, node), - filter_fun=node -> node[:models] !== nothing - ) - - # The inputs should not have changed: - @test attr_before_sim[:var1] == leaf.attributes[:var1] - @test attr_before_sim[:var2] == leaf.attributes[:var2] - # The outputs should have changed: - @test attr_before_sim[:var3] != leaf.attributes[:var3] - - # And they should have changed according to the models: - @test leaf.attributes[:var3] == models["Leaf"].models.process1.a .+ attr_before_sim[:var1] .* attr_before_sim[:var2] - @test leaf.attributes[:var4] == leaf.attributes[:var3] .* 4.0 - @test leaf.attributes[:var5] == (leaf.attributes[:var4] ./ 2.0) .+ 1.0 .* meteo.T .+ 2.0 .* meteo.Wind .+ 3.0 .* meteo.Rh - @test leaf.attributes[:var6] == leaf.attributes[:var5] .+ leaf.attributes[:var4] -end \ No newline at end of file From 01f05fd711ae4ab583e4afad720ce523ff03db00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 13 Nov 2023 14:54:02 +0100 Subject: [PATCH 86/97] Update examples_import.jl export processes --- src/examples_import.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/examples_import.jl b/src/examples_import.jl index bc15d11f..fd99dbb0 100644 --- a/src/examples_import.jl +++ b/src/examples_import.jl @@ -67,6 +67,15 @@ function import_mtg_example() return mtg end +# Processes: +export AbstractProcess1Model, AbstractProcess2Model, AbstractProcess3Model +export AbstractProcess4Model, AbstractProcess5Model, AbstractProcess6Model +export AbstractProcess7Model +export AbstractLight_InterceptionModel, AbstractLai_DynamicModel, AbstractDegreedaysModel +export AbstractPhotosynthesisModel, AbstractCarbon_AllocationModel, AbstractCarbon_DemandModel +export AbstractSoil_WaterModel, AbstractGrowthModel + +# Models: export Beer, ToyLAIModel, ToyDegreeDaysCumulModel export ToyAssimModel, ToyCAllocationModel, ToyCDemandModel, ToySoilWaterModel export ToyAssimGrowthModel, ToyRUEGrowthModel From e3f71cd3237bdf0db5f59ce64f196788ec9b8250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 13 Nov 2023 15:19:53 +0100 Subject: [PATCH 87/97] Remove old MTG simulation functions --- src/PlantSimEngine.jl | 2 - src/mtg/init_mtg_models.jl | 309 ------------------------------------- 2 files changed, 311 deletions(-) delete mode 100644 src/mtg/init_mtg_models.jl diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index d9663ed3..a4cda38e 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -54,7 +54,6 @@ include("dependencies/dependencies.jl") # MTG compatibility: include("mtg/GraphSimulation.jl") -include("mtg/init_mtg_models.jl") include("mtg/mapping.jl") include("mtg/save_results.jl") @@ -82,7 +81,6 @@ include("examples_import.jl") export AbstractModel export ModelList, MultiScaleModel -export init_mtg_models! export RMSE, NRMSE, EF, dr export Status, TimeStepTable, status export init_status! diff --git a/src/mtg/init_mtg_models.jl b/src/mtg/init_mtg_models.jl deleted file mode 100644 index b3775aed..00000000 --- a/src/mtg/init_mtg_models.jl +++ /dev/null @@ -1,309 +0,0 @@ -""" - init_mtg_models!( - mtg::MultiScaleTreeGraph.Node, - models::Dict{String,<:ModelList}, - i=nothing; - verbose=true, - attr_name=Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models")), - ) - -initialize the components of an MTG (*i.e.* nodes) with the corresponding models. - -The function checks if the models associated to each component of the MTG are fully initialized, -and if not, tries to initialize the variables using the MTG attributes with the exact same name, -and if not found, returns an error. - -# Arguments - -- `mtg::MultiScaleTreeGraph.Node`: the MTG tree. -- `models::Dict{String,ModelList}`: a dictionary of models named by components names -- `i=nothing`: the time-step to initialize. If `nothing`, initialize all the time-steps. -- `verbose = true`: return information during the processes -- `attr_name = Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models"))`: the node attribute name used to store the models, default to -Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models")) - -# Examples - -```@example -using PlantSimEngine, MultiScaleTreeGraph - -# Including example processes and models: -using PlantSimEngine.Examples; - -# Make an MTG: -mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) -internode = Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) -leaf = Node(mtg, MultiScaleTreeGraph.NodeMTG("<", "Leaf", 1, 2)) -leaf[:var1] = [15.0, 16.0] -leaf[:var2] = 0.3 - -# Declare our models: -models = Dict( - "Leaf" => ModelList( - process1=Process1Model(1.0), - process2=Process2Model(), - process3=Process3Model() - ) -) - -# Checking which variables are needed for our models: -[component => to_initialize(model) for (component, model) in models] -# OK we need to initialize :var1 and :var2 - -# We could compute them directly inside the MTG from available variables instead of -# giving them as initialisations: -transform!( - mtg, - :var1 => (x -> x .+ 2.0) => :var2, - ignore_nothing = true -) - -# Initialising all components with their corresponding models and initialisations at time-step 1: -init_mtg_models!(mtg, models, 1) -``` -Note that this is possible only because the initialisation values are found in the MTG. -If the initialisations are constant values between components, we can directly initialize -them in the models definition (as we do in the begining). -""" -function init_mtg_models!( - mtg, - models::Dict{String,<:ModelList}, - nsteps; - verbose=true, - attr_name=Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models")), - force=false -) - error( - "The function `init_mtg_models!` is not implemented for the type $(typeof(mtg)).", - ". At the moment, only `MultiScaleTreeGraph.Node` with `Dict` attributes are supported." - ) -end - -function init_mtg_models!( - mtg::MultiScaleTreeGraph.Node{N,A}, - models::Dict{String,<:ModelList}, - nsteps; - verbose=true, - attr_name=Symbol(MultiScaleTreeGraph.cache_name("PlantSimEngine models")), - force=false -) where {N<:MultiScaleTreeGraph.AbstractNodeMTG,A<:AbstractDict} - - attr_name_sym = Symbol(attr_name) - # Check if all components have a model - component_no_models = setdiff(MultiScaleTreeGraph.components(mtg), keys(models)) - if verbose && length(component_no_models) > 0 - @info string("No model found for component(s) ", join(component_no_models, ", ", ", and ")) maxlog = 1 - end - - # Get the variables in the models that has values that are not initialized: - to_init = Dict{String,Set{Symbol}}() - for (key, value) in models - inits = to_initialize(value) - vars = Set{Symbol}() - for init_ in inits - for j in init_ - push!(vars, j) - end - end - if length(vars) > 0 - push!(to_init, key => vars) - end - end - - # If some values need initialisation, check first if they are found as MTG attributes, and if they do, use them: - attrs_missing = Dict{String,Set{Symbol}}() - - MultiScaleTreeGraph.traverse!(mtg) do node - # If the component has models associated to it - # node = get_node(mtg, 3) - if haskey(models, node.MTG.symbol) - # Search if any variable is missing from the models *and* the attributes: - attr_not_found = setdiff( - to_init[node.MTG.symbol], - collect(keys(node.attributes)) - ) - - # If not, pre-allocate the node attributes with missing variables: - if length(attr_not_found) == 0 - # Get the status of the model (needed variables for the simulation): - st = status(models[node.MTG.symbol]) - # Get the variables that should be taken from the input models (already initialized): - vars_default = setdiff(keys(st), to_init[node.MTG.symbol]) - - # Put the default values found in the models' status into the attributes: - for var in vars_default - # var = :var4 - # Make a copy of the default value: - default_value = copy(models[node.MTG.symbol][var]) - - # If the value already exist but is not an array, make an array out of it. - # This happens when dealing with variables initialized with only one value. - if length(default_value) == 1 && nsteps > 1 - default_value = fill(default_value[1], nsteps) - end - - # If the variable is already defined in the node, is different than default_value, and we don't force overwrite, raise an error: - if !force && node[var] !== nothing && node[var] != default_value - error("The attribute $(var) is already defined in node $(node.id). Remove it from the `models` or set `force=true`.") - end - - # If the default models only have one time-sep, we'll reference it for all time-steps. Otherwise, we use the time-step value. - # Check if we don't go out of bounds: - if length(default_value) > 1 && nsteps > length(default_value) - error("The default value for $(var) in `models` for $(node.MTG.symbol) type is only defined for $(length(default_value)) time-steps, you required $(nsteps).") - end - - node[var] = default_value - end - - # Pre-allocate the warning information if a variable is found but has length == 1 and is changed to length nsteps: - verbose && (attr_one_value = Symbol[]) - - # Pre-allocate the node attributes for all time-step: - for var in keys(st) - # var = :var2 - if node[var] === nothing - # If the attribute does not exist, create a vector of n-steps values - node[var] = fill(st[var], nsteps) - elseif typeof(node[var]) <: AbstractArray - # If it does exist and is already an n-steps array, do nothing - length(node[var]) == nsteps && continue - - if length(node[var]) != 1 - error("Attribute $var is already stored in node $(node.id) but as length", - "!= number of steps to simulate ($nsteps).") - end - - if !force - error("The attribute $(var) is already defined in node $(node.id) but has length != nsteps ($nsteps). Update it or set `force=true`.") - elseif verbose - push!(attr_one_value, var) # Store the variable name for the warning message at the end (return all variables at once) - end - - node[var] = fill(node[var], nsteps) - else - # If the value already exist but is not an array, make an array out of it. - # This happens when dealing with variables initialized with only one value. - if !force && length(node[var]) > 1 - error("The attribute $(var) is already defined in node $(node.id) but has length != nsteps ($nsteps). Update it or set `force=true`.") - elseif verbose - push!(attr_one_value, var) # Store the variable name for the warning message at the end (return all variables at once) - end - node[var] = fill(node[var], nsteps) - end - end - - if verbose && length(attr_one_value) > 0 - @info "The attributes $(attr_one_value) were already defined in node $(node.id) but had length == 1. Extending it to nsteps ($nsteps)." maxlog = 3 - end - - # Initialize the ModelList using attributes: - # as_default = NamedTuple(begin - # # if the default models only have one time-sep, we use it for all time-steps. Otherwise, we use the time-step value. - # default_value = models[node.MTG.symbol][var] - # # Check if we don't go out of bounds: - # if length(default_value) > 1 && i > length(default_value) - # error("The default value for $(var) in `models` for $(node.MTG.symbol) type is not defined for time-step $(i) and is not a constant value. Please provide one time-sep, or $i.") - # end - # i_var = length(default_value) > 1 ? i : 1 - # var => get_Ref_attr(default_value, i_var) - # end for var in vars_default) - - # Finally, use references to the attributes values as the status of the ModelList: - node[attr_name_sym] = - copy( - models[node.MTG.symbol], - TimeStepTable([ - Status(NamedTuple(var => get_Ref_i(node, var, i) for var in keys(st))) for i in 1:nsteps - ]) - ) - else - # If some initialisations are not available from the node attributes: - if length(attr_not_found) > 0 - for i in attr_not_found - !haskey(attrs_missing, node.MTG.symbol) && (attrs_missing[node.MTG.symbol] = Set{Symbol}()) - push!(attrs_missing[node.MTG.symbol], i) - end - end - end - end - end - if any([length(value) > 0 for (key, value) in attrs_missing]) - err_msg = [string("\n", key, ": [", join(value, ", ", " and "), "]") for (key, value) in attrs_missing] - error( - string( - "Some variables need to be initialized for some components before simulation:", - join(err_msg, ", ", " and ") - ) - ) - end - - return to_init -end - -""" - get_Ref_i(node, attr, i<:Nothing) - get_Ref_i(node, attr, i) - -Get reference to node attribute at ith value or value if `i<:Nothing`. -""" -function get_Ref_i(node, attr, i::T) where {T<:Nothing} - Ref(node[attr]) -end - -function get_Ref_i(node, attr, i) - node_attr = node[attr] - # Throw a bound error if the index is out of bounds: - length(node_attr) >= i || error("Indexing out of bounds for attribute $attr in node $(node.id)") - - get_Ref_attr(node_attr, i) -end - -function get_Ref_attr(attr::T, i) where {T<:AbstractVector} - length(attr) >= i || error("Indexing ($i) out of bounds vector $attr") - Ref(attr, i) -end - -function get_Ref_attr(attr, i) - Ref(attr) -end - - -""" - update_mtg_models!(mtg::MultiScaleTreeGraph.Node, i, attr_name::Symbol) - -Update the mtg models initialisations by using the ith time-step. The mtg is considered fully -initialized already once, so [`init_mtg_models!`](@ref) must be called before -`update_mtg_models!`. - -The values are updated only for node attributes in `to_init`. Those attributes must have -several time-steps, *i.e.* indexable by 1:n time-steps. -""" -function update_mtg_models!(mtg::MultiScaleTreeGraph.Node, i, to_init, attr_name::Symbol) - - MultiScaleTreeGraph.traverse!(mtg) do node - # If the component has models associated to it - if haskey(to_init, node.MTG.symbol) - # Set the initialisation value of the model at the ith value of the node attribute - - # Get the default initialisation values from the previous time-step: - as_default = NamedTuple(var => Ref(status(node[attr_name], var)[i]) for var in setdiff(keys(status(node[attr_name])), to_init[node.MTG.symbol])) - - # Make a new ModelList with the updated values pointing to the ith value in the attributes - node[attr_name] = copy( - node[attr_name], - TimeStepTable([ - Status( - merge( - as_default, - NamedTuple(j => get_Ref_i(node, j, i) for j in to_init[node.MTG.symbol]) - ) - ) - ]) - ) - # Note that it is mandantory to copy the ModeList as it is immutable - end - end - - return nothing -end From 3bb1bdc361b594c405af5f019478d75c11669f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 13 Nov 2023 15:20:13 +0100 Subject: [PATCH 88/97] Update multiscale.md Reduce the size of this help file, by removing some outputs --- docs/src/model_coupling/multiscale.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/src/model_coupling/multiscale.md b/docs/src/model_coupling/multiscale.md index 8e4d13ff..dec121b9 100644 --- a/docs/src/model_coupling/multiscale.md +++ b/docs/src/model_coupling/multiscale.md @@ -77,7 +77,8 @@ mapping = Dict( ), Status(aPPFD=1300.0), ), -) +); +nothing # hide ``` The `MultiScaleModel` takes two arguments: the model and the mapping between the model and the scales. The mapping is a vector of pairs mapping the variable's name with the name of the scale its value comes from. In this example, we map the `soil_water_content` variable to the `"Soil"` scale. @@ -141,6 +142,7 @@ mapping = Dict( ToySoilWaterModel(), ), ); +nothing # hide ``` In this example, we expect to make a simulation at five different scales: `"Scene"`, `"Plant"`, `"Internode"`, `"Leaf"`, and `"Soil"`. The `"Scene"` scale represents the whole scene, where one or several plants can live. The `"Plant"` scale is, well, the whole plant scale, `"Internode"` and `"Leaf"` are organ scales, and `"Soil"` is the soil scale. This mapping is used to compute the carbon allocation (`ToyCAllocationModel`) to the different organs of the plant (`"Leaf"` and `"Internode"`) from the assimilation at the `"Leaf"` scale (*i.e.* the offer) and their carbon demand (`ToyCDemandModel`). The `"Soil"` scale is used to compute the soil water content (`ToySoilWaterModel`), which is needed to calculate the assimilation at the `"Leaf"` scale (`ToyAssimModel`). @@ -216,7 +218,8 @@ And that's it! We can now access the outputs for each scale as a dictionary of vectors of values per variable and scale like this: ```@example usepkg -outputs(sim) +outputs(sim); +nothing # hide ``` Or as a `DataFrame` using the `DataFrames` package: @@ -229,7 +232,8 @@ outputs(sim, DataFrame) The values for the last time-step of the simulation are also available from the statuses: ```@example usepkg -status(sim) +status(sim); +nothing # hide ``` This is a dictionary with the scale as the key and a vector of `Status` as values, one per node of that scale. So, in this example, the `"Leaf"` scale has two nodes, so the value is a vector of two `Status` objects, and the `"Soil"` scale has only one node, so the value is a vector of one `Status` object. From da64154aa536ea6a6902f38419a6ab99e4634113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 13 Nov 2023 15:55:45 +0100 Subject: [PATCH 89/97] Update make.jl increase size_threshold to 250kB. --- docs/make.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 5303faa6..2dda33fd 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -9,13 +9,14 @@ DocMeta.setdocmeta!(PlantSimEngine, :DocTestSetup, :(using PlantSimEngine, Plant makedocs(; modules=[PlantSimEngine], authors="Rémi Vezy and contributors", - repo="https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/{commit}{path}#{line}", + repo=Documenter.Remotes.GitHub("VirtualPlantLab", "PlantSimEngine.jl"), sitename="PlantSimEngine.jl", format=Documenter.HTML(; prettyurls=get(ENV, "CI", "false") == "true", canonical="https://VirtualPlantLab.github.io/PlantSimEngine.jl", edit_link="main", - assets=String[] + assets=String[], + size_threshold=250000 ), pages=[ "Home" => "index.md", From 0c6a90993786b27f01f868783fe1e4bdd60a1a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 22 Nov 2023 16:09:49 +0100 Subject: [PATCH 90/97] Reformat mtg initialisation --- src/PlantSimEngine.jl | 1 + src/mtg/GraphSimulation.jl | 2 +- src/mtg/initialisation.jl | 367 ++++++++++++++++++++++++++++++++++++ src/mtg/mapping.jl | 331 -------------------------------- src/mtg/save_results.jl | 19 +- test/test-mtg-multiscale.jl | 2 +- 6 files changed, 375 insertions(+), 347 deletions(-) create mode 100644 src/mtg/initialisation.jl diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index a4cda38e..bd53d18d 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -55,6 +55,7 @@ include("dependencies/dependencies.jl") # MTG compatibility: include("mtg/GraphSimulation.jl") include("mtg/mapping.jl") +include("mtg/initialisation.jl") include("mtg/save_results.jl") # Model evaluation (statistics): diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index 270b2de2..b48408fc 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -60,7 +60,7 @@ julia> using PlantSimEngine.Examples; $MAPPING_EXAMPLE ```@example -import_mtg_example(); +mtg = import_mtg_example(); ``` ```@example diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl new file mode 100644 index 00000000..c5d4e6cd --- /dev/null +++ b/src/mtg/initialisation.jl @@ -0,0 +1,367 @@ +""" + init_statuses(mtg, mapping; type_promotion=nothing, check=true) + +Get the status of each node in the MTG by node type, pre-initialised considering multi-scale variables. +""" +function init_statuses(mtg, mapping; type_promotion=nothing, check=true) + # We make a pre-initialised status for each kind of organ (this is a template for each node type): + status_templates = status_template(mapping, type_promotion) + # Get the reverse mapping, i.e. the variables that are mapped to other scales. This is used to initialise + # the RefVectors properly: + map_other_scales = reverse_mapping(mapping, all=false) + #NB: we use all=false because we only want the variables that are mapped as RefVectors. + + # We need to know which variables are not initialized, and not computed by other models: + var_need_init = to_initialize(mapping, mtg) + + # If we find some, we return an error: + check && error_mtg_init(var_need_init) + + nodes_with_models = collect(keys(status_templates)) + # We traverse the MTG to initialise the statuses linked to the nodes: + statuses = Dict(i => Status[] for i in nodes_with_models) + MultiScaleTreeGraph.traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) + init_status!(node, statuses, status_templates, map_other_scales, var_need_init, nodes_with_models) + end + + return statuses +end + + +""" + init_status!( + node, + statuses, + status_templates, + map_other_scales, + var_need_init=Dict{String,Any}(), + nodes_with_models=collect(keys(status_templates)) + ) + +Initialise the status of a node, taking into account the multiscale mapping, and add it to the +statuses dictionary. + +# Arguments + +- `node`: the node to initialise +- `statuses`: the dictionary of statuses by node type +- `status_templates`: the template of status for each node type +- `map_other_scales`: the variables that are mapped to other scales +- `var_need_init`: the variables that are not initialised or computed by other models +- `nodes_with_models`: the nodes that have a model defined for their symbol + +# Details + +Most arguments can be computed from the graph and the mapping: +- `statuses` is given by the first initialisation: `statuses = Dict(i => Status[] for i in nodes_with_models)` +- `status_templates` is computed usin `status_template(mappinxg, type_promotion)` +- `map_other_scales` is computed using `reverse_mapping(mapping, all=false)`. We use `all=false` because we only +want the variables that are mapped as `RefVectors` +- `var_need_init` is computed using `to_initialize(mapping, mtg)` +- `nodes_with_models` is computed using `collect(keys(status_templates))` +""" +function init_status!(node, statuses, status_templates, map_other_scales, var_need_init=Dict{String,Any}(), nodes_with_models=collect(keys(status_templates))) + # Check if the node has a model defined for its symbol, if not, no need to compute + node.MTG.symbol ∉ nodes_with_models && return + + # We make a copy of the template status for this node: + st_template = copy(status_templates[node.MTG.symbol]) + + # We add a reference to the node into the status, so that we can access it from the models if needed. + push!(st_template, :node => Ref(node)) + + # If some variables still need to be instantiated in the status, look into the MTG node if we can find them, + # and if so, use their value in the status: + if haskey(var_need_init, node.MTG.symbol) && length(var_need_init[node.MTG.symbol].need_var_from_mtg) > 0 + for i in var_need_init[node.MTG.symbol].need_var_from_mtg + @assert typeof(node[i.var]) == typeof(st_template[i.var]) string( + "Initializing variable $(i.var) using MTG node $(node.id): expected type $(typeof(st_template[i.var])), found $(typeof(node[i.var])). ", + "Please check the type of the variable in the MTG, and make it a $(typeof(st_template[i.var]))." + ) + st_template[i.var] = node[i.var] + # NB: the variable is not a reference to the value in the MTG, but a copy of it. + # This is because we can't reference a value in a Dict. If we need a ref, the user can use a RefValue in the MTG directly, + # and it will be automatically passed as is. + end + end + + # Make the node status from the template: + st = status_from_template(st_template) + + push!(statuses[node.MTG.symbol], st) + + # Instantiate the RefVectors on the fly for other scales that map into this scale, *i.e.* + # add a reference to the value of any variable that is used by another scale into its RefVector: + if haskey(map_other_scales, node.MTG.symbol) + for (organ, vars) in map_other_scales[node.MTG.symbol] + for var in vars # e.g.: var = :carbon_demand + push!(status_templates[organ][var], refvalue(st, var)) + end + end + end +end + + +""" + status_template(models::Dict{String,Any}, type_promotion) + +Create a status template for a given set of models and type promotion. + +# Arguments +- `models::Dict{String,Any}`: A dictionary of models. +- `type_promotion`: The type promotion to use. + +# Returns + +- A dictionary with the organ types as keys, and a dictionary of variables => default values as values. + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine; +``` + +Import example models (can be found in the `examples` folder of the package, or in the `Examples` sub-modules): + +```jldoctest mylabel +julia> using PlantSimEngine.Examples; +``` + +```jldoctest mylabel +julia> models = Dict( \ + "Plant" => \ + MultiScaleModel( \ + model=ToyCAllocationModel(), \ + mapping=[ \ + :A => ["Leaf"], \ + :carbon_demand => ["Leaf", "Internode"], \ + :carbon_allocation => ["Leaf", "Internode"] \ + ], \ + ), \ + "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + "Leaf" => ( \ + MultiScaleModel( \ + model=ToyAssimModel(), \ + mapping=[:soil_water_content => "Soil",], \ + ), \ + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + Status(aPPFD=1300.0, TT=10.0), \ + ), \ + "Soil" => ( \ + ToySoilWaterModel(), \ + ), \ + ); +``` + +```jldoctest mylabel +julia> organs_statuses = PlantSimEngine.status_template(models, nothing) +Dict{String, Dict{Symbol, Any}} with 4 entries: + "Soil" => Dict(:soil_water_content=>RefValue{Float64}(-Inf)) + "Internode" => Dict(:carbon_allocation=>-Inf, :TT=>-Inf, :carbon_demand=>-Inf) + "Plant" => Dict(:carbon_allocation=>RefVector{Float64}[], :A=>RefVector{F… + "Leaf" => Dict(:carbon_allocation=>-Inf, :A=>-Inf, :TT=>10.0, :aPPFD=>13… +``` + +Note that variables that are multiscale (*i.e.* defined in a mapping) are linked between scales, so if we write at a scale, the value will be +automatically updated at the other scale: + +```jldoctest mylabel +julia> organs_statuses["Soil"][:soil_water_content] === organs_statuses["Leaf"][:soil_water_content] +true +``` +""" +function status_template(mapping::Dict{String,T}, type_promotion) where {T} + organs_mapping, var_outputs_from_mapping = compute_mapping(mapping, type_promotion) + # Vector of pre-initialised variables with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: + organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() + dict_mapped_vars = Dict{Pair,Any}() + + for organ in keys(mapping) # e.g.: organ = "Internode" + # Parsing the models into a NamedTuple to get the process name: + node_models = parse_models(get_models(mapping[organ])) + + # Get the status if any was given by the user (this can be used as default values in the mapping): + st = get_status(mapping[organ]) # User status + + if isnothing(st) + st = NamedTuple() + else + st = NamedTuple(st) + end + + # Add the variables that are defined as multiscale (coming from other scales): + if haskey(organs_mapping, organ) + st_vars_mapped = (; zip(vars_from_mapping(organs_mapping[organ]), vars_type_from_mapping(organs_mapping[organ]))...) + !isnothing(st_vars_mapped) && (st = merge(st, st_vars_mapped)) + end + + # Add the variable(s) written by other scales into this node scale: + haskey(var_outputs_from_mapping, organ) && (st = merge(st, var_outputs_from_mapping[organ])) + + # Then we initialise a status taking into account the status given by the user. + # This step is done to get default values for each variables: + if length(st) == 0 + st = nothing + else + st = Status(st) + end + + st = add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) + + # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable + # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. + val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) + if any(x -> isa(x, MappedVar), values(st)) + for (k, v) in val_pointers # e.g.: k = :soil_water_content; v = val_pointers[k] + if isa(v, MappedVar) + # First time we encounter this variable as a MappedVar, we create its value into the dict_mapped_vars Dict: + if !haskey(dict_mapped_vars, v.organ => v.var) + push!(dict_mapped_vars, Pair(v.organ, v.var) => Ref(st[k].default)) + end + + # Then we replace the MappedVar by a RefValue to the actual variable: + val_pointers[k] = dict_mapped_vars[v.organ=>v.var] + else + val_pointers[k] = st[k] + end + end + end + organs_statuses_dict[organ] = val_pointers + end + + return organs_statuses_dict +end + +""" + status_from_template(d::Dict{Symbol,Any}) + +Create a status from a template dictionary of variables and values. If the values +are already RefValues or RefVectors, they are used as is, else they are converted to Refs. + +# Arguments + +- `d::Dict{Symbol,Any}`: A dictionary of variables and values. + +# Returns + +- A [`Status`](@ref). + +# Examples + +```jldoctest mylabel +julia> using PlantSimEngine +``` + +```jldoctest mylabel +julia> a, b = PlantSimEngine.status_from_template(Dict(:a => 1.0, :b => 2.0)); +``` + +```jldoctest mylabel +julia> a +1.0 +``` + +```jldoctest mylabel +julia> b +2.0 +``` +""" +function status_from_template(d::Dict{Symbol,T} where {T}) + Status(NamedTuple(first(i) => ref_var(last(i)) for i in d)) +end + + +# Return an error if some variables are not initialized or computed by other models in the output +# from to_initialize(models, organs_statuses) +function error_mtg_init(var_need_init) + if length(var_need_init) > 0 + error_string = String[] + for need_init in var_need_init + organ_init = first(need_init) + need_initialisation = last(need_init).need_initialisation + + # A model needs initialisations: + if length(need_initialisation) > 0 + push!( + error_string, + "Nodes of type $organ_init need variable(s) $(join(need_initialisation, ", ")) to be initialized or computed by a model." + ) + end + + # The mapping is wrong: + need_models_from_scales = last(need_init).need_models_from_scales + for er in need_models_from_scales + var, scale, need_scales = er + push!( + error_string, + "Nodes of type $need_scales should provide a model to compute variable `:$var` as input for nodes of type $scale, but none is provided." + ) + end + end + + if length(error_string) > 0 + error(join(error_string, "\n")) + end + end +end + + +""" + init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) + +Initialise the simulation. Returns: + +- the mtg +- a status for each node by organ type, considering multi-scale variables +- the dependency graph of the models +- the models parsed as a Dict of organ type => NamedTuple of process => model mapping +- the pre-allocated outputs + +# Arguments + +- `mtg`: the MTG +- `mapping::Dict{String,Any}`: a dictionary of model mapping +- `nsteps`: the number of steps of the simulation +- `outputs`: the dynamic outputs needed for the simulation +- `type_promotion`: the type promotion to use for the variables +- `check`: whether to check the mapping for errors +- `verbose`: print information about errors in the mapping + +# Details + +The function first computes a template of status for each organ type that has a model in the mapping. +This template is used to initialise the status of each node of the MTG, taking into account the user-defined +initialisation, and the (multiscale) mapping. The mapping is used to make references to the variables +that are defined at another scale, so that the values are automatically updated when the variable is changed at +the other scale. Two types of multiscale variables are available: `RefVector` and `MappedVar`. The first one is +used when the variable is mapped to a vector of nodes, and the second one when it is mapped to a single node. This +is given by the user through the mapping, using a string for a single node (*e.g.* `=> "Leaf"`), and a vector of strings for a vector of +nodes (*e.g.* `=> ["Leaf"]` for one type of node or `=> ["Leaf", "Internode"]` for several). + +The function also computes the dependency graph of the models, i.e. the order in which the models should be +called, considering the dependencies between them. The dependency graph is used to call the models in the right order +when the simulation is run. + +Note that if a variable is not computed by models or initialised from the mapping, it is searched in the MTG attributes. +The value is not a reference to the one in the attribute of the MTG, but a copy of it. This is because we can't reference +a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be +automatically passed as is. +""" +function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) + # Get the status of each node by node type, pre-initialised considering multi-scale variables: + statuses = init_statuses(mtg, mapping; type_promotion=type_promotion, check=check) + # Print an info if models are declared for nodes that don't exist in the MTG: + if check && any(x -> length(last(x)) == 0, statuses) + model_no_node = join(findall(x -> length(x) == 0, statuses), ", ") + @info "Models given for $model_no_node, but no node with this symbol was found in the MTG." maxlog = 1 + end + + # Compute the multi-scale dependency graph of the models: + dependency_graph = dep(mapping, verbose=verbose) + + models = Dict(first(m) => parse_models(get_models(last(m))) for m in mapping) + + outputs = pre_allocate_outputs(statuses, outputs, nsteps, check=check) + + return (; mtg, statuses, dependency_graph, models, outputs) +end \ No newline at end of file diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index eb3eff46..b5d1bfb6 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -432,80 +432,6 @@ function outputs_from_other_scale!(var_outputs_from_mapping, multi_scale_outs, m end end -""" - init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) - -Initialise the simulation. Returns: - -- the mtg -- a status for each node by organ type, considering multi-scale variables -- the dependency graph of the models -- the models parsed as a Dict of organ type => NamedTuple of process => model mapping -- the pre-allocated outputs - -# Arguments - -- `mtg`: the MTG -- `mapping::Dict{String,Any}`: a dictionary of model mapping -- `nsteps`: the number of steps of the simulation -- `outputs`: the dynamic outputs needed for the simulation -- `type_promotion`: the type promotion to use for the variables -- `check`: whether to check the mapping for errors -- `verbose`: print information about errors in the mapping - -# Details - -The function first computes a template of status for each organ type that has a model in the mapping. -This template is used to initialise the status of each node of the MTG, taking into account the user-defined -initialisation, and the (multiscale) mapping. The mapping is used to make references to the variables -that are defined at another scale, so that the values are automatically updated when the variable is changed at -the other scale. Two types of multiscale variables are available: `RefVector` and `MappedVar`. The first one is -used when the variable is mapped to a vector of nodes, and the second one when it is mapped to a single node. This -is given by the user through the mapping, using a string for a single node (*e.g.* `=> "Leaf"`), and a vector of strings for a vector of -nodes (*e.g.* `=> ["Leaf"]` for one type of node or `=> ["Leaf", "Internode"]` for several). - -The function also computes the dependency graph of the models, i.e. the order in which the models should be -called, considering the dependencies between them. The dependency graph is used to call the models in the right order -when the simulation is run. - -Note that if a variable is not computed by models or initialised from the mapping, it is searched in the MTG attributes. -The value is not a reference to the one in the attribute of the MTG, but a copy of it. This is because we can't reference -a value in a Dict. If you need a reference, you can use a `Ref` for your variable in the MTG directly, and it will be -automatically passed as is. -""" -function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) - # We make a pre-initialised status for each kind of organ (this is a template for each node type): - organs_statuses = status_template(mapping, type_promotion) - # Get the reverse mapping, i.e. the variables that are mapped to other scales. This is used to initialise - # the RefVectors properly: - var_refvector = reverse_mapping(mapping, all=false) - #NB: we use all=false because we only want the variables that are mapped as RefVectors. - - # We need to know which variables are not initialized, and not computed by other models: - var_need_init = to_initialize(mapping, mtg) - - # If we find some, we return an error: - check && error_mtg_init(var_need_init) - - # Get the status of each node by node type, pre-initialised considering multi-scale variables: - statuses = init_statuses(mtg, organs_statuses, var_refvector, var_need_init) - - # Print an info if models are declared for nodes that don't exist in the MTG: - if check && any(x -> length(last(x)) == 0, statuses) - model_no_node = join(findall(x -> length(x) == 0, statuses), ", ") - @info "Models given for $model_no_node, but no node with this symbol was found in the MTG." maxlog = 1 - end - - # Compute the multi-scale dependency graph of the models: - dependency_graph = dep(mapping, verbose=verbose) - - models = Dict(first(m) => parse_models(get_models(last(m))) for m in mapping) - - outputs = pre_allocate_outputs(statuses, outputs, nsteps, check=check) - - return (; mtg, statuses, dependency_graph, models, outputs) -end - function map_scale(f, m, scale::String) map_scale(f, m, [scale]) end @@ -514,209 +440,6 @@ function map_scale(f, m, scales::AbstractVector{String}) map(s -> f(m, s), scales) end -# Return an error if some variables are not initialized or computed by other models in the output -# from to_initialize(models, organs_statuses) -function error_mtg_init(var_need_init) - if length(var_need_init) > 0 - error_string = String[] - for need_init in var_need_init - organ_init = first(need_init) - need_initialisation = last(need_init).need_initialisation - - # A model needs initialisations: - if length(need_initialisation) > 0 - push!( - error_string, - "Nodes of type $organ_init need variable(s) $(join(need_initialisation, ", ")) to be initialized or computed by a model." - ) - end - - # The mapping is wrong: - need_models_from_scales = last(need_init).need_models_from_scales - for er in need_models_from_scales - var, scale, need_scales = er - push!( - error_string, - "Nodes of type $need_scales should provide a model to compute variable `:$var` as input for nodes of type $scale, but none is provided." - ) - end - end - - if length(error_string) > 0 - error(join(error_string, "\n")) - end - end -end - - -""" - status_template(models::Dict{String,Any}, type_promotion) - -Create a status template for a given set of models and type promotion. - -# Arguments -- `models::Dict{String,Any}`: A dictionary of models. -- `type_promotion`: The type promotion to use. - -# Returns - -- A dictionary with the organ types as keys, and a dictionary of variables => default values as values. - -# Examples - -```jldoctest mylabel -julia> using PlantSimEngine; -``` - -Import example models (can be found in the `examples` folder of the package, or in the `Examples` sub-modules): - -```jldoctest mylabel -julia> using PlantSimEngine.Examples; -``` - -```jldoctest mylabel -julia> models = Dict( \ - "Plant" => \ - MultiScaleModel( \ - model=ToyCAllocationModel(), \ - mapping=[ \ - :A => ["Leaf"], \ - :carbon_demand => ["Leaf", "Internode"], \ - :carbon_allocation => ["Leaf", "Internode"] \ - ], \ - ), \ - "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ - "Leaf" => ( \ - MultiScaleModel( \ - model=ToyAssimModel(), \ - mapping=[:soil_water_content => "Soil",], \ - ), \ - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ - Status(aPPFD=1300.0, TT=10.0), \ - ), \ - "Soil" => ( \ - ToySoilWaterModel(), \ - ), \ - ); -``` - -```jldoctest mylabel -julia> organs_statuses = PlantSimEngine.status_template(models, nothing) -Dict{String, Dict{Symbol, Any}} with 4 entries: - "Soil" => Dict(:soil_water_content=>RefValue{Float64}(-Inf)) - "Internode" => Dict(:carbon_allocation=>-Inf, :TT=>-Inf, :carbon_demand=>-Inf) - "Plant" => Dict(:carbon_allocation=>RefVector{Float64}[], :A=>RefVector{F… - "Leaf" => Dict(:carbon_allocation=>-Inf, :A=>-Inf, :TT=>10.0, :aPPFD=>13… -``` - -Note that variables that are multiscale (*i.e.* defined in a mapping) are linked between scales, so if we write at a scale, the value will be -automatically updated at the other scale: - -```jldoctest mylabel -julia> organs_statuses["Soil"][:soil_water_content] === organs_statuses["Leaf"][:soil_water_content] -true -``` -""" -function status_template(mapping::Dict{String,T}, type_promotion) where {T} - organs_mapping, var_outputs_from_mapping = compute_mapping(mapping, type_promotion) - # Vector of pre-initialised variables with the default values for each variable, taking into account user-defined initialisation, and multiscale mapping: - organs_statuses_dict = Dict{String,Dict{Symbol,Any}}() - dict_mapped_vars = Dict{Pair,Any}() - - for organ in keys(mapping) # e.g.: organ = "Internode" - # Parsing the models into a NamedTuple to get the process name: - node_models = parse_models(get_models(mapping[organ])) - - # Get the status if any was given by the user (this can be used as default values in the mapping): - st = get_status(mapping[organ]) # User status - - if isnothing(st) - st = NamedTuple() - else - st = NamedTuple(st) - end - - # Add the variables that are defined as multiscale (coming from other scales): - if haskey(organs_mapping, organ) - st_vars_mapped = (; zip(vars_from_mapping(organs_mapping[organ]), vars_type_from_mapping(organs_mapping[organ]))...) - !isnothing(st_vars_mapped) && (st = merge(st, st_vars_mapped)) - end - - # Add the variable(s) written by other scales into this node scale: - haskey(var_outputs_from_mapping, organ) && (st = merge(st, var_outputs_from_mapping[organ])) - - # Then we initialise a status taking into account the status given by the user. - # This step is done to get default values for each variables: - if length(st) == 0 - st = nothing - else - st = Status(st) - end - - st = add_model_vars(st, node_models, type_promotion; init_fun=x -> Status(x)) - - # For the variables that are RefValues of other variables at a different scale, we need to actually create a reference to this variable - # in the status. So we replace the RefValue by a RefValue to the actual variable, and instantiate a Status directly with the actual Refs. - val_pointers = Dict{Symbol,Any}(zip(keys(st), values(st))) - if any(x -> isa(x, MappedVar), values(st)) - for (k, v) in val_pointers # e.g.: k = :soil_water_content; v = val_pointers[k] - if isa(v, MappedVar) - # First time we encounter this variable as a MappedVar, we create its value into the dict_mapped_vars Dict: - if !haskey(dict_mapped_vars, v.organ => v.var) - push!(dict_mapped_vars, Pair(v.organ, v.var) => Ref(st[k].default)) - end - - # Then we replace the MappedVar by a RefValue to the actual variable: - val_pointers[k] = dict_mapped_vars[v.organ=>v.var] - else - val_pointers[k] = st[k] - end - end - end - organs_statuses_dict[organ] = val_pointers - end - - return organs_statuses_dict -end - -""" - status_from_template(d::Dict{Symbol,Any}) - -Create a status from a template dictionary of variables and values. If the values -are already RefValues or RefVectors, they are used as is, else they are converted to Refs. - -# Arguments - -- `d::Dict{Symbol,Any}`: A dictionary of variables and values. - -# Returns - -- A [`Status`](@ref). - -# Examples - -```jldoctest mylabel -julia> using PlantSimEngine -``` - -```jldoctest mylabel -julia> a, b = PlantSimEngine.status_from_template(Dict(:a => 1.0, :b => 2.0)); -``` - -```jldoctest mylabel -julia> a -1.0 -``` - -```jldoctest mylabel -julia> b -2.0 -``` -""" -function status_from_template(d::Dict{Symbol,T} where {T}) - Status(NamedTuple(first(i) => ref_var(last(i)) for i in d)) -end - """ ref_var(v) @@ -851,60 +574,6 @@ function reverse_mapping(models; all=true) return filter!(x -> length(last(x)) > 0, var_to_ref) end -""" - init_statuses(mtg, models, status_template, var_need_init) - init_statuses(mtg, models, status_template) - -Get the status of each node in the MTG by node type, pre-initialised considering multi-scale variables -using the template given by `status_template`. -""" -function init_statuses(mtg, status_template, var_refvector, var_need_init=Dict{String,Any}()) - nodes_with_models = collect(keys(status_template)) - # We traverse the MTG to initialise the statuses linked to the nodes: - statuses = Dict(i => Status[] for i in nodes_with_models) - MultiScaleTreeGraph.traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) - # Check if the node has a model defined for its symbol - node.MTG.symbol ∉ nodes_with_models && return - - # We make a copy of the template status for this node: - st_template = copy(status_template[node.MTG.symbol]) - - # We add a reference to the node into the status, so that we can access it from the models if needed. - push!(st_template, :node => Ref(node)) - - # If some variables still need to be instantiated in the status, look into the MTG node if we can find them, - # and if so, use their value in the status: - if haskey(var_need_init, node.MTG.symbol) && length(var_need_init[node.MTG.symbol].need_var_from_mtg) > 0 - for i in var_need_init[node.MTG.symbol].need_var_from_mtg - @assert typeof(node[i.var]) == typeof(st_template[i.var]) string( - "Initializing variable $(i.var) using MTG node $(node.id): expected type $(typeof(st_template[i.var])), found $(typeof(node[i.var])). ", - "Please check the type of the variable in the MTG, and make it a $(typeof(st_template[i.var]))." - ) - st_template[i.var] = node[i.var] - # NB: the variable is not a reference to the value in the MTG, but a copy of it. - # This is because we can't reference a value in a Dict. If we need a ref, the user can use a RefValue in the MTG directly, - # and it will be automatically passed as is. - end - end - - # Make the node status from the template: - st = status_from_template(st_template) - - push!(statuses[node.MTG.symbol], st) - - # Instantiate the RefVectors on the fly for other scales that map into this scale, *i.e.* - # add a reference to the value of any variable that is used by another scale into its RefVector: - if haskey(var_refvector, node.MTG.symbol) - for (organ, vars) in var_refvector[node.MTG.symbol] - for var in vars # e.g.: var = :carbon_demand - push!(status_template[organ][var], refvalue(st, var)) - end - end - end - end - return statuses -end - """ variables_multiscale(node, organ, mapping) diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl index 03b05706..6824002f 100644 --- a/src/mtg/save_results.jl +++ b/src/mtg/save_results.jl @@ -44,7 +44,10 @@ julia> mapping = Dict( \ :carbon_allocation => ["Leaf", "Internode"] \ ], \ ), \ - "Internode" => ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + "Internode" => ( \ + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ + Status(aPPFD=1300.0, TT=10.0), \ + ), \ "Leaf" => ( \ MultiScaleModel( \ model=ToyAssimModel(), \ @@ -66,19 +69,7 @@ julia> mtg = import_mtg_example(); ``` ```jldoctest mylabel -julia> organs_statuses = PlantSimEngine.status_template(mapping, nothing); -``` - -```jldoctest mylabel -julia> var_refvector = PlantSimEngine.reverse_mapping(mapping, all=false); -``` - -```jldoctest mylabel -julia> var_need_init = PlantSimEngine.to_initialize(mapping, mtg); -``` - -```jldoctest mylabel -julia> statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector, var_need_init); +julia> statuses = PlantSimEngine.init_statuses(mtg, mapping); ``` ```jldoctest mylabel diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index cc13acfd..057a236c 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -161,7 +161,7 @@ end var_need_init = PlantSimEngine.to_initialize(mapping_1, mtg) @test var_need_init == Dict{String,Any}() - statuses = PlantSimEngine.init_statuses(mtg, organs_statuses, var_refvector_1, var_need_init) + statuses = PlantSimEngine.init_statuses(mtg, mapping_1) @test collect(keys(statuses)) == ["Soil", "Internode", "Plant", "Leaf"] @test length(statuses["Internode"]) == length(statuses["Leaf"]) == 2 From 2b79ea5ead28682c3bc874dc777abff7f00e4a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 23 Nov 2023 19:14:08 +0100 Subject: [PATCH 91/97] Add more info into the GraphSimulation This is done so we can dynamically add organs --- src/mtg/GraphSimulation.jl | 21 ++++++++++++++- src/mtg/initialisation.jl | 37 +++++++++++++++++++-------- src/mtg/mapping.jl | 2 +- src/processes/model_initialisation.jl | 12 +-------- src/run.jl | 11 ++++---- test/test-mtg-multiscale.jl | 2 +- 6 files changed, 54 insertions(+), 31 deletions(-) diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index b48408fc..666200ac 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -1,3 +1,13 @@ +""" + VarFromMTG(var::Symbol, scale::String) + +A strucure to hold the variables that are needed for initialisation, and that must be taken from the MTG attributes. +""" +struct VarFromMTG + var::Symbol + scale::String +end + """ GraphSimulation(graph, mapping) GraphSimulation(graph, statuses, dependency_graph, models, outputs) @@ -9,13 +19,19 @@ A type that holds all information for a simulation over a graph. - `graph`: an graph, such as an MTG - `mapping`: a dictionary of model mapping - `statuses`: a structure that defines the status of each node in the graph +- `status_templates`: a dictionary of status templates +- `map_other_scales`: a dictionary of mapping for other scales +- `var_need_init`: a dictionary indicating if a variable needs to be initialized - `dependency_graph`: the dependency graph of the models applied to the graph - `models`: a dictionary of models - `outputs`: a dictionary of outputs """ -struct GraphSimulation{T,S,U,O} +struct GraphSimulation{T,S,U,O,V} graph::T statuses::S + status_templates::Dict{String,Dict{Symbol,Any}} + map_other_scales::Dict{String,Dict{String,Vector{Symbol}}} + var_need_init::Dict{String,V} dependency_graph::DependencyGraph models::Dict{String,U} outputs::Dict{String,O} @@ -27,6 +43,9 @@ end dep(g::GraphSimulation) = g.dependency_graph status(g::GraphSimulation) = g.statuses +status_template(g::GraphSimulation) = g.status_templates +map_other_scales(g::GraphSimulation) = g.map_other_scales +var_need_init(g::GraphSimulation) = g.var_need_init get_models(g::GraphSimulation) = g.models outputs(g::GraphSimulation) = g.outputs diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index c5d4e6cd..ae9ec4b0 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -2,6 +2,20 @@ init_statuses(mtg, mapping; type_promotion=nothing, check=true) Get the status of each node in the MTG by node type, pre-initialised considering multi-scale variables. + +# Arguments + +- `mtg`: the plant graph +- `mapping`: a dictionary of model mapping +- `type_promotion`: the type promotion to use for the variables +- `check`: whether to check the mapping for errors + +# Return + +A NamedTuple of status by node type, a dictionary of status templates by node type, a dictionary of variables mapped to other scales, +a dictionary of variables that need to be initialised or computed by other models, and a vector of nodes that have a model defined for their symbol: + +`(;statuses, status_templates, map_other_scales, var_need_init, nodes_with_models)` """ function init_statuses(mtg, mapping; type_promotion=nothing, check=true) # We make a pre-initialised status for each kind of organ (this is a template for each node type): @@ -17,14 +31,13 @@ function init_statuses(mtg, mapping; type_promotion=nothing, check=true) # If we find some, we return an error: check && error_mtg_init(var_need_init) - nodes_with_models = collect(keys(status_templates)) # We traverse the MTG to initialise the statuses linked to the nodes: - statuses = Dict(i => Status[] for i in nodes_with_models) + statuses = Dict(i => Status[] for i in collect(keys(status_templates))) MultiScaleTreeGraph.traverse!(mtg) do node # e.g.: node = get_node(mtg, 5) - init_status!(node, statuses, status_templates, map_other_scales, var_need_init, nodes_with_models) + init_status!(node, statuses, status_templates, map_other_scales, var_need_init) end - return statuses + return (; statuses, status_templates, map_other_scales, var_need_init) end @@ -35,7 +48,6 @@ end status_templates, map_other_scales, var_need_init=Dict{String,Any}(), - nodes_with_models=collect(keys(status_templates)) ) Initialise the status of a node, taking into account the multiscale mapping, and add it to the @@ -54,15 +66,14 @@ statuses dictionary. Most arguments can be computed from the graph and the mapping: - `statuses` is given by the first initialisation: `statuses = Dict(i => Status[] for i in nodes_with_models)` -- `status_templates` is computed usin `status_template(mappinxg, type_promotion)` +- `status_templates` is computed usin `status_template(mapping, type_promotion)` - `map_other_scales` is computed using `reverse_mapping(mapping, all=false)`. We use `all=false` because we only want the variables that are mapped as `RefVectors` - `var_need_init` is computed using `to_initialize(mapping, mtg)` -- `nodes_with_models` is computed using `collect(keys(status_templates))` """ -function init_status!(node, statuses, status_templates, map_other_scales, var_need_init=Dict{String,Any}(), nodes_with_models=collect(keys(status_templates))) +function init_status!(node, statuses, status_templates, map_other_scales, var_need_init=Dict{String,Any}()) # Check if the node has a model defined for its symbol, if not, no need to compute - node.MTG.symbol ∉ nodes_with_models && return + node.MTG.symbol ∉ collect(keys(status_templates)) && return # We make a copy of the template status for this node: st_template = copy(status_templates[node.MTG.symbol]) @@ -99,6 +110,8 @@ function init_status!(node, statuses, status_templates, map_other_scales, var_ne end end end + + return st end @@ -349,7 +362,9 @@ automatically passed as is. """ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion=nothing, check=true, verbose=true) # Get the status of each node by node type, pre-initialised considering multi-scale variables: - statuses = init_statuses(mtg, mapping; type_promotion=type_promotion, check=check) + statuses, status_templates, map_other_scales, var_need_init = + init_statuses(mtg, mapping; type_promotion=type_promotion, check=check) + # Print an info if models are declared for nodes that don't exist in the MTG: if check && any(x -> length(last(x)) == 0, statuses) model_no_node = join(findall(x -> length(x) == 0, statuses), ", ") @@ -363,5 +378,5 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion outputs = pre_allocate_outputs(statuses, outputs, nsteps, check=check) - return (; mtg, statuses, dependency_graph, models, outputs) + return (; mtg, statuses, status_templates, map_other_scales, var_need_init, dependency_graph, models, outputs) end \ No newline at end of file diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index b5d1bfb6..ca1697b2 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -549,7 +549,7 @@ Dict{String, Any} with 3 entries: ``` """ function reverse_mapping(models; all=true) - var_to_ref = Dict{String,Any}(i => Dict{String,Vector{Symbol}}() for i in keys(models)) + var_to_ref = Dict{String,Dict{String,Vector{Symbol}}}(i => Dict{String,Vector{Symbol}}() for i in keys(models)) for organ in keys(models) # organ = "Plant" map_vars = get_mapping(models[organ]) diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 64f6e516..586fc8fa 100644 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -141,16 +141,6 @@ function to_initialize(; verbose=true, vars...) return NamedTuple(to_init) end -""" - VarFromMTG(var::Symbol, scale::String) - -A strucure to hold the variables that are needed for initialisation, and that must be taken from the MTG attributes. -""" -struct VarFromMTG - var::Symbol - scale::String -end - # For the list of mapping given to an MTG: function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} @@ -161,7 +151,7 @@ function to_initialize(mapping::Dict{String,T}, graph=nothing) where {T} vars_in_mtg = names(graph) end - var_need_init = Dict{String,Any}() + var_need_init = Dict{String,NamedTuple{(:need_initialisation, :need_models_from_scales, :need_var_from_mtg),Tuple{Vector{Symbol},Vector{NamedTuple{(:var, :scale, :need_scales),Tuple{Symbol,String,Union{String,Vector{String}}}}},Vector{VarFromMTG}}}}() for organ in keys(mapping) # organ = "Plant" # Get all mapping for the organ: diff --git a/src/run.jl b/src/run.jl index 5554d0fc..770cb149 100644 --- a/src/run.jl +++ b/src/run.jl @@ -13,7 +13,7 @@ If several time-steps are given, the models are run sequentially for each time-s - `meteo`: a [`PlantMeteo.TimeStepTable`](https://palmstudio.github.io/PlantMeteo.jl/stable/API/#PlantMeteo.TimeStepTable) of [`PlantMeteo.Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/API/#PlantMeteo.Atmosphere) or a single `PlantMeteo.Atmosphere`. - `constants`: a [`PlantMeteo.Constants`](https://palmstudio.github.io/PlantMeteo.jl/stable/API/#PlantMeteo.Constants) object, or a `NamedTuple` of constant keys and values. -- `extra`: extra parameters. +- `extra`: extra parameters, not available for simulation of plant graphs (the simulation object is passed using this). - `check`: if `true`, check the validity of the model list before running the simulation (takes a little bit of time), and return more information while running. - `executor`: the [`Floops`](https://juliafolds.github.io/FLoops.jl/stable/) executor used to run the simulation either in sequential (`executor=SequentialEx()`), in a multi-threaded way (`executor=ThreadedEx()`, the default), or in a distributed way (`executor=DistributedEx()`). @@ -353,12 +353,11 @@ function run!( executor=ThreadedEx() ) models = get_models(object) - st = status(object) # Run the simulation of each soft-coupled model in the dependency graph: # Note: hard-coupled processes handle themselves already @floop executor for (process_key, dependency_node) in collect(dep(object).roots) - run!(object, dependency_node, 1, models, meteo, constants, st, check, executor) + run!(object, dependency_node, 1, models, meteo, constants, object, check, executor) end end @@ -376,7 +375,7 @@ function run!( dep_graph = dep(object) models = get_models(object) - st = status(object) + # st = status(object) !isnothing(extra) && error("Extra parameters are not allowed for the simulation of an MTG (already used for statuses).") @@ -387,7 +386,7 @@ function run!( # In parallel over dependency root, i.e. for independant computations: @floop executor for (process_key, dependency_node) in collect(dep_graph.roots) # Note: parallelization over objects is handled by the run! method below - run!(object, dependency_node, i, models, meteo_i, constants, st, check, executor) + run!(object, dependency_node, i, models, meteo_i, constants, object, check, executor) end # At the end of the time-step, we save the results of the simulation in the object: save_results!(object, i) @@ -445,7 +444,7 @@ function run!( models, meteo, constants, - extra, + extra::T, # we pass the simulation object as extra so we can access its parameters during simulation check, executor ) where {T<:GraphSimulation} # T is the status of each node by organ type diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index 057a236c..886e20a1 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -161,7 +161,7 @@ end var_need_init = PlantSimEngine.to_initialize(mapping_1, mtg) @test var_need_init == Dict{String,Any}() - statuses = PlantSimEngine.init_statuses(mtg, mapping_1) + statuses, = PlantSimEngine.init_statuses(mtg, mapping_1) @test collect(keys(statuses)) == ["Soil", "Internode", "Plant", "Leaf"] @test length(statuses["Internode"]) == length(statuses["Leaf"]) == 2 From 906f0c15fd7f6591aba68b513c27b7fb17167efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 23 Nov 2023 19:14:24 +0100 Subject: [PATCH 92/97] Add ToyInternodeEmergence example --- examples/ToyInternodeEmergence.jl | 38 +++++++++++++++++++++++++++++++ src/examples_import.jl | 4 ++++ 2 files changed, 42 insertions(+) create mode 100644 examples/ToyInternodeEmergence.jl diff --git a/examples/ToyInternodeEmergence.jl b/examples/ToyInternodeEmergence.jl new file mode 100644 index 00000000..d83d4e10 --- /dev/null +++ b/examples/ToyInternodeEmergence.jl @@ -0,0 +1,38 @@ + +# Declaring the process of LAI dynamic: +PlantSimEngine.@process "organ_emergence" verbose = false + +# Declaring the model of LAI dynamic with its parameter values: + +""" + ToyInternodeEmergence(;init_TT=0.0, TT_emergence = 300) + +Computes the organ emergence based on cumulated thermal time since last event. +""" +struct ToyInternodeEmergence <: AbstractOrgan_EmergenceModel + TT_emergence::Float64 +end + +# Defining default values: +ToyInternodeEmergence(; TT_emergence=300.0) = ToyInternodeEmergence(TT_emergence) + +# Defining the inputs and outputs of the model: +PlantSimEngine.inputs_(m::ToyInternodeEmergence) = (TT_cu=-Inf,) +PlantSimEngine.outputs_(m::ToyInternodeEmergence) = (TT_cu_emergence=0.0,) + +# Implementing the actual algorithm by adding a method to the run! function for our model: +function PlantSimEngine.run!(m::ToyInternodeEmergence, models, status, meteo, constants=nothing, sim_object=nothing) + + if length(status.node.children) == 1 && status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + # NB: the node can produce one leaf, and one internode only, so we check that it did not produce + # any internode yet. + new_node = MultiScaleTreeGraph.Node(status.node, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) + status_new_node = PlantSimEngine.init_status!(new_node, sim_object.statuses, sim_object.status_templates, sim_object.map_other_scales, sim_object.var_need_init) + leaf_node = MultiScaleTreeGraph.Node(new_node, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + PlantSimEngine.init_status!(leaf_node, sim_object.statuses, sim_object.status_templates, sim_object.map_other_scales, sim_object.var_need_init) + + status_new_node.TT_cu_emergence = status.TT_cu + end + + return nothing +end \ No newline at end of file diff --git a/src/examples_import.jl b/src/examples_import.jl index fd99dbb0..4b5b3a43 100644 --- a/src/examples_import.jl +++ b/src/examples_import.jl @@ -32,6 +32,8 @@ include(joinpath(@__DIR__, "../examples/ToyAssimGrowthModel.jl")) include(joinpath(@__DIR__, "../examples/ToyRUEGrowthModel.jl")) include(joinpath(@__DIR__, "../examples/ToyCAllocationModel.jl")) include(joinpath(@__DIR__, "../examples/ToySoilModel.jl")) +include(joinpath(@__DIR__, "../examples/ToyInternodeEmergence.jl")) + """ import_mtg_example() @@ -74,6 +76,7 @@ export AbstractProcess7Model export AbstractLight_InterceptionModel, AbstractLai_DynamicModel, AbstractDegreedaysModel export AbstractPhotosynthesisModel, AbstractCarbon_AllocationModel, AbstractCarbon_DemandModel export AbstractSoil_WaterModel, AbstractGrowthModel +export AbstractOrgan_EmergenceModel # Models: export Beer, ToyLAIModel, ToyDegreeDaysCumulModel @@ -82,5 +85,6 @@ export ToyAssimGrowthModel, ToyRUEGrowthModel export Process1Model, Process2Model, Process3Model, Process4Model, Process5Model export Process6Model, Process7Model +export ToyInternodeEmergence export import_mtg_example end \ No newline at end of file From dbded6cf83dce34ae2fd78f72fabc2d96341c38d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 24 Nov 2023 00:02:16 +0100 Subject: [PATCH 93/97] Remove info from copute_mapping (happens too often) --- src/mtg/mapping.jl | 7 ++++--- src/mtg/save_results.jl | 10 +++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/mtg/mapping.jl b/src/mtg/mapping.jl index ca1697b2..fb02bab2 100644 --- a/src/mtg/mapping.jl +++ b/src/mtg/mapping.jl @@ -309,8 +309,9 @@ function compute_mapping(models::Dict{String,T}, type_promotion) where {T} push!(organs_mapping[organs_mapped], organs_mapped => organ_mapping[organs_mapped]) elseif !haskey(organs_mapping[organs_mapped][organs_mapped], variable) push!(organs_mapping[organs_mapped][organs_mapped], variable => organ_mapping[organs_mapped][variable]) - else - @info "Variable $variable already mapped from scale $organs_mapped to scale $organs_mapped. Skipping." + # else + #@info "Variable $variable already mapped from scale $organs_mapped to scale $organs_mapped. Skipping." + # NB: I removed this else because it happens a lot when we have a variable mapped a lot to a scale (e.g. :TT) end end end @@ -542,7 +543,7 @@ to get the value as a singleton instead of a vector of values. ```jldoctest mylabel julia> PlantSimEngine.reverse_mapping(models) -Dict{String, Any} with 3 entries: +Dict{String, Dict{String, Vector{Symbol}}} with 3 entries: "Soil" => Dict("Leaf"=>[:soil_water_content]) "Internode" => Dict("Plant"=>[:carbon_demand, :carbon_allocation]) "Leaf" => Dict("Plant"=>[:A, :carbon_demand, :carbon_allocation]) diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl index 6824002f..716e12de 100644 --- a/src/mtg/save_results.jl +++ b/src/mtg/save_results.jl @@ -69,23 +69,23 @@ julia> mtg = import_mtg_example(); ``` ```jldoctest mylabel -julia> statuses = PlantSimEngine.init_statuses(mtg, mapping); +julia> statuses, = PlantSimEngine.init_statuses(mtg, mapping); ``` ```jldoctest mylabel -julia> outputs = Dict("Leaf" => (:A, :carbon_demand), "Soil" => (:soil_water_content,)); +julia> outs = Dict("Leaf" => (:A, :carbon_demand), "Soil" => (:soil_water_content,)); ``` Pre-allocate the outputs as a dictionary: ```jldoctest mylabel -julia> outs = PlantSimEngine.pre_allocate_outputs(statuses, outputs, 2); +julia> preallocated_vars = PlantSimEngine.pre_allocate_outputs(statuses, outs, 2); ``` The dictionary has a key for each organ from which we want outputs: ```jldoctest mylabel -julia> collect(keys(outs)) +julia> collect(keys(preallocated_vars)) 2-element Vector{String}: "Soil" "Leaf" @@ -95,7 +95,7 @@ Each organ has a dictionary of variables for which we want outputs from, with the pre-allocated empty vectors (one per time-step that will be filled with one value per node): ```jldoctest mylabel -julia> collect(keys(outs["Leaf"])) +julia> collect(keys(preallocated_vars["Leaf"])) 3-element Vector{Symbol}: :A :node From fa528e1b97e792c56baa602c3c26953b608f1d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 24 Nov 2023 00:02:34 +0100 Subject: [PATCH 94/97] Add add_organ! --- examples/ToyInternodeEmergence.jl | 8 +++----- src/PlantSimEngine.jl | 2 ++ src/mtg/add_organ.jl | 34 +++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 src/mtg/add_organ.jl diff --git a/examples/ToyInternodeEmergence.jl b/examples/ToyInternodeEmergence.jl index d83d4e10..7226a45d 100644 --- a/examples/ToyInternodeEmergence.jl +++ b/examples/ToyInternodeEmergence.jl @@ -26,12 +26,10 @@ function PlantSimEngine.run!(m::ToyInternodeEmergence, models, status, meteo, co if length(status.node.children) == 1 && status.TT_cu - status.TT_cu_emergence >= m.TT_emergence # NB: the node can produce one leaf, and one internode only, so we check that it did not produce # any internode yet. - new_node = MultiScaleTreeGraph.Node(status.node, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) - status_new_node = PlantSimEngine.init_status!(new_node, sim_object.statuses, sim_object.status_templates, sim_object.map_other_scales, sim_object.var_need_init) - leaf_node = MultiScaleTreeGraph.Node(new_node, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) - PlantSimEngine.init_status!(leaf_node, sim_object.statuses, sim_object.status_templates, sim_object.map_other_scales, sim_object.var_need_init) + status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 1, 2) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 1, 2) - status_new_node.TT_cu_emergence = status.TT_cu + status_new_internode.TT_cu_emergence = status.TT_cu end return nothing diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index bd53d18d..fbdbb12f 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -57,6 +57,7 @@ include("mtg/GraphSimulation.jl") include("mtg/mapping.jl") include("mtg/initialisation.jl") include("mtg/save_results.jl") +include("mtg/add_organ.jl") # Model evaluation (statistics): include("evaluation/statistics.jl") @@ -85,6 +86,7 @@ export ModelList, MultiScaleModel export RMSE, NRMSE, EF, dr export Status, TimeStepTable, status export init_status! +export add_organ! export @process, process export to_initialize, is_initialized, init_variables, dep export inputs, outputs, variables diff --git a/src/mtg/add_organ.jl b/src/mtg/add_organ.jl new file mode 100644 index 00000000..ec437bcc --- /dev/null +++ b/src/mtg/add_organ.jl @@ -0,0 +1,34 @@ +""" + add_organ!(node::MultiScaleTreeGraph.Node, sim_object, link, symbol, index, scale) + +Add an organ to the graph, automatically taking care of initialising the status of the organ (multiscale-)variables. + +This function should be called from a model that implements organ emergence, for example in function of thermal time. + +# Arguments + +* `node`: the node to which the organ is added (the parent organ of the new organ) +* `sim_object`: the simulation object, e.g. the `GraphSimulation` object from the `extra` argument of a model. +* `link`: the link type between the new node and the organ: + * `"<"`: the new node is following the parent organ + * `"+"`: the new node is branching the parent organ + * `"/"`: the new node is decomposing the parent organ, *i.e.* we change scale +* `symbol`: the symbol of the organ, *e.g.* `"Leaf"` +* `index`: the index of the organ, *e.g.* `1`. The index may be used to easily identify branching order, or growth unit index on the axis. It is different from the node `id` that is unique. +* `scale`: the scale of the organ, *e.g.* `2`. + +# Returns + +* `status`: the status of the new node + +# Examples + +See the `ToyInternodeEmergence` example model from the `Examples` module (also found in the `examples` folder), +or the `test-mtg-dynamic.jl` test file for an example usage. +""" +function add_organ!(node::MultiScaleTreeGraph.Node, sim_object, link, symbol, index, scale) + new_node = MultiScaleTreeGraph.Node(node, MultiScaleTreeGraph.NodeMTG(link, symbol, index, scale)) + st = PlantSimEngine.init_status!(new_node, sim_object.statuses, sim_object.status_templates, sim_object.map_other_scales, sim_object.var_need_init) + + return st +end \ No newline at end of file From 4288505735168dc9f9c86f4cc4e029daa4c69939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 24 Nov 2023 00:02:44 +0100 Subject: [PATCH 95/97] Add test-mtg-dynamic.jl --- test/runtests.jl | 2 +- test/test-mtg-dynamic.jl | 77 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 test/test-mtg-dynamic.jl diff --git a/test/runtests.jl b/test/runtests.jl index dedb79f9..f21af168 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -46,9 +46,9 @@ using PlantSimEngine.Examples @testset "MTG with multiscale mapping" begin include("test-mtg-multiscale.jl") + include("test-mtg-dynamic.jl") end - if VERSION >= v"1.8" # Error formating changed in Julia 1.8 (or was it 1.7?), so the doctest # that returns an error in PlantSimEngine.check_dimensions(models, w) diff --git a/test/test-mtg-dynamic.jl b/test/test-mtg-dynamic.jl new file mode 100644 index 00000000..a049f158 --- /dev/null +++ b/test/test-mtg-dynamic.jl @@ -0,0 +1,77 @@ +# using PlantSimEngine, DataFrames, MultiScaleTreeGraph +# using PlantSimEngine.Examples; +mtg = import_mtg_example(); +# Example meteo: +meteo = Weather( + [ + Atmosphere(T=20.0, Wind=1.0, Rh=0.65), + Atmosphere(T=25.0, Wind=0.5, Rh=0.8) +] +) + +mapping = Dict( + "Scene" => ToyDegreeDaysCumulModel(), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapping=[ + :TT_cu => "Scene", + ], + ), + Beer(0.6), + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + :A => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + ), + "Internode" => ( + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapping=[:TT => "Scene",], + ), + MultiScaleModel( + model=ToyInternodeEmergence(TT_emergence=20.0), + mapping=[:TT_cu => "Scene"], + ), + ), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapping=[:soil_water_content => "Soil", :aPPFD => "Plant"], + ), + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapping=[:TT => "Scene",], + ), + ), + "Soil" => ( + ToySoilWaterModel(), + ), +) + +out_vars = Dict( + "Leaf" => (:A, :carbon_demand, :soil_water_content, :carbon_allocation), + "Internode" => (:carbon_allocation, :TT_cu_emergence), + "Plant" => (:carbon_allocation,), + "Soil" => (:soil_water_content,), +) + +out = run!(mtg, mapping, meteo, outputs=out_vars, executor=SequentialEx()) + +@testset "MTG with dynamic growth" begin + st = out.statuses + @test length(mtg) == 9 + @test length(st["Scene"]) == length(st["Soil"]) == length(st["Plant"]) == 1 + @test length(st["Internode"]) == length(st["Leaf"]) == 3 + @test st["Internode"][1].TT_cu_emergence == 0.0 + @test st["Internode"][end].TT_cu_emergence == 25.0 + + out_df = outputs(out, DataFrame) + @test unique(out_df[:, :organ]) |> sort == ["Internode", "Leaf", "Plant", "Soil"] + @test filter(row -> row.organ == "Internode", out_df)[:, :TT_cu_emergence] == [0.0, 0.0, 0.0, 0.0, 25.0] + @test filter(row -> row.organ == "Leaf", out_df)[:, :carbon_demand] == [0.5, 0.5, 0.75, 0.75, 0.75] +end \ No newline at end of file From 96a0277df9af050193bf86a0bae8b4ae03520e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 24 Nov 2023 00:25:15 +0100 Subject: [PATCH 96/97] Add compat for stdlibs --- Project.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Project.toml b/Project.toml index abca1c12..2a104c8b 100644 --- a/Project.toml +++ b/Project.toml @@ -22,10 +22,13 @@ CSV = "0.10" DataAPI = "1.15" DataFrames = "1" FLoops = "0.2" +Markdown = "<0.0.1, 1" MultiScaleTreeGraph = "0.12" PlantMeteo = "0.6" +Statistics = "<0.0.1, 1" Tables = "1" Term = "1, 2" +Test = "<0.0.1, 1" julia = "1.7" [extras] From 70477a14cf67d7aedc8c10bfdab685bedb360796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Fri, 24 Nov 2023 00:28:54 +0100 Subject: [PATCH 97/97] Update Project.toml --- Project.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index 2a104c8b..95ac1f1d 100644 --- a/Project.toml +++ b/Project.toml @@ -22,13 +22,13 @@ CSV = "0.10" DataAPI = "1.15" DataFrames = "1" FLoops = "0.2" -Markdown = "<0.0.1, 1" +Markdown = "1.7" MultiScaleTreeGraph = "0.12" PlantMeteo = "0.6" -Statistics = "<0.0.1, 1" +Statistics = "1.7" Tables = "1" Term = "1, 2" -Test = "<0.0.1, 1" +Test = "1.7" julia = "1.7" [extras]