Addition: Cleanup logic.

.github/workflows/FormatCheck.yml

@@ -23,7 +23,7 @@ jobs:
       - uses: julia-actions/cache@v2
       - name: Install JuliaFormatter and format
         run: |
-          julia  -e 'using Pkg; Pkg.add(PackageSpec(name = "JuliaFormatter", version="1.0.45"))'
+          julia  -e 'using Pkg; Pkg.add(PackageSpec(name = "JuliaFormatter", version="1.0.60"))'
           julia  -e 'using JuliaFormatter; format(["examples", "src/T8code.jl", "test"])'
       - name: Format check
         run: |

Project.toml

@@ -1,7 +1,7 @@
 name = "T8code"
 uuid = "d0cc0030-9a40-4274-8435-baadcfd54fa1"
 authors = ["Johannes Markert <[email protected]>"]
-version = "0.7.0"
+version = "0.7.1"
 
 [deps]
 CEnum = "fa961155-64e5-5f13-b03f-caf6b980ea82"
@@ -16,8 +16,8 @@ t8code_jll = "4ee9bed8-4011-53f7-90c2-22363c2f500d"
 [compat]
 CEnum = "0.4, 0.5"
 Libdl = "1"
-MPI = "0.20"
-MPIPreferences = "0.1.3"
+MPI = "0.20.6"
+MPIPreferences = "0.1.6"
 Preferences = "1.2"
 Reexport = "0.2, 1.0"
 UUIDs = "1"

examples/t8_step6_stencil.jl

@@ -348,7 +348,7 @@ function t8_step6_output_data_to_vtu(forest, element_data, prefix)
                             pointer(schlieren)),
         t8_vtk_data_field_t(T8_VTK_SCALAR,
                             NTuple{8192, Cchar}(rpad("curvature\0", 8192, ' ')),
-                            pointer(curvature)),
+                            pointer(curvature))
     ]
 
     # The number of user defined data fields to write.

examples/t8_tutorial_build_cmesh.jl

@@ -154,7 +154,7 @@ function t8_cmesh_new_periodic_hybrid_2d(comm)
         1, 1, 0,
         0.5, 0.5, 0,                # tree 5, triangle
         1, 1, 0,
-        0.5, 1, 0,
+        0.5, 1, 0
     ]
 
     # 2. Initialization of the mesh.

src/T8code.jl

@@ -1,5 +1,7 @@
 module T8code
 
+using MPI
+
 using Reexport: @reexport
 using Libdl: Libdl
 
@@ -157,6 +159,85 @@ not a system-provided `t8code` installation. In this case, T8code.jl is not usab
 preferences_set_correctly() = !(_PREFERENCE_LIBT8 == "t8code_jll" &&
                                 MPIPreferences.binary == "system")
 
+"""
+    ForestWrapper
+
+Lightweight `t8_forest_t` pointer wrapper which helps to free
+resources allocated by t8code in an orderly fashion.
+
+When initialized with a t8code forest pointer the wrapper
+registers itself with a t8code object tracker called `T8CODE_OBJECT_TRACKER`.
+
+In serial mode the wrapper and in consequence the t8code forest
+can be finalized immediately whenever Julia's garbage collector sees fit.
+
+In (MPI) parallel mode the wrapper (and the t8code forest) is kept till the end
+of the session or when finalized explicitly. At the end of the session (resp.
+when the program shuts down) the object tracker finalizes all registered t8code
+objects in sync with all MPI ranks. This is necessary since t8code internally
+allocates MPI shared arrays.
+"""
+mutable struct ForestWrapper
+    pointer::Ptr{t8_forest} # cpointer to t8code forest
+
+    function ForestWrapper(pointer::Ptr{t8_forest})
+        wrapper = new(pointer)
+
+        # Compute a unique id from the ForestWrapper object.
+        unique_id = UInt64(pointer_from_objref(wrapper))
+
+        if MPI.Comm_size(MPI.Comm(t8_forest_get_mpicomm(pointer))) > 1
+            # Make sure the unique id is identical for each MPI rank.
+            unique_id = MPI.bcast(unique_id, MPI.Comm(t8_forest_get_mpicomm(pointer)))
+        end
+
+        finalizer(wrapper) do wrapper
+            # When finalizing, `forest`, `scheme`, `cmesh`, and `geometry` are
+            # also cleaned up from within `t8code`. The cleanup code for
+            # `cmesh` does some MPI calls for deallocating shared memory
+            # arrays. Due to garbage collection in Julia the order of shutdown
+            # is not deterministic. Hence, deterministic finalization is necessary in
+            # order to avoid MPI-related error output when closing the Julia
+            # program/session.
+            t8_forest_unref(Ref(wrapper.pointer))
+
+            # Deregister from the object tracker.
+            delete!(T8CODE_OBJECT_TRACKER, unique_id)
+        end
+
+        # Register the T8codeForestWrapper with the object tracker.
+        T8CODE_OBJECT_TRACKER[unique_id] = wrapper
+
+        return wrapper
+    end
+end
+
+function clean_up()
+    # Finalize all registered t8code objects before MPI shuts down.
+    while length(T8CODE_OBJECT_TRACKER) > 0
+        unique_id = first(T8CODE_OBJECT_TRACKER).first
+
+        forest_wrapper = T8CODE_OBJECT_TRACKER[unique_id]
+
+        # Make sure all MPI ranks finalize the same object.
+        if MPI.Comm_size(MPI.Comm(t8_forest_get_mpicomm(forest_wrapper.pointer))) > 1
+            unique_id = MPI.bcast(unique_id,
+                                  MPI.Comm(t8_forest_get_mpicomm(forest_wrapper.pointer)))
+        end
+
+        # Finalize the object. The object deregisters itself from the
+        # object tracker automatically.
+        finalize(forest_wrapper)
+    end
+end
+
+# Minimal reference tracker holding active t8code related objects
+# created throughout the life time of a Julia session. t8code objects
+# should remove themselves from the tracker when they get finalized.
+if !@isdefined(T8CODE_OBJECT_TRACKER)
+    T8CODE_OBJECT_TRACKER = Dict{UInt64, ForestWrapper}()
+end
+
 const T8_QUAD_MAXLEVEL = 30
 const T8_HEX_MAXLEVEL = 19
 

test/Project.toml

@@ -6,6 +6,6 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 Aqua = "0.7, 0.8"
-MPI = "0.20"
-MPIPreferences = "0.1.3"
+MPI = "0.20.6"
+MPIPreferences = "0.1.6"
 Test = "1"

test/cmesh/test_readmshfile.jl

@@ -10,22 +10,22 @@ function t8_supported_msh_file(cmesh)
         [4, 0],
         [1, 2],
         [3, 2],
-        [2, 4],
+        [2, 4]
     ]
 
     # 0-based indexing
     elements = [
         [0, 1, 3],
         [1, 4, 3],
         [1, 2, 4],
-        [3, 4, 5],
+        [3, 4, 5]
     ]
 
     face_neigh_elem = [
         [1, -1, -1],
         [3, 0, 2],
         [-1, 1, -1],
-        [-1, -1, 1],
+        [-1, -1, 1]
     ]
 
     @assert cmesh != C_NULL

test/test_all.jl

@@ -23,6 +23,10 @@ end
     end
 end
 
+@testset "forestwrapper" begin
+    include("test_forestwrapper.jl")
+end
+
 @testset "cmesh" begin
     include("cmesh/test_readmshfile.jl")
 end

test/test_forestwrapper.jl

@@ -0,0 +1,29 @@
+@testset "test forestwrapper" begin
+
+    # Clean up t8code before MPI shuts down.
+    MPI.add_finalize_hook!() do
+        T8code.clean_up()
+        status = T8code.Libt8.sc_finalize_noabort()
+        # If the following test fails the allocated objects were not cleaned up
+        # properly before shutting down.
+        @test status == 0
+    end
+
+    # Create a forest and wrap by `ForestWrapper`
+    scheme = t8_scheme_new_default_cxx()
+    cmesh = t8_cmesh_new_hypercube(T8_ECLASS_QUAD, comm, 0, 0, 0)
+    forest = t8_forest_new_uniform(cmesh, scheme, 0, 0, comm)
+    wrapper_A = T8code.ForestWrapper(forest)
+
+    # Create another forest and wrap by `ForestWrapper`
+    scheme = t8_scheme_new_default_cxx()
+    cmesh = t8_cmesh_new_hypercube(T8_ECLASS_TRIANGLE, comm, 0, 0, 0)
+    forest = t8_forest_new_uniform(cmesh, scheme, 0, 0, comm)
+    wrapper_B = T8code.ForestWrapper(forest)
+
+    # Finalize the first wrapper.
+    finalize(wrapper_A)
+
+    # The second wrapper should be finalized automatically when Julia shuts down.
+    # ... finalize(wrapper_B) ...
+end

utils/format.jl

@@ -0,0 +1,31 @@
+#!/usr/bin/env julia
+
+using Pkg
+Pkg.activate(; temp = true, io = devnull)
+Pkg.add(PackageSpec(name = "JuliaFormatter", version = "1.0.60"); preserve = PRESERVE_ALL,
+        io = devnull)
+
+using JuliaFormatter: format
+
+function main()
+    # Show help
+    if "-h" in ARGS || "--help" in ARGS
+        println("usage: trixi-format.jl PATH [PATH...]")
+        println()
+        println("positional arguments:")
+        println()
+        println("    PATH        One or more paths (directories or files) to format. Default: '.'")
+        return nothing
+    end
+
+    # Set default path if none is given on command line
+    if isempty(ARGS)
+        paths = String["./src/T8code.jl", "./test", "./examples"]
+    else
+        paths = ARGS
+    end
+
+    return format(paths)
+end
+
+main()