From 6e01d17e72391ff281f72e77a51db4b5764aca94 Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Thu, 5 Aug 2021 08:47:42 -0600 Subject: [PATCH 01/15] Created a folder in example for 2D Mixed-Hdiv --- backends/ref/ceed-ref-basis.c | 2 +- examples/Hdiv-mixed/2DQuadbasis.h | 78 ++++++++++++ examples/Hdiv-mixed/Makefile | 47 ++++++++ examples/Hdiv-mixed/MixedPoisson.c | 188 +++++++++++++++++++++++++++++ interface/ceed-basis.c | 109 +++++++++++++++++ tests/t330-basis.h | 2 +- tests/t999-Hdiv2D.c | 136 +++++++++++++++++++++ tests/t999-Hdiv2D.h | 89 ++++++++++++++ 8 files changed, 649 insertions(+), 2 deletions(-) create mode 100644 examples/Hdiv-mixed/2DQuadbasis.h create mode 100644 examples/Hdiv-mixed/Makefile create mode 100644 examples/Hdiv-mixed/MixedPoisson.c create mode 100644 tests/t999-Hdiv2D.c create mode 100644 tests/t999-Hdiv2D.h diff --git a/backends/ref/ceed-ref-basis.c b/backends/ref/ceed-ref-basis.c index 18e9dd007d..d492f3910e 100644 --- a/backends/ref/ceed-ref-basis.c +++ b/backends/ref/ceed-ref-basis.c @@ -14,7 +14,7 @@ #include "ceed-ref.h" //------------------------------------------------------------------------------ -// Basis Apply +// Basis Apply H1 //------------------------------------------------------------------------------ static int CeedBasisApply_Ref(CeedBasis basis, CeedInt num_elem, CeedTransposeMode t_mode, CeedEvalMode eval_mode, CeedVector U, CeedVector V) { Ceed ceed; diff --git a/examples/Hdiv-mixed/2DQuadbasis.h b/examples/Hdiv-mixed/2DQuadbasis.h new file mode 100644 index 0000000000..fc94605bd0 --- /dev/null +++ b/examples/Hdiv-mixed/2DQuadbasis.h @@ -0,0 +1,78 @@ +// Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. +// All Rights reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +// Hdiv basis for quadrilateral element in 2D +// Local numbering is as follow (each edge has 2 vector dof) +// b5 b7 +// 2---------3 +// b4| |b6 +// | | +// b0| |b2 +// 0---------1 +// b1 b3 +// Bx[0-->7] = b0_x-->b7_x, By[0-->7] = b0_y-->b7_y +int HdivBasisQuad(CeedScalar *xhat, CeedScalar *Bx, CeedScalar *By) { + Bx[0] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; + By[0] = (xhat[1]*xhat[1] - 1)*0.125; + Bx[1] = (xhat[0]*xhat[0] - 1)*0.125; + By[1] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; + Bx[2] = (-xhat[0]*xhat[1] + xhat[0] - xhat[1] + 1)*0.25; + By[2] = (xhat[1]*xhat[1] - 1)*0.125; + Bx[3] = (-xhat[0]*xhat[0] + 1)*0.125; + By[3] = (xhat[0]*xhat[1] - xhat[0] + xhat[1] - 1)*0.25; + Bx[4] = (xhat[0]*xhat[1] + xhat[0] - xhat[1] - 1)*0.25; + By[4] = (-xhat[1]*xhat[1] + 1)*0.125; + Bx[5] = (xhat[0]*xhat[0] - 1)*0.125; + By[5] = (-xhat[0]*xhat[1] - xhat[0] + xhat[1] + 1)*0.25; + Bx[6] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; + By[6] = (-xhat[1]*xhat[1] + 1)*0.125; + Bx[7] = (-xhat[0]*xhat[0] + 1)*0.125; + By[7] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; + return 0; +} + +static void buildmats(CeedInt Q1d, CeedScalar *q_ref, CeedScalar *q_weights, + CeedScalar *interp, CeedScalar *div) { + + // Get 1D quadrature on [-1,1] + CeedScalar q_ref_1d[Q1d], q_weight_1d[Q1d]; + CeedGaussQuadrature(Q1d, q_ref_1d, q_weight_1d); + + // Divergence operator; Divergence of nodal basis for ref element + CeedScalar D[8] = {0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25}; + // Loop over quadrature points + CeedScalar Bx[8], By[8]; + CeedScalar xhat[2]; + + for (CeedInt i=0; i +#include +#include "2DQuadbasis.h" + +int main(int argc, char **argv) { + PetscInt ierr; + MPI_Comm comm; + // PETSc objects + DM dm; + PetscSection sec; + PetscBool interpolate = PETSC_TRUE; + PetscInt nx = 2, ny = 1, num_elem = nx * ny; + PetscInt num_nodes = (nx+1)*(ny+1); + PetscInt faces[2] = {nx, ny}; + PetscInt dim = 2; + PetscInt pStart, pEnd; + PetscInt cStart, cEnd, c; // cells + PetscInt eStart, eEnd, e; // edges + PetscInt vStart, vEnd, v; // vertices + PetscInt dofs_per_face; + const PetscInt *ornt; + // libCEED objects + Ceed ceed; + const CeedInt loc_node = 4, Q1d = 2, Q = Q1d*Q1d; + CeedInt num_comp = dim; + CeedInt dof_e = dim*num_nodes; // dof per element! dof is vector in Hdiv + CeedBasis b; + CeedScalar q_ref[dim*Q], q_weights[Q]; + CeedScalar div[dof_e*Q], interp[dim*dof_e*Q]; + + ierr = PetscInitialize(&argc, &argv, NULL, help); + if (ierr) return ierr; + + comm = PETSC_COMM_WORLD; + + CeedInit(argv[1], &ceed); + + // --------------------------------------------------------------------------- + // Build 2D Hdiv basis + // --------------------------------------------------------------------------- + buildmats(Q1d, q_ref, q_weights, interp, div); + ierr = CeedBasisCreateHdiv(ceed, CEED_QUAD, num_comp, loc_node, Q, + interp, div, q_ref, q_weights, &b); CHKERRQ(ierr); + //CeedBasisHdivView(b, stdout); + + // --------------------------------------------------------------------------- + // Set-up DM + // --------------------------------------------------------------------------- + ierr = DMPlexCreateBoxMesh(comm, dim, PETSC_FALSE, faces, NULL, + NULL, NULL, interpolate, &dm); CHKERRQ(ierr); + // Get plex limits + ierr = DMPlexGetChart(dm, &pStart, &pEnd); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd); CHKERRQ(ierr); + ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd); CHKERRQ(ierr); + ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd); CHKERRQ(ierr); + + // Create section + ierr = PetscSectionCreate(comm, &sec); CHKERRQ(ierr); + ierr = PetscSectionSetNumFields(sec,1); CHKERRQ(ierr); + ierr = PetscSectionSetFieldName(sec,0,"Velocity"); CHKERRQ(ierr); + ierr = PetscSectionSetFieldComponents(sec,0,1); CHKERRQ(ierr); + ierr = PetscSectionSetChart(sec,pStart,pEnd); CHKERRQ(ierr); + + // Setup dofs_per_face + for (e = eStart; e < eEnd; e++) { + ierr = DMPlexGetConeSize(dm, e, &dofs_per_face); CHKERRQ(ierr); + ierr = PetscSectionSetFieldDof(sec, e, 0, dofs_per_face); CHKERRQ(ierr); + ierr = PetscSectionSetDof (sec, e, dofs_per_face); CHKERRQ(ierr); + } + printf("=============cell========\n"); + CeedInt cone_size; + for (c = cStart; c< cEnd; c++) { + ierr = DMPlexGetConeSize(dm, c, &cone_size); CHKERRQ(ierr); + printf("cell number %d\n", c); + ierr = DMPlexGetConeOrientation(dm, c, &ornt); CHKERRQ(ierr); + for (CeedInt j = 0; j < cone_size; j++){ + printf("%d\n", ornt[j]); + } + } + + //================================== To Check Restrictions =================== + CeedInt P = 2; + CeedInt ind_x[num_elem*P*P]; + CeedScalar x[dim*num_nodes]; + CeedVector X, Y; + CeedElemRestriction r; + + for (CeedInt i=0; iBasisCreateH1(topo, dim, P, Q, interp, grad, q_ref, q_weight, *basis)); return CEED_ERROR_SUCCESS; } +/** + @brief Create a basis for H(div) discretizations + + @param ceed A Ceed object where the CeedBasis will be created + @param topo Topology of element, e.g. hypercube, simplex, ect + @param num_nodes Total number of nodes + @param num_qpts Total number of quadrature points + @param interp Row-major (dim*num_qpts * num_nodes*dim) matrix expressing the values of + nodal basis functions at quadrature points + @param div Row-major (num_qpts * num_nodes*dim) matrix expressing + divergence of nodal basis functions at quadrature points + @param q_ref Array of length num_qpts holding the locations of quadrature + points on the reference element [-1, 1] + @param q_weight Array of length num_qpts holding the quadrature weights on the + reference element + @param[out] basis Address of the variable where the newly created + CeedBasis will be stored. + + @return An error code: 0 - success, otherwise - failure + + @ref User +**/ +int CeedBasisCreateHdiv(Ceed ceed, CeedElemTopology topo, CeedInt num_comp, + CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, + const CeedScalar *div, const CeedScalar *q_ref, + const CeedScalar *q_weight, CeedBasis *basis) { + int ierr; + CeedInt Q = num_qpts, P = num_nodes, dim = 0; + ierr = CeedBasisGetTopologyDimension(topo, &dim); CeedChk(ierr); + CeedInt dof = dim*P; // dof per element! + if (!ceed->BasisCreateHdiv) { + Ceed delegate; + ierr = CeedGetObjectDelegate(ceed, &delegate, "Basis"); CeedChk(ierr); + + if (!delegate) + // LCOV_EXCL_START + return CeedError(ceed, CEED_ERROR_UNSUPPORTED, + "Backend does not support BasisCreateHdiv"); + // LCOV_EXCL_STOP + + ierr = CeedBasisCreateHdiv(delegate, topo, num_comp, num_nodes, + num_qpts, interp, div, q_ref, + q_weight, basis); CeedChk(ierr); + return CEED_ERROR_SUCCESS; + } + + ierr = CeedCalloc(1,basis); CeedChk(ierr); + + (*basis)->ceed = ceed; + ierr = CeedReference(ceed); CeedChk(ierr); + (*basis)->ref_count = 1; + (*basis)->tensor_basis = 0; + (*basis)->dim = dim; + (*basis)->topo = topo; + (*basis)->num_comp = num_comp; + (*basis)->P = P; + (*basis)->Q = Q; + ierr = CeedMalloc(Q*dim,&(*basis)->q_ref_1d); CeedChk(ierr); + ierr = CeedMalloc(Q,&(*basis)->q_weight_1d); CeedChk(ierr); + memcpy((*basis)->q_ref_1d, q_ref, Q*dim*sizeof(q_ref[0])); + memcpy((*basis)->q_weight_1d, q_weight, Q*sizeof(q_weight[0])); + ierr = CeedMalloc(dim*Q*dof, &(*basis)->interp); CeedChk(ierr); + ierr = CeedMalloc(Q*dof, &(*basis)->div); CeedChk(ierr); + memcpy((*basis)->interp, interp, dim*Q*dof*sizeof(interp[0])); + memcpy((*basis)->div, div, Q*dof*sizeof(div[0])); + ierr = ceed->BasisCreateHdiv(topo, dim, P, Q, interp, div, q_ref, + q_weight, *basis); CeedChk(ierr); + return CEED_ERROR_SUCCESS; +} /** @brief Create a non tensor-product basis for \f$H(\mathrm{div})\f$ discretizations @@ -1293,6 +1362,46 @@ int CeedBasisView(CeedBasis basis, FILE *stream) { return CEED_ERROR_SUCCESS; } +/** + @brief View a CeedBasisHdiv + + @param basis CeedBasisHdiv to view + @param stream Stream to view to, e.g., stdout + + @return An error code: 0 - success, otherwise - failure + + @ref User +**/ +int CeedBasisHdivView(CeedBasis basis, FILE *stream) { + int ierr; + + if (basis->tensor_basis) { + fprintf(stream, "CeedBasis: dim=%d P=%d Q=%d\n", basis->dim, basis->P_1d, + basis->Q_1d); + ierr = CeedScalarView("qref1d", "\t% 12.8f", 1, basis->Q_1d, basis->q_ref_1d, + stream); CeedChk(ierr); + ierr = CeedScalarView("qweight1d", "\t% 12.8f", 1, basis->Q_1d, + basis->q_weight_1d, stream); CeedChk(ierr); + ierr = CeedScalarView("interp1d", "\t% 12.8f", basis->Q_1d, basis->P_1d, + basis->interp_1d, stream); CeedChk(ierr); + ierr = CeedScalarView("grad1d", "\t% 12.8f", basis->Q_1d, basis->P_1d, + basis->grad_1d, stream); CeedChk(ierr); + } else { + fprintf(stream, "CeedBasis: dim=%d P=%d Q=%d\n", basis->dim, basis->P, + basis->Q); + ierr = CeedScalarView("qref", "\t% 12.8f", 1, basis->Q*basis->dim, + basis->q_ref_1d, + stream); CeedChk(ierr); + ierr = CeedScalarView("qweight", "\t% 12.8f", 1, basis->Q, basis->q_weight_1d, + stream); CeedChk(ierr); + ierr = CeedScalarView("interp", "\t% 12.8f", basis->dim*basis->Q, basis->dim*basis->P, + basis->interp, stream); CeedChk(ierr); + ierr = CeedScalarView("div", "\t% 12.8f", basis->Q, basis->dim*basis->P, + basis->div, stream); CeedChk(ierr); + } + return CEED_ERROR_SUCCESS; +} + /** @brief Apply basis evaluation from nodes to quadrature points or vice versa diff --git a/tests/t330-basis.h b/tests/t330-basis.h index fd7069cb61..260cfb6ee8 100644 --- a/tests/t330-basis.h +++ b/tests/t330-basis.h @@ -76,4 +76,4 @@ static void BuildHdivQuadrilateral(CeedInt q, CeedScalar *q_ref, CeedScalar *q_w } } } -} \ No newline at end of file +} diff --git a/tests/t999-Hdiv2D.c b/tests/t999-Hdiv2D.c new file mode 100644 index 0000000000..aab44aa578 --- /dev/null +++ b/tests/t999-Hdiv2D.c @@ -0,0 +1,136 @@ +/// @file +/// test H(div)-mixed fem for 2D quadrilateral element +#include +#include +#include +#include + +// ----------------------------------------------------------------------------- +// Nodal Basis (B=[Bx;By]), xhat is in reference element [-1,1]^2 +// ----------------------------------------------------------------------------- +// B = [b1,b2,b3,b4,b5,b6,b7,b8], size(2x8), +// local numbering is as follow (each edge has 2 dof) +// b6 b8 +// 3---------4 +// b5| |b7 +// | | +// b1| |b3 +// 1---------2 +// b2 b4 +// Sience nodal basis are vector, we have 16 componenets +// For example B[0] = b1_x, B[1] = b1_y, and so on +/* +static int HdivBasisQuad(CeedScalar *xhat, CeedScalar *B) { + B[ 0] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; + B[ 1] = (xhat[1]*xhat[1] - 1)*0.125; + B[ 2] = (xhat[0]*xhat[0] - 1)*0.125; + B[ 3] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; + B[ 4] = (-xhat[0]*xhat[1] + xhat[0] - xhat[1] + 1)*0.25; + B[ 5] = (xhat[1]*xhat[1] - 1)*0.125; + B[ 6] = (-xhat[0]*xhat[0] + 1)*0.125; + B[ 7] = (xhat[0]*xhat[1] - xhat[0] + xhat[1] - 1)*0.25; + B[ 8] = (xhat[0]*xhat[1] + xhat[0] - xhat[1] - 1)*0.25; + B[ 9] = (-xhat[1]*xhat[1] + 1)*0.125; + B[10] = (xhat[0]*xhat[0] - 1)*0.125; + B[11] = (-xhat[0]*xhat[1] - xhat[0] + xhat[1] + 1)*0.25; + B[12] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; + B[13] = (-xhat[1]*xhat[1] + 1)*0.125; + B[14] = (-xhat[0]*xhat[0] + 1)*0.125; + B[15] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; + return 0; +} +*/ + +// ----------------------------------------------------------------------------- +// Divergence operator; Divergence of nodal basis +// CeedScalar Dhat[8] ={0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25} +// ----------------------------------------------------------------------------- + +#include "t999-Hdiv2D.h" +int main(int argc, char **argv) { + Ceed ceed; + CeedInt P = 2, Q = 2, dim = 2; + CeedInt nx = 1, ny = 1, num_elem = nx * ny; + CeedInt num_nodes = (nx+1)*(ny+1), num_qpts = num_elem*Q*Q; + CeedInt ind_x[num_elem*P*P]; + CeedScalar x[dim*num_nodes]; + CeedVector X, u, U; + CeedElemRestriction elem_restr_x, elem_restr_mass; + CeedBasis basis_x, basis_xc; + CeedQFunction qf_setup; + CeedOperator op_setup; + + CeedInit(argv[1], &ceed); + + //============= Node Coordinates, mesh [0,1]^2 square ============== + for (CeedInt i=0; i Date: Tue, 2 Nov 2021 09:59:30 -0600 Subject: [PATCH 02/15] WIP: H(div) orientations in restriction Co-authored-by: Rezgar Shakeri --- examples/Hdiv-mixed/Makefile | 63 ++- examples/Hdiv-mixed/MixedPoisson.c | 188 --------- .../{2DQuadbasis.h => basis/quad.h} | 2 +- examples/Hdiv-mixed/include/cl-options.h | 12 + examples/Hdiv-mixed/include/matops.h | 13 + examples/Hdiv-mixed/include/petsc-macros.h | 17 + examples/Hdiv-mixed/include/problems.h | 21 + examples/Hdiv-mixed/include/setup-dm.h | 14 + examples/Hdiv-mixed/include/setup-libceed.h | 24 ++ examples/Hdiv-mixed/include/structs.h | 74 ++++ examples/Hdiv-mixed/main.c | 207 ++++++++++ examples/Hdiv-mixed/main.h | 11 + examples/Hdiv-mixed/problems/poisson-quad2d.c | 79 ++++ .../Hdiv-mixed/qfunctions/poisson-quad2d.h | 152 ++++++++ examples/Hdiv-mixed/qfunctions/setup-geo.h | 104 +++++ examples/Hdiv-mixed/qfunctions/setup-geo2d.h | 92 +++++ examples/Hdiv-mixed/src/cl-options.c | 73 ++++ examples/Hdiv-mixed/src/matops.c | 98 +++++ examples/Hdiv-mixed/src/setup-dm.c | 44 +++ examples/Hdiv-mixed/src/setup-libceed.c | 368 ++++++++++++++++++ interface/ceed-basis.c | 54 +-- 21 files changed, 1463 insertions(+), 247 deletions(-) delete mode 100644 examples/Hdiv-mixed/MixedPoisson.c rename examples/Hdiv-mixed/{2DQuadbasis.h => basis/quad.h} (97%) create mode 100644 examples/Hdiv-mixed/include/cl-options.h create mode 100644 examples/Hdiv-mixed/include/matops.h create mode 100644 examples/Hdiv-mixed/include/petsc-macros.h create mode 100644 examples/Hdiv-mixed/include/problems.h create mode 100644 examples/Hdiv-mixed/include/setup-dm.h create mode 100644 examples/Hdiv-mixed/include/setup-libceed.h create mode 100644 examples/Hdiv-mixed/include/structs.h create mode 100644 examples/Hdiv-mixed/main.c create mode 100644 examples/Hdiv-mixed/main.h create mode 100644 examples/Hdiv-mixed/problems/poisson-quad2d.c create mode 100644 examples/Hdiv-mixed/qfunctions/poisson-quad2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/setup-geo.h create mode 100644 examples/Hdiv-mixed/qfunctions/setup-geo2d.h create mode 100644 examples/Hdiv-mixed/src/cl-options.c create mode 100644 examples/Hdiv-mixed/src/matops.c create mode 100644 examples/Hdiv-mixed/src/setup-dm.c create mode 100644 examples/Hdiv-mixed/src/setup-libceed.c diff --git a/examples/Hdiv-mixed/Makefile b/examples/Hdiv-mixed/Makefile index 12d74fcef6..5912d6f96e 100644 --- a/examples/Hdiv-mixed/Makefile +++ b/examples/Hdiv-mixed/Makefile @@ -1,30 +1,68 @@ +# Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. +# All Rights reserved. See files LICENSE and NOTICE for details. +# +# This file is part of CEED, a collection of benchmarks, miniapps, software +# libraries and APIs for efficient high-order finite element and spectral +# element discretizations for exascale applications. For more information and +# source code availability see http://github.com/ceed. +# +# The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +# a collaborative effort of two U.S. Department of Energy organizations (Office +# of Science and the National Nuclear Security Administration) responsible for +# the planning and preparation of a capable exascale ecosystem, including +# software, applications, hardware, advanced system engineering and early +# testbed platforms, in support of the nation's exascale computing imperative. COMMON ?= ../../common.mk -include $(COMMON) + + PETSc.pc := $(PETSC_DIR)/$(PETSC_ARCH)/lib/pkgconfig/PETSc.pc CEED_DIR ?= ../.. ceed.pc := $(CEED_DIR)/lib/pkgconfig/ceed.pc CC = $(call pkgconf, --variable=ccompiler $(PETSc.pc) $(ceed.pc)) -CFLAGS = -std=c99 $(call pkgconf, --variable=cflags_extra $(PETSc.pc)) $(call pkgconf, --cflags-only-other $(PETSc.pc)) $(OPT) -Wno-unused-variable -CPPFLAGS = $(call pkgconf, --cflags-only-I $(PETSc.pc) $(ceed.pc)) +CFLAGS = -std=c99 \ + $(call pkgconf, --variable=cflags_extra $(PETSc.pc)) \ + $(call pkgconf, --cflags-only-other $(PETSc.pc)) \ + $(OPT) +CPPFLAGS = $(call pkgconf, --cflags-only-I $(PETSc.pc) $(ceed.pc)) \ + $(call pkgconf, --variable=cflags_dep $(PETSc.pc)) LDFLAGS = $(call pkgconf, --libs-only-L --libs-only-other $(PETSc.pc) $(ceed.pc)) LDFLAGS += $(patsubst -L%, $(call pkgconf, --variable=ldflag_rpath $(PETSc.pc))%, $(call pkgconf, --libs-only-L $(PETSc.pc) $(ceed.pc))) LDLIBS = $(call pkgconf, --libs-only-l $(PETSc.pc) $(ceed.pc)) -lm -MixedPoisson.c := $(wildcard MixedPoisson*.c) -MixedPoisson := $(MixedPoisson.c:%.c=%) +OBJDIR := build +SRCDIR := src +PROBLEMDIR := problems + +all: main + +utils.c := $(sort $(wildcard $(PROBLEMDIR)/*.c)) $(sort $(wildcard $(SRCDIR)/*.c)) +utils.o = $(utils.c:%.c=$(OBJDIR)/%.o) +libutils.a: $(utils.o) + $(call quiet,AR) $(ARFLAGS) $@ $^ -all: $(MixedPoisson) -$(MixedPoisson): | $(PETSc.pc) $(ceed.pc) +main.c := main.c +main.o = $(main.c:%.c=$(OBJDIR)/%.o) +main: $(main.o) libutils.a | $(PETSc.pc) $(ceed.pc) + $(call quiet,LINK.o) $(CEED_LDFLAGS) $^ $(LOADLIBES) $(LDLIBS) -o $@ + +.SECONDEXPANSION: # to expand $$(@D)/.DIR +%/.DIR : + @mkdir -p $(@D) + @touch $@ # Quiet, color output quiet ?= $($(1)) -# Rules for building the example -%: %.c - $(call quiet,CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $(CEED_LDFLAGS) $(abspath $<) -o $@ $(LDLIBS) +$(OBJDIR)/%.o : %.c | $$(@D)/.DIR + $(call quiet,CC) $(CPPFLAGS) $(CFLAGS) -c -o $@ $(abspath $<) + +# Rules for building the examples +#%: %.c print: $(PETSc.pc) $(ceed.pc) $(info CC : $(CC)) @@ -32,16 +70,17 @@ print: $(PETSc.pc) $(ceed.pc) $(info CPPFLAGS: $(CPPFLAGS)) $(info LDFLAGS : $(LDFLAGS)) $(info LDLIBS : $(LDLIBS)) - $(info OPT : $(OPT)) @true clean: - $(RM) $(MixedPoisson) *.vtr *.vts *.vtu *.bin* + $(RM) -r $(OBJDIR) *.vtu main libutils.a $(PETSc.pc): $(if $(wildcard $@),,$(error \ - PETSc config not found. Please set PETSC_DIR and PETSC_ARCH)) + PETSc config not found at $@. Please set PETSC_DIR and PETSC_ARCH)) .PHONY: all print clean pkgconf = $(shell pkg-config $1 | sed -e 's/^"//g' -e 's/"$$//g') + +-include $(src.o:%.o=%.d) \ No newline at end of file diff --git a/examples/Hdiv-mixed/MixedPoisson.c b/examples/Hdiv-mixed/MixedPoisson.c deleted file mode 100644 index 8e8f004459..0000000000 --- a/examples/Hdiv-mixed/MixedPoisson.c +++ /dev/null @@ -1,188 +0,0 @@ -/// @file -/// Test creation, use, and destruction of an element restriction for 2D quad Hdiv - -// run with ./MixedPoisson /cpu/self/ref/serial -const char help[] = "Test creation, use, and destruction of an element restriction for 2D quad Hdiv\n"; - -#include -#include -#include "2DQuadbasis.h" - -int main(int argc, char **argv) { - PetscInt ierr; - MPI_Comm comm; - // PETSc objects - DM dm; - PetscSection sec; - PetscBool interpolate = PETSC_TRUE; - PetscInt nx = 2, ny = 1, num_elem = nx * ny; - PetscInt num_nodes = (nx+1)*(ny+1); - PetscInt faces[2] = {nx, ny}; - PetscInt dim = 2; - PetscInt pStart, pEnd; - PetscInt cStart, cEnd, c; // cells - PetscInt eStart, eEnd, e; // edges - PetscInt vStart, vEnd, v; // vertices - PetscInt dofs_per_face; - const PetscInt *ornt; - // libCEED objects - Ceed ceed; - const CeedInt loc_node = 4, Q1d = 2, Q = Q1d*Q1d; - CeedInt num_comp = dim; - CeedInt dof_e = dim*num_nodes; // dof per element! dof is vector in Hdiv - CeedBasis b; - CeedScalar q_ref[dim*Q], q_weights[Q]; - CeedScalar div[dof_e*Q], interp[dim*dof_e*Q]; - - ierr = PetscInitialize(&argc, &argv, NULL, help); - if (ierr) return ierr; - - comm = PETSC_COMM_WORLD; - - CeedInit(argv[1], &ceed); - - // --------------------------------------------------------------------------- - // Build 2D Hdiv basis - // --------------------------------------------------------------------------- - buildmats(Q1d, q_ref, q_weights, interp, div); - ierr = CeedBasisCreateHdiv(ceed, CEED_QUAD, num_comp, loc_node, Q, - interp, div, q_ref, q_weights, &b); CHKERRQ(ierr); - //CeedBasisHdivView(b, stdout); - - // --------------------------------------------------------------------------- - // Set-up DM - // --------------------------------------------------------------------------- - ierr = DMPlexCreateBoxMesh(comm, dim, PETSC_FALSE, faces, NULL, - NULL, NULL, interpolate, &dm); CHKERRQ(ierr); - // Get plex limits - ierr = DMPlexGetChart(dm, &pStart, &pEnd); CHKERRQ(ierr); - ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd); CHKERRQ(ierr); - ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd); CHKERRQ(ierr); - ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd); CHKERRQ(ierr); - - // Create section - ierr = PetscSectionCreate(comm, &sec); CHKERRQ(ierr); - ierr = PetscSectionSetNumFields(sec,1); CHKERRQ(ierr); - ierr = PetscSectionSetFieldName(sec,0,"Velocity"); CHKERRQ(ierr); - ierr = PetscSectionSetFieldComponents(sec,0,1); CHKERRQ(ierr); - ierr = PetscSectionSetChart(sec,pStart,pEnd); CHKERRQ(ierr); - - // Setup dofs_per_face - for (e = eStart; e < eEnd; e++) { - ierr = DMPlexGetConeSize(dm, e, &dofs_per_face); CHKERRQ(ierr); - ierr = PetscSectionSetFieldDof(sec, e, 0, dofs_per_face); CHKERRQ(ierr); - ierr = PetscSectionSetDof (sec, e, dofs_per_face); CHKERRQ(ierr); - } - printf("=============cell========\n"); - CeedInt cone_size; - for (c = cStart; c< cEnd; c++) { - ierr = DMPlexGetConeSize(dm, c, &cone_size); CHKERRQ(ierr); - printf("cell number %d\n", c); - ierr = DMPlexGetConeOrientation(dm, c, &ornt); CHKERRQ(ierr); - for (CeedInt j = 0; j < cone_size; j++){ - printf("%d\n", ornt[j]); - } - } - - //================================== To Check Restrictions =================== - CeedInt P = 2; - CeedInt ind_x[num_elem*P*P]; - CeedScalar x[dim*num_nodes]; - CeedVector X, Y; - CeedElemRestriction r; - - for (CeedInt i=0; i +#include + +#include "structs.h" + +PetscErrorCode MatGetDiag(Mat A, Vec D); +PetscErrorCode ApplyLocal_Ceed(Vec X, Vec Y, User user); +PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y); + +#endif // matops_h diff --git a/examples/Hdiv-mixed/include/petsc-macros.h b/examples/Hdiv-mixed/include/petsc-macros.h new file mode 100644 index 0000000000..dbdf17d2e9 --- /dev/null +++ b/examples/Hdiv-mixed/include/petsc-macros.h @@ -0,0 +1,17 @@ +#ifndef petsc_macros +#define petsc_macros + +#if PETSC_VERSION_LT(3,14,0) +# define DMPlexGetClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexGetClosureIndices(a,b,c,d,f,g,i) +# define DMPlexRestoreClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexRestoreClosureIndices(a,b,c,d,f,g,i) +#endif + +#if PETSC_VERSION_LT(3,14,0) +# define DMAddBoundary(a,b,c,d,e,f,g,h,i,j,k,l,m,n) DMAddBoundary(a,b,c,e,h,i,j,k,f,g,m) +#elif PETSC_VERSION_LT(3,16,0) +# define DMAddBoundary(a,b,c,d,e,f,g,h,i,j,k,l,m,n) DMAddBoundary(a,b,c,e,h,i,j,k,l,f,g,m) +#else +# define DMAddBoundary(a,b,c,d,e,f,g,h,i,j,k,l,m,n) DMAddBoundary(a,b,c,d,f,g,h,i,j,k,l,m,n) +#endif + +#endif diff --git a/examples/Hdiv-mixed/include/problems.h b/examples/Hdiv-mixed/include/problems.h new file mode 100644 index 0000000000..dd7d65b039 --- /dev/null +++ b/examples/Hdiv-mixed/include/problems.h @@ -0,0 +1,21 @@ +#ifndef problems_h +#define problems_h + +#include "../include/structs.h" + +// ----------------------------------------------------------------------------- +// Set up problems function prototype +// ----------------------------------------------------------------------------- +// 1) poisson-quad2d +PetscErrorCode Hdiv_POISSON_QUAD2D(ProblemData *problem_data, void *ctx); + +PetscErrorCode SetupContext_POISSON_QUAD2D(Ceed ceed, CeedData ceed_data, + Physics phys); + +// 2) poisson-hex3d + +// 3) poisson-prism3d + +// 4) richard + +#endif // problems_h diff --git a/examples/Hdiv-mixed/include/setup-dm.h b/examples/Hdiv-mixed/include/setup-dm.h new file mode 100644 index 0000000000..ebdebdaf7c --- /dev/null +++ b/examples/Hdiv-mixed/include/setup-dm.h @@ -0,0 +1,14 @@ +#ifndef setupdm_h +#define setupdm_h + +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Set-up DM +// --------------------------------------------------------------------------- +PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm); + +#endif // setupdm_h diff --git a/examples/Hdiv-mixed/include/setup-libceed.h b/examples/Hdiv-mixed/include/setup-libceed.h new file mode 100644 index 0000000000..5b95f928c1 --- /dev/null +++ b/examples/Hdiv-mixed/include/setup-libceed.h @@ -0,0 +1,24 @@ +#ifndef setuplibceed_h +#define setuplibceed_h + +#include "../include/structs.h" + +// Convert PETSc MemType to libCEED MemType +CeedMemType MemTypeP2C(PetscMemType mtype); +// Destroy libCEED objects +PetscErrorCode CeedDataDestroy(CeedData ceed_data); +// Utility function - essential BC dofs are encoded in closure indices as -(i+1) +PetscInt Involute(PetscInt i); +// Utility function to create local CEED restriction from DMPlex +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt P, + CeedInt topo_dim, CeedElemRestriction *elem_restr); +// Utility function to create local CEED Oriented restriction from DMPlex +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, + CeedInt topo_dim, CeedElemRestriction *elem_restr_oriented); +// Set up libCEED for a given degree +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, + ProblemData *problem_data, + PetscInt U_g_size, PetscInt U_loc_size, + CeedData ceed_data, CeedVector rhs_ceed, + CeedVector *target); +#endif // setuplibceed_h diff --git a/examples/Hdiv-mixed/include/structs.h b/examples/Hdiv-mixed/include/structs.h new file mode 100644 index 0000000000..6a094a41bb --- /dev/null +++ b/examples/Hdiv-mixed/include/structs.h @@ -0,0 +1,74 @@ +#ifndef structs_h +#define structs_h + +#include +#include + +// Application context from user command line options +typedef struct AppCtx_ *AppCtx; +struct AppCtx_ { + // libCEED arguments + PetscInt degree; + PetscInt q_extra; + // Problem type arguments + PetscFunctionList problems; + char problem_name[PETSC_MAX_PATH_LEN]; +}; + +// libCEED data struct +typedef struct CeedData_ *CeedData; +struct CeedData_ { + CeedBasis basis_x, basis_u; + CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_geo_data_i, + elem_restr_u_i; + CeedQFunction qf_residual; + CeedOperator op_residual; + CeedVector geo_data, x_ceed, y_ceed; + CeedQFunctionContext pq2d_context; +}; + +// 1) poisson-quad2d +#ifndef PHYSICS_POISSONQUAD2D_STRUCT +#define PHYSICS_POISSONQUAD2D_STRUCT +typedef struct PQ2DContext_ *PQ2DContext; +struct PQ2DContext_ { + CeedScalar kappa; +}; +#endif + +// 2) poisson-hex3d + +// 3) poisson-prism3d + +// 4) richard + +// Struct that contains all enums and structs used for the physics of all problems +typedef struct Physics_ *Physics; +struct Physics_ { + PQ2DContext pq2d_ctx; +}; + +// PETSc user data +typedef struct User_ *User; +struct User_ { + MPI_Comm comm; + Vec X_loc, Y_loc; + CeedVector x_ceed, y_ceed; + CeedOperator op; + DM dm; + Ceed ceed; + AppCtx app_ctx; + Physics phys; +}; + +// Problem specific data +typedef struct { + CeedQFunctionUser setup_geo, residual, setup_rhs; + const char *setup_geo_loc, *residual_loc, *setup_rhs_loc; + CeedQuadMode quadrature_mode; + CeedInt geo_data_size, elem_node; + PetscErrorCode (*setup_ctx)(Ceed, CeedData, Physics); + +} ProblemData; + +#endif // structs_h \ No newline at end of file diff --git a/examples/Hdiv-mixed/main.c b/examples/Hdiv-mixed/main.c new file mode 100644 index 0000000000..60942485ba --- /dev/null +++ b/examples/Hdiv-mixed/main.c @@ -0,0 +1,207 @@ +/// @file +/// Test creation, use, and destruction of an element restriction for 2D quad Hdiv + +// run with ./main +const char help[] = "Solve H(div)-mixed problem using PETSc and libCEED\n"; + +#include "main.h" + +int main(int argc, char **argv) { + // --------------------------------------------------------------------------- + // Initialize PETSc + // --------------------------------------------------------------------------- + PetscInt ierr; + ierr = PetscInitialize(&argc, &argv, NULL, help); + if (ierr) return ierr; + + // --------------------------------------------------------------------------- + // Create structs + // --------------------------------------------------------------------------- + AppCtx app_ctx; + ierr = PetscCalloc1(1, &app_ctx); CHKERRQ(ierr); + + ProblemData *problem_data = NULL; + ierr = PetscCalloc1(1, &problem_data); CHKERRQ(ierr); + + User user; + ierr = PetscCalloc1(1, &user); CHKERRQ(ierr); + + CeedData ceed_data; + ierr = PetscCalloc1(1, &ceed_data); CHKERRQ(ierr); + + Physics phys_ctx; + ierr = PetscCalloc1(1, &phys_ctx); CHKERRQ(ierr); + + user->app_ctx = app_ctx; + user->phys = phys_ctx; + + // --------------------------------------------------------------------------- + // Process command line options + // --------------------------------------------------------------------------- + // -- Register problems to be available on the command line + ierr = RegisterProblems_Hdiv(app_ctx); CHKERRQ(ierr); + + // -- Process general command line options + MPI_Comm comm = PETSC_COMM_WORLD; + ierr = ProcessCommandLineOptions(comm, app_ctx); CHKERRQ(ierr); + + // --------------------------------------------------------------------------- + // Choose the problem from the list of registered problems + // --------------------------------------------------------------------------- + { + PetscErrorCode (*p)(ProblemData *, void *); + ierr = PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p); + CHKERRQ(ierr); + if (!p) SETERRQ1(PETSC_COMM_SELF, 1, "Problem '%s' not found", + app_ctx->problem_name); + ierr = (*p)(problem_data, &user); CHKERRQ(ierr); + } + + // --------------------------------------------------------------------------- + // Initialize libCEED + // --------------------------------------------------------------------------- + // -- Initialize backend + Ceed ceed; + CeedInit("/cpu/self/ref/serial", &ceed); + CeedMemType mem_type_backend; + CeedGetPreferredMemType(ceed, &mem_type_backend); + // --------------------------------------------------------------------------- + // Set-up DM + // --------------------------------------------------------------------------- + // PETSc objects + DM dm; + VecType vec_type; + ierr = CreateDistributedDM(comm, &dm); CHKERRQ(ierr); + ierr = DMGetVecType(dm, &vec_type); CHKERRQ(ierr); + + // --------------------------------------------------------------------------- + // Create global, local solution, local rhs vector + // --------------------------------------------------------------------------- + Vec U_g, U_loc; + PetscInt U_l_size, U_g_size, U_loc_size; + // Create global and local solution vectors + ierr = DMCreateGlobalVector(dm, &U_g); CHKERRQ(ierr); + ierr = VecGetSize(U_g, &U_g_size); CHKERRQ(ierr); + // Local size for matShell + ierr = VecGetLocalSize(U_g, &U_l_size); CHKERRQ(ierr); + // Create local unknown vector U_loc + ierr = DMCreateLocalVector(dm, &U_loc); CHKERRQ(ierr); + // Local size for libCEED + ierr = VecGetSize(U_loc, &U_loc_size); CHKERRQ(ierr); + + // Operator + Mat mat; + ierr = MatCreateShell(comm, U_l_size, U_l_size, U_g_size, U_g_size, + user, &mat); CHKERRQ(ierr); + ierr = MatShellSetOperation(mat, MATOP_MULT, + (void(*)(void))MatMult_Ceed); CHKERRQ(ierr); + ierr = MatShellSetOperation(mat, MATOP_GET_DIAGONAL, + (void(*)(void))MatGetDiag); CHKERRQ(ierr); + ierr = MatShellSetVecType(mat, vec_type); CHKERRQ(ierr); + // Get RHS vector + Vec rhs_loc; + PetscScalar *r; + CeedVector rhs_ceed, target; + PetscMemType mem_type; + ierr = VecDuplicate(U_loc, &rhs_loc); CHKERRQ(ierr); + ierr = VecZeroEntries(rhs_loc); CHKERRQ(ierr); + ierr = VecGetArrayAndMemType(rhs_loc, &r, &mem_type); CHKERRQ(ierr); + CeedVectorCreate(ceed, U_l_size, &rhs_ceed); + CeedVectorSetArray(rhs_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, r); + + // --------------------------------------------------------------------------- + // Setup libCEED + // --------------------------------------------------------------------------- + // -- Set up libCEED objects + ierr = SetupLibceed(dm, ceed, app_ctx, problem_data, U_g_size, + U_loc_size, ceed_data, rhs_ceed, &target); + CHKERRQ(ierr); + // -- Set up context for QFunctions + ierr = problem_data->setup_ctx(ceed, ceed_data, phys_ctx); + CHKERRQ(ierr); + + // --------------------------------------------------------------------------- + // Gather RHS + // --------------------------------------------------------------------------- + Vec rhs; + ierr = VecDuplicate(U_g, &rhs); CHKERRQ(ierr); + CeedVectorTakeArray(rhs_ceed, MemTypeP2C(mem_type), NULL); + ierr = VecRestoreArrayAndMemType(rhs_loc, &r); CHKERRQ(ierr); + ierr = VecZeroEntries(rhs); CHKERRQ(ierr); + ierr = DMLocalToGlobal(dm, rhs_loc, ADD_VALUES, rhs); CHKERRQ(ierr); + // --------------------------------------------------------------------------- + // Setup Mat, KSP + // --------------------------------------------------------------------------- + user->comm = comm; + user->dm = dm; + user->X_loc = U_loc; + ierr = VecDuplicate(U_loc, &user->Y_loc); CHKERRQ(ierr); + user->x_ceed = ceed_data->x_ceed; + user->y_ceed = ceed_data->y_ceed; + user->op = ceed_data->op_residual; + user->ceed = ceed; + // Ksp + KSP ksp; + PC pc; + ierr = KSPCreate(comm, &ksp); CHKERRQ(ierr); + ierr = KSPGetPC(ksp, &pc); CHKERRQ(ierr); + ierr = PCSetType(pc, PCJACOBI); CHKERRQ(ierr); + ierr = PCJacobiSetType(pc, PC_JACOBI_ROWSUM); CHKERRQ(ierr); + ierr = KSPSetType(ksp, KSPCG); CHKERRQ(ierr); + ierr = KSPSetNormType(ksp, KSP_NORM_NATURAL); CHKERRQ(ierr); + ierr = KSPSetTolerances(ksp, 1e-10, PETSC_DEFAULT, PETSC_DEFAULT, + PETSC_DEFAULT); CHKERRQ(ierr); + ierr = KSPSetOperators(ksp, mat, mat); CHKERRQ(ierr); + ierr = KSPSolve(ksp, rhs, U_g); CHKERRQ(ierr); + ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr); + + // Output results + KSPType ksp_type; + KSPConvergedReason reason; + PetscReal rnorm; + PetscInt its; + ierr = KSPGetType(ksp, &ksp_type); CHKERRQ(ierr); + ierr = KSPGetConvergedReason(ksp, &reason); CHKERRQ(ierr); + ierr = KSPGetIterationNumber(ksp, &its); CHKERRQ(ierr); + ierr = KSPGetResidualNorm(ksp, &rnorm); CHKERRQ(ierr); + ierr = PetscPrintf(comm, + " KSP:\n" + " KSP Type : %s\n" + " KSP Convergence : %s\n" + " Total KSP Iterations : %D\n" + " Final rnorm : %e\n", + ksp_type, KSPConvergedReasons[reason], its, + (double)rnorm); CHKERRQ(ierr); + + // --------------------------------------------------------------------------- + // Free objects + // --------------------------------------------------------------------------- + + // Free PETSc objects + ierr = DMDestroy(&dm); CHKERRQ(ierr); + ierr = VecDestroy(&U_g); CHKERRQ(ierr); + ierr = VecDestroy(&U_loc); CHKERRQ(ierr); + ierr = VecDestroy(&user->Y_loc); CHKERRQ(ierr); + ierr = VecDestroy(&rhs); CHKERRQ(ierr); + ierr = VecDestroy(&rhs_loc); CHKERRQ(ierr); + ierr = MatDestroy(&mat); CHKERRQ(ierr); + ierr = KSPDestroy(&ksp); CHKERRQ(ierr); + + // -- Function list + ierr = PetscFunctionListDestroy(&app_ctx->problems); CHKERRQ(ierr); + + // -- Structs + ierr = PetscFree(app_ctx); CHKERRQ(ierr); + ierr = PetscFree(problem_data); CHKERRQ(ierr); + ierr = PetscFree(user); CHKERRQ(ierr); + ierr = PetscFree(phys_ctx->pq2d_ctx); CHKERRQ(ierr); + ierr = PetscFree(phys_ctx); CHKERRQ(ierr); + // Free libCEED objects + + CeedVectorDestroy(&rhs_ceed); + CeedVectorDestroy(&target); + ierr = CeedDataDestroy(ceed_data); CHKERRQ(ierr); + CeedDestroy(&ceed); + + return PetscFinalize(); +} diff --git a/examples/Hdiv-mixed/main.h b/examples/Hdiv-mixed/main.h new file mode 100644 index 0000000000..da283f2d04 --- /dev/null +++ b/examples/Hdiv-mixed/main.h @@ -0,0 +1,11 @@ + +#ifndef MAIN_H +#define MAIN_H + +#include "include/setup-libceed.h" +#include "include/setup-dm.h" +#include "include/cl-options.h" +#include "include/problems.h" +#include "include/matops.h" + +#endif // MAIN_H \ No newline at end of file diff --git a/examples/Hdiv-mixed/problems/poisson-quad2d.c b/examples/Hdiv-mixed/problems/poisson-quad2d.c new file mode 100644 index 0000000000..9dbbce5057 --- /dev/null +++ b/examples/Hdiv-mixed/problems/poisson-quad2d.c @@ -0,0 +1,79 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Utility functions for setting up POISSON_QUAD2D + +#include "../include/setup-libceed.h" +#include "../include/problems.h" +#include "../qfunctions/setup-geo2d.h" +#include "../qfunctions/poisson-quad2d.h" +// Hdiv_POISSON_QUAD2D is registered in cl-option.c +PetscErrorCode Hdiv_POISSON_QUAD2D(ProblemData *problem_data, void *ctx) { + User user = *(User *)ctx; + MPI_Comm comm = PETSC_COMM_WORLD; + PetscInt ierr; + PetscFunctionBeginUser; + + ierr = PetscCalloc1(1, &user->phys->pq2d_ctx); CHKERRQ(ierr); + + // ------------------------------------------------------ + // SET UP POISSON_QUAD2D + // ------------------------------------------------------ + problem_data->elem_node = 4; + problem_data->geo_data_size = 5; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->setup_geo = SetupGeo2D; + problem_data->setup_geo_loc = SetupGeo2D_loc; + problem_data->residual = PoissonQuadF; + problem_data->residual_loc = PoissonQuadF_loc; + problem_data->setup_rhs = SetupRhs; + problem_data->setup_rhs_loc = SetupRhs_loc; + problem_data->setup_ctx = SetupContext_POISSON_QUAD2D; + + // ------------------------------------------------------ + // Create the libCEED context + // ------------------------------------------------------ + CeedScalar kappa = 1.; + + // ------------------------------------------------------ + // Command line Options + // ------------------------------------------------------ + ierr = PetscOptionsBegin(comm, NULL, "Options for DENSITY_CURRENT problem", + NULL); CHKERRQ(ierr); + // -- Physics + ierr = PetscOptionsScalar("-kappa", "hydraulic conductivity", + NULL, kappa, &kappa, NULL); CHKERRQ(ierr); + + ierr = PetscOptionsEnd(); CHKERRQ(ierr); + + // -- QFunction Context + user->phys->pq2d_ctx->kappa = kappa; + + PetscFunctionReturn(0); +} + +PetscErrorCode SetupContext_POISSON_QUAD2D(Ceed ceed, CeedData ceed_data, + Physics phys) { + PetscFunctionBeginUser; + + CeedQFunctionContextCreate(ceed, &ceed_data->pq2d_context); + CeedQFunctionContextSetData(ceed_data->pq2d_context, CEED_MEM_HOST, + CEED_USE_POINTER, sizeof(*phys->pq2d_ctx), phys->pq2d_ctx); + CeedQFunctionSetContext(ceed_data->qf_residual, ceed_data->pq2d_context); + + PetscFunctionReturn(0); +} \ No newline at end of file diff --git a/examples/Hdiv-mixed/qfunctions/poisson-quad2d.h b/examples/Hdiv-mixed/qfunctions/poisson-quad2d.h new file mode 100644 index 0000000000..41705d6e82 --- /dev/null +++ b/examples/Hdiv-mixed/qfunctions/poisson-quad2d.h @@ -0,0 +1,152 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Mixed poisson 2D quad element using PETSc + +#ifndef POISSON_QUAD2D_H +#define POISSON_QUAD2D_H + +#include + +#ifndef PHYSICS_POISSONQUAD2D_STRUCT +#define PHYSICS_POISSONQUAD2D_STRUCT +typedef struct PQ2DContext_ *PQ2DContext; +struct PQ2DContext_ { + CeedScalar kappa; +}; +#endif + +// ----------------------------------------------------------------------------- +// This QFunction applies the mass operator for a vector field of 2 components. +// +// Inputs: +// w - weight of quadrature +// J - dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// u - Input basis at quadrature points +// +// Output: +// v - Output vector (test functions) at quadrature points +// Note we need to apply Piola map on the basis, which is J*u/detJ +// So (v,u) = \int (v^T * u detJ*w) ==> \int (v^T J^T*J*u*w/detJ) +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(PoissonQuadF)(void *ctx, CeedInt Q, const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*J)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + // *INDENT-ON* + + // Context + const PQ2DContext context = (PQ2DContext)ctx; + const CeedScalar kappa = context->kappa; + // for simplicity we considered kappa as scalar (it should be tensor) + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i \int (v^T J^T* f * w), f=true_soln +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(SetupRhs)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*x)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])in[0], + (*w) = in[1], + (*J)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[2]; + // Outputs + CeedScalar (*true_soln)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], + (*rhs)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[1]; + + // *INDENT-ON* + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; iproblems = NULL; + PetscErrorCode ierr; + PetscFunctionBeginUser; + // 1) poisson-quad2d (Hdiv_POISSON_QUAD2D is created in poisson-quad2d.c) + ierr = PetscFunctionListAdd(&app_ctx->problems, "poisson_quad2d", + Hdiv_POISSON_QUAD2D); CHKERRQ(ierr); + // 2) poisson-hex3d + + // 3) poisson-prism3d + + // 4) richard + + PetscFunctionReturn(0); +} + +// Process general command line options +PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx) { + + PetscBool problem_flag = PETSC_FALSE; + PetscErrorCode ierr; + PetscFunctionBeginUser; + + ierr = PetscOptionsBegin(comm, NULL, + "H(div) examples in PETSc with libCEED", + NULL); CHKERRQ(ierr); + + ierr = PetscOptionsFList("-problem", "Problem to solve", NULL, + app_ctx->problems, + app_ctx->problem_name, app_ctx->problem_name, sizeof(app_ctx->problem_name), + &problem_flag); CHKERRQ(ierr); + + app_ctx->degree = 1; + ierr = PetscOptionsInt("-degree", "Polynomial degree of finite elements", + NULL, app_ctx->degree, &app_ctx->degree, NULL); CHKERRQ(ierr); + + app_ctx->q_extra = 0; + ierr = PetscOptionsInt("-q_extra", "Number of extra quadrature points", + NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL); CHKERRQ(ierr); + + // Provide default problem if not specified + if (!problem_flag) { + const char *problem_name = "poisson_quad2d"; + strncpy(app_ctx->problem_name, problem_name, 16); + } + + ierr = PetscOptionsEnd(); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mixed/src/matops.c b/examples/Hdiv-mixed/src/matops.c new file mode 100644 index 0000000000..ce3451341c --- /dev/null +++ b/examples/Hdiv-mixed/src/matops.c @@ -0,0 +1,98 @@ +#include "../include/matops.h" +#include "../include/setup-libceed.h" + +// ----------------------------------------------------------------------------- +// This function returns the computed diagonal of the operator +// ----------------------------------------------------------------------------- +PetscErrorCode MatGetDiag(Mat A, Vec D) { + PetscErrorCode ierr; + User user; + + PetscFunctionBeginUser; + + ierr = MatShellGetContext(A, &user); CHKERRQ(ierr); + + // Compute Diagonal via libCEED + PetscScalar *x; + PetscMemType mem_type; + + // -- Place PETSc vector in libCEED vector + ierr = VecGetArrayAndMemType(user->X_loc, &x, &mem_type); CHKERRQ(ierr); + CeedVectorSetArray(user->x_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, x); + + // -- Compute Diagonal + CeedOperatorLinearAssembleDiagonal(user->op, user->x_ceed, + CEED_REQUEST_IMMEDIATE); + + // -- Local-to-Global + CeedVectorTakeArray(user->x_ceed, MemTypeP2C(mem_type), NULL); + ierr = VecRestoreArrayAndMemType(user->X_loc, &x); CHKERRQ(ierr); + ierr = VecZeroEntries(D); CHKERRQ(ierr); + ierr = DMLocalToGlobal(user->dm, user->X_loc, ADD_VALUES, D); CHKERRQ(ierr); + + // Cleanup + ierr = VecZeroEntries(user->X_loc); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// This function uses libCEED to compute the action of the Laplacian with +// Dirichlet boundary conditions +// ----------------------------------------------------------------------------- +PetscErrorCode ApplyLocal_Ceed(Vec X, Vec Y, User user) { + PetscErrorCode ierr; + PetscScalar *x, *y; + PetscMemType x_mem_type, y_mem_type; + + PetscFunctionBeginUser; + + // Global-to-local + ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); CHKERRQ(ierr); + + // Setup libCEED vectors + ierr = VecGetArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x, + &x_mem_type); CHKERRQ(ierr); + ierr = VecGetArrayAndMemType(user->Y_loc, &y, &y_mem_type); CHKERRQ(ierr); + CeedVectorSetArray(user->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, x); + CeedVectorSetArray(user->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); + + // Apply libCEED operator + printf("==== x"); + CeedVectorView(user->x_ceed,"%12.8f", stdout); + CeedOperatorApply(user->op, user->x_ceed, user->y_ceed, CEED_REQUEST_IMMEDIATE); + printf("==== y"); + CeedVectorView(user->y_ceed,"%12.8f", stdout); + + // Restore PETSc vectors + CeedVectorTakeArray(user->x_ceed, MemTypeP2C(x_mem_type), NULL); + CeedVectorTakeArray(user->y_ceed, MemTypeP2C(y_mem_type), NULL); + ierr = VecRestoreArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x); + CHKERRQ(ierr); + ierr = VecRestoreArrayAndMemType(user->Y_loc, &y); CHKERRQ(ierr); + + // Local-to-global + ierr = VecZeroEntries(Y); CHKERRQ(ierr); + ierr = DMLocalToGlobal(user->dm, user->Y_loc, ADD_VALUES, Y); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// This function wraps the libCEED operator for a MatShell +// ----------------------------------------------------------------------------- +PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y) { + PetscErrorCode ierr; + User user; + + PetscFunctionBeginUser; + + ierr = MatShellGetContext(A, &user); CHKERRQ(ierr); + + // libCEED for local action of residual evaluator + ierr = ApplyLocal_Ceed(X, Y, user); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/src/setup-dm.c b/examples/Hdiv-mixed/src/setup-dm.c new file mode 100644 index 0000000000..d15e431750 --- /dev/null +++ b/examples/Hdiv-mixed/src/setup-dm.c @@ -0,0 +1,44 @@ +#include "../include/setup-dm.h" + +// --------------------------------------------------------------------------- +// Set-up DM +// --------------------------------------------------------------------------- +PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm) { + PetscErrorCode ierr; + PetscSection sec; + PetscBool interpolate = PETSC_TRUE; + PetscInt nx = 1, ny = 2; + PetscInt faces[2] = {nx, ny}; + PetscInt dim = 2, num_comp_u = dim; + PetscInt p_start, p_end; + PetscInt c_start, c_end; // cells + PetscInt e_start, e_end; // edges + PetscInt v_start, v_end, v; // vertices + + PetscFunctionBeginUser; + + ierr = DMPlexCreateBoxMesh(comm, dim, PETSC_FALSE, faces, NULL, + NULL, NULL, interpolate, dm); CHKERRQ(ierr); + // Get plex limits + ierr = DMPlexGetChart(*dm, &p_start, &p_end); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(*dm, 0, &c_start, &c_end); CHKERRQ(ierr); + ierr = DMPlexGetDepthStratum(*dm, 1, &e_start, &e_end); CHKERRQ(ierr); + ierr = DMPlexGetDepthStratum(*dm, 0, &v_start, &v_end); CHKERRQ(ierr); + // Create section + ierr = PetscSectionCreate(comm, &sec); CHKERRQ(ierr); + ierr = PetscSectionSetNumFields(sec,1); CHKERRQ(ierr); + ierr = PetscSectionSetFieldName(sec,0,"Velocity"); CHKERRQ(ierr); + ierr = PetscSectionSetFieldComponents(sec,0,num_comp_u); CHKERRQ(ierr); + ierr = PetscSectionSetChart(sec,p_start,p_end); CHKERRQ(ierr); + // Setup dofs + for (v = v_start; v < v_end; v++) { + ierr = PetscSectionSetFieldDof(sec, v, 0, num_comp_u); CHKERRQ(ierr); + ierr = PetscSectionSetDof (sec, v, num_comp_u); CHKERRQ(ierr); + } + ierr = PetscSectionSetUp(sec); CHKERRQ(ierr); + ierr = DMSetSection(*dm,sec); CHKERRQ(ierr); + ierr = DMViewFromOptions(*dm, NULL, "-dm_view"); CHKERRQ(ierr); + ierr = PetscSectionDestroy(&sec); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; \ No newline at end of file diff --git a/examples/Hdiv-mixed/src/setup-libceed.c b/examples/Hdiv-mixed/src/setup-libceed.c new file mode 100644 index 0000000000..aa08502bbf --- /dev/null +++ b/examples/Hdiv-mixed/src/setup-libceed.c @@ -0,0 +1,368 @@ +#include "../include/setup-libceed.h" +#include "../include/petsc-macros.h" +#include "../basis/quad.h" + +// ----------------------------------------------------------------------------- +// Convert PETSc MemType to libCEED MemType +// ----------------------------------------------------------------------------- +CeedMemType MemTypeP2C(PetscMemType mem_type) { + return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; +} +// Destroy libCEED objects +PetscErrorCode CeedDataDestroy(CeedData ceed_data) { + PetscErrorCode ierr; + + PetscFunctionBegin; + + // Vectors + CeedVectorDestroy(&ceed_data->x_ceed); + CeedVectorDestroy(&ceed_data->y_ceed); + CeedVectorDestroy(&ceed_data->geo_data); + // Restrictions + CeedElemRestrictionDestroy(&ceed_data->elem_restr_x); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_u); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_geo_data_i); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_u_i); + // Bases + CeedBasisDestroy(&ceed_data->basis_x); + CeedBasisDestroy(&ceed_data->basis_u); + // QFunctions + CeedQFunctionDestroy(&ceed_data->qf_residual); + // Operators + CeedOperatorDestroy(&ceed_data->op_residual); + + ierr = PetscFree(ceed_data); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// Utility function - essential BC dofs are encoded in closure indices as -(i+1) +// ----------------------------------------------------------------------------- +PetscInt Involute(PetscInt i) { + return i >= 0 ? i : -(i + 1); +}; +// ----------------------------------------------------------------------------- +// Get CEED restriction data from DMPlex +// ----------------------------------------------------------------------------- +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt P, + CeedInt topo_dim, CeedElemRestriction *elem_restr) { + PetscSection section; + PetscInt p, num_elem, num_dof, *elem_restr_offsets, e_offset, num_fields, + c_start, c_end; + Vec U_loc; + PetscErrorCode ierr; + + PetscFunctionBeginUser; + ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); + ierr = PetscSectionGetNumFields(section, &num_fields); CHKERRQ(ierr); + PetscInt num_comp[num_fields], field_off[num_fields+1]; + field_off[0] = 0; + for (PetscInt f = 0; f < num_fields; f++) { + ierr = PetscSectionGetFieldComponents(section, f, &num_comp[f]); CHKERRQ(ierr); + field_off[f+1] = field_off[f] + num_comp[f]; + } + ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); + num_elem = c_end - c_start; + ierr = PetscMalloc1(num_elem*PetscPowInt(P, topo_dim), &elem_restr_offsets); + CHKERRQ(ierr); + CHKERRQ(ierr); + for (p = 0, e_offset = 0; p < num_elem; p++) { + PetscInt num_indices, *indices, num_nodes; + ierr = DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, + &num_indices, &indices, NULL, NULL); + CHKERRQ(ierr); + num_nodes = num_indices / field_off[num_fields]; + for (PetscInt i = 0; i < num_nodes; i++) { + PetscInt ii = i; + // Check that indices are blocked by node and thus can be coalesced as a single field with + // field_off[num_fields] = sum(num_comp) components. + for (PetscInt f = 0; f < num_fields; f++) { + for (PetscInt j = 0; j < num_comp[f]; j++) { + if (Involute(indices[field_off[f]*num_nodes + ii*num_comp[f] + j]) + != Involute(indices[ii*num_comp[0]]) + field_off[f] + j) + SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, + "Cell %D closure indices not interlaced for node %D field %D component %D", + p, ii, f, j); + } + } + // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. + PetscInt loc = Involute(indices[ii*num_comp[0]]); + elem_restr_offsets[e_offset++] = loc; + } + ierr = DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, + &num_indices, &indices, NULL, NULL); + CHKERRQ(ierr); + } + if (e_offset != num_elem*PetscPowInt(P, topo_dim)) + SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, + "ElemRestriction of size (%D,%D) initialized %D nodes", num_elem, + PetscPowInt(P, topo_dim),e_offset); + + ierr = DMGetLocalVector(dm, &U_loc); CHKERRQ(ierr); + ierr = VecGetLocalSize(U_loc, &num_dof); CHKERRQ(ierr); + ierr = DMRestoreLocalVector(dm, &U_loc); CHKERRQ(ierr); + CeedElemRestrictionCreate(ceed, num_elem, PetscPowInt(P, topo_dim), + field_off[num_fields], 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, + elem_restr_offsets, elem_restr); + ierr = PetscFree(elem_restr_offsets); CHKERRQ(ierr); + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// Get Oriented CEED restriction data from DMPlex +// ----------------------------------------------------------------------------- +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, + CeedInt topo_dim, CeedElemRestriction *elem_restr_oriented) { + PetscSection section; + PetscInt p, num_elem, num_dof, *elem_restr_offsets, e_offset, num_fields, + c_start, c_end; + Vec U_loc; + PetscErrorCode ierr; + const PetscInt *ornt; + + PetscFunctionBeginUser; + ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); + ierr = PetscSectionGetNumFields(section, &num_fields); CHKERRQ(ierr); + PetscInt num_comp[num_fields], field_off[num_fields+1]; + field_off[0] = 0; + for (PetscInt f = 0; f < num_fields; f++) { + ierr = PetscSectionGetFieldComponents(section, f, &num_comp[f]); CHKERRQ(ierr); + field_off[f+1] = field_off[f] + num_comp[f]; + } + ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); + num_elem = c_end - c_start; + ierr = PetscMalloc1(num_elem*PetscPowInt(P, topo_dim), &elem_restr_offsets); + CHKERRQ(ierr); + bool *orient; + ierr = PetscMalloc1(num_elem*PetscPowInt(P, topo_dim), &orient); + CHKERRQ(ierr); + for (p = 0, e_offset = 0; p < num_elem; p++) { + PetscInt num_indices, *indices, num_nodes; + ierr = DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, + &num_indices, &indices, NULL, NULL); + CHKERRQ(ierr); + num_nodes = num_indices / field_off[num_fields]; + for (PetscInt i = 0; i < num_nodes; i++) { + PetscInt ii = i; + // Check that indices are blocked by node and thus can be coalesced as a single field with + // field_off[num_fields] = sum(num_comp) components. + for (PetscInt f = 0; f < num_fields; f++) { + for (PetscInt j = 0; j < num_comp[f]; j++) { + if (Involute(indices[field_off[f]*num_nodes + ii*num_comp[f] + j]) + != Involute(indices[ii*num_comp[0]]) + field_off[f] + j) + SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, + "Cell %D closure indices not interlaced for node %D field %D component %D", + p, ii, f, j); + } + } + // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. + PetscInt loc = Involute(indices[ii*num_comp[0]]); + elem_restr_offsets[e_offset] = loc; + // Set orientation + ierr = DMPlexGetConeOrientation(dm, p, &ornt); CHKERRQ(ierr); + orient[e_offset] = ornt[i] < 0; + e_offset++; + } + ierr = DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, + &num_indices, &indices, NULL, NULL); + CHKERRQ(ierr); + } + if (e_offset != num_elem*PetscPowInt(P, topo_dim)) + SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, + "ElemRestriction of size (%D,%D) initialized %D nodes", num_elem, + PetscPowInt(P, topo_dim),e_offset); + + ierr = DMGetLocalVector(dm, &U_loc); CHKERRQ(ierr); + ierr = VecGetLocalSize(U_loc, &num_dof); CHKERRQ(ierr); + ierr = DMRestoreLocalVector(dm, &U_loc); CHKERRQ(ierr); + CeedElemRestrictionCreateOriented(ceed, num_elem, PetscPowInt(P, topo_dim), + field_off[num_fields], + 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, + elem_restr_offsets, orient, elem_restr_oriented); + ierr = PetscFree(elem_restr_offsets); CHKERRQ(ierr); + ierr = PetscFree(orient); CHKERRQ(ierr); + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// Set up libCEED on the fine grid for a given degree +// ----------------------------------------------------------------------------- +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, + ProblemData *problem_data, + PetscInt U_g_size, PetscInt U_loc_size, + CeedData ceed_data, CeedVector rhs_ceed, + CeedVector *target) { + int ierr; + CeedInt P = app_ctx->degree + 1; + CeedInt Q = P + 1 + app_ctx->q_extra; // Number of quadratures in 1D + CeedInt num_qpts = Q*Q; // Number of quadratures per element + CeedInt dim, num_comp_x, num_comp_u; + CeedInt geo_data_size = problem_data->geo_data_size; + CeedInt elem_node = problem_data->elem_node; + DM dm_coord; + Vec coords; + PetscInt c_start, c_end, num_elem; + const PetscScalar *coordArray; + CeedVector x_coord; + CeedQFunction qf_setup_geo, qf_residual; + CeedOperator op_setup_geo, op_residual; + + PetscFunctionBeginUser; + // --------------------------------------------------------------------------- + // libCEED bases:Hdiv basis_u and Lagrange basis_x + // --------------------------------------------------------------------------- + ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr); + num_comp_x = dim; + num_comp_u = dim; + CeedInt elem_dof = dim*elem_node; // dof per element + CeedScalar q_ref[dim*num_qpts], q_weights[num_qpts]; + CeedScalar div[elem_dof*num_qpts], interp[dim*elem_dof*num_qpts]; + QuadBasis(Q, q_ref, q_weights, interp, div); + CeedBasisCreateHdiv(ceed, CEED_QUAD, num_comp_u, elem_node, num_qpts, + interp, div, q_ref, q_weights, &ceed_data->basis_u); + CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, + problem_data->quadrature_mode, &ceed_data->basis_x); + // --------------------------------------------------------------------------- + // libCEED restrictions + // --------------------------------------------------------------------------- + ierr = DMGetCoordinateDM(dm, &dm_coord); CHKERRQ(ierr); + ierr = DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL); + CHKERRQ(ierr); + // -- Coordinate restriction + ierr = CreateRestrictionFromPlex(ceed, dm_coord, 2, dim, + &ceed_data->elem_restr_x); + CHKERRQ(ierr); + // -- Solution restriction + ierr = CreateRestrictionFromPlexOriented(ceed, dm, P, dim, + &ceed_data->elem_restr_u); + // ---- Geometric ceed_data restriction + ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); + num_elem = c_end - c_start; + CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, geo_data_size, + num_elem*num_qpts*geo_data_size, + CEED_STRIDES_BACKEND, &ceed_data->elem_restr_geo_data_i); + CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, num_comp_u, + num_comp_u*num_elem*num_qpts, + CEED_STRIDES_BACKEND, &ceed_data->elem_restr_u_i); + //CeedElemRestrictionView(ceed_data->elem_restr_x, stdout); + //CeedElemRestrictionView(ceed_data->elem_restr_u, stdout); + //CeedElemRestrictionView(ceed_data->elem_restr_geo_data_i, stdout); + //CeedElemRestrictionView(ceed_data->elem_restr_u_i, stdout); + + // --------------------------------------------------------------------------- + // Element coordinates + // --------------------------------------------------------------------------- + ierr = DMGetCoordinatesLocal(dm, &coords); CHKERRQ(ierr); + ierr = VecGetArrayRead(coords, &coordArray); CHKERRQ(ierr); + + CeedElemRestrictionCreateVector(ceed_data->elem_restr_x, &x_coord, NULL); + CeedVectorSetArray(x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, + (PetscScalar *)coordArray); + ierr = VecRestoreArrayRead(coords, &coordArray); CHKERRQ(ierr); + + // --------------------------------------------------------------------------- + // Persistent libCEED vectors + // --------------------------------------------------------------------------- + // -- Operator action variables + CeedVectorCreate(ceed, U_loc_size, &ceed_data->x_ceed); + CeedVectorCreate(ceed, U_loc_size, &ceed_data->y_ceed); + // -- Geometric data vector + CeedVectorCreate(ceed, num_elem*num_qpts*geo_data_size, + &ceed_data->geo_data); + + // --------------------------------------------------------------------------- + // Geometric factor computation + // --------------------------------------------------------------------------- + // Create the QFunction and Operator that computes the quadrature data + // geo_data returns dXdx_i,j and w * det. + // --------------------------------------------------------------------------- + // -- QFunction + CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_geo, + problem_data->setup_geo_loc, &qf_setup_geo); + CeedQFunctionAddInput(qf_setup_geo, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_setup_geo, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddOutput(qf_setup_geo, "geo_data", geo_data_size, CEED_EVAL_NONE); + // -- Operator + CeedOperatorCreate(ceed, qf_setup_geo, CEED_QFUNCTION_NONE, + CEED_QFUNCTION_NONE, &op_setup_geo); + CeedOperatorSetField(op_setup_geo, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_setup_geo, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup_geo, "geo_data", + ceed_data->elem_restr_geo_data_i, + CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + // -- Compute the quadrature data + CeedOperatorApply(op_setup_geo, x_coord, ceed_data->geo_data, + CEED_REQUEST_IMMEDIATE); + // -- Cleanup + CeedQFunctionDestroy(&qf_setup_geo); + CeedOperatorDestroy(&op_setup_geo); + + // --------------------------------------------------------------------------- + // Local residual evaluator + // --------------------------------------------------------------------------- + // Create the QFunction and Operator that computes the residual of the PDE. + // --------------------------------------------------------------------------- + // -- QFunction + CeedQFunctionCreateInterior(ceed, 1, problem_data->residual, + problem_data->residual_loc, &qf_residual); + CeedQFunctionAddInput(qf_residual, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_residual, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_residual, "u", num_comp_u, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_residual, "v", num_comp_u, CEED_EVAL_INTERP); + + // -- Operator + CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_residual); + CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_residual, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "u", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "v", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + + // -- Save libCEED data + ceed_data->qf_residual = qf_residual; + ceed_data->op_residual = op_residual; + // --------------------------------------------------------------------------- + // Setup RHS and true solution + // --------------------------------------------------------------------------- + CeedQFunction qf_setup_rhs; + CeedOperator op_setup_rhs; + CeedVectorCreate(ceed, num_elem*num_qpts*num_comp_u, target); + // Create the q-function that sets up the RHS and true solution + CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_rhs, + problem_data->setup_rhs_loc, &qf_setup_rhs); + CeedQFunctionAddInput(qf_setup_rhs, "x", num_comp_x, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_setup_rhs, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_setup_rhs, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddOutput(qf_setup_rhs, "true_soln", num_comp_u, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_setup_rhs, "rhs", num_comp_u, CEED_EVAL_INTERP); + // Create the operator that builds the RHS and true solution + CeedOperatorCreate(ceed, qf_setup_rhs, NULL, NULL, &op_setup_rhs); + CeedOperatorSetField(op_setup_rhs, "x", ceed_data->elem_restr_x, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup_rhs, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_setup_rhs, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup_rhs, "true_soln", ceed_data->elem_restr_u_i, + CEED_BASIS_COLLOCATED, *target); + CeedOperatorSetField(op_setup_rhs, "rhs", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + + // Setup RHS and target + CeedOperatorApply(op_setup_rhs, x_coord, rhs_ceed, CEED_REQUEST_IMMEDIATE); + + // Cleanup + CeedQFunctionDestroy(&qf_setup_rhs); + CeedOperatorDestroy(&op_setup_rhs); + CeedVectorDestroy(&x_coord); + + PetscFunctionReturn(0); +}; +// ----------------------------------------------------------------------------- \ No newline at end of file diff --git a/interface/ceed-basis.c b/interface/ceed-basis.c index 05eb0207c9..714a0a2e24 100644 --- a/interface/ceed-basis.c +++ b/interface/ceed-basis.c @@ -1098,6 +1098,7 @@ int CeedBasisCreateH1(Ceed ceed, CeedElemTopology topo, CeedInt num_comp, CeedIn @param topo Topology of element, e.g. hypercube, simplex, ect @param num_nodes Total number of nodes @param num_qpts Total number of quadrature points + @param basis_space 2 for H(div) discretization (1 for H^1, 3 for H(curl)) @param interp Row-major (dim*num_qpts * num_nodes*dim) matrix expressing the values of nodal basis functions at quadrature points @param div Row-major (num_qpts * num_nodes*dim) matrix expressing @@ -1118,9 +1119,9 @@ int CeedBasisCreateHdiv(Ceed ceed, CeedElemTopology topo, CeedInt num_comp, const CeedScalar *div, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis *basis) { int ierr; - CeedInt Q = num_qpts, P = num_nodes, dim = 0; + CeedInt Q = num_qpts, dim = 0; ierr = CeedBasisGetTopologyDimension(topo, &dim); CeedChk(ierr); - CeedInt dof = dim*P; // dof per element! + CeedInt P = dim*num_nodes; // dof per element! if (!ceed->BasisCreateHdiv) { Ceed delegate; ierr = CeedGetObjectDelegate(ceed, &delegate, "Basis"); CeedChk(ierr); @@ -1148,14 +1149,15 @@ int CeedBasisCreateHdiv(Ceed ceed, CeedElemTopology topo, CeedInt num_comp, (*basis)->num_comp = num_comp; (*basis)->P = P; (*basis)->Q = Q; + (*basis)->basis_space = 2; // 2 for H(div) space ierr = CeedMalloc(Q*dim,&(*basis)->q_ref_1d); CeedChk(ierr); ierr = CeedMalloc(Q,&(*basis)->q_weight_1d); CeedChk(ierr); memcpy((*basis)->q_ref_1d, q_ref, Q*dim*sizeof(q_ref[0])); memcpy((*basis)->q_weight_1d, q_weight, Q*sizeof(q_weight[0])); - ierr = CeedMalloc(dim*Q*dof, &(*basis)->interp); CeedChk(ierr); - ierr = CeedMalloc(Q*dof, &(*basis)->div); CeedChk(ierr); - memcpy((*basis)->interp, interp, dim*Q*dof*sizeof(interp[0])); - memcpy((*basis)->div, div, Q*dof*sizeof(div[0])); + ierr = CeedMalloc(dim*Q*P, &(*basis)->interp); CeedChk(ierr); + ierr = CeedMalloc(Q*P, &(*basis)->div); CeedChk(ierr); + memcpy((*basis)->interp, interp, dim*Q*P*sizeof(interp[0])); + memcpy((*basis)->div, div, Q*P*sizeof(div[0])); ierr = ceed->BasisCreateHdiv(topo, dim, P, Q, interp, div, q_ref, q_weight, *basis); CeedChk(ierr); return CEED_ERROR_SUCCESS; @@ -1362,46 +1364,6 @@ int CeedBasisView(CeedBasis basis, FILE *stream) { return CEED_ERROR_SUCCESS; } -/** - @brief View a CeedBasisHdiv - - @param basis CeedBasisHdiv to view - @param stream Stream to view to, e.g., stdout - - @return An error code: 0 - success, otherwise - failure - - @ref User -**/ -int CeedBasisHdivView(CeedBasis basis, FILE *stream) { - int ierr; - - if (basis->tensor_basis) { - fprintf(stream, "CeedBasis: dim=%d P=%d Q=%d\n", basis->dim, basis->P_1d, - basis->Q_1d); - ierr = CeedScalarView("qref1d", "\t% 12.8f", 1, basis->Q_1d, basis->q_ref_1d, - stream); CeedChk(ierr); - ierr = CeedScalarView("qweight1d", "\t% 12.8f", 1, basis->Q_1d, - basis->q_weight_1d, stream); CeedChk(ierr); - ierr = CeedScalarView("interp1d", "\t% 12.8f", basis->Q_1d, basis->P_1d, - basis->interp_1d, stream); CeedChk(ierr); - ierr = CeedScalarView("grad1d", "\t% 12.8f", basis->Q_1d, basis->P_1d, - basis->grad_1d, stream); CeedChk(ierr); - } else { - fprintf(stream, "CeedBasis: dim=%d P=%d Q=%d\n", basis->dim, basis->P, - basis->Q); - ierr = CeedScalarView("qref", "\t% 12.8f", 1, basis->Q*basis->dim, - basis->q_ref_1d, - stream); CeedChk(ierr); - ierr = CeedScalarView("qweight", "\t% 12.8f", 1, basis->Q, basis->q_weight_1d, - stream); CeedChk(ierr); - ierr = CeedScalarView("interp", "\t% 12.8f", basis->dim*basis->Q, basis->dim*basis->P, - basis->interp, stream); CeedChk(ierr); - ierr = CeedScalarView("div", "\t% 12.8f", basis->Q, basis->dim*basis->P, - basis->div, stream); CeedChk(ierr); - } - return CEED_ERROR_SUCCESS; -} - /** @brief Apply basis evaluation from nodes to quadrature points or vice versa From d23c35b6d0d83e0bf48303a3d26d9e8ed4c53459 Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Thu, 16 Dec 2021 11:48:02 -0700 Subject: [PATCH 03/15] fixed element restriction. Co-authored-by: Jed Brown --- examples/Hdiv-mixed/src/setup-dm.c | 12 ++++++------ examples/Hdiv-mixed/src/setup-libceed.c | 23 ++++++++++++++++------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/examples/Hdiv-mixed/src/setup-dm.c b/examples/Hdiv-mixed/src/setup-dm.c index d15e431750..1e5edb77c0 100644 --- a/examples/Hdiv-mixed/src/setup-dm.c +++ b/examples/Hdiv-mixed/src/setup-dm.c @@ -12,8 +12,8 @@ PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm) { PetscInt dim = 2, num_comp_u = dim; PetscInt p_start, p_end; PetscInt c_start, c_end; // cells - PetscInt e_start, e_end; // edges - PetscInt v_start, v_end, v; // vertices + PetscInt e_start, e_end, e; // edges + PetscInt v_start, v_end; // vertices PetscFunctionBeginUser; @@ -28,12 +28,12 @@ PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm) { ierr = PetscSectionCreate(comm, &sec); CHKERRQ(ierr); ierr = PetscSectionSetNumFields(sec,1); CHKERRQ(ierr); ierr = PetscSectionSetFieldName(sec,0,"Velocity"); CHKERRQ(ierr); - ierr = PetscSectionSetFieldComponents(sec,0,num_comp_u); CHKERRQ(ierr); + ierr = PetscSectionSetFieldComponents(sec,0,1); CHKERRQ(ierr); ierr = PetscSectionSetChart(sec,p_start,p_end); CHKERRQ(ierr); // Setup dofs - for (v = v_start; v < v_end; v++) { - ierr = PetscSectionSetFieldDof(sec, v, 0, num_comp_u); CHKERRQ(ierr); - ierr = PetscSectionSetDof (sec, v, num_comp_u); CHKERRQ(ierr); + for (e = e_start; e < e_end; e++) { + ierr = PetscSectionSetFieldDof(sec, e, 0, num_comp_u); CHKERRQ(ierr); + ierr = PetscSectionSetDof (sec, e, num_comp_u); CHKERRQ(ierr); } ierr = PetscSectionSetUp(sec); CHKERRQ(ierr); ierr = DMSetSection(*dm,sec); CHKERRQ(ierr); diff --git a/examples/Hdiv-mixed/src/setup-libceed.c b/examples/Hdiv-mixed/src/setup-libceed.c index aa08502bbf..4462072f33 100644 --- a/examples/Hdiv-mixed/src/setup-libceed.c +++ b/examples/Hdiv-mixed/src/setup-libceed.c @@ -8,7 +8,9 @@ CeedMemType MemTypeP2C(PetscMemType mem_type) { return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; } +// ----------------------------------------------------------------------------- // Destroy libCEED objects +// ----------------------------------------------------------------------------- PetscErrorCode CeedDataDestroy(CeedData ceed_data) { PetscErrorCode ierr; @@ -132,7 +134,8 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, } ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); num_elem = c_end - c_start; - ierr = PetscMalloc1(num_elem*PetscPowInt(P, topo_dim), &elem_restr_offsets); + ierr = PetscMalloc1(num_elem*topo_dim*PetscPowInt(P, topo_dim), + &elem_restr_offsets); // doesn't alocate as many entries CHKERRQ(ierr); bool *orient; ierr = PetscMalloc1(num_elem*PetscPowInt(P, topo_dim), &orient); @@ -142,7 +145,8 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, ierr = DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL); CHKERRQ(ierr); - num_nodes = num_indices / field_off[num_fields]; + num_nodes = num_indices / + field_off[num_fields]; // 8 / 2, but I think there are really 8 nodes for (PetscInt i = 0; i < num_nodes; i++) { PetscInt ii = i; // Check that indices are blocked by node and thus can be coalesced as a single field with @@ -158,7 +162,7 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, } // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. PetscInt loc = Involute(indices[ii*num_comp[0]]); - elem_restr_offsets[e_offset] = loc; + elem_restr_offsets[e_offset] = loc; // Are we getting two nodes per edge? yes, // Set orientation ierr = DMPlexGetConeOrientation(dm, p, &ornt); CHKERRQ(ierr); orient[e_offset] = ornt[i] < 0; @@ -168,7 +172,8 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, &num_indices, &indices, NULL, NULL); CHKERRQ(ierr); } - if (e_offset != num_elem*PetscPowInt(P, topo_dim)) + if (e_offset != num_elem*topo_dim*PetscPowInt(P, + topo_dim)) // this probably needs to be like this SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "ElemRestriction of size (%D,%D) initialized %D nodes", num_elem, PetscPowInt(P, topo_dim),e_offset); @@ -176,7 +181,9 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, ierr = DMGetLocalVector(dm, &U_loc); CHKERRQ(ierr); ierr = VecGetLocalSize(U_loc, &num_dof); CHKERRQ(ierr); ierr = DMRestoreLocalVector(dm, &U_loc); CHKERRQ(ierr); - CeedElemRestrictionCreateOriented(ceed, num_elem, PetscPowInt(P, topo_dim), + // dof per element in Hdiv is dim*P^dim, for linear element P=2 + CeedElemRestrictionCreateOriented(ceed, num_elem, topo_dim*PetscPowInt(P, + topo_dim), // as we're using here field_off[num_fields], 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets, orient, elem_restr_oriented); @@ -245,8 +252,10 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, num_comp_u, num_comp_u*num_elem*num_qpts, CEED_STRIDES_BACKEND, &ceed_data->elem_restr_u_i); - //CeedElemRestrictionView(ceed_data->elem_restr_x, stdout); - //CeedElemRestrictionView(ceed_data->elem_restr_u, stdout); + printf("----elem_restr_x:\n"); + CeedElemRestrictionView(ceed_data->elem_restr_x, stdout); + printf("----elem_restr_u:\n"); + CeedElemRestrictionView(ceed_data->elem_restr_u, stdout); //CeedElemRestrictionView(ceed_data->elem_restr_geo_data_i, stdout); //CeedElemRestrictionView(ceed_data->elem_restr_u_i, stdout); From 8e0a56bdff14a69f83ceaabbb9cae1c4ea132033 Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Thu, 16 Dec 2021 14:17:28 -0700 Subject: [PATCH 04/15] setup mass matrix and ksp for Darcy problem --- examples/Hdiv-mass/Makefile | 86 ++++ examples/Hdiv-mass/basis/quad.h | 78 +++ examples/Hdiv-mass/include/cl-options.h | 12 + examples/Hdiv-mass/include/matops.h | 12 + examples/Hdiv-mass/include/petsc-macros.h | 17 + examples/Hdiv-mass/include/problems.h | 18 + examples/Hdiv-mass/include/setup-dm.h | 14 + examples/Hdiv-mass/include/setup-libceed.h | 25 + examples/Hdiv-mass/include/structs.h | 74 +++ examples/Hdiv-mass/main.c | 206 ++++++++ examples/Hdiv-mass/main.h | 11 + examples/Hdiv-mass/problems/poisson-mass2d.c | 54 +++ .../Hdiv-mass/qfunctions/poisson-mass2d.h | 85 ++++ examples/Hdiv-mass/qfunctions/poisson-rhs2d.h | 76 +++ examples/Hdiv-mass/src/cl-options.c | 73 +++ examples/Hdiv-mass/src/matops.c | 59 +++ examples/Hdiv-mass/src/setup-dm.c | 45 ++ examples/Hdiv-mass/src/setup-libceed.c | 454 ++++++++++++++++++ examples/Hdiv-mixed/include/structs.h | 4 +- examples/Hdiv-mixed/problems/poisson-quad2d.c | 5 +- .../Hdiv-mixed/qfunctions/poisson-quad2d.h | 18 +- examples/Hdiv-mixed/qfunctions/setup-geo.h | 104 ---- examples/Hdiv-mixed/qfunctions/setup-geo2d.h | 92 ---- examples/Hdiv-mixed/src/setup-dm.c | 2 +- examples/Hdiv-mixed/src/setup-libceed.c | 16 +- interface/ceed-basis.c | 1 + 26 files changed, 1424 insertions(+), 217 deletions(-) create mode 100644 examples/Hdiv-mass/Makefile create mode 100644 examples/Hdiv-mass/basis/quad.h create mode 100644 examples/Hdiv-mass/include/cl-options.h create mode 100644 examples/Hdiv-mass/include/matops.h create mode 100644 examples/Hdiv-mass/include/petsc-macros.h create mode 100644 examples/Hdiv-mass/include/problems.h create mode 100644 examples/Hdiv-mass/include/setup-dm.h create mode 100644 examples/Hdiv-mass/include/setup-libceed.h create mode 100644 examples/Hdiv-mass/include/structs.h create mode 100644 examples/Hdiv-mass/main.c create mode 100644 examples/Hdiv-mass/main.h create mode 100644 examples/Hdiv-mass/problems/poisson-mass2d.c create mode 100644 examples/Hdiv-mass/qfunctions/poisson-mass2d.h create mode 100644 examples/Hdiv-mass/qfunctions/poisson-rhs2d.h create mode 100644 examples/Hdiv-mass/src/cl-options.c create mode 100644 examples/Hdiv-mass/src/matops.c create mode 100644 examples/Hdiv-mass/src/setup-dm.c create mode 100644 examples/Hdiv-mass/src/setup-libceed.c delete mode 100644 examples/Hdiv-mixed/qfunctions/setup-geo.h delete mode 100644 examples/Hdiv-mixed/qfunctions/setup-geo2d.h diff --git a/examples/Hdiv-mass/Makefile b/examples/Hdiv-mass/Makefile new file mode 100644 index 0000000000..5912d6f96e --- /dev/null +++ b/examples/Hdiv-mass/Makefile @@ -0,0 +1,86 @@ +# Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. +# All Rights reserved. See files LICENSE and NOTICE for details. +# +# This file is part of CEED, a collection of benchmarks, miniapps, software +# libraries and APIs for efficient high-order finite element and spectral +# element discretizations for exascale applications. For more information and +# source code availability see http://github.com/ceed. +# +# The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +# a collaborative effort of two U.S. Department of Energy organizations (Office +# of Science and the National Nuclear Security Administration) responsible for +# the planning and preparation of a capable exascale ecosystem, including +# software, applications, hardware, advanced system engineering and early +# testbed platforms, in support of the nation's exascale computing imperative. + +COMMON ?= ../../common.mk +-include $(COMMON) + + + +PETSc.pc := $(PETSC_DIR)/$(PETSC_ARCH)/lib/pkgconfig/PETSc.pc +CEED_DIR ?= ../.. +ceed.pc := $(CEED_DIR)/lib/pkgconfig/ceed.pc + +CC = $(call pkgconf, --variable=ccompiler $(PETSc.pc) $(ceed.pc)) +CFLAGS = -std=c99 \ + $(call pkgconf, --variable=cflags_extra $(PETSc.pc)) \ + $(call pkgconf, --cflags-only-other $(PETSc.pc)) \ + $(OPT) +CPPFLAGS = $(call pkgconf, --cflags-only-I $(PETSc.pc) $(ceed.pc)) \ + $(call pkgconf, --variable=cflags_dep $(PETSc.pc)) +LDFLAGS = $(call pkgconf, --libs-only-L --libs-only-other $(PETSc.pc) $(ceed.pc)) +LDFLAGS += $(patsubst -L%, $(call pkgconf, --variable=ldflag_rpath $(PETSc.pc))%, $(call pkgconf, --libs-only-L $(PETSc.pc) $(ceed.pc))) +LDLIBS = $(call pkgconf, --libs-only-l $(PETSc.pc) $(ceed.pc)) -lm + +OBJDIR := build +SRCDIR := src +PROBLEMDIR := problems + +all: main + +utils.c := $(sort $(wildcard $(PROBLEMDIR)/*.c)) $(sort $(wildcard $(SRCDIR)/*.c)) +utils.o = $(utils.c:%.c=$(OBJDIR)/%.o) +libutils.a: $(utils.o) + $(call quiet,AR) $(ARFLAGS) $@ $^ + +main.c := main.c +main.o = $(main.c:%.c=$(OBJDIR)/%.o) +main: $(main.o) libutils.a | $(PETSc.pc) $(ceed.pc) + $(call quiet,LINK.o) $(CEED_LDFLAGS) $^ $(LOADLIBES) $(LDLIBS) -o $@ + +.SECONDEXPANSION: # to expand $$(@D)/.DIR +%/.DIR : + @mkdir -p $(@D) + @touch $@ + +# Quiet, color output +quiet ?= $($(1)) + +$(OBJDIR)/%.o : %.c | $$(@D)/.DIR + $(call quiet,CC) $(CPPFLAGS) $(CFLAGS) -c -o $@ $(abspath $<) + +# Rules for building the examples +#%: %.c + +print: $(PETSc.pc) $(ceed.pc) + $(info CC : $(CC)) + $(info CFLAGS : $(CFLAGS)) + $(info CPPFLAGS: $(CPPFLAGS)) + $(info LDFLAGS : $(LDFLAGS)) + $(info LDLIBS : $(LDLIBS)) + @true + +clean: + $(RM) -r $(OBJDIR) *.vtu main libutils.a + +$(PETSc.pc): + $(if $(wildcard $@),,$(error \ + PETSc config not found at $@. Please set PETSC_DIR and PETSC_ARCH)) + +.PHONY: all print clean + +pkgconf = $(shell pkg-config $1 | sed -e 's/^"//g' -e 's/"$$//g') + +-include $(src.o:%.o=%.d) \ No newline at end of file diff --git a/examples/Hdiv-mass/basis/quad.h b/examples/Hdiv-mass/basis/quad.h new file mode 100644 index 0000000000..22fbde0bab --- /dev/null +++ b/examples/Hdiv-mass/basis/quad.h @@ -0,0 +1,78 @@ +// Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. +// All Rights reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +// Hdiv basis for quadrilateral element in 2D +// Local numbering is as follow (each edge has 2 vector dof) +// b5 b7 +// 2---------3 +// b4| |b6 +// | | +// b0| |b2 +// 0---------1 +// b1 b3 +// Bx[0-->7] = b0_x-->b7_x, By[0-->7] = b0_y-->b7_y +int HdivBasisQuad(CeedScalar *xhat, CeedScalar *Bx, CeedScalar *By) { + Bx[0] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; + By[0] = (xhat[1]*xhat[1] - 1)*0.125; + Bx[1] = (xhat[0]*xhat[0] - 1)*0.125; + By[1] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; + Bx[2] = (-xhat[0]*xhat[1] + xhat[0] - xhat[1] + 1)*0.25; + By[2] = (xhat[1]*xhat[1] - 1)*0.125; + Bx[3] = (-xhat[0]*xhat[0] + 1)*0.125; + By[3] = (xhat[0]*xhat[1] - xhat[0] + xhat[1] - 1)*0.25; + Bx[4] = (xhat[0]*xhat[1] + xhat[0] - xhat[1] - 1)*0.25; + By[4] = (-xhat[1]*xhat[1] + 1)*0.125; + Bx[5] = (xhat[0]*xhat[0] - 1)*0.125; + By[5] = (-xhat[0]*xhat[1] - xhat[0] + xhat[1] + 1)*0.25; + Bx[6] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; + By[6] = (-xhat[1]*xhat[1] + 1)*0.125; + Bx[7] = (-xhat[0]*xhat[0] + 1)*0.125; + By[7] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; + return 0; +} + +static void QuadBasis(CeedInt Q1d, CeedScalar *q_ref, CeedScalar *q_weights, + CeedScalar *interp, CeedScalar *div) { + + // Get 1D quadrature on [-1,1] + CeedScalar q_ref_1d[Q1d], q_weight_1d[Q1d]; + CeedGaussQuadrature(Q1d, q_ref_1d, q_weight_1d); + + // Divergence operator; Divergence of nodal basis for ref element + CeedScalar D[8] = {0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25}; + // Loop over quadrature points + CeedScalar Bx[8], By[8]; + CeedScalar xhat[2]; + + for (CeedInt i=0; i +#include + +#include "structs.h" + +PetscErrorCode ApplyLocal_Ceed(Vec X, Vec Y, User user); +PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y); + +#endif // matops_h diff --git a/examples/Hdiv-mass/include/petsc-macros.h b/examples/Hdiv-mass/include/petsc-macros.h new file mode 100644 index 0000000000..dbdf17d2e9 --- /dev/null +++ b/examples/Hdiv-mass/include/petsc-macros.h @@ -0,0 +1,17 @@ +#ifndef petsc_macros +#define petsc_macros + +#if PETSC_VERSION_LT(3,14,0) +# define DMPlexGetClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexGetClosureIndices(a,b,c,d,f,g,i) +# define DMPlexRestoreClosureIndices(a,b,c,d,e,f,g,h,i) DMPlexRestoreClosureIndices(a,b,c,d,f,g,i) +#endif + +#if PETSC_VERSION_LT(3,14,0) +# define DMAddBoundary(a,b,c,d,e,f,g,h,i,j,k,l,m,n) DMAddBoundary(a,b,c,e,h,i,j,k,f,g,m) +#elif PETSC_VERSION_LT(3,16,0) +# define DMAddBoundary(a,b,c,d,e,f,g,h,i,j,k,l,m,n) DMAddBoundary(a,b,c,e,h,i,j,k,l,f,g,m) +#else +# define DMAddBoundary(a,b,c,d,e,f,g,h,i,j,k,l,m,n) DMAddBoundary(a,b,c,d,f,g,h,i,j,k,l,m,n) +#endif + +#endif diff --git a/examples/Hdiv-mass/include/problems.h b/examples/Hdiv-mass/include/problems.h new file mode 100644 index 0000000000..64eff651f8 --- /dev/null +++ b/examples/Hdiv-mass/include/problems.h @@ -0,0 +1,18 @@ +#ifndef problems_h +#define problems_h + +#include "../include/structs.h" + +// ----------------------------------------------------------------------------- +// Set up problems function prototype +// ----------------------------------------------------------------------------- +// 1) poisson-quad2d +PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx); + +// 2) poisson-hex3d + +// 3) poisson-prism3d + +// 4) richard + +#endif // problems_h diff --git a/examples/Hdiv-mass/include/setup-dm.h b/examples/Hdiv-mass/include/setup-dm.h new file mode 100644 index 0000000000..ebdebdaf7c --- /dev/null +++ b/examples/Hdiv-mass/include/setup-dm.h @@ -0,0 +1,14 @@ +#ifndef setupdm_h +#define setupdm_h + +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Set-up DM +// --------------------------------------------------------------------------- +PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm); + +#endif // setupdm_h diff --git a/examples/Hdiv-mass/include/setup-libceed.h b/examples/Hdiv-mass/include/setup-libceed.h new file mode 100644 index 0000000000..500d47a61c --- /dev/null +++ b/examples/Hdiv-mass/include/setup-libceed.h @@ -0,0 +1,25 @@ +#ifndef setuplibceed_h +#define setuplibceed_h + +#include "../include/structs.h" + +// Convert PETSc MemType to libCEED MemType +CeedMemType MemTypeP2C(PetscMemType mtype); +// Destroy libCEED objects +PetscErrorCode CeedDataDestroy(CeedData ceed_data); +// Utility function - essential BC dofs are encoded in closure indices as -(i+1) +PetscInt Involute(PetscInt i); +// Utility function to create local CEED restriction from DMPlex +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, + CeedInt height, DMLabel domain_label, CeedInt value, CeedInt P, + CeedElemRestriction *elem_restr); +// Utility function to create local CEED Oriented restriction from DMPlex +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, + CeedInt height, DMLabel domain_label, CeedInt value, CeedInt P, + CeedElemRestriction *elem_restr_oriented); +// Set up libCEED for a given degree +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, + ProblemData *problem_data, + PetscInt U_g_size, PetscInt U_loc_size, + CeedData ceed_data, CeedVector rhs_ceed); +#endif // setuplibceed_h diff --git a/examples/Hdiv-mass/include/structs.h b/examples/Hdiv-mass/include/structs.h new file mode 100644 index 0000000000..391ae98b39 --- /dev/null +++ b/examples/Hdiv-mass/include/structs.h @@ -0,0 +1,74 @@ +#ifndef structs_h +#define structs_h + +#include +#include + +// Application context from user command line options +typedef struct AppCtx_ *AppCtx; +struct AppCtx_ { + // libCEED arguments + PetscInt degree; + PetscInt q_extra; + // Problem type arguments + PetscFunctionList problems; + char problem_name[PETSC_MAX_PATH_LEN]; +}; + +// libCEED data struct +typedef struct CeedData_ *CeedData; +struct CeedData_ { + CeedBasis basis_x, basis_u; + CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_geo_data_i, + elem_restr_u_i; + CeedQFunction qf_residual; + CeedOperator op_residual; + CeedVector x_ceed, y_ceed; + CeedQFunctionContext pq2d_context; +}; + +// 1) poisson-quad2d +#ifndef PHYSICS_POISSONQUAD2D_STRUCT +#define PHYSICS_POISSONQUAD2D_STRUCT +typedef struct PQ2DContext_ *PQ2DContext; +struct PQ2DContext_ { + CeedScalar kappa; +}; +#endif + +// 2) poisson-hex3d + +// 3) poisson-prism3d + +// 4) richard + +// Struct that contains all enums and structs used for the physics of all problems +typedef struct Physics_ *Physics; +struct Physics_ { + PQ2DContext pq2d_ctx; +}; + +// PETSc user data +typedef struct User_ *User; +struct User_ { + MPI_Comm comm; + Vec X_loc, Y_loc; + CeedVector x_ceed, y_ceed; + CeedOperator op; + DM dm; + Ceed ceed; + AppCtx app_ctx; + Physics phys; +}; + +// Problem specific data +typedef struct { + CeedQFunctionUser setup_rhs, residual; + const char *setup_rhs_loc, *residual_loc; + CeedQuadMode quadrature_mode; + CeedInt geo_data_size, elem_node; + PetscErrorCode (*setup_ctx)(Ceed, CeedData, Physics); + +} ProblemData; + +#endif // structs_h \ No newline at end of file diff --git a/examples/Hdiv-mass/main.c b/examples/Hdiv-mass/main.c new file mode 100644 index 0000000000..1f7ce93288 --- /dev/null +++ b/examples/Hdiv-mass/main.c @@ -0,0 +1,206 @@ +/// @file +/// Test creation, use, and destruction of an element restriction for 2D quad Hdiv + +// run with ./main +const char help[] = "Solve H(div)-mixed problem using PETSc and libCEED\n"; + +#include "main.h" + +int main(int argc, char **argv) { + // --------------------------------------------------------------------------- + // Initialize PETSc + // --------------------------------------------------------------------------- + PetscInt ierr; + ierr = PetscInitialize(&argc, &argv, NULL, help); + if (ierr) return ierr; + + // --------------------------------------------------------------------------- + // Create structs + // --------------------------------------------------------------------------- + AppCtx app_ctx; + ierr = PetscCalloc1(1, &app_ctx); CHKERRQ(ierr); + + ProblemData *problem_data = NULL; + ierr = PetscCalloc1(1, &problem_data); CHKERRQ(ierr); + + User user; + ierr = PetscCalloc1(1, &user); CHKERRQ(ierr); + + CeedData ceed_data; + ierr = PetscCalloc1(1, &ceed_data); CHKERRQ(ierr); + + Physics phys_ctx; + ierr = PetscCalloc1(1, &phys_ctx); CHKERRQ(ierr); + + user->app_ctx = app_ctx; + user->phys = phys_ctx; + + // --------------------------------------------------------------------------- + // Process command line options + // --------------------------------------------------------------------------- + // -- Register problems to be available on the command line + ierr = RegisterProblems_Hdiv(app_ctx); CHKERRQ(ierr); + + // -- Process general command line options + MPI_Comm comm = PETSC_COMM_WORLD; + ierr = ProcessCommandLineOptions(comm, app_ctx); CHKERRQ(ierr); + + // --------------------------------------------------------------------------- + // Choose the problem from the list of registered problems + // --------------------------------------------------------------------------- + { + PetscErrorCode (*p)(ProblemData *, void *); + ierr = PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p); + CHKERRQ(ierr); + if (!p) SETERRQ1(PETSC_COMM_SELF, 1, "Problem '%s' not found", + app_ctx->problem_name); + ierr = (*p)(problem_data, &user); CHKERRQ(ierr); + } + + // --------------------------------------------------------------------------- + // Initialize libCEED + // --------------------------------------------------------------------------- + // -- Initialize backend + Ceed ceed; + CeedInit("/cpu/self/opt/serial", &ceed); + CeedMemType mem_type_backend; + CeedGetPreferredMemType(ceed, &mem_type_backend); + // --------------------------------------------------------------------------- + // Set-up DM + // --------------------------------------------------------------------------- + // PETSc objects + DM dm; + VecType vec_type; + ierr = CreateDistributedDM(comm, &dm); CHKERRQ(ierr); + ierr = DMGetVecType(dm, &vec_type); CHKERRQ(ierr); + if (!vec_type) { // Not yet set by user -dm_vec_type + switch (mem_type_backend) { + case CEED_MEM_HOST: vec_type = VECSTANDARD; break; + case CEED_MEM_DEVICE: { + const char *resolved; + CeedGetResource(ceed, &resolved); + if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; + else if (strstr(resolved, "/gpu/hip/occa")) + vec_type = VECSTANDARD; // https://github.com/CEED/libCEED/issues/678 + else if (strstr(resolved, "/gpu/hip")) vec_type = VECHIP; + else vec_type = VECSTANDARD; + } + } + ierr = DMSetVecType(dm, vec_type); CHKERRQ(ierr); + } + // --------------------------------------------------------------------------- + // Create global, local solution, local rhs vector + // --------------------------------------------------------------------------- + Vec U_g, U_loc; + PetscInt U_l_size, U_g_size, U_loc_size; + // Create global and local solution vectors + ierr = DMCreateGlobalVector(dm, &U_g); CHKERRQ(ierr); + ierr = VecGetSize(U_g, &U_g_size); CHKERRQ(ierr); + // Local size for matShell + ierr = VecGetLocalSize(U_g, &U_l_size); CHKERRQ(ierr); + // Create local unknown vector U_loc + ierr = DMCreateLocalVector(dm, &U_loc); CHKERRQ(ierr); + // Local size for libCEED + ierr = VecGetSize(U_loc, &U_loc_size); CHKERRQ(ierr); + + // Get RHS vector + Vec rhs_loc; + PetscScalar *r; + CeedVector rhs_ceed; + PetscMemType mem_type; + ierr = VecDuplicate(U_loc, &rhs_loc); CHKERRQ(ierr); + ierr = VecZeroEntries(rhs_loc); CHKERRQ(ierr); + ierr = VecGetArrayAndMemType(rhs_loc, &r, &mem_type); CHKERRQ(ierr); + CeedVectorCreate(ceed, U_l_size, &rhs_ceed); + CeedVectorSetArray(rhs_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, r); + // --------------------------------------------------------------------------- + // Setup libCEED + // --------------------------------------------------------------------------- + // -- Set up libCEED objects + ierr = SetupLibceed(dm, ceed, app_ctx, problem_data, U_g_size, + U_loc_size, ceed_data, rhs_ceed); CHKERRQ(ierr); + //CeedVectorView(rhs_ceed, "%12.8f", stdout); + // --------------------------------------------------------------------------- + // Gather RHS + // --------------------------------------------------------------------------- + Vec rhs; + CeedVectorTakeArray(rhs_ceed, MemTypeP2C(mem_type), NULL); + ierr = VecRestoreArrayAndMemType(rhs_loc, &r); CHKERRQ(ierr); + ierr = VecDuplicate(U_g, &rhs); CHKERRQ(ierr); + ierr = VecZeroEntries(rhs); CHKERRQ(ierr); + ierr = DMLocalToGlobal(dm, rhs_loc, ADD_VALUES, rhs); CHKERRQ(ierr); + // --------------------------------------------------------------------------- + // Setup Mat, KSP + // --------------------------------------------------------------------------- + user->comm = comm; + user->dm = dm; + user->X_loc = U_loc; + ierr = VecDuplicate(U_loc, &user->Y_loc); CHKERRQ(ierr); + user->x_ceed = ceed_data->x_ceed; + user->y_ceed = ceed_data->y_ceed; + user->op = ceed_data->op_residual; + user->ceed = ceed; + // Operator + Mat mat; + ierr = MatCreateShell(comm, U_l_size, U_l_size, U_g_size, U_g_size, + user, &mat); CHKERRQ(ierr); + ierr = MatShellSetOperation(mat, MATOP_MULT, + (void(*)(void))MatMult_Ceed); CHKERRQ(ierr); + ierr = MatShellSetVecType(mat, vec_type); CHKERRQ(ierr); + + KSP ksp; + ierr = KSPCreate(comm, &ksp); CHKERRQ(ierr); + ierr = KSPSetOperators(ksp, mat, mat); CHKERRQ(ierr); + ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr); + ierr = KSPSetUp(ksp); CHKERRQ(ierr); + ierr = KSPSolve(ksp, rhs, U_g); CHKERRQ(ierr); + + // Output results + KSPType ksp_type; + KSPConvergedReason reason; + PetscReal rnorm; + PetscInt its; + ierr = KSPGetType(ksp, &ksp_type); CHKERRQ(ierr); + ierr = KSPGetConvergedReason(ksp, &reason); CHKERRQ(ierr); + ierr = KSPGetIterationNumber(ksp, &its); CHKERRQ(ierr); + ierr = KSPGetResidualNorm(ksp, &rnorm); CHKERRQ(ierr); + ierr = PetscPrintf(comm, + " KSP:\n" + " KSP Type : %s\n" + " KSP Convergence : %s\n" + " Total KSP Iterations : %D\n" + " Final rnorm : %e\n", + ksp_type, KSPConvergedReasons[reason], its, + (double)rnorm); CHKERRQ(ierr); + + // --------------------------------------------------------------------------- + // Free objects + // --------------------------------------------------------------------------- + + // Free PETSc objects + ierr = DMDestroy(&dm); CHKERRQ(ierr); + ierr = VecDestroy(&U_g); CHKERRQ(ierr); + ierr = VecDestroy(&U_loc); CHKERRQ(ierr); + ierr = VecDestroy(&rhs); CHKERRQ(ierr); + ierr = VecDestroy(&rhs_loc); CHKERRQ(ierr); + ierr = VecDestroy(&user->Y_loc); CHKERRQ(ierr); + ierr = MatDestroy(&mat); CHKERRQ(ierr); + ierr = KSPDestroy(&ksp); CHKERRQ(ierr); + + // -- Function list + ierr = PetscFunctionListDestroy(&app_ctx->problems); CHKERRQ(ierr); + + // -- Structs + ierr = PetscFree(app_ctx); CHKERRQ(ierr); + ierr = PetscFree(problem_data); CHKERRQ(ierr); + ierr = PetscFree(user); CHKERRQ(ierr); + ierr = PetscFree(phys_ctx->pq2d_ctx); CHKERRQ(ierr); + ierr = PetscFree(phys_ctx); CHKERRQ(ierr); + // Free libCEED objects + + CeedVectorDestroy(&rhs_ceed); + ierr = CeedDataDestroy(ceed_data); CHKERRQ(ierr); + CeedDestroy(&ceed); + + return PetscFinalize(); +} diff --git a/examples/Hdiv-mass/main.h b/examples/Hdiv-mass/main.h new file mode 100644 index 0000000000..da283f2d04 --- /dev/null +++ b/examples/Hdiv-mass/main.h @@ -0,0 +1,11 @@ + +#ifndef MAIN_H +#define MAIN_H + +#include "include/setup-libceed.h" +#include "include/setup-dm.h" +#include "include/cl-options.h" +#include "include/problems.h" +#include "include/matops.h" + +#endif // MAIN_H \ No newline at end of file diff --git a/examples/Hdiv-mass/problems/poisson-mass2d.c b/examples/Hdiv-mass/problems/poisson-mass2d.c new file mode 100644 index 0000000000..3faa056345 --- /dev/null +++ b/examples/Hdiv-mass/problems/poisson-mass2d.c @@ -0,0 +1,54 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Utility functions for setting up POISSON_QUAD2D + +#include "../include/setup-libceed.h" +#include "../include/problems.h" +#include "../qfunctions/poisson-rhs2d.h" +#include "../qfunctions/poisson-mass2d.h" + +// Hdiv_POISSON_MASS2D is registered in cl-option.c +PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx) { + User user = *(User *)ctx; + MPI_Comm comm = PETSC_COMM_WORLD; + PetscInt ierr; + PetscFunctionBeginUser; + + ierr = PetscCalloc1(1, &user->phys->pq2d_ctx); CHKERRQ(ierr); + + // ------------------------------------------------------ + // SET UP POISSON_QUAD2D + // ------------------------------------------------------ + problem_data->elem_node = 4; + problem_data->geo_data_size = 1; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->setup_rhs = SetupRhs; + problem_data->setup_rhs_loc = SetupRhs_loc; + problem_data->residual = SetupMass; + problem_data->residual_loc = SetupMass_loc; + + // ------------------------------------------------------ + // Command line Options + // ------------------------------------------------------ + ierr = PetscOptionsBegin(comm, NULL, "Options for DENSITY_CURRENT problem", + NULL); CHKERRQ(ierr); + + ierr = PetscOptionsEnd(); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mass/qfunctions/poisson-mass2d.h b/examples/Hdiv-mass/qfunctions/poisson-mass2d.h new file mode 100644 index 0000000000..7951b22ebb --- /dev/null +++ b/examples/Hdiv-mass/qfunctions/poisson-mass2d.h @@ -0,0 +1,85 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Mixed poisson 2D quad element using PETSc + +#ifndef POISSON_MASS2D_H +#define POISSON_MASS2D_H + +#include + +// ----------------------------------------------------------------------------- +// This QFunction applies the mass operator for a vector field of 2 components. +// +// Inputs: +// w - weight of quadrature +// J - dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// u - Input basis at quadrature points +// +// Output: +// v - Output vector (test functions) at quadrature points +// Note we need to apply Piola map on the basis, which is J*u/detJ +// So (v,u) = \int (v^T * u detJ*w) ==> \int (v^T J^T*J*u*w/detJ) +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(SetupMass)(void *ctx, CeedInt Q, const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*J)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + // *INDENT-ON* + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i + +// ----------------------------------------------------------------------------- +// This QFunction sets up the rhs for the problem +// Inputs: +// x - interpolation of the physical coordinate +// w - weight of quadrature +// J - dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// +// Output: +// rhs - Output vector (test functions) at quadrature points +// Note we need to apply Piola map on the basis, which is J*u/detJ +// So (v,ue) = \int (v^T * ue detJ*w) ==> \int (v^T J^T* ue * w) +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(SetupRhs)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*x) = in[0], + (*w) = in[1], + (*J)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[2]; + // Outputs + //CeedScalar (*rhs)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + CeedScalar (*rhs) = out[0]; + + // Quadrature Point Loop + //printf("--------------------\n"); + //printf("inside qfunction poisson-rhs2d.h CEED_Q_VLA: %d, Q: %d \n",CEED_Q_VLA, Q); + //printf("--------------------\n"); + CeedPragmaSIMD + for (CeedInt i=0; iproblems = NULL; + PetscErrorCode ierr; + PetscFunctionBeginUser; + // 1) poisson-quad2d (Hdiv_POISSON_MASS2D is created in poisson-mass2d.c) + ierr = PetscFunctionListAdd(&app_ctx->problems, "poisson_mass2d", + Hdiv_POISSON_MASS2D); CHKERRQ(ierr); + // 2) poisson-hex3d + + // 3) poisson-prism3d + + // 4) richard + + PetscFunctionReturn(0); +} + +// Process general command line options +PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx) { + + PetscBool problem_flag = PETSC_FALSE; + PetscErrorCode ierr; + PetscFunctionBeginUser; + + ierr = PetscOptionsBegin(comm, NULL, + "H(div) examples in PETSc with libCEED", + NULL); CHKERRQ(ierr); + + ierr = PetscOptionsFList("-problem", "Problem to solve", NULL, + app_ctx->problems, + app_ctx->problem_name, app_ctx->problem_name, sizeof(app_ctx->problem_name), + &problem_flag); CHKERRQ(ierr); + + app_ctx->degree = 1; + ierr = PetscOptionsInt("-degree", "Polynomial degree of finite elements", + NULL, app_ctx->degree, &app_ctx->degree, NULL); CHKERRQ(ierr); + + app_ctx->q_extra = 0; + ierr = PetscOptionsInt("-q_extra", "Number of extra quadrature points", + NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL); CHKERRQ(ierr); + + // Provide default problem if not specified + if (!problem_flag) { + const char *problem_name = "poisson_mass2d"; + strncpy(app_ctx->problem_name, problem_name, 16); + } + + ierr = PetscOptionsEnd(); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mass/src/matops.c b/examples/Hdiv-mass/src/matops.c new file mode 100644 index 0000000000..328f10407e --- /dev/null +++ b/examples/Hdiv-mass/src/matops.c @@ -0,0 +1,59 @@ +#include "../include/matops.h" +#include "../include/setup-libceed.h" + +// ----------------------------------------------------------------------------- +// This function uses libCEED to compute the action of the Laplacian with +// Dirichlet boundary conditions +// ----------------------------------------------------------------------------- +PetscErrorCode ApplyLocal_Ceed(Vec X, Vec Y, User user) { + PetscErrorCode ierr; + PetscScalar *x, *y; + PetscMemType x_mem_type, y_mem_type; + + PetscFunctionBeginUser; + + // Global-to-local + ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); CHKERRQ(ierr); + + // Setup libCEED vectors + ierr = VecGetArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x, + &x_mem_type); CHKERRQ(ierr); + ierr = VecGetArrayAndMemType(user->Y_loc, &y, &y_mem_type); CHKERRQ(ierr); + CeedVectorSetArray(user->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, x); + CeedVectorSetArray(user->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); + + // Apply libCEED operator + CeedOperatorApply(user->op, user->x_ceed, user->y_ceed, CEED_REQUEST_IMMEDIATE); + + // Restore PETSc vectors + CeedVectorTakeArray(user->x_ceed, MemTypeP2C(x_mem_type), NULL); + CeedVectorTakeArray(user->y_ceed, MemTypeP2C(y_mem_type), NULL); + ierr = VecRestoreArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x); + CHKERRQ(ierr); + ierr = VecRestoreArrayAndMemType(user->Y_loc, &y); CHKERRQ(ierr); + + // Local-to-global + ierr = VecZeroEntries(Y); CHKERRQ(ierr); + ierr = DMLocalToGlobal(user->dm, user->Y_loc, ADD_VALUES, Y); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// This function wraps the libCEED operator for a MatShell +// ----------------------------------------------------------------------------- +PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y) { + PetscErrorCode ierr; + User user; + + PetscFunctionBeginUser; + + ierr = MatShellGetContext(A, &user); CHKERRQ(ierr); + + // libCEED for local action of residual evaluator + ierr = ApplyLocal_Ceed(X, Y, user); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mass/src/setup-dm.c b/examples/Hdiv-mass/src/setup-dm.c new file mode 100644 index 0000000000..9bc908a8c4 --- /dev/null +++ b/examples/Hdiv-mass/src/setup-dm.c @@ -0,0 +1,45 @@ +#include "../include/setup-dm.h" + +// --------------------------------------------------------------------------- +// Set-up DM +// --------------------------------------------------------------------------- +PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm) { + PetscErrorCode ierr; + PetscSection sec; + PetscBool interpolate = PETSC_TRUE; + PetscInt nx = 1, ny = 1; + PetscInt faces[2] = {nx, ny}; + PetscInt dim = 2, dofs_per_edge; + PetscInt p_start, p_end; + PetscInt c_start, c_end; // cells + PetscInt e_start, e_end, e; // edges + PetscInt v_start, v_end; // vertices + + PetscFunctionBeginUser; + + ierr = DMPlexCreateBoxMesh(comm, dim, PETSC_FALSE, faces, NULL, + NULL, NULL, interpolate, dm); CHKERRQ(ierr); + // Get plex limits + ierr = DMPlexGetChart(*dm, &p_start, &p_end); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(*dm, 0, &c_start, &c_end); CHKERRQ(ierr); + ierr = DMPlexGetDepthStratum(*dm, 1, &e_start, &e_end); CHKERRQ(ierr); + ierr = DMPlexGetDepthStratum(*dm, 0, &v_start, &v_end); CHKERRQ(ierr); + // Create section + ierr = PetscSectionCreate(comm, &sec); CHKERRQ(ierr); + ierr = PetscSectionSetNumFields(sec,1); CHKERRQ(ierr); + ierr = PetscSectionSetFieldName(sec,0,"Velocity"); CHKERRQ(ierr); + ierr = PetscSectionSetFieldComponents(sec,0,1); CHKERRQ(ierr); + ierr = PetscSectionSetChart(sec,p_start,p_end); CHKERRQ(ierr); + // Setup dofs per edge + for (e = e_start; e < e_end; e++) { + ierr = DMPlexGetConeSize(*dm, e, &dofs_per_edge); CHKERRQ(ierr); + ierr = PetscSectionSetFieldDof(sec, e, 0, dofs_per_edge); CHKERRQ(ierr); + ierr = PetscSectionSetDof (sec, e, dofs_per_edge); CHKERRQ(ierr); + } + ierr = PetscSectionSetUp(sec); CHKERRQ(ierr); + ierr = DMSetSection(*dm,sec); CHKERRQ(ierr); + ierr = DMViewFromOptions(*dm, NULL, "-dm_view"); CHKERRQ(ierr); + ierr = PetscSectionDestroy(&sec); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; \ No newline at end of file diff --git a/examples/Hdiv-mass/src/setup-libceed.c b/examples/Hdiv-mass/src/setup-libceed.c new file mode 100644 index 0000000000..c7332534be --- /dev/null +++ b/examples/Hdiv-mass/src/setup-libceed.c @@ -0,0 +1,454 @@ +#include "../include/setup-libceed.h" +#include "../include/petsc-macros.h" +#include "../basis/quad.h" + +// ----------------------------------------------------------------------------- +// Convert PETSc MemType to libCEED MemType +// ----------------------------------------------------------------------------- +CeedMemType MemTypeP2C(PetscMemType mem_type) { + return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; +} +// ----------------------------------------------------------------------------- +// Destroy libCEED objects +// ----------------------------------------------------------------------------- +PetscErrorCode CeedDataDestroy(CeedData ceed_data) { + PetscErrorCode ierr; + + PetscFunctionBegin; + + // Vectors + CeedVectorDestroy(&ceed_data->x_ceed); + CeedVectorDestroy(&ceed_data->y_ceed); + // Restrictions + CeedElemRestrictionDestroy(&ceed_data->elem_restr_x); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_u); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_geo_data_i); + // Bases + CeedBasisDestroy(&ceed_data->basis_x); + CeedBasisDestroy(&ceed_data->basis_u); + // QFunctions + CeedQFunctionDestroy(&ceed_data->qf_residual); + // Operators + CeedOperatorDestroy(&ceed_data->op_residual); + ierr = PetscFree(ceed_data); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// Utility function - essential BC dofs are encoded in closure indices as -(i+1) +// ----------------------------------------------------------------------------- +PetscInt Involute(PetscInt i) { + return i >= 0 ? i : -(i + 1); +}; +// ----------------------------------------------------------------------------- +// Get CEED restriction data from DMPlex +// ----------------------------------------------------------------------------- +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, + CeedInt height, DMLabel domain_label, CeedInt value, CeedInt P, + CeedElemRestriction *elem_restr) { + PetscSection section; + PetscInt p, num_elem, num_dof, *restr_indices, elem_offset, num_fields, + dim, depth; + Vec U_loc; + DMLabel depth_label; + IS depth_is, iter_is; + const PetscInt *iter_indices; + PetscErrorCode ierr; + + PetscFunctionBeginUser; + + ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr); + dim -= height; + ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); + ierr = PetscSectionGetNumFields(section, &num_fields); CHKERRQ(ierr); + PetscInt num_comp[num_fields], field_offsets[num_fields+1]; + field_offsets[0] = 0; + for (PetscInt f = 0; f < num_fields; f++) { + ierr = PetscSectionGetFieldComponents(section, f, &num_comp[f]); CHKERRQ(ierr); + field_offsets[f+1] = field_offsets[f] + num_comp[f]; + } + + ierr = DMPlexGetDepth(dm, &depth); CHKERRQ(ierr); + ierr = DMPlexGetDepthLabel(dm, &depth_label); CHKERRQ(ierr); + ierr = DMLabelGetStratumIS(depth_label, depth - height, &depth_is); + CHKERRQ(ierr); + if (domain_label) { + IS domain_is; + ierr = DMLabelGetStratumIS(domain_label, value, &domain_is); CHKERRQ(ierr); + if (domain_is) { // domainIS is non-empty + ierr = ISIntersect(depth_is, domain_is, &iter_is); CHKERRQ(ierr); + ierr = ISDestroy(&domain_is); CHKERRQ(ierr); + } else { // domainIS is NULL (empty) + iter_is = NULL; + } + ierr = ISDestroy(&depth_is); CHKERRQ(ierr); + } else { + iter_is = depth_is; + } + if (iter_is) { + ierr = ISGetLocalSize(iter_is, &num_elem); CHKERRQ(ierr); + ierr = ISGetIndices(iter_is, &iter_indices); CHKERRQ(ierr); + } else { + num_elem = 0; + iter_indices = NULL; + } + ierr = PetscMalloc1(num_elem*PetscPowInt(P, dim), &restr_indices); + CHKERRQ(ierr); + for (p = 0, elem_offset = 0; p < num_elem; p++) { + PetscInt c = iter_indices[p]; + PetscInt num_indices, *indices, num_nodes; + ierr = DMPlexGetClosureIndices(dm, section, section, c, PETSC_TRUE, + &num_indices, &indices, NULL, NULL); + CHKERRQ(ierr); + bool flip = false; + if (height > 0) { + PetscInt num_cells, num_faces, start = -1; + const PetscInt *orients, *faces, *cells; + ierr = DMPlexGetSupport(dm, c, &cells); CHKERRQ(ierr); + ierr = DMPlexGetSupportSize(dm, c, &num_cells); CHKERRQ(ierr); + if (num_cells != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, + "Expected one cell in support of exterior face, but got %D cells", + num_cells); + ierr = DMPlexGetCone(dm, cells[0], &faces); CHKERRQ(ierr); + ierr = DMPlexGetConeSize(dm, cells[0], &num_faces); CHKERRQ(ierr); + for (PetscInt i=0; i 0) { + PetscInt num_cells, num_faces, start = -1; + const PetscInt *orients, *faces, *cells; + ierr = DMPlexGetSupport(dm, c, &cells); CHKERRQ(ierr); + ierr = DMPlexGetSupportSize(dm, c, &num_cells); CHKERRQ(ierr); + if (num_cells != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, + "Expected one cell in support of exterior face, but got %D cells", + num_cells); + ierr = DMPlexGetCone(dm, cells[0], &faces); CHKERRQ(ierr); + ierr = DMPlexGetConeSize(dm, cells[0], &num_faces); CHKERRQ(ierr); + for (PetscInt i=0; idegree + 1; + CeedInt Q = P + 1 + app_ctx->q_extra; // Number of quadratures in 1D + CeedInt num_qpts = Q*Q; // Number of quadratures per element + CeedInt dim, num_comp_x, num_comp_u; + CeedInt geo_data_size = problem_data->geo_data_size; + CeedInt elem_node = problem_data->elem_node; + DM dm_coord; + Vec coords; + PetscInt c_start, c_end, num_elem; + const PetscScalar *coordArray; + CeedVector x_coord; + CeedQFunction qf_setup_rhs, qf_residual; + CeedOperator op_setup_rhs, op_residual; + + PetscFunctionBeginUser; + // --------------------------------------------------------------------------- + // libCEED bases:Hdiv basis_u and Lagrange basis_x + // --------------------------------------------------------------------------- + ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr); + num_comp_x = dim; + num_comp_u = 1; // one vector dof + CeedInt elem_dof = dim*elem_node; // dof per element + CeedScalar q_ref[dim*num_qpts], q_weights[num_qpts]; + CeedScalar div[elem_dof*num_qpts], interp[dim*elem_dof*num_qpts]; + QuadBasis(Q, q_ref, q_weights, interp, div); + CeedBasisCreateHdiv(ceed, CEED_QUAD, num_comp_u, elem_node, num_qpts, + interp, div, q_ref, q_weights, &ceed_data->basis_u); + CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, + problem_data->quadrature_mode, &ceed_data->basis_x); + // --------------------------------------------------------------------------- + // libCEED restrictions + // --------------------------------------------------------------------------- + ierr = DMGetCoordinateDM(dm, &dm_coord); CHKERRQ(ierr); + ierr = DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL); + CHKERRQ(ierr); + CeedInt height = 0; // No BC + DMLabel domain_label = 0; + PetscInt value = 0; + // -- Coordinate restriction + ierr = CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, + value, 2, &ceed_data->elem_restr_x); CHKERRQ(ierr); + // -- Solution restriction + ierr = CreateRestrictionFromPlexOriented(ceed, dm, height, domain_label, + value, P, &ceed_data->elem_restr_u); CHKERRQ(ierr); + // ---- Geometric ceed_data restriction + ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); + num_elem = c_end - c_start; + + CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, geo_data_size, + num_elem*num_qpts*geo_data_size, + CEED_STRIDES_BACKEND, &ceed_data->elem_restr_geo_data_i); + + // --------------------------------------------------------------------------- + // Element coordinates + // --------------------------------------------------------------------------- + ierr = DMGetCoordinatesLocal(dm, &coords); CHKERRQ(ierr); + ierr = VecGetArrayRead(coords, &coordArray); CHKERRQ(ierr); + + CeedElemRestrictionCreateVector(ceed_data->elem_restr_x, &x_coord, NULL); + CeedVectorSetArray(x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, + (PetscScalar *)coordArray); + ierr = VecRestoreArrayRead(coords, &coordArray); CHKERRQ(ierr); + + // --------------------------------------------------------------------------- + // Setup RHS and true solution + // --------------------------------------------------------------------------- + // Create the q-function that sets up the RHS and true solution + CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_rhs, + problem_data->setup_rhs_loc, &qf_setup_rhs); + CeedQFunctionAddInput(qf_setup_rhs, "x", num_comp_x, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_setup_rhs, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_setup_rhs, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddOutput(qf_setup_rhs, "rhs", dim, CEED_EVAL_INTERP); + // Create the operator that builds the RHS and true solution + CeedOperatorCreate(ceed, qf_setup_rhs, NULL, NULL, &op_setup_rhs); + CeedOperatorSetField(op_setup_rhs, "x", ceed_data->elem_restr_x, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup_rhs, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_setup_rhs, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup_rhs, "rhs", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + + // Setup RHS and target + CeedOperatorApply(op_setup_rhs, x_coord, rhs_ceed, CEED_REQUEST_IMMEDIATE); + + + // --------------------------------------------------------------------------- + // Persistent libCEED vectors + // --------------------------------------------------------------------------- + // -- Operator action variables + CeedVectorCreate(ceed, U_loc_size, &ceed_data->x_ceed); + CeedVectorCreate(ceed, U_loc_size, &ceed_data->y_ceed); + + // Local residual evaluator + // --------------------------------------------------------------------------- + // Create the QFunction and Operator that computes the residual of the PDE. + // --------------------------------------------------------------------------- + // -- QFunction + CeedQFunctionCreateInterior(ceed, 1, problem_data->residual, + problem_data->residual_loc, &qf_residual); + CeedQFunctionAddInput(qf_residual, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_residual, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_residual, "u", dim, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_residual, "v", dim, CEED_EVAL_INTERP); + + // -- Operator + CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_residual); + CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_residual, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "u", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "v", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + + // -- Save libCEED data + ceed_data->qf_residual = qf_residual; + ceed_data->op_residual = op_residual; + + // Cleanup + CeedQFunctionDestroy(&qf_setup_rhs); + CeedOperatorDestroy(&op_setup_rhs); + CeedVectorDestroy(&x_coord); + + PetscFunctionReturn(0); +}; +// ----------------------------------------------------------------------------- \ No newline at end of file diff --git a/examples/Hdiv-mixed/include/structs.h b/examples/Hdiv-mixed/include/structs.h index 6a094a41bb..4f75262e28 100644 --- a/examples/Hdiv-mixed/include/structs.h +++ b/examples/Hdiv-mixed/include/structs.h @@ -63,8 +63,8 @@ struct User_ { // Problem specific data typedef struct { - CeedQFunctionUser setup_geo, residual, setup_rhs; - const char *setup_geo_loc, *residual_loc, *setup_rhs_loc; + CeedQFunctionUser residual, setup_rhs; + const char *residual_loc, *setup_rhs_loc; CeedQuadMode quadrature_mode; CeedInt geo_data_size, elem_node; PetscErrorCode (*setup_ctx)(Ceed, CeedData, Physics); diff --git a/examples/Hdiv-mixed/problems/poisson-quad2d.c b/examples/Hdiv-mixed/problems/poisson-quad2d.c index 9dbbce5057..609696d6ad 100644 --- a/examples/Hdiv-mixed/problems/poisson-quad2d.c +++ b/examples/Hdiv-mixed/problems/poisson-quad2d.c @@ -19,7 +19,6 @@ #include "../include/setup-libceed.h" #include "../include/problems.h" -#include "../qfunctions/setup-geo2d.h" #include "../qfunctions/poisson-quad2d.h" // Hdiv_POISSON_QUAD2D is registered in cl-option.c PetscErrorCode Hdiv_POISSON_QUAD2D(ProblemData *problem_data, void *ctx) { @@ -34,10 +33,8 @@ PetscErrorCode Hdiv_POISSON_QUAD2D(ProblemData *problem_data, void *ctx) { // SET UP POISSON_QUAD2D // ------------------------------------------------------ problem_data->elem_node = 4; - problem_data->geo_data_size = 5; + problem_data->geo_data_size = 1; problem_data->quadrature_mode = CEED_GAUSS; - problem_data->setup_geo = SetupGeo2D; - problem_data->setup_geo_loc = SetupGeo2D_loc; problem_data->residual = PoissonQuadF; problem_data->residual_loc = PoissonQuadF_loc; problem_data->setup_rhs = SetupRhs; diff --git a/examples/Hdiv-mixed/qfunctions/poisson-quad2d.h b/examples/Hdiv-mixed/qfunctions/poisson-quad2d.h index 41705d6e82..b1c1a489e1 100644 --- a/examples/Hdiv-mixed/qfunctions/poisson-quad2d.h +++ b/examples/Hdiv-mixed/qfunctions/poisson-quad2d.h @@ -70,6 +70,7 @@ CEED_QFUNCTION(PoissonQuadF)(void *ctx, CeedInt Q, const CeedScalar *const *in, const CeedScalar detJ = JJ[0][0]*JJ[1][1] - JJ[0][1]*JJ[1][0]; const CeedScalar u1[2] = {u[0][i], u[1][i]}; + printf("===u1 %f\n",u1[0]); // *INDENT-ON* // Piola map: J^T*J*u*w/detJ // Compute J^T * J @@ -110,12 +111,12 @@ CEED_QFUNCTION(SetupRhs)(void *ctx, const CeedInt Q, CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*x)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])in[0], + const CeedScalar (*x) = in[0], (*w) = in[1], (*J)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[2]; // Outputs - CeedScalar (*true_soln)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], - (*rhs)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[1]; + CeedScalar (*true_soln) = out[0], + (*rhs) = out[1]; // *INDENT-ON* @@ -123,16 +124,16 @@ CEED_QFUNCTION(SetupRhs)(void *ctx, const CeedInt Q, CeedPragmaSIMD for (CeedInt i=0; ielem_restr_geo_data_i); @@ -256,8 +257,10 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, CeedElemRestrictionView(ceed_data->elem_restr_x, stdout); printf("----elem_restr_u:\n"); CeedElemRestrictionView(ceed_data->elem_restr_u, stdout); - //CeedElemRestrictionView(ceed_data->elem_restr_geo_data_i, stdout); - //CeedElemRestrictionView(ceed_data->elem_restr_u_i, stdout); + printf("----elem_restr_geo_data_i:\n"); + CeedElemRestrictionView(ceed_data->elem_restr_geo_data_i, stdout); + printf("----elem_restr_u_i:\n"); + CeedElemRestrictionView(ceed_data->elem_restr_u_i, stdout); // --------------------------------------------------------------------------- // Element coordinates @@ -276,6 +279,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // -- Operator action variables CeedVectorCreate(ceed, U_loc_size, &ceed_data->x_ceed); CeedVectorCreate(ceed, U_loc_size, &ceed_data->y_ceed); + /* // -- Geometric data vector CeedVectorCreate(ceed, num_elem*num_qpts*geo_data_size, &ceed_data->geo_data); @@ -308,7 +312,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // -- Cleanup CeedQFunctionDestroy(&qf_setup_geo); CeedOperatorDestroy(&op_setup_geo); - + */ // --------------------------------------------------------------------------- // Local residual evaluator // --------------------------------------------------------------------------- diff --git a/interface/ceed-basis.c b/interface/ceed-basis.c index 714a0a2e24..5a9358e072 100644 --- a/interface/ceed-basis.c +++ b/interface/ceed-basis.c @@ -1096,6 +1096,7 @@ int CeedBasisCreateH1(Ceed ceed, CeedElemTopology topo, CeedInt num_comp, CeedIn @param ceed A Ceed object where the CeedBasis will be created @param topo Topology of element, e.g. hypercube, simplex, ect + @param num_comp Number of componenet, we have 1 vector componenet in H(div) @param num_nodes Total number of nodes @param num_qpts Total number of quadrature points @param basis_space 2 for H(div) discretization (1 for H^1, 3 for H(curl)) From bae6bd6d36c3631e9e806ca9fbe8761341cc3190 Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Tue, 28 Dec 2021 11:13:45 -0700 Subject: [PATCH 05/15] Fixed the error of multiple elements! Authored-by: Jed Brown --- examples/Hdiv-mass/include/matops.h | 4 +- examples/Hdiv-mass/include/setup-libceed.h | 7 +- examples/Hdiv-mass/include/structs.h | 11 +- examples/Hdiv-mass/main.c | 35 +++++- examples/Hdiv-mass/problems/poisson-mass2d.c | 7 +- .../Hdiv-mass/qfunctions/poisson-error2d.h | 56 +++++++++ .../Hdiv-mass/qfunctions/poisson-mass2d.h | 13 +-- examples/Hdiv-mass/qfunctions/poisson-rhs2d.h | 22 ++-- .../Hdiv-mass/qfunctions/poisson-true2d.h | 80 +++++++++++++ examples/Hdiv-mass/src/cl-options.c | 2 +- examples/Hdiv-mass/src/matops.c | 44 ++++++- examples/Hdiv-mass/src/setup-libceed.c | 109 +++++++++++++++--- 12 files changed, 338 insertions(+), 52 deletions(-) create mode 100644 examples/Hdiv-mass/qfunctions/poisson-error2d.h create mode 100644 examples/Hdiv-mass/qfunctions/poisson-true2d.h diff --git a/examples/Hdiv-mass/include/matops.h b/examples/Hdiv-mass/include/matops.h index 5685636a78..aa960557db 100644 --- a/examples/Hdiv-mass/include/matops.h +++ b/examples/Hdiv-mass/include/matops.h @@ -6,7 +6,9 @@ #include "structs.h" -PetscErrorCode ApplyLocal_Ceed(Vec X, Vec Y, User user); +PetscErrorCode ApplyLocal_Ceed(User user, Vec X, Vec Y); PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y); +PetscErrorCode ComputeError(User user, Vec X, CeedVector target, + CeedScalar *l2_error); #endif // matops_h diff --git a/examples/Hdiv-mass/include/setup-libceed.h b/examples/Hdiv-mass/include/setup-libceed.h index 500d47a61c..0e9c6f08e4 100644 --- a/examples/Hdiv-mass/include/setup-libceed.h +++ b/examples/Hdiv-mass/include/setup-libceed.h @@ -19,7 +19,8 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedElemRestriction *elem_restr_oriented); // Set up libCEED for a given degree PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, - ProblemData *problem_data, - PetscInt U_g_size, PetscInt U_loc_size, - CeedData ceed_data, CeedVector rhs_ceed); + ProblemData *problem_data, PetscInt U_g_size, + PetscInt U_loc_size, CeedData ceed_data, + CeedVector rhs_ceed, CeedVector *target, + CeedVector true_ceed); #endif // setuplibceed_h diff --git a/examples/Hdiv-mass/include/structs.h b/examples/Hdiv-mass/include/structs.h index 391ae98b39..41c3ecd864 100644 --- a/examples/Hdiv-mass/include/structs.h +++ b/examples/Hdiv-mass/include/structs.h @@ -21,8 +21,8 @@ struct CeedData_ { CeedBasis basis_x, basis_u; CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_geo_data_i, elem_restr_u_i; - CeedQFunction qf_residual; - CeedOperator op_residual; + CeedQFunction qf_residual, qf_error; + CeedOperator op_residual, op_error; CeedVector x_ceed, y_ceed; CeedQFunctionContext pq2d_context; }; @@ -54,7 +54,7 @@ struct User_ { MPI_Comm comm; Vec X_loc, Y_loc; CeedVector x_ceed, y_ceed; - CeedOperator op; + CeedOperator op_apply, op_error; DM dm; Ceed ceed; AppCtx app_ctx; @@ -63,8 +63,9 @@ struct User_ { // Problem specific data typedef struct { - CeedQFunctionUser setup_rhs, residual; - const char *setup_rhs_loc, *residual_loc; + CeedQFunctionUser setup_rhs, residual, setup_error, setup_true; + const char *setup_rhs_loc, *residual_loc, *setup_error_loc, + *setup_true_loc; CeedQuadMode quadrature_mode; CeedInt geo_data_size, elem_node; PetscErrorCode (*setup_ctx)(Ceed, CeedData, Physics); diff --git a/examples/Hdiv-mass/main.c b/examples/Hdiv-mass/main.c index 1f7ce93288..15b41871c5 100644 --- a/examples/Hdiv-mass/main.c +++ b/examples/Hdiv-mass/main.c @@ -62,7 +62,7 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // -- Initialize backend Ceed ceed; - CeedInit("/cpu/self/opt/serial", &ceed); + CeedInit("/cpu/self/ref/serial", &ceed); CeedMemType mem_type_backend; CeedGetPreferredMemType(ceed, &mem_type_backend); // --------------------------------------------------------------------------- @@ -106,20 +106,36 @@ int main(int argc, char **argv) { // Get RHS vector Vec rhs_loc; PetscScalar *r; - CeedVector rhs_ceed; + CeedVector rhs_ceed, target; PetscMemType mem_type; ierr = VecDuplicate(U_loc, &rhs_loc); CHKERRQ(ierr); ierr = VecZeroEntries(rhs_loc); CHKERRQ(ierr); ierr = VecGetArrayAndMemType(rhs_loc, &r, &mem_type); CHKERRQ(ierr); CeedVectorCreate(ceed, U_l_size, &rhs_ceed); CeedVectorSetArray(rhs_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, r); + // Get True vector + Vec true_loc; + PetscScalar *t; + CeedVector true_ceed; + PetscMemType t_mem_type; + ierr = VecDuplicate(U_loc, &true_loc); CHKERRQ(ierr); + ierr = VecZeroEntries(true_loc); CHKERRQ(ierr); + ierr = VecGetArrayAndMemType(true_loc, &t, &t_mem_type); CHKERRQ(ierr); + CeedVectorCreate(ceed, U_l_size, &true_ceed); + CeedVectorSetArray(true_ceed, MemTypeP2C(t_mem_type), CEED_USE_POINTER, t); + // --------------------------------------------------------------------------- // Setup libCEED // --------------------------------------------------------------------------- // -- Set up libCEED objects ierr = SetupLibceed(dm, ceed, app_ctx, problem_data, U_g_size, - U_loc_size, ceed_data, rhs_ceed); CHKERRQ(ierr); + U_loc_size, ceed_data, rhs_ceed, &target, true_ceed); CHKERRQ(ierr); //CeedVectorView(rhs_ceed, "%12.8f", stdout); + printf("------------------------------------------------------------\n"); + printf("True solution projected into H(div) space; main.c:\n"); + printf("------------------------------------------------------------\n"); + CeedVectorView(true_ceed, "%12.8f", stdout); + // --------------------------------------------------------------------------- // Gather RHS // --------------------------------------------------------------------------- @@ -138,7 +154,8 @@ int main(int argc, char **argv) { ierr = VecDuplicate(U_loc, &user->Y_loc); CHKERRQ(ierr); user->x_ceed = ceed_data->x_ceed; user->y_ceed = ceed_data->y_ceed; - user->op = ceed_data->op_residual; + user->op_apply = ceed_data->op_residual; + user->op_error = ceed_data->op_error; user->ceed = ceed; // Operator Mat mat; @@ -154,7 +171,13 @@ int main(int argc, char **argv) { ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); ierr = KSPSolve(ksp, rhs, U_g); CHKERRQ(ierr); - + //VecView(U_g, PETSC_VIEWER_STDOUT_WORLD); + // --------------------------------------------------------------------------- + // Compute pointwise L2 error + // --------------------------------------------------------------------------- + CeedScalar l2_error; + ierr = ComputeError(user, U_g, target, &l2_error); CHKERRQ(ierr); + //printf("l2_error: %f\n",l2_error); // Output results KSPType ksp_type; KSPConvergedReason reason; @@ -198,7 +221,9 @@ int main(int argc, char **argv) { ierr = PetscFree(phys_ctx); CHKERRQ(ierr); // Free libCEED objects + CeedVectorDestroy(&true_ceed); CeedVectorDestroy(&rhs_ceed); + CeedVectorDestroy(&target); ierr = CeedDataDestroy(ceed_data); CHKERRQ(ierr); CeedDestroy(&ceed); diff --git a/examples/Hdiv-mass/problems/poisson-mass2d.c b/examples/Hdiv-mass/problems/poisson-mass2d.c index 3faa056345..1ac1bf9e5b 100644 --- a/examples/Hdiv-mass/problems/poisson-mass2d.c +++ b/examples/Hdiv-mass/problems/poisson-mass2d.c @@ -21,6 +21,8 @@ #include "../include/problems.h" #include "../qfunctions/poisson-rhs2d.h" #include "../qfunctions/poisson-mass2d.h" +#include "../qfunctions/poisson-error2d.h" +#include "../qfunctions/poisson-true2d.h" // Hdiv_POISSON_MASS2D is registered in cl-option.c PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx) { @@ -41,7 +43,10 @@ PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx) { problem_data->setup_rhs_loc = SetupRhs_loc; problem_data->residual = SetupMass; problem_data->residual_loc = SetupMass_loc; - + problem_data->setup_error = SetupError2D; + problem_data->setup_error_loc = SetupError2D_loc; + problem_data->setup_true = SetupTrueSoln2D; + problem_data->setup_true_loc = SetupTrueSoln2D_loc; // ------------------------------------------------------ // Command line Options // ------------------------------------------------------ diff --git a/examples/Hdiv-mass/qfunctions/poisson-error2d.h b/examples/Hdiv-mass/qfunctions/poisson-error2d.h new file mode 100644 index 0000000000..1bdfd8ad1c --- /dev/null +++ b/examples/Hdiv-mass/qfunctions/poisson-error2d.h @@ -0,0 +1,56 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Compute error of the H(div) example using PETSc + +#ifndef ERROR_H +#define ERROR_H + +#include + +// ----------------------------------------------------------------------------- +// Compuet error +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(SetupError2D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*target) = in[3]; + // Outputs + CeedScalar (*error) = out[0]; + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i + +// ----------------------------------------------------------------------------- +// Compuet true solution +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(SetupTrueSoln2D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*coords) = in[0], + (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1]; + // Outputs + CeedScalar (*true_soln_Hdiv) = out[0]; + // Quadrature Point Loop + printf("True solution projected into H(div) space;Qfunction poisson-true2d.h\n"); + CeedPragmaSIMD + for (CeedInt i=0; idegree, &app_ctx->degree, NULL); CHKERRQ(ierr); - app_ctx->q_extra = 0; + app_ctx->q_extra = 2; ierr = PetscOptionsInt("-q_extra", "Number of extra quadrature points", NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL); CHKERRQ(ierr); diff --git a/examples/Hdiv-mass/src/matops.c b/examples/Hdiv-mass/src/matops.c index 328f10407e..ed1fd4cfeb 100644 --- a/examples/Hdiv-mass/src/matops.c +++ b/examples/Hdiv-mass/src/matops.c @@ -5,7 +5,7 @@ // This function uses libCEED to compute the action of the Laplacian with // Dirichlet boundary conditions // ----------------------------------------------------------------------------- -PetscErrorCode ApplyLocal_Ceed(Vec X, Vec Y, User user) { +PetscErrorCode ApplyLocal_Ceed(User user, Vec X, Vec Y) { PetscErrorCode ierr; PetscScalar *x, *y; PetscMemType x_mem_type, y_mem_type; @@ -23,7 +23,8 @@ PetscErrorCode ApplyLocal_Ceed(Vec X, Vec Y, User user) { CeedVectorSetArray(user->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); // Apply libCEED operator - CeedOperatorApply(user->op, user->x_ceed, user->y_ceed, CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(user->op_apply, user->x_ceed, user->y_ceed, + CEED_REQUEST_IMMEDIATE); // Restore PETSc vectors CeedVectorTakeArray(user->x_ceed, MemTypeP2C(x_mem_type), NULL); @@ -51,7 +52,44 @@ PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y) { ierr = MatShellGetContext(A, &user); CHKERRQ(ierr); // libCEED for local action of residual evaluator - ierr = ApplyLocal_Ceed(X, Y, user); CHKERRQ(ierr); + ierr = ApplyLocal_Ceed(user, X, Y); CHKERRQ(ierr); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// This function calculates the error in the final solution +// ----------------------------------------------------------------------------- +PetscErrorCode ComputeError(User user, Vec X, CeedVector target, + CeedScalar *l2_error) { + PetscErrorCode ierr; + PetscScalar *x; + PetscMemType mem_type; + CeedVector collocated_error; + CeedInt length; + + PetscFunctionBeginUser; + CeedVectorGetLength(target, &length); + CeedVectorCreate(user->ceed, length, &collocated_error); + + // Global-to-local + ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); CHKERRQ(ierr); + + // Setup CEED vector + ierr = VecGetArrayAndMemType(user->X_loc, &x, &mem_type); CHKERRQ(ierr); + CeedVectorSetArray(user->x_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, x); + + // Apply CEED operator + CeedOperatorApply(user->op_error, user->x_ceed, collocated_error, + CEED_REQUEST_IMMEDIATE); + // Restore PETSc vector + CeedVectorTakeArray(user->x_ceed, MemTypeP2C(mem_type), NULL); + ierr = VecRestoreArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x); + CHKERRQ(ierr); + + CeedVectorNorm(collocated_error, CEED_NORM_2, l2_error); + // Cleanup + CeedVectorDestroy(&collocated_error); PetscFunctionReturn(0); }; diff --git a/examples/Hdiv-mass/src/setup-libceed.c b/examples/Hdiv-mass/src/setup-libceed.c index c7332534be..8a5d3a995a 100644 --- a/examples/Hdiv-mass/src/setup-libceed.c +++ b/examples/Hdiv-mass/src/setup-libceed.c @@ -23,13 +23,16 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data) { CeedElemRestrictionDestroy(&ceed_data->elem_restr_x); CeedElemRestrictionDestroy(&ceed_data->elem_restr_u); CeedElemRestrictionDestroy(&ceed_data->elem_restr_geo_data_i); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_u_i); // Bases CeedBasisDestroy(&ceed_data->basis_x); CeedBasisDestroy(&ceed_data->basis_u); // QFunctions CeedQFunctionDestroy(&ceed_data->qf_residual); + CeedQFunctionDestroy(&ceed_data->qf_error); // Operators CeedOperatorDestroy(&ceed_data->op_residual); + CeedOperatorDestroy(&ceed_data->op_error); ierr = PetscFree(ceed_data); CHKERRQ(ierr); PetscFunctionReturn(0); @@ -305,10 +308,15 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, ierr = VecGetLocalSize(U_loc, &num_dof); CHKERRQ(ierr); ierr = DMRestoreLocalVector(dm, &U_loc); CHKERRQ(ierr); // dof per element in Hdiv is dim*P^dim, for linear element P=2 - CeedElemRestrictionCreateOriented(ceed, num_elem, dim*PetscPowInt(P, dim), - field_offsets[num_fields], - 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, - restr_indices, orient_dof, elem_restr_oriented); + //CeedElemRestrictionCreateOriented(ceed, num_elem, dim*PetscPowInt(P, dim), + // field_offsets[num_fields], + // 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, + // restr_indices, orient_dof, elem_restr_oriented); + // Test without orientation + CeedElemRestrictionCreate(ceed, num_elem, dim*PetscPowInt(P, dim), + field_offsets[num_fields], + 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, + restr_indices, elem_restr_oriented); ierr = PetscFree(restr_indices); CHKERRQ(ierr); ierr = PetscFree(orient_dof); CHKERRQ(ierr); PetscFunctionReturn(0); @@ -318,11 +326,14 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, // Set up libCEED on the fine grid for a given degree // ----------------------------------------------------------------------------- PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, - ProblemData *problem_data, PetscInt U_g_size, PetscInt U_loc_size, - CeedData ceed_data, CeedVector rhs_ceed) { + ProblemData *problem_data, PetscInt U_g_size, + PetscInt U_loc_size, CeedData ceed_data, + CeedVector rhs_ceed, CeedVector *target, + CeedVector true_ceed) { int ierr; CeedInt P = app_ctx->degree + 1; - CeedInt Q = P + 1 + app_ctx->q_extra; // Number of quadratures in 1D + // Number of quadratures in 1D, q_extra defined in cl-options.c + CeedInt Q = P + 1 + app_ctx->q_extra; CeedInt num_qpts = Q*Q; // Number of quadratures per element CeedInt dim, num_comp_x, num_comp_u; CeedInt geo_data_size = problem_data->geo_data_size; @@ -332,8 +343,8 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, PetscInt c_start, c_end, num_elem; const PetscScalar *coordArray; CeedVector x_coord; - CeedQFunction qf_setup_rhs, qf_residual; - CeedOperator op_setup_rhs, op_residual; + CeedQFunction qf_setup_rhs, qf_residual, qf_error; + CeedOperator op_setup_rhs, op_residual, op_error; PetscFunctionBeginUser; // --------------------------------------------------------------------------- @@ -350,13 +361,14 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, interp, div, q_ref, q_weights, &ceed_data->basis_u); CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, problem_data->quadrature_mode, &ceed_data->basis_x); + // --------------------------------------------------------------------------- // libCEED restrictions // --------------------------------------------------------------------------- ierr = DMGetCoordinateDM(dm, &dm_coord); CHKERRQ(ierr); ierr = DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL); CHKERRQ(ierr); - CeedInt height = 0; // No BC + CeedInt height = 0; // 0 means no boundary conditions DMLabel domain_label = 0; PetscInt value = 0; // -- Coordinate restriction @@ -365,7 +377,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // -- Solution restriction ierr = CreateRestrictionFromPlexOriented(ceed, dm, height, domain_label, value, P, &ceed_data->elem_restr_u); CHKERRQ(ierr); - // ---- Geometric ceed_data restriction + // -- Geometric ceed_data restriction ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); num_elem = c_end - c_start; @@ -373,6 +385,10 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, num_elem*num_qpts*geo_data_size, CEED_STRIDES_BACKEND, &ceed_data->elem_restr_geo_data_i); + CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, dim, + dim*num_elem*num_qpts, + CEED_STRIDES_BACKEND, &ceed_data->elem_restr_u_i); + // --------------------------------------------------------------------------- // Element coordinates // --------------------------------------------------------------------------- @@ -387,25 +403,30 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // --------------------------------------------------------------------------- // Setup RHS and true solution // --------------------------------------------------------------------------- + CeedVectorCreate(ceed, num_elem*num_qpts*dim, target); // Create the q-function that sets up the RHS and true solution CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_rhs, problem_data->setup_rhs_loc, &qf_setup_rhs); CeedQFunctionAddInput(qf_setup_rhs, "x", num_comp_x, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_setup_rhs, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddInput(qf_setup_rhs, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddOutput(qf_setup_rhs, "true_soln", dim, CEED_EVAL_NONE); CeedQFunctionAddOutput(qf_setup_rhs, "rhs", dim, CEED_EVAL_INTERP); // Create the operator that builds the RHS and true solution - CeedOperatorCreate(ceed, qf_setup_rhs, NULL, NULL, &op_setup_rhs); + CeedOperatorCreate(ceed, qf_setup_rhs, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_setup_rhs); CeedOperatorSetField(op_setup_rhs, "x", ceed_data->elem_restr_x, ceed_data->basis_x, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_setup_rhs, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); CeedOperatorSetField(op_setup_rhs, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup_rhs, "true_soln", ceed_data->elem_restr_u_i, + CEED_BASIS_COLLOCATED, *target); CeedOperatorSetField(op_setup_rhs, "rhs", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); - // Setup RHS and target + // Setup RHS and true solution CeedOperatorApply(op_setup_rhs, x_coord, rhs_ceed, CEED_REQUEST_IMMEDIATE); @@ -434,16 +455,74 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); CeedOperatorSetField(op_residual, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); + ceed_data->basis_x, x_coord); CeedOperatorSetField(op_residual, "u", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_residual, "v", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); - // -- Save libCEED data + // -- Save libCEED data to apply operator in matops.c ceed_data->qf_residual = qf_residual; ceed_data->op_residual = op_residual; + // --------------------------------------------------------------------------- + // Setup Error Qfunction + // --------------------------------------------------------------------------- + // Create the q-function that sets up the error + CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_error, + problem_data->setup_error_loc, &qf_error); + CeedQFunctionAddInput(qf_error, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_error, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_error, "u", dim, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_error, "true_soln", dim, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_error, "error", dim, CEED_EVAL_NONE); + // Create the operator that builds the error + CeedOperatorCreate(ceed, qf_error, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_error); + CeedOperatorSetField(op_error, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_error, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, x_coord); + CeedOperatorSetField(op_error, "u", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_error, "true_soln", ceed_data->elem_restr_u_i, + CEED_BASIS_COLLOCATED, *target); + CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_u_i, + CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + // -- Save libCEED data to apply operator in matops.c + ceed_data->qf_error = qf_error; + ceed_data->op_error = op_error; + + // --------------------------------------------------------------------------- + // Setup True Qfunction: True solution projected to H(div) space + // --------------------------------------------------------------------------- + CeedBasis basis_true; + CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, 2, + CEED_GAUSS_LOBATTO, &basis_true); + CeedQFunction qf_true; + CeedOperator op_true; + // Create the q-function that sets up the true solution in H(div) space + CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_true, + problem_data->setup_true_loc, &qf_true); + CeedQFunctionAddInput(qf_true, "x", num_comp_x, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_true, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddOutput(qf_true, "true_soln_Hdiv", dim, CEED_EVAL_NONE); + // Create the operator that builds the true solution in H(div) space + CeedOperatorCreate(ceed, qf_true, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_true); + CeedOperatorSetField(op_true, "x", ceed_data->elem_restr_x, + basis_true, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_true, "dx", ceed_data->elem_restr_x, + basis_true, x_coord); + CeedOperatorSetField(op_true, "true_soln_Hdiv", ceed_data->elem_restr_u, + CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + + CeedOperatorApply(op_true, x_coord, true_ceed, CEED_REQUEST_IMMEDIATE); + // Cleanup + CeedBasisDestroy(&basis_true); + CeedQFunctionDestroy(&qf_true); + CeedOperatorDestroy(&op_true); + // Cleanup CeedQFunctionDestroy(&qf_setup_rhs); CeedOperatorDestroy(&op_setup_rhs); From f9e5d75da603dbc30b7cb8d04bf938f2330b5029 Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Sun, 2 Jan 2022 20:14:28 -0700 Subject: [PATCH 06/15] Added snes,got convergence for 2D/3D Darcy problem --- examples/Hdiv-mass/Makefile | 20 +- examples/Hdiv-mass/basis/Hdiv-hex.h | 162 +++++ examples/Hdiv-mass/basis/Hdiv-quad.h | 87 +++ examples/Hdiv-mass/basis/quad.h | 78 --- examples/Hdiv-mass/conv_plot.py | 62 ++ examples/Hdiv-mass/conv_test.sh | 68 +++ examples/Hdiv-mass/conv_test_result.csv | 5 + examples/Hdiv-mass/convrate_mass.png | Bin 0 -> 25544 bytes examples/Hdiv-mass/include/problems.h | 1 + examples/Hdiv-mass/include/setup-dm.h | 4 +- examples/Hdiv-mass/include/setup-libceed.h | 8 +- examples/Hdiv-mass/include/structs.h | 17 +- examples/Hdiv-mass/main.c | 77 ++- .../problems/{poisson-mass2d.c => mass2d.c} | 19 +- examples/Hdiv-mass/problems/mass3d.c | 56 ++ .../Hdiv-mass/qfunctions/poisson-error2d.h | 28 +- .../Hdiv-mass/qfunctions/poisson-error3d.h | 80 +++ .../Hdiv-mass/qfunctions/poisson-mass2d.h | 6 +- .../Hdiv-mass/qfunctions/poisson-mass3d.h | 100 ++++ examples/Hdiv-mass/qfunctions/poisson-rhs2d.h | 16 +- examples/Hdiv-mass/qfunctions/poisson-rhs3d.h | 88 +++ examples/Hdiv-mass/src/cl-options.c | 15 +- examples/Hdiv-mass/src/matops.c | 6 +- examples/Hdiv-mass/src/setup-dm.c | 34 +- examples/Hdiv-mass/src/setup-libceed.c | 359 +++-------- examples/Hdiv-mixed/Makefile | 20 +- examples/Hdiv-mixed/basis/Hdiv-hex.h | 164 +++++ examples/Hdiv-mixed/basis/Hdiv-quad.h | 89 +++ examples/Hdiv-mixed/basis/L2-P0.h | 64 ++ examples/Hdiv-mixed/basis/quad.h | 78 --- examples/Hdiv-mixed/conv_plot.py | 69 +++ examples/Hdiv-mixed/conv_test.sh | 70 +++ examples/Hdiv-mixed/conv_test_result.csv | 12 + examples/Hdiv-mixed/convrate_mixed.png | Bin 0 -> 28667 bytes examples/Hdiv-mixed/include/cl-options.h | 7 +- examples/Hdiv-mixed/include/matops.h | 13 - examples/Hdiv-mixed/include/problems.h | 21 - .../Hdiv-mixed/include/register-problem.h | 24 + examples/Hdiv-mixed/include/setup-boundary.h | 25 + examples/Hdiv-mixed/include/setup-dm.h | 6 +- examples/Hdiv-mixed/include/setup-fe.h | 15 + examples/Hdiv-mixed/include/setup-libceed.h | 15 +- examples/Hdiv-mixed/include/setup-matops.h | 14 + examples/Hdiv-mixed/include/setup-solvers.h | 28 + examples/Hdiv-mixed/include/setup-ts.h | 17 + examples/Hdiv-mixed/include/structs.h | 112 ++-- examples/Hdiv-mixed/main.c | 296 ++++----- examples/Hdiv-mixed/main.h | 8 +- examples/Hdiv-mixed/problems/darcy2d.c | 85 +++ examples/Hdiv-mixed/problems/darcy3d.c | 85 +++ examples/Hdiv-mixed/problems/poisson-quad2d.c | 76 --- .../Hdiv-mixed/problems/register-problem.c | 46 ++ examples/Hdiv-mixed/problems/richard2d.c | 95 +++ .../qfunctions/darcy-error2d.h} | 80 ++- .../Hdiv-mixed/qfunctions/darcy-error3d.h | 81 +++ .../Hdiv-mixed/qfunctions/darcy-system2d.h | 232 ++++++++ .../Hdiv-mixed/qfunctions/darcy-system3d.h | 239 ++++++++ examples/Hdiv-mixed/qfunctions/darcy-true2d.h | 102 ++++ examples/Hdiv-mixed/qfunctions/darcy-true3d.h | 106 ++++ .../Hdiv-mixed/qfunctions/poisson-quad2d.h | 154 ----- .../qfunctions/pressure-boundary2d.h | 58 ++ .../qfunctions/pressure-boundary3d.h | 57 ++ .../Hdiv-mixed/qfunctions/richard-mms2d.h | 157 +++++ .../Hdiv-mixed/qfunctions/richard-system2d.h | 274 +++++++++ examples/Hdiv-mixed/qfunctions/utils.h | 205 +++++++ examples/Hdiv-mixed/src/cl-options.c | 59 +- examples/Hdiv-mixed/src/matops.c | 98 --- examples/Hdiv-mixed/src/setup-boundary.c | 114 ++++ examples/Hdiv-mixed/src/setup-dm.c | 85 +-- examples/Hdiv-mixed/src/setup-fe.c | 46 ++ examples/Hdiv-mixed/src/setup-libceed.c | 563 ++++++++++-------- examples/Hdiv-mixed/src/setup-matops.c | 61 ++ examples/Hdiv-mixed/src/setup-solvers.c | 354 +++++++++++ examples/Hdiv-mixed/src/setup-ts.c | 152 +++++ interface/ceed-basis.c | 72 --- tests/t330-basis.h | 2 +- tests/t999-Hdiv2D.c | 136 ----- tests/t999-Hdiv2D.h | 89 --- 78 files changed, 4736 insertions(+), 1790 deletions(-) create mode 100644 examples/Hdiv-mass/basis/Hdiv-hex.h create mode 100644 examples/Hdiv-mass/basis/Hdiv-quad.h delete mode 100644 examples/Hdiv-mass/basis/quad.h create mode 100644 examples/Hdiv-mass/conv_plot.py create mode 100755 examples/Hdiv-mass/conv_test.sh create mode 100644 examples/Hdiv-mass/conv_test_result.csv create mode 100644 examples/Hdiv-mass/convrate_mass.png rename examples/Hdiv-mass/problems/{poisson-mass2d.c => mass2d.c} (78%) create mode 100644 examples/Hdiv-mass/problems/mass3d.c create mode 100644 examples/Hdiv-mass/qfunctions/poisson-error3d.h create mode 100644 examples/Hdiv-mass/qfunctions/poisson-mass3d.h create mode 100644 examples/Hdiv-mass/qfunctions/poisson-rhs3d.h create mode 100644 examples/Hdiv-mixed/basis/Hdiv-hex.h create mode 100644 examples/Hdiv-mixed/basis/Hdiv-quad.h create mode 100644 examples/Hdiv-mixed/basis/L2-P0.h delete mode 100644 examples/Hdiv-mixed/basis/quad.h create mode 100644 examples/Hdiv-mixed/conv_plot.py create mode 100755 examples/Hdiv-mixed/conv_test.sh create mode 100644 examples/Hdiv-mixed/conv_test_result.csv create mode 100644 examples/Hdiv-mixed/convrate_mixed.png delete mode 100644 examples/Hdiv-mixed/include/matops.h delete mode 100644 examples/Hdiv-mixed/include/problems.h create mode 100644 examples/Hdiv-mixed/include/register-problem.h create mode 100644 examples/Hdiv-mixed/include/setup-boundary.h create mode 100644 examples/Hdiv-mixed/include/setup-fe.h create mode 100644 examples/Hdiv-mixed/include/setup-matops.h create mode 100644 examples/Hdiv-mixed/include/setup-solvers.h create mode 100644 examples/Hdiv-mixed/include/setup-ts.h create mode 100644 examples/Hdiv-mixed/problems/darcy2d.c create mode 100644 examples/Hdiv-mixed/problems/darcy3d.c delete mode 100644 examples/Hdiv-mixed/problems/poisson-quad2d.c create mode 100644 examples/Hdiv-mixed/problems/register-problem.c create mode 100644 examples/Hdiv-mixed/problems/richard2d.c rename examples/{Hdiv-mass/qfunctions/poisson-true2d.h => Hdiv-mixed/qfunctions/darcy-error2d.h} (52%) create mode 100644 examples/Hdiv-mixed/qfunctions/darcy-error3d.h create mode 100644 examples/Hdiv-mixed/qfunctions/darcy-system2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/darcy-system3d.h create mode 100644 examples/Hdiv-mixed/qfunctions/darcy-true2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/darcy-true3d.h delete mode 100644 examples/Hdiv-mixed/qfunctions/poisson-quad2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/pressure-boundary2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/pressure-boundary3d.h create mode 100644 examples/Hdiv-mixed/qfunctions/richard-mms2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/richard-system2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/utils.h delete mode 100644 examples/Hdiv-mixed/src/matops.c create mode 100644 examples/Hdiv-mixed/src/setup-boundary.c create mode 100644 examples/Hdiv-mixed/src/setup-fe.c create mode 100644 examples/Hdiv-mixed/src/setup-matops.c create mode 100644 examples/Hdiv-mixed/src/setup-solvers.c create mode 100644 examples/Hdiv-mixed/src/setup-ts.c delete mode 100644 tests/t999-Hdiv2D.c delete mode 100644 tests/t999-Hdiv2D.h diff --git a/examples/Hdiv-mass/Makefile b/examples/Hdiv-mass/Makefile index 5912d6f96e..6cfc950525 100644 --- a/examples/Hdiv-mass/Makefile +++ b/examples/Hdiv-mass/Makefile @@ -17,8 +17,8 @@ COMMON ?= ../../common.mk -include $(COMMON) - - +# Note: PETSC_ARCH can be undefined or empty for installations which do not use +# PETSC_ARCH - for example when using PETSc installed through Spack. PETSc.pc := $(PETSC_DIR)/$(PETSC_ARCH)/lib/pkgconfig/PETSc.pc CEED_DIR ?= ../.. ceed.pc := $(CEED_DIR)/lib/pkgconfig/ceed.pc @@ -38,16 +38,12 @@ OBJDIR := build SRCDIR := src PROBLEMDIR := problems -all: main +src.c := main.c $(sort $(wildcard $(PROBLEMDIR)/*.c)) $(sort $(wildcard $(SRCDIR)/*.c)) +src.o = $(src.c:%.c=$(OBJDIR)/%.o) -utils.c := $(sort $(wildcard $(PROBLEMDIR)/*.c)) $(sort $(wildcard $(SRCDIR)/*.c)) -utils.o = $(utils.c:%.c=$(OBJDIR)/%.o) -libutils.a: $(utils.o) - $(call quiet,AR) $(ARFLAGS) $@ $^ +all: main -main.c := main.c -main.o = $(main.c:%.c=$(OBJDIR)/%.o) -main: $(main.o) libutils.a | $(PETSc.pc) $(ceed.pc) +main: $(src.o) | $(PETSc.pc) $(ceed.pc) $(call quiet,LINK.o) $(CEED_LDFLAGS) $^ $(LOADLIBES) $(LDLIBS) -o $@ .SECONDEXPANSION: # to expand $$(@D)/.DIR @@ -73,7 +69,7 @@ print: $(PETSc.pc) $(ceed.pc) @true clean: - $(RM) -r $(OBJDIR) *.vtu main libutils.a + $(RM) -r $(OBJDIR) main *.vtu $(PETSc.pc): $(if $(wildcard $@),,$(error \ @@ -83,4 +79,4 @@ $(PETSc.pc): pkgconf = $(shell pkg-config $1 | sed -e 's/^"//g' -e 's/"$$//g') --include $(src.o:%.o=%.d) \ No newline at end of file +-include $(src.o:%.o=%.d) diff --git a/examples/Hdiv-mass/basis/Hdiv-hex.h b/examples/Hdiv-mass/basis/Hdiv-hex.h new file mode 100644 index 0000000000..a01b1a3ef3 --- /dev/null +++ b/examples/Hdiv-mass/basis/Hdiv-hex.h @@ -0,0 +1,162 @@ +// Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. +// All Rights reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +// To see how the nodal basis is constructed visit: +// https://github.com/rezgarshakeri/H-div-Tests +int NodalHdivBasisHex(CeedScalar *x, CeedScalar *Bx, CeedScalar *By, + CeedScalar *Bz) { + + Bx[ 0] = 0.0625*x[0]*x[0] - 0.0625 ; + By[ 0] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; + Bz[ 0] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bx[ 1] = 0.0625 - 0.0625*x[0]*x[0] ; + By[ 1] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; + Bz[ 1] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bx[ 2] = 0.0625*x[0]*x[0] - 0.0625 ; + By[ 2] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; + Bz[ 2] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bx[ 3] = 0.0625 - 0.0625*x[0]*x[0] ; + By[ 3] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; + Bz[ 3] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] + + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bx[ 4] = 0.0625*x[0]*x[0] - 0.0625 ; + By[ 4] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; + Bz[ 4] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] - + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bx[ 5] = 0.0625 - 0.0625*x[0]*x[0] ; + By[ 5] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; + Bz[ 5] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bx[ 6] = 0.0625*x[0]*x[0] - 0.0625 ; + By[ 6] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; + Bz[ 6] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bx[ 7] = 0.0625 - 0.0625*x[0]*x[0] ; + By[ 7] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; + Bz[ 7] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bx[ 8] = 0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] - 0.0625*x[2] + 0.0625 ; + By[ 8] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bz[ 8] = 0.0625*x[2]*x[2] - 0.0625 ; + Bx[ 9] = -0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] + 0.0625*x[2] - 0.0625 ; + By[ 9] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bz[ 9] = 0.0625*x[2]*x[2] - 0.0625 ; + Bx[10] = -0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] + 0.0625*x[2] + 0.0625 ; + By[10] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; + Bz[10] = 0.0625 - 0.0625*x[2]*x[2] ; + Bx[11] = 0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] - 0.0625*x[2] - 0.0625 ; + By[11] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; + Bz[11] = 0.0625 - 0.0625*x[2]*x[2] ; + Bx[12] = 0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] - 0.0625*x[2] + 0.0625 ; + By[12] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; + Bz[12] = 0.0625*x[2]*x[2] - 0.0625 ; + Bx[13] = -0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] + 0.0625*x[2] - 0.0625 ; + By[13] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; + Bz[13] = 0.0625*x[2]*x[2] - 0.0625 ; + Bx[14] = -0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] + 0.0625*x[2] + 0.0625 ; + By[14] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bz[14] = 0.0625 - 0.0625*x[2]*x[2] ; + Bx[15] = 0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] - 0.0625*x[2] - 0.0625 ; + By[15] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bz[15] = 0.0625 - 0.0625*x[2]*x[2] ; + Bx[16] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] - 0.125*x[1] - 0.125*x[2] + 0.125 ; + By[16] = 0.0625*x[1]*x[1] - 0.0625 ; + Bz[16] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; + Bx[17] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; + By[17] = 0.0625 - 0.0625*x[1]*x[1] ; + Bz[17] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; + Bx[18] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; + By[18] = 0.0625*x[1]*x[1] - 0.0625 ; + Bz[18] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; + Bx[19] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + By[19] = 0.0625 - 0.0625*x[1]*x[1] ; + Bz[19] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; + Bx[20] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + By[20] = 0.0625*x[1]*x[1] - 0.0625 ; + Bz[20] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; + Bx[21] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; + By[21] = 0.0625 - 0.0625*x[1]*x[1] ; + Bz[21] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; + Bx[22] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; + By[22] = 0.0625*x[1]*x[1] - 0.0625 ; + Bz[22] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; + Bx[23] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] - 0.125*x[1] - 0.125*x[2] - 0.125 ; + By[23] = 0.0625 - 0.0625*x[1]*x[1] ; + Bz[23] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; + return 0; +} +static void HdivBasisHex(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, + CeedScalar *interp, CeedScalar *div, CeedQuadMode quad_mode) { + + // Get 1D quadrature on [-1,1] + CeedScalar q_ref_1d[Q], q_weight_1d[Q]; + switch (quad_mode) { + case CEED_GAUSS: + CeedGaussQuadrature(Q, q_ref_1d, q_weight_1d); + break; + case CEED_GAUSS_LOBATTO: + CeedLobattoQuadrature(Q, q_ref_1d, q_weight_1d); + break; + } + + // Divergence operator; Divergence of nodal basis for ref element + CeedScalar D = 0.125; + // Loop over quadrature points + CeedScalar Bx[24], By[24], Bz[24]; + CeedScalar x[3]; + for (CeedInt k=0; k7] = b0_x-->b7_x, By[0-->7] = b0_y-->b7_y +// To see how the nodal basis is constructed visit: +// https://github.com/rezgarshakeri/H-div-Tests +int NodalHdivBasisQuad(CeedScalar *x, CeedScalar *Bx, CeedScalar *By) { + + Bx[0] = 0.125*x[0]*x[0] - 0.125 ; + By[0] = -0.25*x[0]*x[1] + 0.25*x[0] + 0.25*x[1] - 0.25 ; + Bx[1] = 0.125 - 0.125*x[0]*x[0] ; + By[1] = 0.25*x[0]*x[1] - 0.25*x[0] + 0.25*x[1] - 0.25 ; + Bx[2] = -0.25*x[0]*x[1] + 0.25*x[0] - 0.25*x[1] + 0.25 ; + By[2] = 0.125*x[1]*x[1] - 0.125 ; + Bx[3] = 0.25*x[0]*x[1] + 0.25*x[0] + 0.25*x[1] + 0.25 ; + By[3] = 0.125 - 0.125*x[1]*x[1] ; + Bx[4] = 0.125*x[0]*x[0] - 0.125 ; + By[4] = -0.25*x[0]*x[1] - 0.25*x[0] + 0.25*x[1] + 0.25 ; + Bx[5] = 0.125 - 0.125*x[0]*x[0] ; + By[5] = 0.25*x[0]*x[1] + 0.25*x[0] + 0.25*x[1] + 0.25 ; + Bx[6] = -0.25*x[0]*x[1] + 0.25*x[0] + 0.25*x[1] - 0.25 ; + By[6] = 0.125*x[1]*x[1] - 0.125 ; + Bx[7] = 0.25*x[0]*x[1] + 0.25*x[0] - 0.25*x[1] - 0.25 ; + By[7] = 0.125 - 0.125*x[1]*x[1] ; + return 0; +} +static void HdivBasisQuad(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, + CeedScalar *interp, CeedScalar *div, CeedQuadMode quad_mode) { + + // Get 1D quadrature on [-1,1] + CeedScalar q_ref_1d[Q], q_weight_1d[Q]; + switch (quad_mode) { + case CEED_GAUSS: + CeedGaussQuadrature(Q, q_ref_1d, q_weight_1d); + break; + case CEED_GAUSS_LOBATTO: + CeedLobattoQuadrature(Q, q_ref_1d, q_weight_1d); + break; + } + + // Divergence operator; Divergence of nodal basis for ref element + CeedScalar D = 0.25; + // Loop over quadrature points + CeedScalar Bx[8], By[8]; + CeedScalar x[2]; + + for (CeedInt i=0; i7] = b0_x-->b7_x, By[0-->7] = b0_y-->b7_y -int HdivBasisQuad(CeedScalar *xhat, CeedScalar *Bx, CeedScalar *By) { - Bx[0] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; - By[0] = (xhat[1]*xhat[1] - 1)*0.125; - Bx[1] = (xhat[0]*xhat[0] - 1)*0.125; - By[1] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; - Bx[2] = (-xhat[0]*xhat[1] + xhat[0] - xhat[1] + 1)*0.25; - By[2] = (xhat[1]*xhat[1] - 1)*0.125; - Bx[3] = (-xhat[0]*xhat[0] + 1)*0.125; - By[3] = (xhat[0]*xhat[1] - xhat[0] + xhat[1] - 1)*0.25; - Bx[4] = (xhat[0]*xhat[1] + xhat[0] - xhat[1] - 1)*0.25; - By[4] = (-xhat[1]*xhat[1] + 1)*0.125; - Bx[5] = (xhat[0]*xhat[0] - 1)*0.125; - By[5] = (-xhat[0]*xhat[1] - xhat[0] + xhat[1] + 1)*0.25; - Bx[6] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; - By[6] = (-xhat[1]*xhat[1] + 1)*0.125; - Bx[7] = (-xhat[0]*xhat[0] + 1)*0.125; - By[7] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; - return 0; -} - -static void QuadBasis(CeedInt Q1d, CeedScalar *q_ref, CeedScalar *q_weights, - CeedScalar *interp, CeedScalar *div) { - - // Get 1D quadrature on [-1,1] - CeedScalar q_ref_1d[Q1d], q_weight_1d[Q1d]; - CeedGaussQuadrature(Q1d, q_ref_1d, q_weight_1d); - - // Divergence operator; Divergence of nodal basis for ref element - CeedScalar D[8] = {0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25}; - // Loop over quadrature points - CeedScalar Bx[8], By[8]; - CeedScalar xhat[2]; - - for (CeedInt i=0; i $file_name + +i=0 + +for ((res=${test_flags[res_start]}; res<=${test_flags[res_end]}; res+=${test_flags[res_stride]})); do + if [[ $dim -eq 2 ]]; then + run_flags[dm_plex_box_faces]=$res,$res + else + run_flags[dm_plex_box_faces]=$res,$res,$res + fi + args='' + for arg in "${!run_flags[@]}"; do + if ! [[ -z ${run_flags[$arg]} ]]; then + args="$args -$arg ${run_flags[$arg]}" + fi + done + ./main $args | grep "L2 Error" | awk -v i="$i" -v res="$res" '{ printf "%d,%d,%.5f\n", i, res, $4}' >> $file_name + i=$((i+1)) +done + diff --git a/examples/Hdiv-mass/conv_test_result.csv b/examples/Hdiv-mass/conv_test_result.csv new file mode 100644 index 0000000000..8dcc28a8f8 --- /dev/null +++ b/examples/Hdiv-mass/conv_test_result.csv @@ -0,0 +1,5 @@ +run,mesh_res,error_u +0,2,0.55094 +1,3,0.23091 +2,4,0.12473 +3,5,0.07812 diff --git a/examples/Hdiv-mass/convrate_mass.png b/examples/Hdiv-mass/convrate_mass.png new file mode 100644 index 0000000000000000000000000000000000000000..94490982785c5ba47db7a2aa09bfa34525c1ae02 GIT binary patch literal 25544 zcmbSz30RJ8yY7={o@r2$MwBK*nl&he25*BjCqn}jB{XW#Kncm5CK^zoXr2v)NGT@BcsEdf(^0@9Vy<^E%J-x}!{X=(Dp3vQQMozSTh2 zjG`9!QWRY?69fLne|>=j{%_L}J*y)-5A8j2)Y038+U|J7)BVs9_XAEVd|kW`A2@Vy zot%=K;+hrvj~wwltRgS(@vk3{JLJ7jzSVKG6mP=pX|U@sMR7Wkf9WzbZy%s2XQi#W zn`wU6zIPn6+3{^obGXGebN}M##=SymXD(Fgwp%^rVsKtP%@_Ja;LEyd_jgOBnyj8! z1PGjYCg2>A7MUj@<{b0f|NfF>vp3$IEkm4|zm8ssHuU-N!>-|JPoUbDvR6JyV;a|s zzxAqoRneCU<60pmCYEAttkb_B68~hlDztLrpG;;3RlFdkROz6vtE-zPB``={Zdw>B zj925YIOK}sRkOuxzIavF^<-r%UNzWGez}?J=+UFC@86pbY&h6awCM7-h(qNw1=f#t zntuECO+e+yD#48h+3>d&68gW4T)7 zB38Z?Yu2y@{2tG&%5qDRG`i?8^s&?|akHy|5x!wxbKuNR%aUIoPrQEfCL||EesFNm z#Ka`H=V@F|YMe$@QqXMF8KI4awo;k#oEnpqoSfXMHEU{WWBBs|XO81>yBpJu_P@Kc zE4=0C=V#n(%apkee|=?MP&)EN$nW>~>WBIH+4sk-9iN|*B;S;QT~NYjb=qDm9?izk zkY&qh0XjD~w`H5W7w+A=x6WmDcJ{GX4+D87W*Ps0h8yak1qB<{ulM8GZ2q{g(6R4z za@{YF_JZ1n2i|fC3Pv9r`(o}F@N1}3zdC|FCjY?x{aTimzkG}&Q-ZqY^rC1VqIqS` zozeSLj5bxv`#-ghy|YbaH2esuWj4*%cZZtwqCvPp3L{@}qucdFu^ZLx!! z8>_9YEy6-{?OGn^r?KH@&TP;7xoQc^LI#Euyu<6}X1%w!wkx{SEl~^n^SEkyN_+2K zsV^@s(+dd+;ZrKAsuq-$l|6g*%w*zuL&MV)y_nZUK5pcd#zvm8FE3?A#sV&fg)KZX z@Q(fGK#P71pKLDntoQ!4Ys>n+edATVCI6u;KUe1cG1Yl zNbkLo&JQ0xP-^88T}gI6FCKZcGhE)rAiD5mXlTN1Ke`LY&ve8LaLL=3v2k*8uHXNf z!V*8lY8y7uK00HO6r7*1^&ZU*ocZ0?mM?Se+&PL8+TgA=@a}GMb~Y;x#D^lE?3#id zNg8J~0)AIKd&a~rxLykz?$uorUkn6FP~$j zURYSj#LR3utM(*Tc)CASi?;k^B+VLMTCe`1Y<)*(LxVBJg+=78tgNhRX|bfZLK)bl z-!-(gSwHk>w^;AdM)MzizEW(}DuW~aO4EL<*iz^ zic6a*{^CWcHEY&{rKhJ)j@HZ97WoX+6c3fjP?1Ar(>taLFI^I#(`s#NV_CU!Wu9xZ zbl#C~X7N|A%1jJ(%I{fw+0)aLE8F5OPg20dWrYnJ_O%S|?WSxVGxy!-q9# zThG`0{y9+lR3vE4`1p8OOiaxEThW7G&#yENyMF!p>!zk9=cTsuVDT2)2mDO%^z!2V z+K|Mr%_N#wua{$2s*7(|Gx+{NZ3Mee&FQ72gq9Qc{oN5?1wH za(DM5zWPLYe)976E9<%W`1ooAH>4l)^7sATW_~ANZgyrt@Z@AqT!KgMrY{>0eO%(` z=(r%*Z~FTqJu9nAOsuRj1$Qr9y2PU?Ze(Pn==N^S2tI+;*w~m(YnigA*83c>rwf9~ zBH^)}-rsj2|5iPFW?;2zSJcg$D;(Y2WGwF4tk#$r-(uB@7xT6BPhii-7qH@6rZ zrRW41%V@F@%F4=29v&WEzsC=_lg_>(K0c`< zM~;MDy?S-@>j;*-Wc)E#0)MD*JWG9D9S>EaCVA3N2z`_BGs)?|2g9 z(1KuhdOG+uDTt4%kuU!<+SpL8tg*YmottyH8rRXGkG$n`v&H*6it^7s-_!l#^18aR zUiB~9YMhqH*XaibIx9JXnb_H7C)T;XyK`A!?ViZ{cgdEbq;RiZy}GV|U1$Rnm9p!> zUNKeG-S5jBDi_dcy|+v@iQM4P7WuLCgiJXj4q0O2O6!OFMLz~kd$_d6IQ(kpWEBk? z9vR_A0g`)n>-O!HfByVgXlmY*X(h%x(LJ6P60*7Qw}`SQMR9R)ncSJF=G0sm%+u2S zzO7B#IFBwkDvG(Sz$2!7_P5dd4gQm!OZfPla31OkkQqeRty^4uij$FI63spF>mZwe z0;SEgXV0Eh_N6C&ZLzT6dv#sa(!D(}FfhZuylix)qxfk2g$oPq?d|&p24vc?3(O}@ zOgklY6NDCM^7^r)OvP*WcYupAf3A z%K}b3bZfcS>Z`755mUMeIc(^=j+7xZJS9&xr@N=gbpte>P2a5f@s5`WiSKRs zI(~l2Qrj+36cZ<>K2qC-AYBnPzu*@ywz+?JocH2=AxgUZL>bPoq${II$^^g192@iV z=g*(j)t$`DM6oizF;jAEOe%tHsagN;*}@J4dqqu6noYaxnZm+N$f42s1cFdEIzxZF z+}Ifw5y7zKj9|FWz&jKWPL!GoEG#Nf43N)ILOW&nV=U(EuOT&9q=kCmP*T&TS;#KFHn>{A5bSWLRMpo9yBWUEX;f@{H z8xb*==XG_`d@1%54}wNix6^D1*twLPEc)W*OH`S3mVh6fK}+gWGBbC%7IUSvPGd!# zN4jfK$%8{e=0LLzx3#xBH)olr%gr4%@Dm0snEZ0}Q0~zoyOO!t>DD}F#*fFo3cC#ZCA4(x zJ8|NK--#3S6tGxWNr~FFE9;q25%};eWPD!KXRDDQxsdHHy0 z{Y&)^c3|5b^wYY@yJ4faR~p~YZMkP_-`3LBCdd-L06Vn5v#j(!f?N3V<;%GR1@~y| zIAu@y?fpY=f>IB>z4h$Xt0$4%;uq?-Z`j@}wz_aswPGx2k9lvP z#lDxcMGJ!ea>F*?c2ArEnT~?_~?+F(lT|w;jV;s9L!VKuM2DI=)|8t zpX#o8d^}^cV2M1N&;~Z|!eyxE=21VF@bV%dJy>w!*9V+Q&rem0bhm7|Xf@%K=F*z$ z`1tTwj=;%Jr+5NxXaqz9mRTOWosq!=JY`ZQziL$ls=kAh(}I2bzB&awJ-sxPq&Ecx zE~L#Zon-+sMZ3*!VF8hsnF>7GBLFz0H+{)Tm5$XrrmU)(ikBvly}KUF1;*I+|28)b$(T>u&M%*C%$l5>-m+llRcb+ZTUvWIM?Xw>rYNiIrcm`^X$b7 zT>{;{yjU6*7UuB!=H}ty;fy1~Wb3QKS;;wLUMzLt@#W$BF7;fC7l$IDIW(qi&CSoR zP1TR{{PAJaG7W!$xaI1{dpBHMv+HCCJ!8s=Uq1vk9$dCuGw?J(0Dv;@bZXm!ea562 zGD?)wb8-?p{MxoQAgOZ1>%zdHepv+@Gm2r{S*36m%Bah-a(ayJH-y zBcl5H_PZqUKTPCd6Ae6|LT|gZQ&^F&9wJ_(4gjn;J&z~1A zUc6XWPj9QO;hC|Kk`|+6AaLtZArRq6q`?^+eU?Kl6)6=;lh$@ z%3ceuZt&3d_g6dku_S8m%PU)L`M8@GTvOYuD_z~%Du9h?eRyE;ay35+IT@Kt1dua( zT{4HC#$rD-SGo7^Ya19a=?HTt2(c zbp3i-|5`$c#CB)4N`LvR1Hq_(1@NQFi3=hYvQ&Dk@Gm1p;b* zo|yye$mUvF)Vmz}IAo|5$e{p(Px9SbP;gngrb-{q2FNf_%18RRP|IbC1{4$&C@SsN zt*Y0rbE}e8;>-U`yndY>U5?8^zopMt8ox-=6hV`xWn=R}W?Fg)3z|)4kG6-EIQYq# znatE<-|vFQCkCT}gQ*u+Hx8u_0ANR+K3#Kjb7*w_(Ql25z_&25u$UdJM^C?Q->VwL zYkjFEVhDdL-SRmKbOORcR7NIdo^UcWk1ts<^*JhFdLkJ-jNee{FFblQQuxK+Ci}Xk z)=hIPQw|>c+ECrn!jFQDk{1SC4Ja^PXJUB~hf+d9qN2UMxV509wA5?#^K-+yV=n0V z(Sxz_^CwE%Zv@-bYR4XF;|&l9I)+tm&1BL2g4+S=NN-wz>qiR;I9A^8ey z@|I9jSO4_=`x$^Q^P-5;r`ewMj`H00!Y@Xn7J279ZEqLCgD%ITQLL;nYju*#%QfZf z%T7;DP8zmsLna?Tbk5%Sw#lm3Kx1om-Csz(&U4ncDLTT8ihc^DO=eOa0@=z+fUG47 zz7v=LX~pbuzzDNMExduxrO%bGUc6u{C@83HYkPDk4qzm4>hk#bk!?5BmeMn@Y2gTg zTM5NpTNFh|ZNH+mbLV2vbCLzAEJytl>wCd^T_|GveG27fbw^uc zzPxT$%F)6YQ>5b6uO+g4~6S+;J8t^a4Jp9qCm zfQgBxrhg65`)TxJThdr0$~%~;!On6KR=zb_-rj4`zJ(x^QKT}P%1lj7Jr5ray?5_m z-UNxbnwo`R4kR@e3kk7ua>ns|$EUUycw8L~FfrjOoBAS@mY!bu{5dlyip_~~wswcU z+I73UP3-mFL>71F&bKN*hsWAJ11*yt>7!o{`;Ci|VkxdW1aC;bqLyQb`<&O3sjRkEfB=bWg1V!{D{;c>vUZ{CFJ(I4aw=fYJPRFx!~ z`43iwF-Jy6OXt~AtwlaEDo1~?F*2TVeeb&qxepjDRIoz|Zz5l46N~hn{7@*SWdBw5 z69p3co(l{%oxNcH_~^%DjA$Kee{eFXm2VwKz$$>RSf&}c3||_3uLhcekxE996&p7` zdf_uW6f{>|5kjAwlf$W_qXPhYr>3V9$b}!>CU{qVkV7GHadF=7HH~ZAT3ewF`1`Lo z{~QYvy8F?=%{n^tU>sx%UI8Yc(CQ?c&}JF6L6)zSle^bMcbY$p%V7Iw!?%klTxGeA4&`}anQdRZ!1NpAY*YD?A zucXAibSX1M4z2fnW^V2`nqUbktEy672sPL5S!-o&UG?(i-3S@y+?_^Q!{JvI92fWw zeLRCM(E*iWe{&Wu$o+;zA4$TKD2+R5L)o$%veUMooJ z5%_1a#ZDe^EP~=sWDl+9x4BCFL<9s@J)apQfh_I+2^|mvJ3ISeYaTtIwt!}!+Mcyb zRebuxz+0xG?nv6TwY5pUdkgvnk_h?DDD-g z$vE)3Ute9Pqrlcz#c9q)f&%$emmp(cXjnDcTVH?Y<&!52Z{EByFfyu!Mw9TJ6q>7t z%A)YI0z>6@@@4S|$r-HkO$EWUYF`v~y6lq%o1|_Lzmd{>C(B?88DzK+$O-DDhoq zco%bX&m`Tsv*r%gU7IP>h_1pxzgYroKceFls4#S0jvgLo@XLg`tNksDBty^05eBs> zz2N-Wv*2GiaPX;$+ev}ZkoCl6WCnbWhD1j8r30Hd9y%0Pc=)Rm5QkYJeQ;%LR;7K= zOgI8o96dMUTKwRL$01OcNJ6zO_BD^a*W26c2>5kL!BIQCV65+rI62SY2f`DkG%2GuMEw4^{wPm*;`y++p$Er4{- z%KnoIKRy(O&i(nSNd@DJXaQOsDx2CHdGhNfJ+(yD*fQx-LbAO6(FL7ojRq%6oQftMkTT6#@Vx6V3I!J zm#p%f7M1-OH9}`dM}=_-sQRt}90mb#i6_a&$EOxJj>wLmen&+{*6}P^5=I~;&LQdh zaGqn4yf&i%_BUP-7<3f{o4eLtCxUJDyOyS=hc5v7CS>2WJb7|UHz^wQV+7X%ebYnHz~v_@4Y0f)f$vbG$$lo5jGl$!^qZ*xuk>HJZAI-$8ku#M|^yY zY`=7MnGBX*zPuFr2q)z?aQ6}D<2sw6nD{D?ccT}LaQIHz$mwfqF96!4Q{(i-r@3(0 zWh`JQ2HCOl_6rx}iQ13lOgiarUa!({m~Pk|OS3>dfTrl#TgQu@;DS{k{sVk6ft0`; zl?ZhsTjSE>Kch<%pYM@pKRh??LqvM#&!w&{`It8o?>T0f`<3sT*mS zcaE;4D%~j20bC<`1=>_mRHQ-aYG}8NjMZ2{RJ<@$Y4qf)eRlOeanLuHTeXU5YHF&l zqj)3UBLbP?&+PONk;vYZ96L-ZKlQr!=%GJ|T9?>1a|;`rW#}}_(+ALL{7k%ZB^nEk zK4Iyz`ufmQrx>B94*w_~GPJcWRBSa(aC>`;{nZWi7}ENHFC>*#%J>FRX0NFoBW?Er z3cO6}&f7Z(IhEz!4n$IocT_ykm?ph@C2~|b;->oJ$K|Mc5pWrx4xRydR}H`RSPIM683s}uwzEcyF;NAV+@c7m9&t45_8mjg!c!jX>l4iFx?Y^^n>*X7gjN9>N#UV2g{3?{RTiU2Y|>c4C4J64QQgUY8#&6g@B^fcmfl z-18+RH$Ji?5Wo}Wf+p)ntI7$whQ@5`pX%cI(!y`6?9r`7=uS)F)^)RPdWr{Z!BcJeKAU1O{nV z$YGS$(jNx-M7BJ38oQtZZUASl8X60FK1QJZJG&k*prV>3TJ+#oE^U(v#UY2|1iaQN zpH1VOKl__|41~GkSpeZ{(f4~z4m(o8iHZCL?(Y}K$jC&=oM%8vy_uQ$oNF^WdU-uP zJ&?o|unm}&2L`OMFI$F$U$-x=MbGDG1Cn8e#oNZls&kS?uOU?dhdQA>_fcu0^S8K@ zCOta15nH)n;ztK7>T1c@E7iIU5E9<)E08|N?T8gjHz$W_VuL>#Y28T@Wzpesu=?62=J^=$8(zyvI^{>8=hSLa!x_ z7KWTt1%t+7scjW8s(DKV1S(ovt-AYN`5B4{A;&>aPoG&215=g~p6Jt;FA>HXQwu@5 zl3s3fbTmya`<+>)!y^P0T2I7Pv0=-J!B$10euC>p0JS0sWzG=72VE@uDM!FT?R#}C zvrz?)*tZEp18OBWvu;#NfHhr^hhpO5n>KeVWo*lhiHmy*Bn~t^+!#n`FK97Q69gcs ztCu!4eE#yK>dhO|O}0hrK~pd>F;&2`Lb!jtUkX%a1ajH^IA9%kkPJ(Hd@SAlsubxj z1&l+gWsV&=me4;~zvI5EI6sW?aQ?!??T23SX- zqjkF+JZKkHDup$@imoXhsTkE+9O)d58wiwyCYQ`bcmf1(UuU@{STp-JV|2QV9Lp-E z#=d?U7+}SsUEO%7>h0UbuxV7`NJ;0<{AJpfw3k7c{J%48@zQm$_i)q^e+`xl;zJmy zG4QZru^Ee=2b}{WP=k_|i(T^mxeHzU>Sy zQUKfuqF4a|*^-rt+5ltB8pSCwsEzpH6L)<8hMV(?s;&HC5W)+#oc zuiL6WX!qje)NN4x$tXs|uwtY>xpe7L=-k=>OnGiCayI3G!a&|IE>u414BVZm0{Bei zBj^Eag6o+o#-aq#d|gs>-je=qE1(R2@0Tx5A4`r+qL|ZB0Pb6GAh6_8UM@bNX_a6) zD42a}+aO54Mv(D*uNcd1esstl^oO{b+A?cvYe-;(r~oEJtr#=YAM*D0_Uw7W-a8E* z3u!OKG<8eeE@e78ItV8^0BxW8`=ddi;4nFXz&9^x!2N=$IriSb4C-}fn)&0VoV_P&e!LqPZMC&8C z5!p!B+FJ11wQIw~fO5$MSzEW3S@c3nQ9IOT5Db7B%`!v2o6U`-?9|SlakcUF-2f{D z90-e0Fy_^RtX#Z}19&Q+68dY`ucyoD%bH8b8w3_FCZ#xlR5^A}i9a*r+Qi`dP+;r@ezOnL(YwMMXV#t7mo)F*%&=kLTz;C6V6aD4f~}c6ZOL0B z4oXTly$DtH&&&)YTl%gk4`Rb$#0iN|;jQH+C_>s1pLA)h3k!@r6a|+Vb(5!I@9q1F z2N8quW+*-W0Rhk~xL_%%sH~)elj~y98tIkuWn%$_ZlDrq3;FT&>(>=07qNOmIz<{Z zv=vNPY+TC&jDcLn*!#=|EIIs!Cqc~_+VGk$ng3QyLnUJzSJodmxpdv$E+_(QKKXxN z=)d{rk3S)W1U9$}P!%9AkvH|wd`aFrZO1tOyZx82BgEu~Fwoz#C++_Aeb932GM`vsk5}zcnNN2B&P%30_A|;>p^ThWd!^Fb)8)^+x#0heE>S_`usTz zWsr39sFkg7!022TL~lzRvg;HTEAbjp0{+>E{&N{u=H0cKOGOtFwgQ4DMg9IY6o>PM zPI_!c z6*u1odja4aiarvRiXNT0@`;HxB==<+r3wO@GBB;edqNTbxIbW;6>Ha?OO7@C84{L zqKQ!_L=X|&q86{r^&4@DP|4!C#w#QG3LBE8y5pKM9jZ{t)R(Iig({8mHL^5k+>rM< z>wJ#M+Jcsi)Cu5aPQvN}X21f^&W^Y7X7e?FhKK|nhp{=MDV>X8-)-9ndza@_%K|on zB;8~a16#gKMZh!2>TA$)7}_#$b8o!^`vweM5iv2W02vFRu?%(;^S2jx^rUa3-Bp$; zU@0sv7AH1v2(AX&=x6%g-sXf^mYsr9sQ`hJ6KZpmOlpzVD0eY90%RaMtOS8%fIA~- zV9md6Ae|;1JwpW;zO|RxKyl0F+Y=hg-TfvU8TjBXux94gjOhO~(Ne8H(72Gq1}iJ@ z=0Y9eW`XCS*TcgX5eE(0sB0(}ICk)ru9!uN`Kzh|TS~UU1$-J6&P?ENVNTF^g0 zA3Z+ujb}^Ta`wyHq&&Q6kC=G(xVx``TacDwE0;(An5JdR>JfPGmBjb{9y}%;V8>`y#b4? zgdNKRT~2Hj$QR;1o6>IGzkk2(I=_TS-tX``Ogf__7i`3=}wT6(^H`I=u1XL80mpAUepAow=${`V+L(s|~rybzH10-8B zM)wR*9bGF^tcYeT03QLCFbi5eI%u2zC@DUk4v%_k`&B--Lv0UNH#Robl`cPY=n%2^ z!x2ou#+)U&1l{F0sCjl!a)a%UI6;nE6}T@&ygZq1jQ>n?vH%qDh`)3x63ilwYeho? zCzvnUbsxadiej9A(A01e#0zXAxs_0Nb!5ZrbxiTkz=C20(pyqQ7R9YOK!TfjI zfpm~L3D<~!CL4lgL%|*@G{qa5MsXJh?Oe6wSwn-o&#o<77(g}=uO*yHtf1*mfvkpk zW%&D}P7Bu)cnZS59yziex_8RmdqT3U;kxQdP%%a` zPz6s@Cf%?ETi)q}Y8{U51r?qT%pPgya`F%5n8kmqM=?8$K1&)J$4Z#jsTIqXEeo@e zO9T)|MR&aiu5;TvciQhict9-q2oMd`vyldR*EEBcgP}70xGD&W10>Nf*e7ceOl@IF zxXE*VCD2%!1zK1jRU*AjqFBR~9F5XI)E+h;nwvr383GifNqyFs9$ZINpav4-i$+g^ ztMEK>qCoXZ5g#~Q{KQgX7B!v9W4?qSCe!QVkZXE6j&|09Q%LY2B?TGerHOt%9|0&C zJGiwS5Y>bjBIgXplTdJ|aWqL%12bL`O$A{%QZHXlQ2_A~Xmz8q#j;f9;g1hs;X;Bk z@UqtK-9kXwPYE-GnTt##@vOD;6(gU2`SMt9@hubuml=%mPxh6ZvvwlrDu3ht*DHwq z10}JRc;e7n`E1o~o*xuy+OcB?2Yg&HXy#JvN{^HNBXj3%6BID=-WCMOq*>s+a{#6P z0SqDH7@B7)I61a|;f@>V`16Mc->)P5xBBt%3UKs`K3Q4=7T8Nc1eK_#0*g(&`ks%?F_K&zaJ@4Ns2Zt+XrL-4s12C zC}La!Rq*ZKcaGLhu7E0@ToQN>!#Tv>Wc}#i%Y7wg*Pjx!oTW57JADYS2jRP18N)31 zOF=@j`RJ0NsT6B1rVs)MfrlYe5vBss3M63AJ4{=H(u9_9Ju#_*24JL)41E+PWpbTe zlwl=#Z?jzpv)l&{$RG~Z^bES6p1wD!cypI70-G}L5Xv6X2W+g`zz~DqzXP-xq$>h( z#T!0{aS^zOjsgv&4+2@isfGzt1xC9COl2#`mrmnBKc?W*PXm8Sx>yQ&cGti_y@@_; z2iX;gLi}bmOwBN3RsflLbyd)18i{2kkdQK1OL$hab;KJ=wvE`Wkao7jE?W)RC!c2? zENvy_kyv(dBJdzj&QxRB8Bvd;WW=*d=ii41_U0NgiiNyRv=sj*GN!!qizg@^W}@-i zjr7VbEl+_C-=u8?E)X~4EO~-W&AP6gto65`(TgsmZUUE*PE1GGly@GLn`c9qcSIi< zdFIR_3QA218iqzby|i2N8{4eS6oSK44R8UPcmL~~bP!Qak@!I*M1F+hAlyzj+_c9Se5$JjWhbu{a@h>q>qf3eJ!I2Qfhkr^DV5B70Ja0o{S&E?G1etQFh9MOfV z24+xIhM{u-AR4}h1>t6LrBPv&ou%HWcKE*94Up0y*km5@La?y^wPGYN`Bcnx*dIi1 z3N+w{LX-;fNSUIb?{R(+Usmz^Z4ba**%o<>f`pAeZaI|M5yJ zgORX4Ujum|EiH|rq;_131nFsRUZ_iBPAH4(zyJ))07{e=PK~3JlbMSi=<8+xFS~?s ztj|)CX#Z-5P}m&tL&Uioy*8>(iKg62*bPuM=oZ4YlPPMyt1E`7_`%idfyv4$F@U%d zuy^m?oQ@nwMYcspDj^`wpQWb=;im&(Q8e9f`C!8I0YQ_zx&lWCV3uggUgLeH{^0AH zO~tvn&FR2iSy@?KI3}_QV`j)*2!w<}4%h*RX%X3?7Ft&>;6{Xs7_Zr*|JbNI zpEMm1LeHK*54mw;IppA|=jT_VXV%Tv3fCVD{(9RP#>?1j%bW!ivAlsuN`@)Q-0r3d z(atl^!Dod$HF#wOQ!Iabccel0+MX01$->GV8MF+M55Dgy;TJO}28+LiU zH7331jUzlU7O88>=j|b`T0r( z2Rm4xs-Y;SNYj$t22qMB!WUgSRUl5yEahNxPGsE%Pc# zlM^1n`LCN5shu1g@nkHh96hV^-(z1LOh=E!kIXW{kw+Ilf{f4&Bn_}Cr+BQYvNE|Y z8uhZS(0^(zj7c##Y(2<54Af}>MeXja$c!R$Ka0Vbpc&rXQDjsT7MRc8#}asN|4^2> z7@2kDZz<52`n(v=OT<2YIh(GYib&Ty*Xz8~PEx$6mKVV~o)%W+gLi|T!ZR>}e{#wY z&(e`zHt6R8ea96j3RE)Av=%T6y1rcRUPCz7E+@xk#IF8zrxln}eC100saZH*bPpc9 zH;N1)1kq@8{^MUZc&Y9`$puca7Dws=;EG2z=!>QOk^PTZI&NOFd)3h)%)r2a329x| z!HN)if-Q)sF*KCp{`G*R*aF&$R$1GkC2;1#{u`W`xta{pASeF6G~A{x>WEs9f?H7J zFkKn}E6pd&Tc0ahBW9+^dxf_c{u{#jQ)Cz$8%?+u3R0Y9<^qyAKt+Nxy$Kc}$$yj? zN^v;gXyW53MT%I$0k1KQXqQK%msFH|w8sac{*hm}iQolXZFH9q&|rSaNJP<*ZzsVi zkN@ZpLh0FQXW0iH4=0l+N&O$M^Mh$!-JByx*(5k%EISFl2qioZoJvy;u_r_t)Lu9- z8dh99+SdswtbePCkIpMRx<^uxjDrHz7G+8DjG+pGA zV1(&D>L${mVJ|=w`KUro zP~rZuWS8XLFkuaUt9~A^n#IT_`0nv%&j#N%PIAIwq$*7Gb$g|z?zHm5rV=tU{YzX* ziUDO%{`=J%H)2s_oGXIq;6$i~C)v)eo!CBc{Qy#*k^5~Nw4D5dVkwsK^}SYw-jc+G z2l5DR4HAiDEAr2NuH3;C0`408B@bq3h|$Zg4Ga{$ot+)#$J7aD4+a>X8fSQY67GT5 z6N2^}85?U+0F#BTzWx|Zn%g{^zl9miEkE&9fE^>Y6p*~2(7Btn&m~3wgan9kO<3V- z=rEAIC`w#Bm1pM^FUz9e6W>`8A;h+W@&sErjI<{q5@JxruD{O~sSU=aWdEdd=T4%l zUhmq(gjJ-1VQvP+O04ysK0bN?*4~E?E8-_zKb#QDst06HIQT*N^XJc6#>Si|rB$e# zwC+Y8v8?NL0mbL}VfbFZ-jHJ)UjpWW-Agaj*=jAFqi_G~G~Jz|4nZ2uTa5n8z6+fF za||5sh2w%7F)|LFMzU?@&!1e6_6@z$XiWXVKBHkJ)-c?@KnAL4`-7xUHbhHiwL=dH z2cAwvO`kVdzi-#ooukEy8~;kp3?W^>lX=>p9x+$h!HraT_GCkwhm7yH$ywB8J-(;k;e4WXY;& zn&)sFx@o`3VNr6u1DqDn6?eV=@#V`C#FhE250wXDa3*aCLDS@6kVr5oN5{Y>5Dfi- z@@G=wHa$S50RRWEL#Y`VIKBQmbQdD^Vy2FI_wHTYFJk<`Nhed&;}9Ym@y*4|EHQXe ziA5(oHtgidXttH%@xrL-3N&14x9Wrkaa7?WN^tT(k|&cKL{b(JNs^Y2h32Vpe0&w? zj@$!prHDX&c^xA&GcyGK2Y@Z{f-*;Q2b#0_0RpU3Z0+qukl@xR8YM(#$f?3a1p#9b zu4bsIsd2z4rsTm-zyr;70V2g1R9OIdTh`|?#ygQ!A$b={%bdo?tzNT67qE$8+L8`Gh2;to zU3Yi)&77P#0O4-@f=dPlnM{6g7gvTdmLEB`ckh>hciOr%nLD<{d~lbLCIaU^4W3Pd z%ny~Xo;~YJs7D)SUkozC`~hywAUXubVkrve6f*|)Ksb?x(noV;zJI9!5jcp=CpNsm z-``eJT0qQQU%p6rnIj?hd}bC)G5hm-Am@^@mlzsSm`ETXmiYSh7aIH|Q>;TkhV;!X)~*1pip_@%atT+d5U@+)-bLVe;N$o-H!uAkE&05FxG5@MvEgC3oCVHBMhXpDFkC$~=J+BS5KBz2 z!{smnAXtUzL8uQp;Kg#WeuEzh7sL8>u1HNRrR53vn8jo~0ti=BPL6{jmL~VMhs@~T zcr3T*(tJqA?{gC7FBE8c?8F~M!G!a;t!mpwexc1fZm=nq#H`J`ACt~k%4gW|; zb_O;}xlO0M5Gu{b;7GuTpzxD8kvRUjK&AET_aF@NiOS|Gi3E?>BnIHPr*R@Zhzq|z zlX+xAaFdX#n3)hd4KgW9c69%N1D}w%pQ6}o!m{_XL9t1(md8*w7V-(GNjeHin>41F z8blm&H?JW}arZVly}yGMq_>3S4=r?xN));;nv8nD?OZwzXdkPnV@c_6VqqoF<~GVrT_=9(bxq0~d&x zrgw*Ne?m8=5o>U<%px@UpaECl&G3n4iCc8j)`H0~NPphzO~?p9`X|JTm|-f81lL2L zGJ1e?xj6iYC#Fy`AHuFhR7fGpHEgYuy|YYG%yZOe8P>5U}`)#7c|FlZZ_?!h}zH z?y_f(1IANeUgoE6qRe51$wAIqK_twr-!bX~nUuVW&WBiRd*D0QLT)0+BjX(rDz{3Z zh{0w{QMeFA?C+@(JI;>hmRJhw4hU-EF$F`ZgjN|;9E@I&pHl?Mpbl9YbQZyAXJK3* zz5^Yjm^m4Ay)1vqS93Q<;#omb5!)VWpANJvN}&mjnkXpH6jyUVWi%8iNG*?#|CA;( z*(d{e{QekHopqRkobmhcM2m#Cgb5B(8g&YO_zZ>7(vr~0zWaGPEa?Raz}tO zNa>eIjAkm4DBvRmG<-mB^4%=S0Z~}DZhr0apCZJ~L6;#L06#9E@aditgS#+^G|bGS ztE&_S2D~r7qwp}}%D+>Xb>K3%w-R=N=16WN!tVlw5DN{dIzfL%7;${# zYslH=wZNBxmhyL7$jq);A{Z3Hk|w-G%D*tD4gm(?C@v}*g0MFTM_9ixBBustjvxU~ zh~F;LxsKH!BxD?2$Xp(_m39jv%uQU1LT2SbdC&$eq`(9oN4#f*fo1ofc)t&>IUyie z1~7cYaReJcA_|Pyab~h{PDBEQ$C)Sy)G#=C;;tf@>+xFr2ys+O??r(m7#BBXorc=^ z36tIP%RXxOa6q zfI~{Ljs(PsEy83ypkNFrpik%naNEOe;+A~l2F^aqIgE=~f5G{a0Y@+y-MI6o8qq=0 zMWo$gq~BEQ0`;B$ec1#BOh$-=5NFL0M6LK*{;Q|+oWv)8p0STf%9DD7$ z3N{BAun1z+nCzmLkdl&41gnW*FY>VqaIXL%3XcvgM;M|mGHl=e(ro(1ro#;2naRv4 zXpEUX68Z@B;#pZ+DqKZv8)8#{pCmnQ0{?Wp`&Y36l~QR=-x=?TU`fN zT@)rU4D^r|UtCej*3F>5>5dV#4VdyG<}J{AWLTFt#z2!*q9D>3aZEEakm5g-1r)#N z)n+2|Jfxl}VEhDEpDn^9sonMj1Gvs2D)G)3^vtoD!N3)EUrKCeMu5ThGunasI)r<$)R!ulZi)PP?)`Uci@k9T>-fN!vsAi_!!*YModA(RSAP& za7aj{OcG|bb$t@IUGTYaD$DjiTBvxvaH|@oF*zVYV#qJMsTU$;&0mwNfgVvyf&R%w zEpxNKG?%IRvVmPC8ACUpiCkzztU6@CLCNhMMcvwQJsNz=O?ZYI`7i~O&w?KbV_{(# zhABJ+LICNaF)BdZ10?f7QAh-M5X*AGQI;47&?a~Rn~A8dfy?FfZDSFVT~n;1A2>z1 zUIL>9Tktk19h9;>p8*SEQA0qG>ZhB(3>uv4YhXj7g`fiMeE$wUIGjYWEFLVL5=0q$G=~bQ3%vzVorcmH*~A5o!2CSXBj5|6sDF2+!#9nibs8z*3RGz^jKsp% zaY-{sM9LlD+-q_5#R`gjaxOvPzAe%R@{qRYC^_Q zMt~*=y^6}ofR{ARR(_+mJpuoNNHlf#Cp?{1_on@Xt|ASiIWVs z2p+@r7K9CfWu1%%V>W?^iNvKxR?n>aGjob1@zPwqDnt|nY_4=WD6vR`X4HU$%}gOU zw#y+>g8z`fpudBQi=+?sM%Se~(<;RbM^T6fXz(#o704S1rSSaxd;-h36V4!C&5a_Y zDse{(yqRcjYjJRhKb-8o*)hPb--&>cD>OAl(9{qX8kc#b0U1PQ*x-9n z6E^Agj_Yvh;cJ`2>fQek5;U4}^7g2Ar|x$&Hz$xaC6aO0gMWsYAmT8Idg5X*h;t|^ zvTVMzujKSV-GEDW3)v3HSV$}A`SihN;oP!@cDRj>jD+OnDI%9i`>dvMfFRidP64-2 zETk@N@DM~tMJ{86-bR!VHs72BIE{cz2}Nwuw-IJ!ppIPpN6NYX+{~}}6NUXsk*>_H zwm%P70&ZJKmkYHM!*>Tzq=L~7T9x*hOI3YXk8rPza;wK|Xho``pllc&mxhPUANH+#wH7lH2oD^(QckUmD zEhicvC8h42z9h3~?!gWff83rz23*i4V;$+CY0ZacmanV2)33F-83K4d_|c}@+SF`O_iGP+1>z+;2RB2`t0cx%wx{#$xS&)_ON zh*=s;?Q`}{g94+-d*C}o;Fm}nXZ6@q6m2#!f0DcQYQk8~fyrC0F+1tpyUuPm4i@I& zNc$4mC$guzc99{i=il8U0)m2uwy}mMNMpR%oTOKDHuG7Jwvr|xpYbzNI5Q5;g&ri0 z){$zU?YQ?x|KYH>Gx`G)pHlXkCHl!4{(N7 zPck`hIh3#w4PTzD>|c4sL30|2IdXLmr4<;cF$?1nnbae#Ij}SeKAuKN8iXlMrGSRN zGW05nLdqbWovt}1GhKBN4#afqH|#+C+n{@Ojvp@~tqXT1Kr}O%O@rG9x`kPynK*bT zBAMW-xc&S0J77K%eph{WapQK-7z<{g_wnKT`ln{^UK@SAl@L#%g&b_AC6Gr(u-jIs$afj zpPOn3G65-u$qh`VvAS+r8b%5i=Hk-HZK{hFk$w{;<0SY_Jbv!JhUJ&8TrspIGiujy ziydt>xr7HE>|u1EXiptsY=gl9@lzu(m|+G+I2L>>Q5n&;oQhyz3_@i}ds6;{CWl+< zh?asm%De7kV`J4IOEIJt>!)<1So&zy7Z{5NY z6IU6O4s!3^ya$83`tLS$#24dzOIt-f8}AmbI#`n-UMbjy#uGrap^MV$#6DExrWae+$}4zk3#F# zjI7Y>Hgc!P?Y!|tLFb!iUYfW!V>NjgAn{+{|AX4f!sqBj$WE4b0 zqs;o}Y01fSxGu=GBLa(r#wy!KNY{rh zC@~{zL^a(mYuc~OHSNcu9XCu4gY1GmHH~kwSl9ea=jxv+(ofI-+xMQIUHBh-f75jM z5HM{gK-`r@OH!=QlpE26Crh)g;=DEfGiQj|yR4u1l;)7m>>`vW>6iZ!lW7AZ_%`(A zmOhI^ZP|ANq2c(D`wt;llhGpRuGx(qq=N%@7L37t988Qi?shHzPcSd*AxBC-uCi89 zR#vXSs4Vn9a>sHYDyVb^Lr`LW0j{3g3?DTTka>|^U`~dLVrz3QD4wd$P7!26GPeab z^G&v;JbJW?ZF4nIi$OH5c|&f;f*m>W_dhpdL12CSLIqR+Hy0O~9sh(fXs@~1 z02dK36PVaf@arf&@#y4isQD=XXiih3y*W0T6s{lHl>&Tk z4%~&?Rv|%hZ1OP0OdGA;UHn%qq}K#Bi1Ie62R4Ib8aO7Zq8#z##Eo8Nm_Fc2fUw z3@|X~e^qvNAx(v09Ji1NCfGz_30*i6hS zENYaV8*ds3hxtW_7ImW%B8rF&onB~nA|s-k4NXLU&$`PTc|X{D-t!)w=lOrVrGvk0 zW9m6*W2eW^9}EW9@E3>dw`wZCZETdF2Nu(vm|ejG=*yKU>l1CS1z}GohecGhIOiB- z=fMCzI9JkP3yKs?`+Zv$kh8g-laVM(z|1Sj@ImZ05+E3yQ zxA$hvJU=)*1O>eo3rul1T*y`vTt1&saw5hii{BN}pi2}7Ll^6vqZMNs<_l5jy2_8V zrhj+raXDvyU@DFvo%8?LWHP<|wDSNu8t6rd+-YTpGRq2Xe4MyY9qwSGN21oQz1OJT z+dNTDiibG8`&LSNZ8(<_J`id6@@O2{YKrK~`S6C}^SgUq=?ju>jf>GBL6oR;sJ&m| zDwlq&a{Rs8_D}UP{}fq#cEw-v<~nuCSZ8Q#mLhnbn-k6`IW`xQvy&*P?yDHDM9KTt z58s=4d5}aecuykd+Q+XVR6{#BirJjT(0Y+JUw~J9`?GN29$mKpW%=!DtW_vdZ6ORe z;x$l|s;ynZC-Ep`H_TMbFS+$xp*eg51eRN(J9zBq0yRC|9C!&>;kF5(8_S`nwxp)i x-BqW-Us&Ji8zR8WVwLxQE6}N@jI)==$;&t0jirU7+1#p;P #include #include +#include "../include/structs.h" // --------------------------------------------------------------------------- // Set-up DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm); +PetscErrorCode CreateDistributedDM(MPI_Comm comm, ProblemData *problem_data, + DM *dm); #endif // setupdm_h diff --git a/examples/Hdiv-mass/include/setup-libceed.h b/examples/Hdiv-mass/include/setup-libceed.h index 0e9c6f08e4..bced0f69cc 100644 --- a/examples/Hdiv-mass/include/setup-libceed.h +++ b/examples/Hdiv-mass/include/setup-libceed.h @@ -11,16 +11,14 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data); PetscInt Involute(PetscInt i); // Utility function to create local CEED restriction from DMPlex PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, - CeedInt height, DMLabel domain_label, CeedInt value, CeedInt P, + CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr); // Utility function to create local CEED Oriented restriction from DMPlex PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, - CeedInt height, DMLabel domain_label, CeedInt value, CeedInt P, - CeedElemRestriction *elem_restr_oriented); + CeedInt P, CeedElemRestriction *elem_restr_oriented); // Set up libCEED for a given degree PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *problem_data, PetscInt U_g_size, PetscInt U_loc_size, CeedData ceed_data, - CeedVector rhs_ceed, CeedVector *target, - CeedVector true_ceed); + CeedVector rhs_ceed, CeedVector *target); #endif // setuplibceed_h diff --git a/examples/Hdiv-mass/include/structs.h b/examples/Hdiv-mass/include/structs.h index 41c3ecd864..b4a8754e65 100644 --- a/examples/Hdiv-mass/include/structs.h +++ b/examples/Hdiv-mass/include/structs.h @@ -19,7 +19,7 @@ struct AppCtx_ { typedef struct CeedData_ *CeedData; struct CeedData_ { CeedBasis basis_x, basis_u; - CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_geo_data_i, + CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_u_i; CeedQFunction qf_residual, qf_error; CeedOperator op_residual, op_error; @@ -37,6 +37,13 @@ struct PQ2DContext_ { #endif // 2) poisson-hex3d +#ifndef PHYSICS_POISSONHEX3D_STRUCT +#define PHYSICS_POISSONHEX3D_STRUCT +typedef struct PH3DContext_ *PH3DContext; +struct PH3DContext_ { + CeedScalar kappa; +}; +#endif // 3) poisson-prism3d @@ -46,6 +53,7 @@ struct PQ2DContext_ { typedef struct Physics_ *Physics; struct Physics_ { PQ2DContext pq2d_ctx; + PH3DContext ph3d_ctx; }; // PETSc user data @@ -63,11 +71,10 @@ struct User_ { // Problem specific data typedef struct { - CeedQFunctionUser setup_rhs, residual, setup_error, setup_true; - const char *setup_rhs_loc, *residual_loc, *setup_error_loc, - *setup_true_loc; + CeedQFunctionUser setup_rhs, residual, setup_error; + const char *setup_rhs_loc, *residual_loc, *setup_error_loc; CeedQuadMode quadrature_mode; - CeedInt geo_data_size, elem_node; + CeedInt elem_node, dim; PetscErrorCode (*setup_ctx)(Ceed, CeedData, Physics); } ProblemData; diff --git a/examples/Hdiv-mass/main.c b/examples/Hdiv-mass/main.c index 15b41871c5..ddd66fbb70 100644 --- a/examples/Hdiv-mass/main.c +++ b/examples/Hdiv-mass/main.c @@ -1,7 +1,32 @@ -/// @file -/// Test creation, use, and destruction of an element restriction for 2D quad Hdiv +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +// libCEED + PETSc Example: Mixed-Poisson in H(div) space +// +// This example demonstrates a simple usage of libCEED with PETSc to solve +// elasticity problems. +// +// The code uses higher level communication protocols in DMPlex. +// +// Build with: make +// Run with: +// ./main pc_type svd +// ./main -pc_type svd -problem mass2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 +// ./main -pc_type svd -problem mass3d -dm_plex_dim 3 -dm_plex_box_faces 4,4,4 -// run with ./main const char help[] = "Solve H(div)-mixed problem using PETSc and libCEED\n"; #include "main.h" @@ -52,8 +77,8 @@ int main(int argc, char **argv) { PetscErrorCode (*p)(ProblemData *, void *); ierr = PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p); CHKERRQ(ierr); - if (!p) SETERRQ1(PETSC_COMM_SELF, 1, "Problem '%s' not found", - app_ctx->problem_name); + if (!p) SETERRQ(PETSC_COMM_SELF, 1, "Problem '%s' not found", + app_ctx->problem_name); ierr = (*p)(problem_data, &user); CHKERRQ(ierr); } @@ -71,7 +96,7 @@ int main(int argc, char **argv) { // PETSc objects DM dm; VecType vec_type; - ierr = CreateDistributedDM(comm, &dm); CHKERRQ(ierr); + ierr = CreateDistributedDM(comm, problem_data, &dm); CHKERRQ(ierr); ierr = DMGetVecType(dm, &vec_type); CHKERRQ(ierr); if (!vec_type) { // Not yet set by user -dm_vec_type switch (mem_type_backend) { @@ -113,28 +138,13 @@ int main(int argc, char **argv) { ierr = VecGetArrayAndMemType(rhs_loc, &r, &mem_type); CHKERRQ(ierr); CeedVectorCreate(ceed, U_l_size, &rhs_ceed); CeedVectorSetArray(rhs_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, r); - // Get True vector - Vec true_loc; - PetscScalar *t; - CeedVector true_ceed; - PetscMemType t_mem_type; - ierr = VecDuplicate(U_loc, &true_loc); CHKERRQ(ierr); - ierr = VecZeroEntries(true_loc); CHKERRQ(ierr); - ierr = VecGetArrayAndMemType(true_loc, &t, &t_mem_type); CHKERRQ(ierr); - CeedVectorCreate(ceed, U_l_size, &true_ceed); - CeedVectorSetArray(true_ceed, MemTypeP2C(t_mem_type), CEED_USE_POINTER, t); // --------------------------------------------------------------------------- // Setup libCEED // --------------------------------------------------------------------------- // -- Set up libCEED objects ierr = SetupLibceed(dm, ceed, app_ctx, problem_data, U_g_size, - U_loc_size, ceed_data, rhs_ceed, &target, true_ceed); CHKERRQ(ierr); - //CeedVectorView(rhs_ceed, "%12.8f", stdout); - printf("------------------------------------------------------------\n"); - printf("True solution projected into H(div) space; main.c:\n"); - printf("------------------------------------------------------------\n"); - CeedVectorView(true_ceed, "%12.8f", stdout); + U_loc_size, ceed_data, rhs_ceed, &target); CHKERRQ(ierr); // --------------------------------------------------------------------------- // Gather RHS @@ -145,6 +155,8 @@ int main(int argc, char **argv) { ierr = VecDuplicate(U_g, &rhs); CHKERRQ(ierr); ierr = VecZeroEntries(rhs); CHKERRQ(ierr); ierr = DMLocalToGlobal(dm, rhs_loc, ADD_VALUES, rhs); CHKERRQ(ierr); + //VecView(rhs, PETSC_VIEWER_STDOUT_WORLD); + // --------------------------------------------------------------------------- // Setup Mat, KSP // --------------------------------------------------------------------------- @@ -171,14 +183,17 @@ int main(int argc, char **argv) { ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); ierr = KSPSolve(ksp, rhs, U_g); CHKERRQ(ierr); + //printf("U_g\n"); //VecView(U_g, PETSC_VIEWER_STDOUT_WORLD); // --------------------------------------------------------------------------- - // Compute pointwise L2 error + // Compute pointwise L2 maximum error // --------------------------------------------------------------------------- CeedScalar l2_error; ierr = ComputeError(user, U_g, target, &l2_error); CHKERRQ(ierr); - //printf("l2_error: %f\n",l2_error); + + // --------------------------------------------------------------------------- // Output results + // --------------------------------------------------------------------------- KSPType ksp_type; KSPConvergedReason reason; PetscReal rnorm; @@ -189,12 +204,13 @@ int main(int argc, char **argv) { ierr = KSPGetResidualNorm(ksp, &rnorm); CHKERRQ(ierr); ierr = PetscPrintf(comm, " KSP:\n" - " KSP Type : %s\n" - " KSP Convergence : %s\n" - " Total KSP Iterations : %D\n" - " Final rnorm : %e\n", + " KSP Type : %s\n" + " KSP Convergence : %s\n" + " Total KSP Iterations : %" PetscInt_FMT "\n" + " Final rnorm : %e\n" + " L2 Error : %e\n", ksp_type, KSPConvergedReasons[reason], its, - (double)rnorm); CHKERRQ(ierr); + (double)rnorm, (double)l2_error); CHKERRQ(ierr); // --------------------------------------------------------------------------- // Free objects @@ -219,9 +235,8 @@ int main(int argc, char **argv) { ierr = PetscFree(user); CHKERRQ(ierr); ierr = PetscFree(phys_ctx->pq2d_ctx); CHKERRQ(ierr); ierr = PetscFree(phys_ctx); CHKERRQ(ierr); - // Free libCEED objects - CeedVectorDestroy(&true_ceed); + // Free libCEED objects CeedVectorDestroy(&rhs_ceed); CeedVectorDestroy(&target); ierr = CeedDataDestroy(ceed_data); CHKERRQ(ierr); diff --git a/examples/Hdiv-mass/problems/poisson-mass2d.c b/examples/Hdiv-mass/problems/mass2d.c similarity index 78% rename from examples/Hdiv-mass/problems/poisson-mass2d.c rename to examples/Hdiv-mass/problems/mass2d.c index 1ac1bf9e5b..4049b42d5d 100644 --- a/examples/Hdiv-mass/problems/poisson-mass2d.c +++ b/examples/Hdiv-mass/problems/mass2d.c @@ -22,7 +22,6 @@ #include "../qfunctions/poisson-rhs2d.h" #include "../qfunctions/poisson-mass2d.h" #include "../qfunctions/poisson-error2d.h" -#include "../qfunctions/poisson-true2d.h" // Hdiv_POISSON_MASS2D is registered in cl-option.c PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx) { @@ -36,24 +35,22 @@ PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx) { // ------------------------------------------------------ // SET UP POISSON_QUAD2D // ------------------------------------------------------ + problem_data->dim = 2; problem_data->elem_node = 4; - problem_data->geo_data_size = 1; problem_data->quadrature_mode = CEED_GAUSS; - problem_data->setup_rhs = SetupRhs; - problem_data->setup_rhs_loc = SetupRhs_loc; - problem_data->residual = SetupMass; - problem_data->residual_loc = SetupMass_loc; + problem_data->setup_rhs = SetupRhs2D; + problem_data->setup_rhs_loc = SetupRhs2D_loc; + problem_data->residual = SetupMass2D; + problem_data->residual_loc = SetupMass2D_loc; problem_data->setup_error = SetupError2D; problem_data->setup_error_loc = SetupError2D_loc; - problem_data->setup_true = SetupTrueSoln2D; - problem_data->setup_true_loc = SetupTrueSoln2D_loc; + // ------------------------------------------------------ // Command line Options // ------------------------------------------------------ - ierr = PetscOptionsBegin(comm, NULL, "Options for DENSITY_CURRENT problem", - NULL); CHKERRQ(ierr); + PetscOptionsBegin(comm, NULL, "Options for Hdiv-mass problem", NULL); - ierr = PetscOptionsEnd(); CHKERRQ(ierr); + PetscOptionsEnd(); PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mass/problems/mass3d.c b/examples/Hdiv-mass/problems/mass3d.c new file mode 100644 index 0000000000..a86d3c5600 --- /dev/null +++ b/examples/Hdiv-mass/problems/mass3d.c @@ -0,0 +1,56 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Utility functions for setting up POISSON_QUAD2D + +#include "../include/setup-libceed.h" +#include "../include/problems.h" +#include "../qfunctions/poisson-rhs3d.h" +#include "../qfunctions/poisson-mass3d.h" +#include "../qfunctions/poisson-error3d.h" + +// Hdiv_POISSON_MASS2D is registered in cl-option.c +PetscErrorCode Hdiv_POISSON_MASS3D(ProblemData *problem_data, void *ctx) { + User user = *(User *)ctx; + MPI_Comm comm = PETSC_COMM_WORLD; + PetscInt ierr; + PetscFunctionBeginUser; + + ierr = PetscCalloc1(1, &user->phys->ph3d_ctx); CHKERRQ(ierr); + + // ------------------------------------------------------ + // SET UP POISSON_QUAD3D + // ------------------------------------------------------ + problem_data->dim = 3; + problem_data->elem_node = 8; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->setup_rhs = SetupRhs3D; + problem_data->setup_rhs_loc = SetupRhs3D_loc; + problem_data->residual = SetupMass3D; + problem_data->residual_loc = SetupMass3D_loc; + problem_data->setup_error = SetupError3D; + problem_data->setup_error_loc = SetupError3D_loc; + + // ------------------------------------------------------ + // Command line Options + // ------------------------------------------------------ + PetscOptionsBegin(comm, NULL, "Options for Hdiv-mass problem", NULL); + + PetscOptionsEnd(); + + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mass/qfunctions/poisson-error2d.h b/examples/Hdiv-mass/qfunctions/poisson-error2d.h index 1bdfd8ad1c..da156742a2 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-error2d.h +++ b/examples/Hdiv-mass/qfunctions/poisson-error2d.h @@ -15,10 +15,10 @@ // testbed platforms, in support of the nation's exascale computing imperative. /// @file -/// Compute error of the H(div) example using PETSc +/// Compute pointwise error of the H(div) example using PETSc -#ifndef ERROR_H -#define ERROR_H +#ifndef ERROR2D_H +#define ERROR2D_H #include @@ -30,10 +30,9 @@ CEED_QFUNCTION(SetupError2D)(void *ctx, const CeedInt Q, CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], - (*target) = in[3]; + const CeedScalar (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[0], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[1], + (*target) = in[2], (*w) = in[3]; // Outputs CeedScalar (*error) = out[0]; // Quadrature Point Loop @@ -42,10 +41,17 @@ CEED_QFUNCTION(SetupError2D)(void *ctx, const CeedInt Q, // Setup, J = dx/dX const CeedScalar J[2][2] = {{dxdX[0][0][i], dxdX[1][0][i]}, {dxdX[0][1][i], dxdX[1][1][i]}}; - const CeedScalar detJ = J[0][0]*J[1][1] - J[0][1]*J[1][0]; + const CeedScalar detJ = J[0][0]*J[1][1] - J[0][1]*J[1][0]; + // Compute Piola map:uh = J*u/detJ + CeedScalar uh[2]; + for (CeedInt k = 0; k < 2; k++) { + uh[k] = 0; + for (CeedInt m = 0; m < 2; m++) + uh[k] += J[k][m] * u[m][i]/detJ; + } // Error - error[i+0*Q] = fabs(u[0][i] - target[i+0*Q]) * w[i] * detJ; - error[i+1*Q] = fabs(u[1][i] - target[i+1*Q]) * w[i] * detJ; + error[i+0*Q] = (uh[0] - target[i+0*Q])*(uh[0] - target[i+0*Q])*w[i]*detJ; + error[i+1*Q] = (uh[1] - target[i+1*Q])*(uh[1] - target[i+1*Q])*w[i]*detJ; } // End of Quadrature Point Loop @@ -53,4 +59,4 @@ CEED_QFUNCTION(SetupError2D)(void *ctx, const CeedInt Q, } // ----------------------------------------------------------------------------- -#endif // End ERROR_H +#endif // End ERROR2D_H diff --git a/examples/Hdiv-mass/qfunctions/poisson-error3d.h b/examples/Hdiv-mass/qfunctions/poisson-error3d.h new file mode 100644 index 0000000000..dd8f0fbc66 --- /dev/null +++ b/examples/Hdiv-mass/qfunctions/poisson-error3d.h @@ -0,0 +1,80 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Compute pointwise error of the H(div) example using PETSc + +#ifndef ERROR3D_H +#define ERROR3D_H + +#include + +// ----------------------------------------------------------------------------- +// Compute determinant of 3x3 matrix +// ----------------------------------------------------------------------------- +#ifndef DetMat +#define DetMat +CEED_QFUNCTION_HELPER CeedScalar ComputeDetMat(const CeedScalar A[3][3]) { + // Compute det(A) + const CeedScalar B11 = A[1][1]*A[2][2] - A[1][2]*A[2][1]; + const CeedScalar B12 = A[0][2]*A[2][1] - A[0][1]*A[2][2]; + const CeedScalar B13 = A[0][1]*A[1][2] - A[0][2]*A[1][1]; + CeedScalar detA = A[0][0]*B11 + A[1][0]*B12 + A[2][0]*B13; + + return detA; +}; +#endif + +// ----------------------------------------------------------------------------- +// Compuet error +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(SetupError3D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[0], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[1], + (*target) = in[2], (*w) = in[3]; + // Outputs + CeedScalar (*error) = out[0]; + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i \int (v^T J^T*J*u*w/detJ) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupMass)(void *ctx, CeedInt Q, const CeedScalar *const *in, - CeedScalar *const *out) { +CEED_QFUNCTION(SetupMass2D)(void *ctx, CeedInt Q, const CeedScalar *const *in, + CeedScalar *const *out) { // *INDENT-OFF* // Inputs const CeedScalar (*w) = in[0], @@ -68,7 +68,7 @@ CEED_QFUNCTION(SetupMass)(void *ctx, CeedInt Q, const CeedScalar *const *in, JTJ[j][k] += J[m][j] * J[m][k]; } } - // 2) Compute Piola map J^T*J*u * w /detJ + // 2) Compute J^T*J*u * w /detJ for (CeedInt k = 0; k < 2; k++) { v[k][i] = 0; for (CeedInt m = 0; m < 2; m++) diff --git a/examples/Hdiv-mass/qfunctions/poisson-mass3d.h b/examples/Hdiv-mass/qfunctions/poisson-mass3d.h new file mode 100644 index 0000000000..4b50b86d0a --- /dev/null +++ b/examples/Hdiv-mass/qfunctions/poisson-mass3d.h @@ -0,0 +1,100 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Mixed poisson 3D hex element using PETSc + +#ifndef POISSON_MASS3D_H +#define POISSON_MASS3D_H + +#include + +// ----------------------------------------------------------------------------- +// Compute determinant of 3x3 matrix +// ----------------------------------------------------------------------------- +#ifndef DetMat +#define DetMat +CEED_QFUNCTION_HELPER CeedScalar ComputeDetMat(const CeedScalar A[3][3]) { + // Compute det(A) + const CeedScalar B11 = A[1][1]*A[2][2] - A[1][2]*A[2][1]; + const CeedScalar B12 = A[0][2]*A[2][1] - A[0][1]*A[2][2]; + const CeedScalar B13 = A[0][1]*A[1][2] - A[0][2]*A[1][1]; + CeedScalar detA = A[0][0]*B11 + A[1][0]*B12 + A[2][0]*B13; + + return detA; +}; +#endif + +// ----------------------------------------------------------------------------- +// This QFunction applies the mass operator for a vector field of 2 components. +// +// Inputs: +// w - weight of quadrature +// J - dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// u - Input basis at quadrature points +// +// Output: +// v - Output vector (test functions) at quadrature points +// Note we need to apply Piola map on the basis, which is J*u/detJ +// So (v,u) = \int (v^T * u detJ*w) ==> \int (v^T J^T*J*u*w/detJ) +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(SetupMass3D)(void *ctx, CeedInt Q, const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + // *INDENT-ON* + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif // ----------------------------------------------------------------------------- -// This QFunction sets up the rhs for the problem +// This QFunction sets up the rhs and true solution for the problem // Inputs: // x - interpolation of the physical coordinate // w - weight of quadrature // J - dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim // // Output: +// true_soln - True solution that we use it in poisson-error2d.h +// to compute pointwise max error // rhs - Output vector (test functions) at quadrature points // Note we need to apply Piola map on the basis, which is J*u/detJ // So (v,ue) = \int (v^T * ue detJ*w) ==> \int (v^T J^T* ue * w) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupRhs)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { +CEED_QFUNCTION(SetupRhs2D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { // *INDENT-OFF* // Inputs const CeedScalar (*coords) = in[0], @@ -55,7 +60,8 @@ CEED_QFUNCTION(SetupRhs)(void *ctx, const CeedInt Q, {dxdX[0][1][i], dxdX[1][1][i]}}; // *INDENT-ON* // Compute J^T*ue - CeedScalar ue[2] = {x-y, x+y}; + CeedScalar ue[2] = {-M_PI*cos(M_PI*x) *sin(M_PI*y), -M_PI*sin(M_PI*x) *cos(M_PI*y)}; + //CeedScalar ue[2] = {x-y, x+y}; CeedScalar rhs1[2]; for (CeedInt k = 0; k < 2; k++) { rhs1[k] = 0; diff --git a/examples/Hdiv-mass/qfunctions/poisson-rhs3d.h b/examples/Hdiv-mass/qfunctions/poisson-rhs3d.h new file mode 100644 index 0000000000..81d42d323a --- /dev/null +++ b/examples/Hdiv-mass/qfunctions/poisson-rhs3d.h @@ -0,0 +1,88 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Mixed poisson 3D Hex element using PETSc + +#ifndef POISSON_RHS3D_H +#define POISSON_RHS3D_H + +#include + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif +// ----------------------------------------------------------------------------- +// This QFunction sets up the rhs and true solution for the problem +// Inputs: +// x - interpolation of the physical coordinate +// w - weight of quadrature +// J - dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// +// Output: +// true_soln - True solution that we use it in poisson-error2d.h +// to compute pointwise max error +// rhs - Output vector (test functions) at quadrature points +// Note we need to apply Piola map on the basis, which is J*u/detJ +// So (v,ue) = \int (v^T * ue detJ*w) ==> \int (v^T J^T* ue * w) +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(SetupRhs3D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*coords) = in[0], + (*w) = in[1], + (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[2]; + // Outputs + //CeedScalar (*rhs)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + CeedScalar (*true_soln) = out[0], (*rhs) = out[1]; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; iproblems, "poisson_mass2d", + ierr = PetscFunctionListAdd(&app_ctx->problems, "mass2d", Hdiv_POISSON_MASS2D); CHKERRQ(ierr); // 2) poisson-hex3d + ierr = PetscFunctionListAdd(&app_ctx->problems, "mass3d", + Hdiv_POISSON_MASS3D); CHKERRQ(ierr); // 3) poisson-prism3d @@ -44,9 +46,8 @@ PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx) { PetscErrorCode ierr; PetscFunctionBeginUser; - ierr = PetscOptionsBegin(comm, NULL, - "H(div) examples in PETSc with libCEED", - NULL); CHKERRQ(ierr); + PetscOptionsBegin(comm, NULL, + "H(div) examples in PETSc with libCEED", NULL); ierr = PetscOptionsFList("-problem", "Problem to solve", NULL, app_ctx->problems, @@ -57,17 +58,17 @@ PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx) { ierr = PetscOptionsInt("-degree", "Polynomial degree of finite elements", NULL, app_ctx->degree, &app_ctx->degree, NULL); CHKERRQ(ierr); - app_ctx->q_extra = 2; + app_ctx->q_extra = 0; ierr = PetscOptionsInt("-q_extra", "Number of extra quadrature points", NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL); CHKERRQ(ierr); // Provide default problem if not specified if (!problem_flag) { - const char *problem_name = "poisson_mass2d"; + const char *problem_name = "mass2d"; strncpy(app_ctx->problem_name, problem_name, 16); } - ierr = PetscOptionsEnd(); CHKERRQ(ierr); + PetscOptionsEnd(); PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mass/src/matops.c b/examples/Hdiv-mass/src/matops.c index ed1fd4cfeb..5a31719a35 100644 --- a/examples/Hdiv-mass/src/matops.c +++ b/examples/Hdiv-mass/src/matops.c @@ -66,7 +66,7 @@ PetscErrorCode ComputeError(User user, Vec X, CeedVector target, PetscScalar *x; PetscMemType mem_type; CeedVector collocated_error; - CeedInt length; + CeedSize length; PetscFunctionBeginUser; CeedVectorGetLength(target, &length); @@ -87,7 +87,9 @@ PetscErrorCode ComputeError(User user, Vec X, CeedVector target, ierr = VecRestoreArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x); CHKERRQ(ierr); - CeedVectorNorm(collocated_error, CEED_NORM_2, l2_error); + CeedScalar error; + CeedVectorNorm(collocated_error, CEED_NORM_1, &error); + *l2_error = sqrt(error); // Cleanup CeedVectorDestroy(&collocated_error); diff --git a/examples/Hdiv-mass/src/setup-dm.c b/examples/Hdiv-mass/src/setup-dm.c index 9bc908a8c4..7023fdbe6d 100644 --- a/examples/Hdiv-mass/src/setup-dm.c +++ b/examples/Hdiv-mass/src/setup-dm.c @@ -3,26 +3,31 @@ // --------------------------------------------------------------------------- // Set-up DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm) { +PetscErrorCode CreateDistributedDM(MPI_Comm comm, ProblemData *problem_data, + DM *dm) { PetscErrorCode ierr; PetscSection sec; - PetscBool interpolate = PETSC_TRUE; - PetscInt nx = 1, ny = 1; - PetscInt faces[2] = {nx, ny}; - PetscInt dim = 2, dofs_per_edge; + PetscInt dofs_per_face; PetscInt p_start, p_end; PetscInt c_start, c_end; // cells - PetscInt e_start, e_end, e; // edges + PetscInt f_start, f_end; // faces PetscInt v_start, v_end; // vertices PetscFunctionBeginUser; - ierr = DMPlexCreateBoxMesh(comm, dim, PETSC_FALSE, faces, NULL, - NULL, NULL, interpolate, dm); CHKERRQ(ierr); + // Create DMPLEX + ierr = DMCreate(comm, dm); CHKERRQ(ierr); + ierr = DMSetType(*dm, DMPLEX); CHKERRQ(ierr); + // Set Tensor elements + ierr = PetscOptionsSetValue(NULL, "-dm_plex_simplex", "0"); CHKERRQ(ierr); + // Set CL options + ierr = DMSetFromOptions(*dm); CHKERRQ(ierr); + ierr = DMViewFromOptions(*dm, NULL, "-dm_view"); CHKERRQ(ierr); + // Get plex limits ierr = DMPlexGetChart(*dm, &p_start, &p_end); CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(*dm, 0, &c_start, &c_end); CHKERRQ(ierr); - ierr = DMPlexGetDepthStratum(*dm, 1, &e_start, &e_end); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(*dm, 1, &f_start, &f_end); CHKERRQ(ierr); ierr = DMPlexGetDepthStratum(*dm, 0, &v_start, &v_end); CHKERRQ(ierr); // Create section ierr = PetscSectionCreate(comm, &sec); CHKERRQ(ierr); @@ -30,15 +35,14 @@ PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm) { ierr = PetscSectionSetFieldName(sec,0,"Velocity"); CHKERRQ(ierr); ierr = PetscSectionSetFieldComponents(sec,0,1); CHKERRQ(ierr); ierr = PetscSectionSetChart(sec,p_start,p_end); CHKERRQ(ierr); - // Setup dofs per edge - for (e = e_start; e < e_end; e++) { - ierr = DMPlexGetConeSize(*dm, e, &dofs_per_edge); CHKERRQ(ierr); - ierr = PetscSectionSetFieldDof(sec, e, 0, dofs_per_edge); CHKERRQ(ierr); - ierr = PetscSectionSetDof (sec, e, dofs_per_edge); CHKERRQ(ierr); + // Setup dofs per face + for (PetscInt f = f_start; f < f_end; f++) { + ierr = DMPlexGetConeSize(*dm, f, &dofs_per_face); CHKERRQ(ierr); + ierr = PetscSectionSetFieldDof(sec, f, 0, dofs_per_face); CHKERRQ(ierr); + ierr = PetscSectionSetDof (sec, f, dofs_per_face); CHKERRQ(ierr); } ierr = PetscSectionSetUp(sec); CHKERRQ(ierr); ierr = DMSetSection(*dm,sec); CHKERRQ(ierr); - ierr = DMViewFromOptions(*dm, NULL, "-dm_view"); CHKERRQ(ierr); ierr = PetscSectionDestroy(&sec); CHKERRQ(ierr); PetscFunctionReturn(0); diff --git a/examples/Hdiv-mass/src/setup-libceed.c b/examples/Hdiv-mass/src/setup-libceed.c index 8a5d3a995a..767ceca67a 100644 --- a/examples/Hdiv-mass/src/setup-libceed.c +++ b/examples/Hdiv-mass/src/setup-libceed.c @@ -1,6 +1,7 @@ #include "../include/setup-libceed.h" #include "../include/petsc-macros.h" -#include "../basis/quad.h" +#include "../basis/Hdiv-quad.h" +#include "../basis/Hdiv-hex.h" // ----------------------------------------------------------------------------- // Convert PETSc MemType to libCEED MemType @@ -22,7 +23,6 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data) { // Restrictions CeedElemRestrictionDestroy(&ceed_data->elem_restr_x); CeedElemRestrictionDestroy(&ceed_data->elem_restr_u); - CeedElemRestrictionDestroy(&ceed_data->elem_restr_geo_data_i); CeedElemRestrictionDestroy(&ceed_data->elem_restr_u_i); // Bases CeedBasisDestroy(&ceed_data->basis_x); @@ -44,134 +44,26 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data) { PetscInt Involute(PetscInt i) { return i >= 0 ? i : -(i + 1); }; + // ----------------------------------------------------------------------------- // Get CEED restriction data from DMPlex // ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, - CeedInt height, DMLabel domain_label, CeedInt value, CeedInt P, - CeedElemRestriction *elem_restr) { - PetscSection section; - PetscInt p, num_elem, num_dof, *restr_indices, elem_offset, num_fields, - dim, depth; - Vec U_loc; - DMLabel depth_label; - IS depth_is, iter_is; - const PetscInt *iter_indices; +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, + DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { + PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; PetscErrorCode ierr; PetscFunctionBeginUser; - ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr); - dim -= height; - ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); - ierr = PetscSectionGetNumFields(section, &num_fields); CHKERRQ(ierr); - PetscInt num_comp[num_fields], field_offsets[num_fields+1]; - field_offsets[0] = 0; - for (PetscInt f = 0; f < num_fields; f++) { - ierr = PetscSectionGetFieldComponents(section, f, &num_comp[f]); CHKERRQ(ierr); - field_offsets[f+1] = field_offsets[f] + num_comp[f]; - } - - ierr = DMPlexGetDepth(dm, &depth); CHKERRQ(ierr); - ierr = DMPlexGetDepthLabel(dm, &depth_label); CHKERRQ(ierr); - ierr = DMLabelGetStratumIS(depth_label, depth - height, &depth_is); + ierr = DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, + &elem_size, &num_comp, &num_dof, &elem_restr_offsets); CHKERRQ(ierr); - if (domain_label) { - IS domain_is; - ierr = DMLabelGetStratumIS(domain_label, value, &domain_is); CHKERRQ(ierr); - if (domain_is) { // domainIS is non-empty - ierr = ISIntersect(depth_is, domain_is, &iter_is); CHKERRQ(ierr); - ierr = ISDestroy(&domain_is); CHKERRQ(ierr); - } else { // domainIS is NULL (empty) - iter_is = NULL; - } - ierr = ISDestroy(&depth_is); CHKERRQ(ierr); - } else { - iter_is = depth_is; - } - if (iter_is) { - ierr = ISGetLocalSize(iter_is, &num_elem); CHKERRQ(ierr); - ierr = ISGetIndices(iter_is, &iter_indices); CHKERRQ(ierr); - } else { - num_elem = 0; - iter_indices = NULL; - } - ierr = PetscMalloc1(num_elem*PetscPowInt(P, dim), &restr_indices); - CHKERRQ(ierr); - for (p = 0, elem_offset = 0; p < num_elem; p++) { - PetscInt c = iter_indices[p]; - PetscInt num_indices, *indices, num_nodes; - ierr = DMPlexGetClosureIndices(dm, section, section, c, PETSC_TRUE, - &num_indices, &indices, NULL, NULL); - CHKERRQ(ierr); - bool flip = false; - if (height > 0) { - PetscInt num_cells, num_faces, start = -1; - const PetscInt *orients, *faces, *cells; - ierr = DMPlexGetSupport(dm, c, &cells); CHKERRQ(ierr); - ierr = DMPlexGetSupportSize(dm, c, &num_cells); CHKERRQ(ierr); - if (num_cells != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, - "Expected one cell in support of exterior face, but got %D cells", - num_cells); - ierr = DMPlexGetCone(dm, cells[0], &faces); CHKERRQ(ierr); - ierr = DMPlexGetConeSize(dm, cells[0], &num_faces); CHKERRQ(ierr); - for (PetscInt i=0; i 0) { - PetscInt num_cells, num_faces, start = -1; - const PetscInt *orients, *faces, *cells; - ierr = DMPlexGetSupport(dm, c, &cells); CHKERRQ(ierr); - ierr = DMPlexGetSupportSize(dm, c, &num_cells); CHKERRQ(ierr); - if (num_cells != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, - "Expected one cell in support of exterior face, but got %D cells", - num_cells); - ierr = DMPlexGetCone(dm, cells[0], &faces); CHKERRQ(ierr); - ierr = DMPlexGetConeSize(dm, cells[0], &num_faces); CHKERRQ(ierr); - for (PetscInt i=0; idegree + 1; - // Number of quadratures in 1D, q_extra defined in cl-options.c + // Number of quadratures in 1D, q_extra is set in cl-options.c CeedInt Q = P + 1 + app_ctx->q_extra; - CeedInt num_qpts = Q*Q; // Number of quadratures per element CeedInt dim, num_comp_x, num_comp_u; - CeedInt geo_data_size = problem_data->geo_data_size; - CeedInt elem_node = problem_data->elem_node; + //CeedInt elem_node = problem_data->elem_node; DM dm_coord; Vec coords; PetscInt c_start, c_end, num_elem; @@ -350,15 +164,25 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // --------------------------------------------------------------------------- // libCEED bases:Hdiv basis_u and Lagrange basis_x // --------------------------------------------------------------------------- - ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr); + dim = problem_data->dim; num_comp_x = dim; num_comp_u = 1; // one vector dof - CeedInt elem_dof = dim*elem_node; // dof per element + // Number of quadratures per element + CeedInt num_qpts = PetscPowInt(Q, dim); + CeedInt P_u = dim*PetscPowInt(P, dim); // dof per element CeedScalar q_ref[dim*num_qpts], q_weights[num_qpts]; - CeedScalar div[elem_dof*num_qpts], interp[dim*elem_dof*num_qpts]; - QuadBasis(Q, q_ref, q_weights, interp, div); - CeedBasisCreateHdiv(ceed, CEED_QUAD, num_comp_u, elem_node, num_qpts, - interp, div, q_ref, q_weights, &ceed_data->basis_u); + CeedScalar div[P_u*num_qpts], interp[dim*P_u*num_qpts]; + + if (dim == 2) { + HdivBasisQuad(Q, q_ref, q_weights, interp, div, problem_data->quadrature_mode); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, + interp, div, q_ref, q_weights, &ceed_data->basis_u); + } else { + HdivBasisHex(Q, q_ref, q_weights, interp, div, problem_data->quadrature_mode); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, + interp, div, q_ref, q_weights, &ceed_data->basis_u); + } + CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, problem_data->quadrature_mode, &ceed_data->basis_x); @@ -373,18 +197,15 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, PetscInt value = 0; // -- Coordinate restriction ierr = CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, - value, 2, &ceed_data->elem_restr_x); CHKERRQ(ierr); - // -- Solution restriction - ierr = CreateRestrictionFromPlexOriented(ceed, dm, height, domain_label, - value, P, &ceed_data->elem_restr_u); CHKERRQ(ierr); + value, &ceed_data->elem_restr_x); CHKERRQ(ierr); + // -- Solution and projected true solution restriction + ierr = CreateRestrictionFromPlexOriented(ceed, dm, + P, &ceed_data->elem_restr_u); + CHKERRQ(ierr); // -- Geometric ceed_data restriction ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); num_elem = c_end - c_start; - CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, geo_data_size, - num_elem*num_qpts*geo_data_size, - CEED_STRIDES_BACKEND, &ceed_data->elem_restr_geo_data_i); - CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, dim, dim*num_elem*num_qpts, CEED_STRIDES_BACKEND, &ceed_data->elem_restr_u_i); @@ -429,7 +250,6 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // Setup RHS and true solution CeedOperatorApply(op_setup_rhs, x_coord, rhs_ceed, CEED_REQUEST_IMMEDIATE); - // --------------------------------------------------------------------------- // Persistent libCEED vectors // --------------------------------------------------------------------------- @@ -471,59 +291,28 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // Create the q-function that sets up the error CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_error, problem_data->setup_error_loc, &qf_error); - CeedQFunctionAddInput(qf_error, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddInput(qf_error, "dx", dim*dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_error, "u", dim, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_error, "true_soln", dim, CEED_EVAL_NONE); + CeedQFunctionAddInput(qf_error, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddOutput(qf_error, "error", dim, CEED_EVAL_NONE); // Create the operator that builds the error CeedOperatorCreate(ceed, qf_error, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_error); - CeedOperatorSetField(op_error, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); CeedOperatorSetField(op_error, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, x_coord); CeedOperatorSetField(op_error, "u", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_error, "true_soln", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, *target); + CeedOperatorSetField(op_error, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in matops.c ceed_data->qf_error = qf_error; ceed_data->op_error = op_error; - // --------------------------------------------------------------------------- - // Setup True Qfunction: True solution projected to H(div) space - // --------------------------------------------------------------------------- - CeedBasis basis_true; - CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, 2, - CEED_GAUSS_LOBATTO, &basis_true); - CeedQFunction qf_true; - CeedOperator op_true; - // Create the q-function that sets up the true solution in H(div) space - CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_true, - problem_data->setup_true_loc, &qf_true); - CeedQFunctionAddInput(qf_true, "x", num_comp_x, CEED_EVAL_INTERP); - CeedQFunctionAddInput(qf_true, "dx", dim*dim, CEED_EVAL_GRAD); - CeedQFunctionAddOutput(qf_true, "true_soln_Hdiv", dim, CEED_EVAL_NONE); - // Create the operator that builds the true solution in H(div) space - CeedOperatorCreate(ceed, qf_true, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_true); - CeedOperatorSetField(op_true, "x", ceed_data->elem_restr_x, - basis_true, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_true, "dx", ceed_data->elem_restr_x, - basis_true, x_coord); - CeedOperatorSetField(op_true, "true_soln_Hdiv", ceed_data->elem_restr_u, - CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); - - CeedOperatorApply(op_true, x_coord, true_ceed, CEED_REQUEST_IMMEDIATE); - // Cleanup - CeedBasisDestroy(&basis_true); - CeedQFunctionDestroy(&qf_true); - CeedOperatorDestroy(&op_true); - - // Cleanup CeedQFunctionDestroy(&qf_setup_rhs); CeedOperatorDestroy(&op_setup_rhs); CeedVectorDestroy(&x_coord); diff --git a/examples/Hdiv-mixed/Makefile b/examples/Hdiv-mixed/Makefile index 5912d6f96e..6cfc950525 100644 --- a/examples/Hdiv-mixed/Makefile +++ b/examples/Hdiv-mixed/Makefile @@ -17,8 +17,8 @@ COMMON ?= ../../common.mk -include $(COMMON) - - +# Note: PETSC_ARCH can be undefined or empty for installations which do not use +# PETSC_ARCH - for example when using PETSc installed through Spack. PETSc.pc := $(PETSC_DIR)/$(PETSC_ARCH)/lib/pkgconfig/PETSc.pc CEED_DIR ?= ../.. ceed.pc := $(CEED_DIR)/lib/pkgconfig/ceed.pc @@ -38,16 +38,12 @@ OBJDIR := build SRCDIR := src PROBLEMDIR := problems -all: main +src.c := main.c $(sort $(wildcard $(PROBLEMDIR)/*.c)) $(sort $(wildcard $(SRCDIR)/*.c)) +src.o = $(src.c:%.c=$(OBJDIR)/%.o) -utils.c := $(sort $(wildcard $(PROBLEMDIR)/*.c)) $(sort $(wildcard $(SRCDIR)/*.c)) -utils.o = $(utils.c:%.c=$(OBJDIR)/%.o) -libutils.a: $(utils.o) - $(call quiet,AR) $(ARFLAGS) $@ $^ +all: main -main.c := main.c -main.o = $(main.c:%.c=$(OBJDIR)/%.o) -main: $(main.o) libutils.a | $(PETSc.pc) $(ceed.pc) +main: $(src.o) | $(PETSc.pc) $(ceed.pc) $(call quiet,LINK.o) $(CEED_LDFLAGS) $^ $(LOADLIBES) $(LDLIBS) -o $@ .SECONDEXPANSION: # to expand $$(@D)/.DIR @@ -73,7 +69,7 @@ print: $(PETSc.pc) $(ceed.pc) @true clean: - $(RM) -r $(OBJDIR) *.vtu main libutils.a + $(RM) -r $(OBJDIR) main *.vtu $(PETSc.pc): $(if $(wildcard $@),,$(error \ @@ -83,4 +79,4 @@ $(PETSc.pc): pkgconf = $(shell pkg-config $1 | sed -e 's/^"//g' -e 's/"$$//g') --include $(src.o:%.o=%.d) \ No newline at end of file +-include $(src.o:%.o=%.d) diff --git a/examples/Hdiv-mixed/basis/Hdiv-hex.h b/examples/Hdiv-mixed/basis/Hdiv-hex.h new file mode 100644 index 0000000000..e14b873dfb --- /dev/null +++ b/examples/Hdiv-mixed/basis/Hdiv-hex.h @@ -0,0 +1,164 @@ +#ifndef Hdiv_hex_h +#define Hdiv_hex_h +// Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. +// All Rights reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +// To see how the nodal basis is constructed visit: +// https://github.com/rezgarshakeri/H-div-Tests +int NodalHdivBasisHex(CeedScalar *x, CeedScalar *Bx, CeedScalar *By, + CeedScalar *Bz) { + + Bx[ 0] = 0.0625*x[0]*x[0] - 0.0625 ; + By[ 0] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; + Bz[ 0] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bx[ 1] = 0.0625 - 0.0625*x[0]*x[0] ; + By[ 1] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; + Bz[ 1] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bx[ 2] = 0.0625*x[0]*x[0] - 0.0625 ; + By[ 2] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; + Bz[ 2] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bx[ 3] = 0.0625 - 0.0625*x[0]*x[0] ; + By[ 3] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; + Bz[ 3] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] + + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bx[ 4] = 0.0625*x[0]*x[0] - 0.0625 ; + By[ 4] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; + Bz[ 4] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] - + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bx[ 5] = 0.0625 - 0.0625*x[0]*x[0] ; + By[ 5] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; + Bz[ 5] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bx[ 6] = 0.0625*x[0]*x[0] - 0.0625 ; + By[ 6] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; + Bz[ 6] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bx[ 7] = 0.0625 - 0.0625*x[0]*x[0] ; + By[ 7] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; + Bz[ 7] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bx[ 8] = 0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] - 0.0625*x[2] + 0.0625 ; + By[ 8] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bz[ 8] = 0.0625*x[2]*x[2] - 0.0625 ; + Bx[ 9] = -0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] + 0.0625*x[2] - 0.0625 ; + By[ 9] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + Bz[ 9] = 0.0625*x[2]*x[2] - 0.0625 ; + Bx[10] = -0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] + 0.0625*x[2] + 0.0625 ; + By[10] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; + Bz[10] = 0.0625 - 0.0625*x[2]*x[2] ; + Bx[11] = 0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] - 0.0625*x[2] - 0.0625 ; + By[11] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; + Bz[11] = 0.0625 - 0.0625*x[2]*x[2] ; + Bx[12] = 0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] - 0.0625*x[2] + 0.0625 ; + By[12] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; + Bz[12] = 0.0625*x[2]*x[2] - 0.0625 ; + Bx[13] = -0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] + 0.0625*x[2] - 0.0625 ; + By[13] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; + Bz[13] = 0.0625*x[2]*x[2] - 0.0625 ; + Bx[14] = -0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] + 0.0625*x[2] + 0.0625 ; + By[14] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bz[14] = 0.0625 - 0.0625*x[2]*x[2] ; + Bx[15] = 0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] - 0.0625*x[2] - 0.0625 ; + By[15] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + Bz[15] = 0.0625 - 0.0625*x[2]*x[2] ; + Bx[16] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] - 0.125*x[1] - 0.125*x[2] + 0.125 ; + By[16] = 0.0625*x[1]*x[1] - 0.0625 ; + Bz[16] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; + Bx[17] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; + By[17] = 0.0625 - 0.0625*x[1]*x[1] ; + Bz[17] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; + Bx[18] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; + By[18] = 0.0625*x[1]*x[1] - 0.0625 ; + Bz[18] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; + Bx[19] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; + By[19] = 0.0625 - 0.0625*x[1]*x[1] ; + Bz[19] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; + Bx[20] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; + By[20] = 0.0625*x[1]*x[1] - 0.0625 ; + Bz[20] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; + Bx[21] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; + By[21] = 0.0625 - 0.0625*x[1]*x[1] ; + Bz[21] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; + Bx[22] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; + By[22] = 0.0625*x[1]*x[1] - 0.0625 ; + Bz[22] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; + Bx[23] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] - 0.125*x[1] - 0.125*x[2] - 0.125 ; + By[23] = 0.0625 - 0.0625*x[1]*x[1] ; + Bz[23] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; + return 0; +} +static void HdivBasisHex(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, + CeedScalar *interp, CeedScalar *div, CeedQuadMode quad_mode) { + + // Get 1D quadrature on [-1,1] + CeedScalar q_ref_1d[Q], q_weight_1d[Q]; + switch (quad_mode) { + case CEED_GAUSS: + CeedGaussQuadrature(Q, q_ref_1d, q_weight_1d); + break; + case CEED_GAUSS_LOBATTO: + CeedLobattoQuadrature(Q, q_ref_1d, q_weight_1d); + break; + } + + // Divergence operator; Divergence of nodal basis for ref element + CeedScalar D = 0.125; + // Loop over quadrature points + CeedScalar Bx[24], By[24], Bz[24]; + CeedScalar x[3]; + for (CeedInt k=0; k7] = b0_x-->b7_x, By[0-->7] = b0_y-->b7_y +// To see how the nodal basis is constructed visit: +// https://github.com/rezgarshakeri/H-div-Tests +int NodalHdivBasisQuad(CeedScalar *x, CeedScalar *Bx, CeedScalar *By) { + + Bx[0] = 0.125*x[0]*x[0] - 0.125 ; + By[0] = -0.25*x[0]*x[1] + 0.25*x[0] + 0.25*x[1] - 0.25 ; + Bx[1] = 0.125 - 0.125*x[0]*x[0] ; + By[1] = 0.25*x[0]*x[1] - 0.25*x[0] + 0.25*x[1] - 0.25 ; + Bx[2] = -0.25*x[0]*x[1] + 0.25*x[0] - 0.25*x[1] + 0.25 ; + By[2] = 0.125*x[1]*x[1] - 0.125 ; + Bx[3] = 0.25*x[0]*x[1] + 0.25*x[0] + 0.25*x[1] + 0.25 ; + By[3] = 0.125 - 0.125*x[1]*x[1] ; + Bx[4] = 0.125*x[0]*x[0] - 0.125 ; + By[4] = -0.25*x[0]*x[1] - 0.25*x[0] + 0.25*x[1] + 0.25 ; + Bx[5] = 0.125 - 0.125*x[0]*x[0] ; + By[5] = 0.25*x[0]*x[1] + 0.25*x[0] + 0.25*x[1] + 0.25 ; + Bx[6] = -0.25*x[0]*x[1] + 0.25*x[0] + 0.25*x[1] - 0.25 ; + By[6] = 0.125*x[1]*x[1] - 0.125 ; + Bx[7] = 0.25*x[0]*x[1] + 0.25*x[0] - 0.25*x[1] - 0.25 ; + By[7] = 0.125 - 0.125*x[1]*x[1] ; + return 0; +} +static void HdivBasisQuad(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, + CeedScalar *interp, CeedScalar *div, CeedQuadMode quad_mode) { + + // Get 1D quadrature on [-1,1] + CeedScalar q_ref_1d[Q], q_weight_1d[Q]; + switch (quad_mode) { + case CEED_GAUSS: + CeedGaussQuadrature(Q, q_ref_1d, q_weight_1d); + break; + case CEED_GAUSS_LOBATTO: + CeedLobattoQuadrature(Q, q_ref_1d, q_weight_1d); + break; + } + + // Divergence operator; Divergence of nodal basis for ref element + CeedScalar D = 0.25; + // Loop over quadrature points + CeedScalar Bx[8], By[8]; + CeedScalar x[2]; + + for (CeedInt i=0; i7] = b0_x-->b7_x, By[0-->7] = b0_y-->b7_y -int HdivBasisQuad(CeedScalar *xhat, CeedScalar *Bx, CeedScalar *By) { - Bx[0] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; - By[0] = (xhat[1]*xhat[1] - 1)*0.125; - Bx[1] = (xhat[0]*xhat[0] - 1)*0.125; - By[1] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; - Bx[2] = (-xhat[0]*xhat[1] + xhat[0] - xhat[1] + 1)*0.25; - By[2] = (xhat[1]*xhat[1] - 1)*0.125; - Bx[3] = (-xhat[0]*xhat[0] + 1)*0.125; - By[3] = (xhat[0]*xhat[1] - xhat[0] + xhat[1] - 1)*0.25; - Bx[4] = (xhat[0]*xhat[1] + xhat[0] - xhat[1] - 1)*0.25; - By[4] = (-xhat[1]*xhat[1] + 1)*0.125; - Bx[5] = (xhat[0]*xhat[0] - 1)*0.125; - By[5] = (-xhat[0]*xhat[1] - xhat[0] + xhat[1] + 1)*0.25; - Bx[6] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; - By[6] = (-xhat[1]*xhat[1] + 1)*0.125; - Bx[7] = (-xhat[0]*xhat[0] + 1)*0.125; - By[7] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; - return 0; -} - -static void QuadBasis(CeedInt Q1d, CeedScalar *q_ref, CeedScalar *q_weights, - CeedScalar *interp, CeedScalar *div) { - - // Get 1D quadrature on [-1,1] - CeedScalar q_ref_1d[Q1d], q_weight_1d[Q1d]; - CeedGaussQuadrature(Q1d, q_ref_1d, q_weight_1d); - - // Divergence operator; Divergence of nodal basis for ref element - CeedScalar D[8] = {0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25}; - // Loop over quadrature points - CeedScalar Bx[8], By[8]; - CeedScalar xhat[2]; - - for (CeedInt i=0; i $file_name + +i=0 + +for ((res=${test_flags[res_start]}; res<=${test_flags[res_end]}; res+=${test_flags[res_stride]})); do + if [[ $dim -eq 2 ]]; then + run_flags[dm_plex_box_faces]=$res,$res + else + run_flags[dm_plex_box_faces]=$res,$res,$res + fi + args='' + for arg in "${!run_flags[@]}"; do + if ! [[ -z ${run_flags[$arg]} ]]; then + args="$args -$arg ${run_flags[$arg]}" + fi + done + ./main $args | grep "L2 error of u and p" | awk -v i="$i" -v res="$res" '{ printf "%d,%d,%.5f,%.5f\n", i, res, $8, $9}' >> $file_name + i=$((i+1)) +done + diff --git a/examples/Hdiv-mixed/conv_test_result.csv b/examples/Hdiv-mixed/conv_test_result.csv new file mode 100644 index 0000000000..9997c7c1cc --- /dev/null +++ b/examples/Hdiv-mixed/conv_test_result.csv @@ -0,0 +1,12 @@ +run,mesh_res,error_u,error_p +0,2,9.13818,0.09469 +1,3,4.60103,0.05186 +2,4,2.75883,0.03199 +3,5,1.81813,0.02132 +4,6,1.27780,0.01505 +5,7,0.93759,0.01108 +6,8,0.71073,0.00842 +7,9,0.55232,0.00655 +8,10,0.43767,0.00520 +9,11,0.35229,0.00418 +10,12,0.28723,0.00340 diff --git a/examples/Hdiv-mixed/convrate_mixed.png b/examples/Hdiv-mixed/convrate_mixed.png new file mode 100644 index 0000000000000000000000000000000000000000..cd9db19a6311e31055a55496e4c11b31e601b850 GIT binary patch literal 28667 zcma&O2V9T;`!;+<+9;txLs2B9NNFgfj7US0c4ilS*`pu<;YACzCkzjj|dpnFlv-r}N*$%Ql2QIm`3&)Q!+Yi-K!eCC3qwf(sr zVp3ufqWo4DFP?Xl6&JVruOASzzhEieZ1S@Z7hybq@RTD(tu!J3&?G57v8E_{t3%3r zwO#KIw_eiM?pdH8tJ!|`s?Ylseh=2!&h69DGl7ZL*V@+Z7_`wd z5Ow&Wqi1-e{<=h!{&~Sn{X^SaG}!iP(9*6pkttTJwxdHf7M z6mFKZPWl|bkPw}inAr7$8-uQ0d)-#(z9HgZy344cqf2;FooTwD$}8beyT|?{#csb*POe1Jo9=TtgW~7*2IPT_)yJXzLd7N^DWLc z6(85ve>x@a?d{FNBfGkIVRq}jefx~C-XYxDg&~ z=joiYb!R_RR+c$l% z4Yh99S*40F9ur)@l%M|C~7O8`YnxeYt4^%3C|NI~> zwYb7flFBJ8I$@=;(iY_q#``|8J2W)cT3%IrRguMIOfUM)$Y9* zij8_`{rU0O_;_i;fom6MwLjPF6NoTw$+C|2K7BNL=iazIp67plKSY(hkK9_9V|Tjw z>sLW}`So|ijT(11@_GNcTFeqz3@$m5Q$9~o4&i@+k zUPkRxQww3`;}b`8AEWp;uVzYf_VV)DH#a@JY~{+8okokZchWEZ(2Q3N=k-+FC_%zxj^V$jGSEZ{;4=+u}xIEPQ+P zDgHlwii>sf_BF8zOQ?9Wim-jhk6$1AVj0KCyjGNziJ3WI+vyyJRl&;rE!nAmzdl+t z_l#UQ-m$ORH!d!&aa>qER#f-&X=bg*YHT`BbvG1DHJorS!YO`%mmNqq=)XhB~pMJh`U+I@Gr>q)MBLD6f z?zHSloR>(!M75l6T3T9dvA1_xpRlknb52gqJ_MY&no-0l>)N$C!xHtS)oDkB7?zOwulr5KQ>_93P$z9J_qqev0C9vAog%O1<>^)x<@?M|JF8f!KS zXN-u5i1sF@@4$fr`vx1+0`~gRS?47C8N3-~-plIGCaA)I5~}xg{rdI(O!x-|}+hsvI6UXMSmE4zsr*G!%oc4-ypH>%isM{9LEO+r3*QC0AX#bVMPF}ds(P%4Pqhc=}oZ<3Zhwj>^r*>3Lr>Aef_~X-o!B@!kKAD+2=$3Ooa(dd) zVZ76}k;Q#_NaB~g_n&dqi-V2J1yw?7FBvu5Ss{J4*{7sL>0*EF-of0ZbSxAV(>-8j zXBUbhV8i>hrGrmcR3=Mf3pPuqV;-E=twKUWMMEESLe){xje}8|g zsn*&W(~gqMdUqz=XGWG&oVzbPYMi;_mOrbToSZD_d0`(r1Kj8s#ZQVRhDq4LFrlNo&4~ZT$i^JN?CreTcH` z9CtUjgi8~>xfgn_HcsoL>HR2B!A)n4XknYz$zJUD@%N`K_|) z>(`1KhCF(-QGRYH^Vh=MsKmwo<44C_@2V>Fi5cYa3kWQs9vt6%TwB|Ji`L_gcfxW0 z&!5|;?U{&9In=&I%;h&sYFd6%@O+oHeRYm z)pK?-IVUYHj_>{Z_vi8av~+YfIqkU)OJBWuH8wGkdV=q52$vt?ptFL{+yBZHdO)Oc zAVaI^e1vRW9h*qxVRkx}#F4K+0vD%Gw5?@j^)bkIj<;_w3SZ5-wYc~7l8Z`8)WV+$ zbwp$O+s;=nkd_{%rysYiANlsmk$T{ioQ^t++9ijx!W^X*n0oSg*ER8?qy~vllg_e$ z<}7RWJ+u9SPjxfOHpl&bxc#lA#^-Kq)Ty`f>RQuLm8=C7VM$?yzkBbUKYzXiHN&C% z?eds;4spZs8=H<$)c!yg8mj!w8|!S1a2xsg$TZr43j6%-U)x;Q`T?bPxu-(@8d{7Qvm zK)5>3^ph|B=_p+a9I)!#xpNFFRy5kol`)GO1AL<6@Vj=6IUyl|MPb`RMh1owq~;8_ zDI4=MXSjNHc&~o@OVma{MuwAyhUVAzo9bB$1}P2G@;vL;+f0q#s%(}q1&Cj=WXTv{ zn%T#@5)3RX^?H}OUbcH7Rj%79m5x&ph=knzLA+QQz_qNb?0kRi29LSnm&Z4+?r$&l zPMedk_{dp+!j=B=Wd+#|Ej3rN8=00X-DXBY5qA$zRvfortzf!;M|pU5?L#%>r-E^Ek9MSm7bAN`g@`; z!{jxc=&2XVsk!Hl7L31L*PQDhh}I|I&`qHYcuJhU=7LK*fV?iBYt8En;4mziof!m7$A^q-I`1QB7L(j8+-@YZ9T3X{hcZ|y3 zzEw#bcC1mjBHix6PO^mJ;&1hsd8c90SmDxa92{2=(A+&co^LbrSSw|6=1jozqFE*l z4GjYRg^t~4;N#;nH#axNC!N1=p$u@wd~A@8p56=x{nt=tO~>$XD7p(

wPaTXs~v z-DER~K0#P5O7g(i%nZj_JG*77s;cCelyMk&Z1`5}ZS?usqesejrR`&D^PFr;af|2_ z*7Saidf1@CS}w_tkI1Taz)nrxu1BUXpp*b@X9u6V&AB4pH|*M_Lw^43*|W6j>gpya z5@mOcyppZ8PQ>iYMiyq|k-f+|G&JOo+hD#YZNC@th zK#F0oFZ%uXJPw6kOdrZc0P-y(nk0hd0J2Ffw~d@|9qYI}HeSKU93$(rid-6)lcBV< zw5;ILgg-!8VW0N+UGlxK^}7X--c=Dw9Dn}&@lQN*N9Xtb4f_KbC>#AgHG>iJ@vb+r za})y-LE7~1m2)0tuU{`iVcH-qeUjBb+Ov1StSRf;A*A1nVRBx7Tx0YCrRGH^dGb?c zl2axWz5e8~^6+fS&CMmig+TW0Zj)9UcJDs*S;b&vqnzC7!B?p|X`e*m1sHruoWuhV z?7Jy+l$H|p+-D)kM%y-jNZARU7XwaW^DDcs zb>qgWTeol9*nQ1+5hpnfDe>MDEsI-GQNhUnU5L!>E+@e*IFyG4srCNs`2ZMDf`qWr z-roNHZbv$Fcm!^sm08BY!QtqVaqLlIVrR+arN7X@pZ_&xqF><31}N4$vX<=>|FKwp zft%KK3CqxHj5nM#+8`&lZpDffuCqU$WSs9TQ(+&V9E8TogW2m+PHZKIm&HlhTylNJ zKf(iP@xPc3LX)(aoeX{PLQ)@CC@d^YUn~&c{kV^n_FJJlM_XH)?3?df&C*lfP7OAN z<}t{CJJGO9eDY+!(9IKH152NA<9DcX=}+(8z57_WK|mnk?Pf(qzM`U{Xnjo{x_p}? z@nYHCyCn!E6LG5Hd(L)p^ZyIpWsFx&@OZK|mHZCfSN_PuRjteMvo}rB-D!RK z`L}LmMmp^sVQM%&(OcE|nc$$xuJb>Ywg%Zb_VDH(vYhRCAk9Cw*e*9RD)F^>QrC@PkPPf1{<5)iZ zO5VFUg4xIU=5P<%eD!n2FTEGNSk~=g!49GwJb3u-zLg+K(8oV?8EumRc4&3U zTDyw<@?Fl37OCN3F0L+Ttw3-ga;|P}W#D)g(IR8*E@LIl%KRBo0)F*B3@0gUdid*7 z>eJ`X5;#w!$w#Z~uP;{7^Wl!v^OqpQ07^?rOKI?ns>s3)J@400CFl`@K{~t!IyiRh z80y;F2tief8ZBPj0IcAklvG@qX=^?i`9UTF80<+IW46Skio1zT(s z-MyR3vL^Nol3d1_4{LATy2X!%0HJ&E(4o@HOX&&vXW^CST*)i1gd)6}X&EVf0E7e# zi5E`Zu%H_N%-nGE9XX8a72H;5TGj-HhKBB)-;t$`HGh0PCMNVms?Of{<0TCZYv$)> zR6#tv%1_i7WnQg%;J^kEk(BrI0CQZA)uMdC#Vw&o*&@jSZCUBim(1m|It~sJ;Dn^Y zCnhFrn|^Q~JXFV%oO+PeXYyi(fotx&Jes|%*YDpCLE`O>JZZSs*?AWls4*mfQbZFq zR=haB^30htBxUS0v1>>2$irwtDO@<(lb4rQ_tk|>SQ}PuZky?@ znJKUGXR=td5&#-RO8ghFc>zH|3|O|>j;7P;#x(b1Vx)|a4&DwuyZuqEJ7-5ds0Om! zgiaC>VM4!FIyYLplIs7Icuh!&rYo_YnM14uL4povf2KuM?Ta%psW#>Sp|QPgxXSyOA9bqffMM~@$O0sn#Q&UAX~Z?7pXkK|OAWB2thT~b7zXR67ForFA$eS|7mx+!0dyD<0j{4>&a)pO-p4w zex>*F79c^JTUwgcB^<~_pNfQ(fkdu4!j8h|gdL$26&2-YzB%d^dnt^x6*7y9i>t-R zu0e~xBj@awgqcfQwrpuwxX(sQ^W(=4q#ubBO{onJ{9@A5wxOIy+`s?yi;j*?;P>z5 zC?gdi3aL^3R>s{bB^9df`D#Mv!ozl{7mJ`oZJ8cz4@U#OK~|Rg^5x5}*nNk^1y7mX zyGa);I#@M5<53Cc4NXeg{PCXjN*|eb zrl8h>!oyWxo@*tbzsmk#_$O?WfD((I@UifqkJG*Aa9rGokybieW5iwjm?UTo6Kwh2JM zhwv%#Vr`WV9%Sh1>Pknl0Txo}Ys-(miu>^Jkk=MjgN%Izm@y{(JRT09C?1#j9$qiNsMmEeDpKMqs13>xIu#lS~)TO9l0q3bx zr$D@wk%NWcQ2d#Z>MqE|$}Qw|PFeZ(FtV?>MNMqz{rl^ycDcK|vvBWZxbSMdgoMO- zXXj^AgQ?mg&;Zc&?H~1Yh6Vv8z}nhcCBLHjy79?TVFDQkJ-bDs5p$1^-qTT4z|tqo zcDxN;_v^=#xZ3ANiX=by$kc(Ep$B`$iFO@ziZE1K2JtzMG`HLqJ>`4%F6Y6GtI|Re zVre-!I4pDAoR`l6p$Z7t=#!N^aBVH=EyjNSEW5QuE8TfmuWF&P^5D?dmn-pfx`l31 ztC<4o_QmL3yts4Mxo-?{ho#?rFqb_0Etjp%qTo}piKe?Spba}9ZKdEK!~x<*Ql}|8`YDwZL%0wy)83-qKe=(H8Ycv-nt4v&bsIO zcVm!E$3-H4p=ayp?F~F~M+`h>2`<9t^=pH^zCP=kHE-go8vaRqnc%9JZrr%>KwBgO zpeSwY83Idmbk-Iuc-vJqH|sh&W$ra0YG8tFeC*zoScN>RIswtsIck~3XgrW`0DMRr zp7k)=?5}8e6GyldP)6#FZ`0FL52L=J%Lvtaa%`{Io&92W1o`<(J{H>j;vAan_#Qm) zdu|Rjjtd1xrO*SLD)}6uKo@$e%bLmMXaytg-An!Q^3sIG(e!mP=lSVYthah>s3$}x zr!e656ZtPC^T#K2=lc&GV)P=su_r#fzv($2X){2QH*BnE>cgVxNd-dXLXgu2?9;IH z1jg=`aT!@p*bI~ys~jL_lV4+Yndm+;Lb-P z^YrvIviut$@||-~<`7K9Cjz>E`Rj54cdcQQ>c01ZaT`>Tr~!9i+vZ3UO2zXt62{hj^3qBk!TmY}Ow z2{WakrPP^DV%- zg0TV5K0i?(EO556N*V(QVkSz+*ry}n*4gTZ4zXcnclHhR^$8PIjSaGv^HB2!h?JdB zAfu_T#fqqX)F8BB!(OYpgaDG~A?kNR110Jn z@;8g@jOzTipdfC)p`o)TQv(fpizkbqv_ZB514-!LfaU9UG5r4h+hXWUw}t_4a>#xj ziyKi<*7eST{k8E+V_mx#JtnH8DIY*f{!N>f;ob$v`620@5a}KVR8$gx43Us!pF7P? zTK}4zEd%|VnFcP4py*?9dzMoNj~?~>eoCDwU@rij)(ntl&%`NUX-$fP&yU)o{`&_A|b&$CHZQHN3^38QKC$xo)=2nuLcB z#|fzimqGZSJZ0t3FQhDA=sCP`)27b)WG%fpPAjp>o0sss=Z`Q zF-R*XL*eb)SqY#~S8qByhwmV*T?3k0LghIRUj+xqJyQkh%04L;NFAUg;LL~H@mX2* z8#4oggG9eT;Q$-o3dVtCa6*FssX$sZ#`7FJa%4XaAC8j6+cr(R?afia+9C}hGACBJ z3{*gN7rJ+-9{ofJ*A6(d7C_@o@N9z*Kp^uw^-q3{jhlhtZI)mg6T_+vwp*%n`Fmz*9E-pqeL^$s) zu#lV|`L>%V7&xV_qlGg&=eFKFxDjO#r)C)t{uT6HS=OKFfYYOQ!GwSgY3VNewvY!8 z_*po&Q|R+jUKquB&2DiX`Njwd_u^=q!IK)&yuk$#G6wU^smHcW&xpbr6cXZp@7}st z&*@X(!TCYxKr(C6%XIqjDbaP|cV8WPvK=$gA$tWF+y?AeDJz{=qTk%Sd9$$y!jj;& zZOhTw1{N013-UY>I~i-$PmCKlq#ekA*=|J=iqgO~A{#d<4>~7c1C&rWj&x2>~%WJGe`XD@7{<99y4$V>b2b~t?8oWpta8pn&m*rIM8PV_A%Xq3 zhKPy2nx>@It82Dyzx30beI^j(7Va{F(V)<6&EdpV_>1sLt8>g1+S;)8@3$fha?jHs zFb-&Ht^!e>J$<6!$}`De}(Cp%a$!8Qh}js=d#@Vd^5N%qCLqL?Tar#{zNAlfUIqXt1vJyY@fr&eR18A69KHW z3TViW^h{Uyyp(W^$8qgWuq3sjM?WLx^|46s4wZtMH>n-W&&v}4i-ab-0)C037uN#X zZUsq+Qy!?9tbE~uBw=&r5K4ZhGA(E*0Mm4|W+FzyIP@1@-5>P7Strr_qmGklb;aAa zW#DiSj7mVo6qTlzy>Diul?@dD;>X{V)Q+C^!Mq51<)(t)8Fhsr~mY#{ni zLwvi=a<+NR*@2*Z!SLqI8$4ZY-uD*B{8esG)#b-%8SK(&YJAW1Yw_a=yVLaauV?Q7mXLNFDkf-CDaZ#QJaTK1UUYNLa>1ie z3(z0aM5tD{QG8(%DS7{%30k&1;aZ*=4tBJe9&p@QCd_@!?NK1qnd#3*?LAp8e)-B^VSJz-6 zC<@-0<$NAvmvz>Wlv)CV3><+;hKN{RlGbnC@<(!Hl(uiHAIe80plmFqNDf6h`PgtY z;fiUxZMn^hXwc#_lVXJpS=L%dla^9w@6#3;XslG0UQ0Rr49Bm8^oQt;t>%q-hK53* zW+;IZeEoWVW=wq8fE=A1`{vC_Zbm(Grd!W!*rpTND{54vKPYBx{WWJB5Im4ZTBiPT z>i!;2?rnNmK1jnLIvo0{d17`&E;=IP%kGsx2S}muvA7?`&Zdw++Mi@?0g!1ZH5B#h z*RKajNet*OfmG0eLap(GJ`?4B#9RiSzWsZwv{rUn7<5GRj{c~_o!L!>pYv+O5V|<# zZ(xStk2qLc=Fsw6_NANsQ_n+7J7=5n0LAP6ru76~b;gB0PH72GC`lLW7o z7&jeMm~J+cfz=P&=10_3etrtP?W)^2y|HV0gPPN!xWmVN~0EjoJ$?29gyz&ztck~X5Mlc@okN&o|{b2Hk znD0G3rSIMyvTetMMclp}h<0u;M&I}lA%)PM0NW=bi9?-F>^6N<>&t*7fP{gmFT#>XlRV-Xv$J?t*vrx)uKpTB> z7uHqEQ0FP?7P_*Olks6e+(kPqt2>Aak1E#r`bzfnmoHz4AOv$M<@4^Q@V$rElr7%G z`*aa+NcWwf zcSudG97GoBC98G;;A}c_n_pJ8guV+SA#mGTD!AjrcWH21ITg+z1vI$ntZZ=xMNR=**OowaePtc!|l+A|H*X zyn=$1$uk^?r(0Dg7iG5IhCU;-X;b3+Y+11VUSR+AotkhAaSxlKnBUM8Ekb(zeGJ7 z7d@%GT;(4=q^bX4o?6DNJ8@_mX?-XysM-PiGA7pI-om_JwG9ryrvYK zQRFLZpBDz=)@%l9!I_N9g{leQ!>On}0;voEA;bT`AkOKn*E&(8jI+ zObGyq#lzkLxl*mHrG*wzX16CVxxliaO&0lT}yGv2% z2Z%6Ez){QG?5u#8SapZ2v$Fa6KT=nD2k_jWk^f3XUm@qhCKDnO*WJ?-0GvvJ#1%%m zLqoija_x<>B>i`EdWG`jX0+gA?TZT>q>QJgZpDF0{KCP>8AOUddNEk_RxmL!k#p*? zI6p(|2WaRNEl@a_T4Lc15hv(w$D+)Iyq?|8*_7_#SO%DZQPr8x8O5h(Ltk*_i;fLyRd!p;I6A}c7kCSj8wodm! z9}@ZicbaH%8iW9W0K_JuvYSIdQopBVk`!3NGa{=&q@5i4!v42qij@yWubQE|U1Mn} z(&3x`0VvRo622A5Z>S}knTv~yBmo+V5YiCm$I#mh7O22|)aEg9Gz(Z3GP5g|Fnb!o z090-a{6fsgS<{FZ7GR5v=n-MP<^R0G-cY{#Xt2sERp-FK)%g8a zx>BDPl{KWE1Vz8=Xg*vWS`&-DZV9w(Qt|~A)wFQ=rhXHSJe=LQIqg{*ESNfDDJg03 zJC4@GDX!eh%F4-^)WeupoWt2RsjxOpO0EC29Epcm7 zz(;TlZ$*eu6mDkk9Weun!f(M9L4zRf=7XN6NLv#O$k{$H_%6~cGys;5~-JlpjaVe{>V!8c&>J|;rk^Z#?MR% zYjAfb1~U-j=h_MwDc7Ixgf*Zj^bDpfwLQ^yOj$~pff!y0;K7k291}(S%1PQHus$vW zV8ETAuy!P#5MX|5+{BO#e>y0%C+cyHf2gE>By-Ymg1GCz8lw~L_vBf(Ze3c%s^CX0 z+fPQgT?o8)@80D?VF8Ma9iY7-p9Jos8&nWgZp7Qr-UN6{c13E(r{Kz;63tOcd-v{* zoU3OxOifnJolMqGD>UVKr3ymP7sgT2kPz zNalZ4CVS+Fg#U(_f~b_?*P;e_tAUzn1q21h(GVm+dH4l#a_sFo`x0yzpXVeSMPy{K z4y6ckVgz`|qll^i5%pG`VR+^JRqX5%p0gMA^PP`o9u>N2Lf9Gor!#=^GOlC%&}SHx z)^ioRO^M*PI#D-FUI#)zHipYcn!3Dr;(9HZRG8PRj|`j4%Aq zrjz}3uvuM3#zZb7*$vecbPEVyUIjN}pgwr2QZ&Lm!KV=t2#mAD`$4uSEmKngQ&ZE6 zBd4nV>VYia`K^xUveVL#9bV16=VCuA7FTk|4ipO->Q{%~N@HYm)aQHV+Sh})#e$E2 zl^b10D5O^I_0GpNp=#=h zni$|PvVHVS;BO-tn3a`vg)W__p6IkO@3IQ2(Sz(!!mprLtwrPWa4IHoXo+ zaf8>z0mejltvGwUNC0{uf2#CzMO;dJ^hlcv4w%VpD0m>tWj~x-vu4d+d?G|;itxW4 zI1&^E?=0G7P!DE6`RJb{zwBm>IAfCjkUL7kEU?bd7Fl1#eVc&e`;Q-YCK(|AL%&;+ zs2L2WvGKy6-=t_#;~YuA`@A?c#PeLDs%%cijuP}zejL<8I>WR<7{pksNZMIoX(?|P zFx05$A#JJmI>uw*75)?2al0(WS&|#>p zR)-_B9~ri8KwAZA79|UR_4kDbsa*xQNLmXfmyiPcleH7jXUcx)erGNTgCjzU^USW@ zY#K)i2f&5Xd%(3X#s$vlgfG45*@-;>jT3k=P6!aTAK5fw%+lSJMVB_Gk4*^dhnbpa z8|X05w&K}o`JcVihwGZ~Hu1-?eu6*?f{x1O&MeN^=4NI~m7X4WbH;;v#Z-^vRG+H$ zDHtj+9718`U|ofTXPr}w_Mh^>fJQfT1S_CqF89OAkFJ$IyA2sDTG~DgE%Xvrjt-r6 z(W6u0|7GW?gALzMK#=F9ef!&|r!v#wTeuaSc{*`RC;=_MnOpy#8`x~j!a6N zx#tod&%S&6Hc-$h6#9K1Fdz8DaWn{}u*x9I6C@oPy8O170TXa8_`0%|=tYH98n0h3 zz6wziiIm*V+TeRQaUa5h94;UpptgtJ;yj98>*0O90{jlJm_7X}H<$mP#7FtSL&-hW z3|=7=8~jl~LV^vHBvD*Op~70cq;fvwQ3SSZYngaSpA*e06K zJM(boRcK9FfGQha!7UF7rvFylyZ`6;&v)!wk#nw<3A+S}xD!FGQtUNPQ4du@%RmCe zAKwg7Z5dGK!&qiE4Nn0!-R9K8pxl9rh&LU$ak}>#Mq<`!)t;wk#8ZC zx(E07m%4_F=fdzAq>}W0#TR_^h3C&y0$nF*;El-Q9jzy1af_jfWThZ-oRF0vss`_vy1$MaUx^ zg)?ohvG5TvQ6acZ>ytEKSyUD}{#)2=AOa7K{@e$2b$7R;5b;i-A_u(B^Q0@wU6 zW#vbyhV=OW$Hc$zYB95B&ub{-+a0=;sS;G4bjWZ-I=ZXizN?_HFk7WxQc{w2)hf-N z!8IHl4>5}~A)~0+&S1G$+vPviJ}r{x-`O;+ZAqQI6&YFjKE>nuBH!XKdX(-ZO80-X zK+K0miZQhH5GKPfilW1gar4}g^S`lDu~PU%UnJoT@Qf2~6N~~tNvcN`H24w>SWWZ{ z4%+^l;FrAppTjNM6d4h58D;>?7IFWH#yJJI3Ll{YJ-TzS7S_`Wu&5AId(qs8+&9c( zqQiU(uuM{?xyqCqY+a;L0zYu&%9TcE%-s>2>fgiyTheOe8%&RomZ8eVvb;KK2R&Gb zdim;_8q3^=bf~ptKFp1IAw@j6`mV&%3$Omf-#N2GQZgM~GGPx8$N-0RLtit@=*cWA z_17@eV*8wHMmZ>`KVoOI@kuRoyTlG)Ti;k+y#fYap%K_PQQiAP>Q8KBSOWGl;>Hc% zv9U3cuk&NCR}x26R!+8kJMTXzjX>O`a|gaoTJE#yjxYdeix4U&>+c)`njDLPH4&H0 zEiAa}iIRZPv2}!)hd5caSs$iF(!zoWS&!0=zes`7r;2g0TwE*@CK#MSfPSP!DoEM| zmSi&!PrbwIHFzd|{!G!xCo{lZ<-x?dL39PG1jl!0=`%P4b1?~Y&6S-^>iCpwGj#6` z5Z}=iXu`pV9{{UDJi8z7?`H3=2q%7ZHtc+Et_&n_8;f_1I{#2AZ0)>LtAhpz{eoCS zNKUp{(o;a`gudu?z(me=*gIpSPIC=FHw9#CP`~VE3{@8%{^IIs2+C&AQ7In?DyHyQ zSH6Fb=||Q*i+>y+w$k57H&HY#)a36xb8t#xwL0Ws2Ju9L)MX9-NR8$k1Sva_8XSHK0!}bTXtH0=f9HUJD_= zeE&}dZv$bQ2ZS*s0|GGfXEuI^0AdSc!x{|Nv0$wA%!^22nB5kTifu?EwiIUsvRakqjxDQ*XM3;{!#jlU0dt1;T=3^Dl!26HovszTPvm?^p}l~ z^_sON#w-L2qSsPm$?kYLJW|KUi!9+AV$ZAU$*bj zL>wfUbbeI@cc~ERaA4Li-@+ z&_O=13*O#~B6r6Ai-k)4`Qj`jE{uP#K>da60Y9i}=4g~7`BRym{F$@;te z((zPRfIVe6?mu!M4+wJ!G_N$)e$Z*7W|yMwo%ufgz%kyl9|9VTOPg zEBD0d$1bp={|*TTYomuruGKybgU)#!(GyP|557nly}8r}`FKAFhJl77?7i@NRkC7? zUgeGZP)eyg9s(>=L)r%>3IQiZ`g5uTcX}PX4If`z-hdD>mhb-=bb@CpxYCpiE1!Dt zsO9(!)Czbzv?p>QBa=@xzks`wf^|L+{s$TgDt5&6>kk~aG@vAdUDyw-1ftkH?A$6g z4Z*kWLqRE+We($0e70q+2Y>-ePB+F#pjUG$xb37IlEpDR;p*=0XrKe54X8_-bhYgJ zO)&VTpKFyp8oFBw`V4$Zm>=K(xlREa!uFH0l2~<~HqX}F{cx$LukUqf>0WkHkKO-* zDMAGk_Xe^6ab!Zp32_0MQdW6;Dx-n?+B?nhZISWdn7K= z(OJQWY<IDm8IzN>~FK1=;u=xf<~% zW%z<2;oFf6vRn4RR|C)wvJl1wzvMhM?3^X%P}1Jy9~TUXm6{CUKwjab2!T(A$Kegc zFL3TUs~dXqI&48UoB#f}H7AUKd{7%gy%*=&|3g*+)*9PYdH=-5ywZJem;2T~OMp$2 zZsg%a*Ph6eWEs<(|L)w6UpK?J{mEHJktKpCnN919bzeaM!3#qaYBGh$<#q@cGhCnX zS#ZxDUUU!2fPx=eBULcQO75V7OxVC}0mhLQh!E~lcATZ8%;^+Ru!JHBl*KJ)>5e?& z7Io2de5=2#>v{1)8L`b1mj$tXQ@DWmcv7?yR^ugF~?^6_|Hj5vJ_Dk;8V%;Mi0fXa-}LB#`w$|IXdj>c-MP&l``X#3j0r9Mxu$c=(&=W zwKL5bJl$LHH>okhhmn{#$2CP881W;np!^)ri&8#?g$hIr0nP$!!Z1SNv%I5NvkYug zqfH)@e_ksYPX%-%kO8JbbbgPAcyXr>UmL4t>wG@JUq5Vksb`IrGaLX(b=?L(v~BU? z1=8ka+o3u@U>!^%ufPah=f~4fiDZ`nP*X7Z>4@&jb39bWw@Tk-%#%5HK#8OMgs0l} z16sibHp8n8WY5OO$1#`DJMv!8{gyxx8M#7xj};+iG*qP}6siw-DFj8qV0sn2tO*E1 zS*Ia5I&HN7j+%t{^&Cs7tNjpDpL0GP4G{JUw&t`o=R7?;wtM_KAF_V;Dm2OUmsXs- zi~yC~b6YN6gqhyb+#K)PP|=dnWu+yaYOs2QZKTFm*}@A4ng?&_VELyQc64-z41RQ2 z!vLe>M0b#Z6!$<->uVlZyYLpYVVfpFm$hPj$dHM6k4yl9GYbJ$T|KJAw=Z zXVS|`DPrYEVMJm1#}j?;>$CMWKy#_qrj8Gg6r+=4*8W^p3VwuOQGAKKsDXXvDjDIZ zdT>E_)22;O981v69c|!q!{|x}PUc(0s%^a&Dj3cp=YAr(2W+=0uZC-JX0%=Ugi+F3 zJkttZd4@zyMpOZu2F1zUtAhOl{E!Mnub19|SI2A|j7T?dns2t$^Y78%wW7Zyfq_=K@}xrUUxS&O{kN# z{K1ku?>yMJ?QCn_8bD@*9G%zPFf3DkeoJ9>SdOBF2t`+c;gls*tlPkrLZ=~J8H}w| zN^xIj2LCxGzdB+?U6Y1>Spxnl|_(I5h;LG^WKYvCTlODQB}CKrrsOxwcMm)YBWz4{ ztO7Fy@<0Jpq|Kb>RM?iLt0$7R$-6?1B`tkF8gma4`JVYnE#e~y2*7nnGYpbn+RYm~ zRe6C^-t;71NI{GQ?Vd9+y`NA=iTX+Wj}Jhg2>nRfeb==7+&|c~xc{pisA~w3WDFtn zf<7_T-eq|wNY{|D)Q5CusQ08AUx)MQ+c$l{24bs)T`|S!$+~ONYgd92*hTgo!cNDL z48?_Cf+W>tcg$Viu%ARw2(JP+lzwT8MbR+4T1De4ev~bl3wH_H_mp~;fQyR@hMc() zwQh_4WsRP`4_*n(hv^DYUK9dlx*amjQ$Ie_gPLy8yZqO|j*<@~ZdPA?O5h)bMEM-c zH5)g^&AY9pQ;#L-jcz4xNLQCB8EU7-(D)M>j7&>{SXKL^;g~ci>-=9Ip{Hh8@eJxR z5sLB56ovA_Dd+t3%{C$X_r~<;uG?|X3isdn8sLnQad1oqVcujcLQbzeN|u<1*%U(F zy^m>bZnkPYO|p{m)Roek!o-J$(0&L@*y?$Vk)iLE%nfUWph=vMs2dNUa$}CSwk7WF zU8}5Wav_Z>uJuENEy5g8bb>g!)+7T9lXGHXB4Dl}iBSwPMW>MY9A#~0x>hqw@FL!S zy7@MXiS0T;f*>gAHcPBL!PtP?_x|e|MHM3;2MEV>9=at6p+U7+@*0pj*}r!zWjXeq zT}LO+&jYLP%TFe{5;eKf9D7&s@r9!k2FGz<&LwQwRWiqPRxR&QGkF_F>nN!I$`1{; z`j>q1$sen(%bSiR>K$*$CGXxbV!)rgX^0|DaZDxW1i25@yKaG6`S%kvyV>&J|E?)HJwcwf zvYh3a*}p}Hs|ia;QjpG&`)J`6c+Qtl7+>%S-!sdAceYSI(51R($f#hD%~RYPv^u;> zhWD>uKKxtqtnW?0@}SCZYMK?~goL+j zlpO(q&q)sHAF8VZF(=xa0gaK^6wwx&qN&g)@^4V(N8BON(m=x5+BsglSWf0eM%xr1 zqLlT=07?oZM5j8WyALEkNw;Ybi6PMBH3Uz*G|0X_jNDFyiH-avQ#f;Tz2@`UJYitM zSg9~BW0Cb=-#2Z7l7ROSCBL7?D=Ww=Lo{U1oW=Vll+f3)va_3wJR@XnR7jmpY%X$2 z(ZZ~)S>b(+qj7t9NFu|lq7YE#nxUqLGX86tki?kKnd$$(bAh+WhQy|he`nO%A}Xp3 zE?_&}M*&B~;Lv}j?=m1_PL8y$gEkDiU^(%Bjf||I7@!kdTPIgLx(qeb;IVJUEycY` z?GagX{l*e=izSAJN>-csBcorcORaF8-FNZ;jOp0}O27mlrBc2vVoJp1{jDuTuAzu^ zhoFt3-@WUImyEg|9j%vYc2WoLd^&YtDQi>3CMHJv_P2X&(2~FfP7CexqcR zcBuFpSvk3rbHMXgVDIG0cKjb>Y8q`OTgkL$M@Oljio#k7Z&Fekybr=K)Qbfdv}yi3 z6FGdGp2A4vbh+}xB=dIT@?%C0rds(@KRneXn_CVg$_8e{&3O5 z1VLzo7-u33kslxfeOx10(S!RFSE&xr?sbXdr&cj;H>2afbFc5k(=VeLsm%Y=I2LZN z{&?b}{xHIID7ob|i^&&M=Ovm$?Uy>PQnRTL~-r8B71+(y_rG>l*0Hr?K4zqjk zA2+&hi6lXMtDqp{YjyKSuPr%v7Y2uzey#kme->N4oU_50C^CUDj#JS~o~YLFoI^o~ z$9oxe9BEt)8wMMONMQzfVTKUYc+Ji&Qtee$S?e7aqbxA8a(UWZ5si;B>!efa2zk*M zux8VjJ^$x9ntKKI|BXgg^O7f%oSN;7<3I3ma!`qZ}@!hzq}GU@YYLx_zcXtM5P zMRgNL(6t?2JaXtWzlgsfnL%o2U7hKnn{@dreyAJL)TwZi6}lHYMtf358n5` zc#Yh@chD*PwT=XF3shfCqx<{Q5wEyzs6HJ`B6Ugo^5JEV9#K(IK6f}+xtR_o_KvKP zy7>2EPXBXUE34jnZd5-;CeAUca_cTbdAZdUS*o&<^`)>-^b;dKtF$8VG;Rror>>fb$_e+bxdTF6}+s?gx6KsRQ5`0@y`q|%^qI!nER|D&?cC!$`gI02`+Y$}W zGV<=f>>GC()Z$j?=j#+56=LUj)#vs^xwjNA98xw&Qd!k1dsEU~N(GQz)h@9tU{Itz`Zz*GCL`k7dJ++o^QC2(PGXyYmeZ!4?Tnb05> z45IRWtBzf^FeP5{?$EcHcfutcM*VwK1!(^K@mVL$IW&+BEQVvh|BYO1a1ys>vhJsi zLMne&7MbSWJXo&S-KAR-mb3?+7~SF`4_Yy~)9&6Ece1V$SWwu^fO*&cS{kc%!#|n( z={qtie1wh;6bv@pVDK$!-@!-$rgXyGiNQpfWHUK7jZL!y&A(@_#)ml#1r1=7MrJ@~ z{L_!99VIjEOI!1T?QV;*S*D%bcr)_KOY6_&#ZC0cW?@)Kg@k@9C9yt)wNvs_wSt+n zGar5)dCbUIV*C7rg2|83;tzfXx}0Mp`8^Nq@zSHJ=!T>1M}v_9D;XYX^2>kwR>=QC zerCivJTJMpI%ereJBRp_Fz!t`PUknTbc&75yK@ZEhCdj4M@x&xjn(S@J%bT-3`?#@CZ3+KtKV$=X=6rJ zj9kgLS1mP}Idn^wl-D*sKKgsQM5&{IJB~jVaH<&XswQZ$yxB{KpUH-sbQwR0$?eYl?9hs<4^dtIgP^IW(Btyiz@>R5= zZ2Pno8GKz=p6;o8xx>=u$Q_?#t;=UVl;?W(AyD^|bk>!BGLIiWcEv*%~Av@PA) zHqo$xY&+)WFHu<^w3n^j{EUuyH9J?_;Z1F0`VKHOOiAinTNj8PF=sk+rUdU$3okLq zrEcB6Iv3~0)Lc|pP_W7Wxyxj#3QIoi%4^=)1~S9Nv#x#|AOnkP>9Kev-hc$<=| zJ9nY)(V>#g*CoFvx{9v0Q#t zY+fV0EopJ&JMH~F25r;zTgI7Lmm1|e_uNt28eX>Tl%`9clG9(6Q`;)Zb)|ty;K+Bz z2Y-0;YisjH*G0Tqz4q!6bA#d)^79vePZxP!Zuf3lynj#n&7qY8t2Xd?@O!B{^;20k zZ``%l5l3l?mS(by9xJZrXnr+>p6T++kfRJu{fQePlzU&KQrR+sZag` zFX{iU&dxNP%C*tsON6q@P{=&((xzxqN~S2HZOTj%N{C3N5R#B0yAXvUGLIQ6Lo5v{ z4WbM!X^TvS%;Eg+YQNWe&b80=p6h(*ix%s7o^?OZy6^w+yV>jckI$@;y}n(d#f`E4 zVfA-|`Fo$9mGI`2mC4d4`?I@wqkkv&pMI*>KyaSvzc_!nGZZn(u9wl+~-V9jTQ;)&`lpMI>?t|K{H8@ZD8% zj#nz~?zY1Mj(n4PM~jA~F57%0tXR*-ESCSzyIv(4^yUxF!$H=M<19aP^^YQa$bTH*LO%8)K#LPRjheMVZ~Bcf~gP^yx8LAahtTZqw&=YLl-V z^z^0&PMtd6v^wP!#~=~=(GC5G+9DyeU^xxj=1v>Fer$gkY*<4PWl8iF&+DkaL#7@v zn&NCD3)Od5A6vEXzMEK1o{UsV>liB^|D<;-js;W-UFIn*<_<%~F z{7g$}?l7z6KucTWk-cduU*3D1l2YfnX_d<@9j$qC&mgfMz$iR~q8KF*<^ZJqZm>!S z;Kb=$C6vfjTaI!N1PZT}_sup>{2<{WrvIoL_Lvtoi)Pr<7PWoQIglPyZ1Y&}jOg!Hm!7|NSu1F(bA1gDpP7dbr+5}$gqidNA`0@SzOBW|CH3DJ zBn5l&#aI`i4j70ujG;h3#k%&)V%T8sCC1|)S*E1YzNY6#911vH&mNYn7aXW4DD$2L(PhrVaUC7Pyk~BrtbbRQ$-t%W<7O{h}D0Tk1~CBABdeF z*O5KI)fhPVetl_iA*a25@2B$;RX~It+=?9MtcUV!&U4+6uq?g&;KYqPr~kT)lz6tj z#oycx4fXWZF-5ob+8UcnF_+*@pPi(=S<$@1i^r+z^X2H9b5j8ejOL{)*(QxGYm&nH z%V5*~B4?z&T2TK1&N$U`SbTGLWsaSTz}B$EUT-8^d1b=l3>e(PbI#YwY|x`R#I)`l zdZFsoaMIMYGm(*#PxtzrSm<-Hm+XyRFHRxf30E#PTSLR&BJU(BN;7Zp zD9>Nj@@>194%L<7+ILAlxI~R*L}~l{8I|U8vwcDeDKlXMMSr$EzFd=E9UE1ZxSi4R zzIoNctJQ^utfD-FC8a!d7Cn#Iz-3x*enmN-M|uqxCj(~Xn^v_RcHW7Jp?$pZJ-^DM z`qTE;E>ZlxMGuc*RH#|dc2ym}C-z?9c2$R#y4_M@0*^8og$@*%^QdxEH(y>>dW<1f zmRGH#rx8;J?u#M^g>jQJpIiQBy-;_5maiZS%jG0yPEHCiV(X#(8GWeCS*E*piR(zp z#NqwpYdg~a=$TZ@Ghe(f!Ek=3z5S5(?Np5j?IT!4aO356HkNTE^OgPoP-1+O2t{IQQE*C6vdN1%v=(y-92Ml+N zastgjIXysJN#DUoHfgsEg^+BmmsC7(?4d7xpGMjuoy1q`1x`%ll;3I@Yl@Ri?irTJ z3(1=I?FnZ~REVq2i{ai7aJn;@oijSkx*(M~q|xV%kSe99=t}+>+`Fq@8Hmc8 zQoUo$EFz@08v6P5T`K?NKs#KFMKBl^8LA&nSfxv_);tJ-q1AI}c(`+PE=t)-7~O%u zBt=?dXN;um9hcm@$Bgy#5)+v*#UWR4DNc>NPjQLl`s<3)kq>eYNS8)y2Fg~ijC*NN zT*Sqj|K)`$^#u3ayl{PP`L0tG2~+ zr&-U9Ihh)C8A}e?h^&hOCc-U?7-{DIF_JPT`)=SOMG1TO3PVUxqI!PQEMKbrEW@zo zO?F$HPE#Udsk3x-=z1{*(-N&&pNWZ{+96zQ>#}3<@`XW=TEv+ZND)gg^R?H;gB;eZED}OXqIswYCs8mD@VqbO{OCB@?%A&st3a0&O#( z0{9bKnSVfv)|}Ic3|9T$tJo#RkfU zp;K(t()#pJ~{=wzoZMszVIGONSsijMBC#$N~yw)9v ztf{E%=lRm3Ma4ND-1yc^en0QX#Yi6QguZNB`z#|-Ev@<+#v?_UGfe>t0#|pG-0MUQ zwaNX&tL4tRN@i90+Pv0{JRc_^s%FUG*9$781OoTwMVuzkAfG9_D2|sO`7jpS+ZXm*;#$vx(L076prr zg$99|l-=F*;~gA~=C9Lxf-p67TYU~weoBT+w>C!667Q%35+)*SEU>6UOLYOgy z_k9_2%^mV9U*sj0-|*kf6<*Fybou7;R)gKXrHw1*>mS7(-m`EU5{o=P5rgE>zhO8E3uFem8K*cMR^sf zQ~zvdN5>uQ(35?lI@TFSXHLFOlpx1Pq*&7jIi@NB<+CFXN6iX{+QJ8}8ABQCyc@@& zphER?alup-FOB}}YJ80G(39?3WXtDf7^>&i@bu?qm~FPM4;w{5B%-8SDypW$Ar&|2 z>_o$75&RAxf#fuodKZXN%3K`cmT&hp#*)I?=kBfFtm##w54plu2?w94e_?EHF5K3V zv++|rd7}y-DpZYV!J9YL4+(5rRCf;ZAc}K)>=*bmQ}l(v>wULi@E(e&mvzwl2uFWtSGFQfwQ-0%Bv&hxiK#HH!596_$FO+o>v3Hrd$H?a-j< z>m(Mu+Z$!7r^nzST5!;)haMhJCT4nZ0`VIey+5y(xrVz}eFosm!UUeh^WAJnJeR~h zJ2klRHQ~pFtz$H4&#{$TFWgbFxpfz@QtCGBXx`{C(se$23wj1@MhA8hEM-PIw6v>+ zQn6bYPSz#PxLIBw>3@f}*8f;*ZO=>;caSBAhBn2Q->)Q+vAVhu!f}fIHo0$!y0xwI zr)oRpn)}vq%U>l#Lw~dhqUL|pd-2}?alaPVy@ie$yglg0Y;)Ho2U}>+dnsWZPlL6| zMVNPsD6n;&b*#Y2ISvvWo4tLnV>>w#1am*2hyxPKSuh9sD(;^ee*nArkLKc;ccEZ` zkx-JNVxrT4L8_5R)nw+x)U;wk!_0jA%)P~`1dRS+XlR*qcK??YGARN71^`^gaZ6yU zCR}avlu#~+pX&&Jnx^FUW&hB~nD3U0<8#>&W2fJ0Ch|+_Ue!eVzT&Q)?2JnD+pn{E zhd%aB9_1gHHA#McEzaig(SHey_x4N+2#`UAF6M;>1Yu$Bhaj=^!^3D^i=X18AMN@f z#{b&B=9hM!!faC?#!0$bNTfgQ zptb%xZ?s(o=^^BsL2J{%B-;%|z0`?1gY~Ebfe`p6#}P*$a;&}dS;oVo>yp8cW`n=H z!M1^#TIFKvv%tTISz!~~e9LZ+!w6P@S3pCECK5fXlv}r$(6CyGVEA~cvTGLw&DE2o zXe@$H{_))VUWAfTP>VwgF~3OUS6-adqhS8rBtM5)i8zp)GoYg{z@JXcNBKg)yCrrv z3R*Y{vWz4ZtDpUek4T%nYEMeOc{-F`q=D>&f_%DT`^cPA7gLHBtX)a$J7b9U5Eq3N zC8t*o#el`35TaX9T)YBX(VRuhqps^C$#ar89OtQhhVEE@uB5Pn;qdz1ZG+6SfPgFk z5{S5_M`n?!U?Q8GV5^SPeFG-riYvId#OoUx9EBR&+bv8BtbwAuuH3@DgBuAzYL}_$)eY#J-5;8om-7SjSBQP~SwyvmzFKTt918{Xy|m{_NI?>i zB?x!&NDYKyoo)%fFv-I0x>*ArsadwIlJ{2%AU}A5oNXz@-Qx=MY+ItHM4CY+2?3Ie zjAFx%*=fFhUIBp!Y%*bu?{kNfZUVOwY?G0!a;P%~vfHT>;fbbdd#6daACpXIX z*V^0L`>6Be*RLd6?f88M{+l4y^-WD#QPx0LwQt$B*gnvE2%vJ9UL8fEX@J0>|D5{1 z^gFc#gqr2ZRo3CAcWO@hawQd&gRn1hGs z9Dr3_NHm>O7tc(!GIj<~;M4G#cXtpyM;e2`5O7sDV!Dq2Q2rCwhfR7{&}cy%%oZOI zZioDlF?=|d_f~2%XAYO;;n~4ot)s5aM9}7-R4~%*F zD>`c=SQkOpI)_b84xws_7?(%M(DBa75aeq~feg_L*vxy-`%A&kS{dJs_prX^V}C0b6$|a?FGtQ&J*F(3BWLHXsOKZ=;tAC-n97 z2>0u#@a?oTF*!MIB$7x%jzalIWg)@WDASQ}q>xA;Q4b8lZuGn#qHR*>KkJ!DM$Yyf zJNi3x`@6a*NPFL(j31pVrhH0@)gBl4w$Fln$r4UnrF5ShgG@rW2)>=O;>}{pscJau zX=QVWdYlDe=kqaTY(;V@DJgGqJ6QEEeJ>DpddVy(3}^AeC1gJwP0cVuHB#>XEVmTr z$^}F^6H1G<&z}8(qTUTOcp|0R$2R9!dGzQo1${ZFCJFFbMx*H7@q;w_J%VT@_a5=! zzkiAWbB?6`d!EE3&?{T=4@Tka`#OAocs!xCkthGJc73axF7d#z7QsUSX1N(`)c>)4 z+CO@?|LME@5z|i?0M*?qIEofg@m yU-_8`Fk>bD-v8@Tkni&!X5Ig~hh6;wgTL0!+^+A*Z?F*`W$$hs^}JoyXZ{PTqh1F9 literal 0 HcmV?d00001 diff --git a/examples/Hdiv-mixed/include/cl-options.h b/examples/Hdiv-mixed/include/cl-options.h index ec92a17cf1..c6b6cbd53f 100644 --- a/examples/Hdiv-mixed/include/cl-options.h +++ b/examples/Hdiv-mixed/include/cl-options.h @@ -1,12 +1,9 @@ #ifndef cloptions_h #define cloptions_h -#include "../include/structs.h" - -// Register problems to be available on the command line -PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx); +#include "structs.h" // Process general command line options -PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx); +PetscErrorCode ProcessCommandLineOptions(AppCtx app_ctx); #endif // cloptions_h diff --git a/examples/Hdiv-mixed/include/matops.h b/examples/Hdiv-mixed/include/matops.h deleted file mode 100644 index 0f9d8c3bc1..0000000000 --- a/examples/Hdiv-mixed/include/matops.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef matops_h -#define matops_h - -#include -#include - -#include "structs.h" - -PetscErrorCode MatGetDiag(Mat A, Vec D); -PetscErrorCode ApplyLocal_Ceed(Vec X, Vec Y, User user); -PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y); - -#endif // matops_h diff --git a/examples/Hdiv-mixed/include/problems.h b/examples/Hdiv-mixed/include/problems.h deleted file mode 100644 index dd7d65b039..0000000000 --- a/examples/Hdiv-mixed/include/problems.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef problems_h -#define problems_h - -#include "../include/structs.h" - -// ----------------------------------------------------------------------------- -// Set up problems function prototype -// ----------------------------------------------------------------------------- -// 1) poisson-quad2d -PetscErrorCode Hdiv_POISSON_QUAD2D(ProblemData *problem_data, void *ctx); - -PetscErrorCode SetupContext_POISSON_QUAD2D(Ceed ceed, CeedData ceed_data, - Physics phys); - -// 2) poisson-hex3d - -// 3) poisson-prism3d - -// 4) richard - -#endif // problems_h diff --git a/examples/Hdiv-mixed/include/register-problem.h b/examples/Hdiv-mixed/include/register-problem.h new file mode 100644 index 0000000000..ec623dd49b --- /dev/null +++ b/examples/Hdiv-mixed/include/register-problem.h @@ -0,0 +1,24 @@ +#ifndef register_problems_h +#define register_problems_h + +#include "structs.h" + +// Register problems to be available on the command line +PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx); +// ----------------------------------------------------------------------------- +// Set up problems function prototype +// ----------------------------------------------------------------------------- +// 1) darcy2d +PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, void *ctx); + +// 2) darcy3d +PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, void *ctx); + +// 3) darcy3dprism + +// 4) richard +PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx); + +extern int FreeContextPetsc(void *); + +#endif // register_problems_h diff --git a/examples/Hdiv-mixed/include/setup-boundary.h b/examples/Hdiv-mixed/include/setup-boundary.h new file mode 100644 index 0000000000..21e62d536f --- /dev/null +++ b/examples/Hdiv-mixed/include/setup-boundary.h @@ -0,0 +1,25 @@ +#ifndef setup_boundary_h +#define setup_boundary_h + +#include +#include +#include +#include +#include "structs.h" + +// --------------------------------------------------------------------------- +// Create boundary label +// --------------------------------------------------------------------------- +PetscErrorCode CreateBCLabel(DM dm, const char name[]); + +// --------------------------------------------------------------------------- +// Add Dirichlet boundaries to DM +// --------------------------------------------------------------------------- +PetscErrorCode DMAddBoundariesDirichlet(DM dm); +PetscErrorCode BoundaryDirichletMMS(PetscInt dim, PetscReal t, + const PetscReal coords[], + PetscInt num_comp_u, PetscScalar *u, void *ctx); +PetscErrorCode DMAddBoundariesPressure(Ceed ceed, CeedData ceed_data, + AppCtx app_ctx, ProblemData problem_data, DM dm, + CeedVector bc_pressure); +#endif // setup_boundary_h diff --git a/examples/Hdiv-mixed/include/setup-dm.h b/examples/Hdiv-mixed/include/setup-dm.h index ebdebdaf7c..610d74efd0 100644 --- a/examples/Hdiv-mixed/include/setup-dm.h +++ b/examples/Hdiv-mixed/include/setup-dm.h @@ -5,10 +5,12 @@ #include #include #include +#include "structs.h" // --------------------------------------------------------------------------- -// Set-up DM +// Setup DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm); +PetscErrorCode CreateDM(MPI_Comm comm, VecType vec_type, DM *dm); +PetscErrorCode PerturbVerticesSmooth(DM dm); #endif // setupdm_h diff --git a/examples/Hdiv-mixed/include/setup-fe.h b/examples/Hdiv-mixed/include/setup-fe.h new file mode 100644 index 0000000000..bb35ff51a0 --- /dev/null +++ b/examples/Hdiv-mixed/include/setup-fe.h @@ -0,0 +1,15 @@ +#ifndef setupfe_h +#define setupfe_h + +#include +#include +#include +#include +#include "structs.h" + +// --------------------------------------------------------------------------- +// Setup FE +// --------------------------------------------------------------------------- +PetscErrorCode SetupFE(MPI_Comm comm, DM dm); + +#endif // setupfe_h diff --git a/examples/Hdiv-mixed/include/setup-libceed.h b/examples/Hdiv-mixed/include/setup-libceed.h index 5b95f928c1..2bc8901c48 100644 --- a/examples/Hdiv-mixed/include/setup-libceed.h +++ b/examples/Hdiv-mixed/include/setup-libceed.h @@ -1,7 +1,7 @@ #ifndef setuplibceed_h #define setuplibceed_h -#include "../include/structs.h" +#include "structs.h" // Convert PETSc MemType to libCEED MemType CeedMemType MemTypeP2C(PetscMemType mtype); @@ -10,15 +10,14 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data); // Utility function - essential BC dofs are encoded in closure indices as -(i+1) PetscInt Involute(PetscInt i); // Utility function to create local CEED restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt P, - CeedInt topo_dim, CeedElemRestriction *elem_restr); +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, + CeedInt height, DMLabel domain_label, CeedInt value, + CeedElemRestriction *elem_restr); // Utility function to create local CEED Oriented restriction from DMPlex PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, - CeedInt topo_dim, CeedElemRestriction *elem_restr_oriented); + CeedElemRestriction *elem_restr_oriented, CeedElemRestriction *elem_restr); // Set up libCEED for a given degree PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, - ProblemData *problem_data, - PetscInt U_g_size, PetscInt U_loc_size, - CeedData ceed_data, CeedVector rhs_ceed, - CeedVector *target); + ProblemData problem_data, + PetscInt U_loc_size, CeedData ceed_data); #endif // setuplibceed_h diff --git a/examples/Hdiv-mixed/include/setup-matops.h b/examples/Hdiv-mixed/include/setup-matops.h new file mode 100644 index 0000000000..7c21ed22d4 --- /dev/null +++ b/examples/Hdiv-mixed/include/setup-matops.h @@ -0,0 +1,14 @@ +#ifndef setup_matops_h +#define setup_matops_h + +#include +#include + +#include "structs.h" + +PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, + OperatorApplyContext op_apply_ctx); +PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, + OperatorApplyContext op_apply_ctx); + +#endif // setup_matops_h diff --git a/examples/Hdiv-mixed/include/setup-solvers.h b/examples/Hdiv-mixed/include/setup-solvers.h new file mode 100644 index 0000000000..3c9490df07 --- /dev/null +++ b/examples/Hdiv-mixed/include/setup-solvers.h @@ -0,0 +1,28 @@ +#ifndef setup_solvers_h +#define setup_solvers_h + +#include +#include + +#include "structs.h" + +PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, + OperatorApplyContext ctx_jacobian); +PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, + OperatorApplyContext ctx_residual); +PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, + OperatorApplyContext ctx_error); +PetscErrorCode ApplyJacobian(Mat A, Vec X, Vec Y); +PetscErrorCode SNESFormResidual(SNES snes, Vec X, Vec Y, void *ctx); +PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, void *ctx); +PetscErrorCode PDESolver(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, + VecType vec_type, SNES snes, KSP ksp, Vec *U); +PetscErrorCode ComputeL2Error(DM dm, Ceed ceed, CeedData ceed_data, Vec U, + CeedScalar *l2_error_u, CeedScalar *l2_error_p); +PetscErrorCode PrintOutput(Ceed ceed, + CeedMemType mem_type_backend, + SNES snes, KSP ksp, + Vec U, CeedScalar l2_error_u, + CeedScalar l2_error_p, AppCtx app_ctx); + +#endif // setup_solvers_h diff --git a/examples/Hdiv-mixed/include/setup-ts.h b/examples/Hdiv-mixed/include/setup-ts.h new file mode 100644 index 0000000000..4d7d7546a3 --- /dev/null +++ b/examples/Hdiv-mixed/include/setup-ts.h @@ -0,0 +1,17 @@ +#ifndef setup_ts_h +#define setup_ts_h + +#include +#include + +#include "structs.h" + +PetscErrorCode CreateInitialConditions(DM dm, CeedData ceed_data, Vec *U0); +PetscErrorCode SetupResidualOperatorCtx_Ut(DM dm, Ceed ceed, CeedData ceed_data, + OperatorApplyContext ctx_residual_ut); +PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, + void *ctx_residual_ut); +PetscErrorCode TSSolveRichard(DM dm, CeedData ceed_data, AppCtx app_ctx, + Vec *U, PetscScalar *f_time, TS *ts); + +#endif // setup_ts_h diff --git a/examples/Hdiv-mixed/include/structs.h b/examples/Hdiv-mixed/include/structs.h index 4f75262e28..8912159106 100644 --- a/examples/Hdiv-mixed/include/structs.h +++ b/examples/Hdiv-mixed/include/structs.h @@ -7,68 +7,96 @@ // Application context from user command line options typedef struct AppCtx_ *AppCtx; struct AppCtx_ { - // libCEED arguments + MPI_Comm comm; + // Degree of polynomial (1 only), extra quadrature pts PetscInt degree; PetscInt q_extra; + // For applying traction BCs + PetscInt bc_pressure_count; + PetscInt bc_faces[16]; // face ID + PetscScalar bc_pressure_value[16]; // Problem type arguments PetscFunctionList problems; char problem_name[PETSC_MAX_PATH_LEN]; }; -// libCEED data struct -typedef struct CeedData_ *CeedData; -struct CeedData_ { - CeedBasis basis_x, basis_u; - CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_geo_data_i, - elem_restr_u_i; - CeedQFunction qf_residual; - CeedOperator op_residual; - CeedVector geo_data, x_ceed, y_ceed; - CeedQFunctionContext pq2d_context; -}; +// 2) richard +// We have 3 experiment parameters as described in Table 1:P1, P2, P3 +// Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 -// 1) poisson-quad2d -#ifndef PHYSICS_POISSONQUAD2D_STRUCT -#define PHYSICS_POISSONQUAD2D_STRUCT -typedef struct PQ2DContext_ *PQ2DContext; -struct PQ2DContext_ { - CeedScalar kappa; +#ifndef PHYSICS_RICHARDP2_STRUCT +#define PHYSICS_RICHARDP2_STRUCT +typedef struct RICHARDP2Context_ *RICHARDP2Context; +struct RICHARDP2Context_ { + CeedScalar K_star; + CeedScalar theta_s; + CeedScalar theta_r; + CeedScalar alpha_v; + CeedScalar n_v; + CeedScalar m_v; + CeedScalar m_r; + CeedScalar rho_0; + CeedScalar beta; }; #endif -// 2) poisson-hex3d - -// 3) poisson-prism3d - -// 4) richard +#ifndef PHYSICS_RICHARDP3_STRUCT +#define PHYSICS_RICHARDP3_STRUCT +typedef struct RICHARDP3Context_ *RICHARDP3Context; +struct RICHARDP3Context_ { + CeedScalar K_star; + CeedScalar theta_s; + CeedScalar theta_r; + CeedScalar alpha_star_v; + CeedScalar n_v; + CeedScalar m_v; + CeedScalar rho_0; + CeedScalar beta; +}; +#endif // Struct that contains all enums and structs used for the physics of all problems typedef struct Physics_ *Physics; struct Physics_ { - PQ2DContext pq2d_ctx; + RICHARDP2Context richard_p2_ctx; + RICHARDP3Context richard_p3_ctx; }; -// PETSc user data -typedef struct User_ *User; -struct User_ { - MPI_Comm comm; - Vec X_loc, Y_loc; - CeedVector x_ceed, y_ceed; - CeedOperator op; - DM dm; - Ceed ceed; - AppCtx app_ctx; - Physics phys; +// PETSc operator contexts +typedef struct OperatorApplyContext_ *OperatorApplyContext; +struct OperatorApplyContext_ { + MPI_Comm comm; + Vec X_loc, Y_loc, X_t_loc; + CeedVector x_ceed, y_ceed, x_t_ceed; + CeedOperator op_apply; + DM dm; + Ceed ceed; +}; + +// libCEED data struct +typedef struct CeedData_ *CeedData; +struct CeedData_ { + CeedBasis basis_x, basis_u, basis_p, basis_u_face; + CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_U_i, + elem_restr_p, elem_restr_p_i; + CeedQFunction qf_residual, qf_jacobian, qf_error, qf_ics; + CeedOperator op_residual, op_jacobian, op_error, op_ics; + CeedVector x_ceed, y_ceed, x_coord, U0_ceed, x_t_ceed; + OperatorApplyContext ctx_residual, ctx_jacobian, ctx_error, ctx_residual_ut; }; // Problem specific data -typedef struct { - CeedQFunctionUser residual, setup_rhs; - const char *residual_loc, *setup_rhs_loc; +typedef struct ProblemData_ *ProblemData; +struct ProblemData_ { + CeedQFunctionUser true_solution, residual, jacobian, error, ics, + bc_pressure; + const char *true_solution_loc, *residual_loc, *jacobian_loc, + *error_loc, *bc_pressure_loc, *ics_loc; CeedQuadMode quadrature_mode; - CeedInt geo_data_size, elem_node; - PetscErrorCode (*setup_ctx)(Ceed, CeedData, Physics); - -} ProblemData; + CeedInt elem_node, dim, q_data_size_face; + CeedQFunctionContext qfunction_context; + PetscBool has_ts; +}; #endif // structs_h \ No newline at end of file diff --git a/examples/Hdiv-mixed/main.c b/examples/Hdiv-mixed/main.c index 60942485ba..fbd47fc1f1 100644 --- a/examples/Hdiv-mixed/main.c +++ b/examples/Hdiv-mixed/main.c @@ -1,7 +1,33 @@ -/// @file -/// Test creation, use, and destruction of an element restriction for 2D quad Hdiv - -// run with ./main +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +// libCEED + PETSc Example: Mixed-Poisson in H(div) space +// +// This example demonstrates a simple usage of libCEED with PETSc to solve +// elasticity problems. +// +// The code uses higher level communication protocols in DMPlex. +// +// Build with: make +// Run with: +// ./main -pc_type svd -problem darcy2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 +// ./main -pc_type svd -problem darcy3d -dm_plex_dim 3 -dm_plex_box_faces 4,4,4 +// ./main -pc_type svd -problem darcy3d -dm_plex_filename /path to the mesh file +// ./main -pc_type svd -problem darcy2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 -bc_pressure 1 +// ./main -pc_type svd -problem darcy2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 -bc_pressure 1,2,3,4 const char help[] = "Solve H(div)-mixed problem using PETSc and libCEED\n"; #include "main.h" @@ -10,197 +36,171 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // Initialize PETSc // --------------------------------------------------------------------------- - PetscInt ierr; - ierr = PetscInitialize(&argc, &argv, NULL, help); - if (ierr) return ierr; + PetscCall( PetscInitialize(&argc, &argv, NULL, help) ); + + // --------------------------------------------------------------------------- + // Initialize libCEED + // --------------------------------------------------------------------------- + // -- Initialize backend + Ceed ceed; + CeedInit("/cpu/self/ref/serial", &ceed); + CeedMemType mem_type_backend; + CeedGetPreferredMemType(ceed, &mem_type_backend); + + VecType vec_type = NULL; + switch (mem_type_backend) { + case CEED_MEM_HOST: vec_type = VECSTANDARD; break; + case CEED_MEM_DEVICE: { + const char *resolved; + CeedGetResource(ceed, &resolved); + if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; + else if (strstr(resolved, "/gpu/hip/occa")) + vec_type = VECSTANDARD; // https://github.com/CEED/libCEED/issues/678 + else if (strstr(resolved, "/gpu/hip")) vec_type = VECHIP; + else vec_type = VECSTANDARD; + } + } // --------------------------------------------------------------------------- // Create structs // --------------------------------------------------------------------------- AppCtx app_ctx; - ierr = PetscCalloc1(1, &app_ctx); CHKERRQ(ierr); - - ProblemData *problem_data = NULL; - ierr = PetscCalloc1(1, &problem_data); CHKERRQ(ierr); + PetscCall( PetscCalloc1(1, &app_ctx) ); - User user; - ierr = PetscCalloc1(1, &user); CHKERRQ(ierr); + ProblemData problem_data = NULL; + PetscCall( PetscCalloc1(1, &problem_data) ); CeedData ceed_data; - ierr = PetscCalloc1(1, &ceed_data); CHKERRQ(ierr); + PetscCall( PetscCalloc1(1, &ceed_data) ); Physics phys_ctx; - ierr = PetscCalloc1(1, &phys_ctx); CHKERRQ(ierr); - - user->app_ctx = app_ctx; - user->phys = phys_ctx; + PetscCall( PetscCalloc1(1, &phys_ctx) ); // --------------------------------------------------------------------------- // Process command line options // --------------------------------------------------------------------------- // -- Register problems to be available on the command line - ierr = RegisterProblems_Hdiv(app_ctx); CHKERRQ(ierr); + PetscCall( RegisterProblems_Hdiv(app_ctx) ); // -- Process general command line options MPI_Comm comm = PETSC_COMM_WORLD; - ierr = ProcessCommandLineOptions(comm, app_ctx); CHKERRQ(ierr); + app_ctx->comm = comm; + PetscCall( ProcessCommandLineOptions(app_ctx) ); // --------------------------------------------------------------------------- // Choose the problem from the list of registered problems // --------------------------------------------------------------------------- { - PetscErrorCode (*p)(ProblemData *, void *); - ierr = PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p); - CHKERRQ(ierr); - if (!p) SETERRQ1(PETSC_COMM_SELF, 1, "Problem '%s' not found", - app_ctx->problem_name); - ierr = (*p)(problem_data, &user); CHKERRQ(ierr); + PetscErrorCode (*p)(Ceed, ProblemData, void *); + PetscCall( PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, + &p) ); + if (!p) SETERRQ(PETSC_COMM_SELF, 1, "Problem '%s' not found", + app_ctx->problem_name); + PetscCall( (*p)(ceed, problem_data, &app_ctx) ); } // --------------------------------------------------------------------------- - // Initialize libCEED + // Create DM // --------------------------------------------------------------------------- - // -- Initialize backend - Ceed ceed; - CeedInit("/cpu/self/ref/serial", &ceed); - CeedMemType mem_type_backend; - CeedGetPreferredMemType(ceed, &mem_type_backend); + DM dm; + PetscCall( CreateDM(comm, vec_type, &dm) ); + // TODO: add mesh option + // perturb to have smooth random mesh + PetscCall( PerturbVerticesSmooth(dm) ); + // --------------------------------------------------------------------------- - // Set-up DM + // Setup FE // --------------------------------------------------------------------------- - // PETSc objects - DM dm; - VecType vec_type; - ierr = CreateDistributedDM(comm, &dm); CHKERRQ(ierr); - ierr = DMGetVecType(dm, &vec_type); CHKERRQ(ierr); - - // --------------------------------------------------------------------------- - // Create global, local solution, local rhs vector - // --------------------------------------------------------------------------- - Vec U_g, U_loc; - PetscInt U_l_size, U_g_size, U_loc_size; - // Create global and local solution vectors - ierr = DMCreateGlobalVector(dm, &U_g); CHKERRQ(ierr); - ierr = VecGetSize(U_g, &U_g_size); CHKERRQ(ierr); - // Local size for matShell - ierr = VecGetLocalSize(U_g, &U_l_size); CHKERRQ(ierr); - // Create local unknown vector U_loc - ierr = DMCreateLocalVector(dm, &U_loc); CHKERRQ(ierr); + SetupFE(comm, dm); + + // --------------------------------------------------------------------------- + // Create local Force vector + // --------------------------------------------------------------------------- + Vec U_loc; + PetscInt U_loc_size; + //CeedVector bc_pressure; + PetscCall( DMCreateLocalVector(dm, &U_loc) ); // Local size for libCEED - ierr = VecGetSize(U_loc, &U_loc_size); CHKERRQ(ierr); - - // Operator - Mat mat; - ierr = MatCreateShell(comm, U_l_size, U_l_size, U_g_size, U_g_size, - user, &mat); CHKERRQ(ierr); - ierr = MatShellSetOperation(mat, MATOP_MULT, - (void(*)(void))MatMult_Ceed); CHKERRQ(ierr); - ierr = MatShellSetOperation(mat, MATOP_GET_DIAGONAL, - (void(*)(void))MatGetDiag); CHKERRQ(ierr); - ierr = MatShellSetVecType(mat, vec_type); CHKERRQ(ierr); - // Get RHS vector - Vec rhs_loc; - PetscScalar *r; - CeedVector rhs_ceed, target; - PetscMemType mem_type; - ierr = VecDuplicate(U_loc, &rhs_loc); CHKERRQ(ierr); - ierr = VecZeroEntries(rhs_loc); CHKERRQ(ierr); - ierr = VecGetArrayAndMemType(rhs_loc, &r, &mem_type); CHKERRQ(ierr); - CeedVectorCreate(ceed, U_l_size, &rhs_ceed); - CeedVectorSetArray(rhs_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, r); + PetscCall( VecGetSize(U_loc, &U_loc_size) ); // --------------------------------------------------------------------------- // Setup libCEED // --------------------------------------------------------------------------- // -- Set up libCEED objects - ierr = SetupLibceed(dm, ceed, app_ctx, problem_data, U_g_size, - U_loc_size, ceed_data, rhs_ceed, &target); - CHKERRQ(ierr); - // -- Set up context for QFunctions - ierr = problem_data->setup_ctx(ceed, ceed_data, phys_ctx); - CHKERRQ(ierr); - - // --------------------------------------------------------------------------- - // Gather RHS - // --------------------------------------------------------------------------- - Vec rhs; - ierr = VecDuplicate(U_g, &rhs); CHKERRQ(ierr); - CeedVectorTakeArray(rhs_ceed, MemTypeP2C(mem_type), NULL); - ierr = VecRestoreArrayAndMemType(rhs_loc, &r); CHKERRQ(ierr); - ierr = VecZeroEntries(rhs); CHKERRQ(ierr); - ierr = DMLocalToGlobal(dm, rhs_loc, ADD_VALUES, rhs); CHKERRQ(ierr); - // --------------------------------------------------------------------------- - // Setup Mat, KSP - // --------------------------------------------------------------------------- - user->comm = comm; - user->dm = dm; - user->X_loc = U_loc; - ierr = VecDuplicate(U_loc, &user->Y_loc); CHKERRQ(ierr); - user->x_ceed = ceed_data->x_ceed; - user->y_ceed = ceed_data->y_ceed; - user->op = ceed_data->op_residual; - user->ceed = ceed; - // Ksp + PetscCall( SetupLibceed(dm, ceed, app_ctx, problem_data, + U_loc_size, ceed_data) ); + //CeedVectorView(force_ceed, "%12.8f", stdout); + //PetscCall( DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, + // bc_pressure) ); + + + // --------------------------------------------------------------------------- + // Setup TSSolve for Richard problem + // --------------------------------------------------------------------------- + if (problem_data->has_ts) { + // --------------------------------------------------------------------------- + // Create global initial conditions + // --------------------------------------------------------------------------- + Vec U0; + CreateInitialConditions(dm, ceed_data, &U0); + PetscCall( VecDestroy(&U0) ); + } + + // --------------------------------------------------------------------------- + // Solve PDE + // --------------------------------------------------------------------------- + // Create SNES + SNES snes; KSP ksp; - PC pc; - ierr = KSPCreate(comm, &ksp); CHKERRQ(ierr); - ierr = KSPGetPC(ksp, &pc); CHKERRQ(ierr); - ierr = PCSetType(pc, PCJACOBI); CHKERRQ(ierr); - ierr = PCJacobiSetType(pc, PC_JACOBI_ROWSUM); CHKERRQ(ierr); - ierr = KSPSetType(ksp, KSPCG); CHKERRQ(ierr); - ierr = KSPSetNormType(ksp, KSP_NORM_NATURAL); CHKERRQ(ierr); - ierr = KSPSetTolerances(ksp, 1e-10, PETSC_DEFAULT, PETSC_DEFAULT, - PETSC_DEFAULT); CHKERRQ(ierr); - ierr = KSPSetOperators(ksp, mat, mat); CHKERRQ(ierr); - ierr = KSPSolve(ksp, rhs, U_g); CHKERRQ(ierr); - ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr); - - // Output results - KSPType ksp_type; - KSPConvergedReason reason; - PetscReal rnorm; - PetscInt its; - ierr = KSPGetType(ksp, &ksp_type); CHKERRQ(ierr); - ierr = KSPGetConvergedReason(ksp, &reason); CHKERRQ(ierr); - ierr = KSPGetIterationNumber(ksp, &its); CHKERRQ(ierr); - ierr = KSPGetResidualNorm(ksp, &rnorm); CHKERRQ(ierr); - ierr = PetscPrintf(comm, - " KSP:\n" - " KSP Type : %s\n" - " KSP Convergence : %s\n" - " Total KSP Iterations : %D\n" - " Final rnorm : %e\n", - ksp_type, KSPConvergedReasons[reason], its, - (double)rnorm); CHKERRQ(ierr); + Vec U; + PetscCall( SNESCreate(comm, &snes) ); + PetscCall( SNESGetKSP(snes, &ksp) ); + PetscCall( PDESolver(comm, dm, ceed, ceed_data, vec_type, snes, ksp, &U) ); + //VecView(U, PETSC_VIEWER_STDOUT_WORLD); + + // --------------------------------------------------------------------------- + // Compute L2 error of mms problem + // --------------------------------------------------------------------------- + CeedScalar l2_error_u, l2_error_p; + PetscCall( ComputeL2Error(dm, ceed,ceed_data, U, &l2_error_u, + &l2_error_p) ); + + // --------------------------------------------------------------------------- + // Print output results + // --------------------------------------------------------------------------- + PetscCall( PrintOutput(ceed, mem_type_backend, + snes, ksp, U, l2_error_u, l2_error_p, app_ctx) ); + + // --------------------------------------------------------------------------- + // Save solution (paraview) + // --------------------------------------------------------------------------- + PetscViewer viewer; + + PetscCall( PetscViewerVTKOpen(comm,"solution.vtu",FILE_MODE_WRITE,&viewer) ); + PetscCall( VecView(U, viewer) ); + PetscCall( PetscViewerDestroy(&viewer) ); // --------------------------------------------------------------------------- // Free objects // --------------------------------------------------------------------------- // Free PETSc objects - ierr = DMDestroy(&dm); CHKERRQ(ierr); - ierr = VecDestroy(&U_g); CHKERRQ(ierr); - ierr = VecDestroy(&U_loc); CHKERRQ(ierr); - ierr = VecDestroy(&user->Y_loc); CHKERRQ(ierr); - ierr = VecDestroy(&rhs); CHKERRQ(ierr); - ierr = VecDestroy(&rhs_loc); CHKERRQ(ierr); - ierr = MatDestroy(&mat); CHKERRQ(ierr); - ierr = KSPDestroy(&ksp); CHKERRQ(ierr); + PetscCall( DMDestroy(&dm) ); + PetscCall( VecDestroy(&U) ); + PetscCall( VecDestroy(&U_loc) ); + PetscCall( SNESDestroy(&snes) ); // -- Function list - ierr = PetscFunctionListDestroy(&app_ctx->problems); CHKERRQ(ierr); + PetscCall( PetscFunctionListDestroy(&app_ctx->problems) ); // -- Structs - ierr = PetscFree(app_ctx); CHKERRQ(ierr); - ierr = PetscFree(problem_data); CHKERRQ(ierr); - ierr = PetscFree(user); CHKERRQ(ierr); - ierr = PetscFree(phys_ctx->pq2d_ctx); CHKERRQ(ierr); - ierr = PetscFree(phys_ctx); CHKERRQ(ierr); - // Free libCEED objects + PetscCall( PetscFree(app_ctx) ); + PetscCall( PetscFree(problem_data) ); + PetscCall( PetscFree(phys_ctx) ); - CeedVectorDestroy(&rhs_ceed); - CeedVectorDestroy(&target); - ierr = CeedDataDestroy(ceed_data); CHKERRQ(ierr); + // Free libCEED objects + //CeedVectorDestroy(&bc_pressure); + PetscCall( CeedDataDestroy(ceed_data) ); CeedDestroy(&ceed); return PetscFinalize(); diff --git a/examples/Hdiv-mixed/main.h b/examples/Hdiv-mixed/main.h index da283f2d04..1b85c9f8be 100644 --- a/examples/Hdiv-mixed/main.h +++ b/examples/Hdiv-mixed/main.h @@ -4,8 +4,12 @@ #include "include/setup-libceed.h" #include "include/setup-dm.h" +#include "include/setup-fe.h" +#include "include/setup-boundary.h" #include "include/cl-options.h" -#include "include/problems.h" -#include "include/matops.h" +#include "include/register-problem.h" +#include "include/setup-matops.h" +#include "include/setup-solvers.h" +#include "include/setup-ts.h" #endif // MAIN_H \ No newline at end of file diff --git a/examples/Hdiv-mixed/problems/darcy2d.c b/examples/Hdiv-mixed/problems/darcy2d.c new file mode 100644 index 0000000000..2a4c3d98f1 --- /dev/null +++ b/examples/Hdiv-mixed/problems/darcy2d.c @@ -0,0 +1,85 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Utility functions for setting up Darcy problem in 2D + +#include "../include/register-problem.h" +#include "../qfunctions/darcy-true2d.h" +#include "../qfunctions/darcy-system2d.h" +#include "../qfunctions/darcy-error2d.h" +#include "../qfunctions/pressure-boundary2d.h" + +PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, void *ctx) { + AppCtx app_ctx = *(AppCtx *)ctx; + DARCYContext darcy_ctx; + CeedQFunctionContext darcy_context; + + PetscFunctionBeginUser; + + PetscCall( PetscCalloc1(1, &darcy_ctx) ); + + // ------------------------------------------------------ + // SET UP POISSON_QUAD2D + // ------------------------------------------------------ + problem_data->dim = 2; + problem_data->elem_node = 4; + problem_data->q_data_size_face = 3; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->true_solution = DarcyTrue2D; + problem_data->true_solution_loc = DarcyTrue2D_loc; + problem_data->residual = DarcySystem2D; + problem_data->residual_loc = DarcySystem2D_loc; + problem_data->jacobian = JacobianDarcySystem2D; + problem_data->jacobian_loc = JacobianDarcySystem2D_loc; + problem_data->error = DarcyError2D; + problem_data->error_loc = DarcyError2D_loc; + problem_data->bc_pressure = BCPressure2D; + problem_data->bc_pressure_loc = BCPressure2D_loc; + problem_data->has_ts = PETSC_FALSE; + + // ------------------------------------------------------ + // Command line Options + // ------------------------------------------------------ + CeedScalar kappa = 1., rho_a0 = 998.2, g = 9.8, alpha_a = 1., b_a = 10.; + PetscOptionsBegin(app_ctx->comm, NULL, "Options for Hdiv-mixed problem", NULL); + PetscCall( PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, + kappa, &kappa, NULL)); + PetscCall( PetscOptionsScalar("-rho_a0", "Density at p0", NULL, + rho_a0, &rho_a0, NULL)); + PetscCall( PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", + NULL, + alpha_a, &alpha_a, NULL)); + PetscCall( PetscOptionsScalar("-b_a", "Parameter for relative permeability", + NULL, + b_a, &b_a, NULL)); + PetscOptionsEnd(); + + darcy_ctx->kappa = kappa; + darcy_ctx->rho_a0 = rho_a0; + darcy_ctx->g = g; + darcy_ctx->alpha_a = alpha_a; + darcy_ctx->b_a = b_a; + + CeedQFunctionContextCreate(ceed, &darcy_context); + CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, + sizeof(*darcy_ctx), darcy_ctx); + problem_data->qfunction_context = darcy_context; + CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, + FreeContextPetsc); + PetscCall( PetscFree(darcy_ctx) ); + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mixed/problems/darcy3d.c b/examples/Hdiv-mixed/problems/darcy3d.c new file mode 100644 index 0000000000..88614f243b --- /dev/null +++ b/examples/Hdiv-mixed/problems/darcy3d.c @@ -0,0 +1,85 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Utility functions for setting up Darcy problem in 3D + +#include "../include/register-problem.h" +#include "../qfunctions/darcy-true3d.h" +#include "../qfunctions/darcy-system3d.h" +#include "../qfunctions/darcy-error3d.h" +#include "../qfunctions/pressure-boundary3d.h" + +PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, void *ctx) { + AppCtx app_ctx = *(AppCtx *)ctx; + DARCYContext darcy_ctx; + CeedQFunctionContext darcy_context; + + PetscFunctionBeginUser; + + PetscCall( PetscCalloc1(1, &darcy_ctx) ); + + // ------------------------------------------------------ + // SET UP POISSON_QUAD2D + // ------------------------------------------------------ + problem_data->dim = 3; + problem_data->elem_node = 8; + problem_data->q_data_size_face = 4; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->true_solution = DarcyTrue3D; + problem_data->true_solution_loc = DarcyTrue3D_loc; + problem_data->residual = DarcySystem3D; + problem_data->residual_loc = DarcySystem3D_loc; + problem_data->jacobian = JacobianDarcySystem3D; + problem_data->jacobian_loc = JacobianDarcySystem3D_loc; + problem_data->error = DarcyError3D; + problem_data->error_loc = DarcyError3D_loc; + problem_data->bc_pressure = BCPressure3D; + problem_data->bc_pressure_loc = BCPressure3D_loc; + problem_data->has_ts = PETSC_FALSE; + + // ------------------------------------------------------ + // Command line Options + // ------------------------------------------------------ + CeedScalar kappa = 1., rho_a0 = 998.2, g = 9.8, alpha_a = 1., b_a = 10.; + PetscOptionsBegin(app_ctx->comm, NULL, "Options for Hdiv-mixed problem", NULL); + PetscCall( PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, + kappa, &kappa, NULL)); + PetscCall( PetscOptionsScalar("-rho_a0", "Density at p0", NULL, + rho_a0, &rho_a0, NULL)); + PetscCall( PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", + NULL, + alpha_a, &alpha_a, NULL)); + PetscCall( PetscOptionsScalar("-b_a", "Parameter for relative permeability", + NULL, + b_a, &b_a, NULL)); + PetscOptionsEnd(); + + darcy_ctx->kappa = kappa; + darcy_ctx->rho_a0 = rho_a0; + darcy_ctx->g = g; + darcy_ctx->alpha_a = alpha_a; + darcy_ctx->b_a = b_a; + + CeedQFunctionContextCreate(ceed, &darcy_context); + CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, + sizeof(*darcy_ctx), darcy_ctx); + problem_data->qfunction_context = darcy_context; + CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, + FreeContextPetsc); + PetscCall( PetscFree(darcy_ctx) ); + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mixed/problems/poisson-quad2d.c b/examples/Hdiv-mixed/problems/poisson-quad2d.c deleted file mode 100644 index 609696d6ad..0000000000 --- a/examples/Hdiv-mixed/problems/poisson-quad2d.c +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at -// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights -// reserved. See files LICENSE and NOTICE for details. -// -// This file is part of CEED, a collection of benchmarks, miniapps, software -// libraries and APIs for efficient high-order finite element and spectral -// element discretizations for exascale applications. For more information and -// source code availability see http://github.com/ceed. -// -// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, -// a collaborative effort of two U.S. Department of Energy organizations (Office -// of Science and the National Nuclear Security Administration) responsible for -// the planning and preparation of a capable exascale ecosystem, including -// software, applications, hardware, advanced system engineering and early -// testbed platforms, in support of the nation's exascale computing imperative. - -/// @file -/// Utility functions for setting up POISSON_QUAD2D - -#include "../include/setup-libceed.h" -#include "../include/problems.h" -#include "../qfunctions/poisson-quad2d.h" -// Hdiv_POISSON_QUAD2D is registered in cl-option.c -PetscErrorCode Hdiv_POISSON_QUAD2D(ProblemData *problem_data, void *ctx) { - User user = *(User *)ctx; - MPI_Comm comm = PETSC_COMM_WORLD; - PetscInt ierr; - PetscFunctionBeginUser; - - ierr = PetscCalloc1(1, &user->phys->pq2d_ctx); CHKERRQ(ierr); - - // ------------------------------------------------------ - // SET UP POISSON_QUAD2D - // ------------------------------------------------------ - problem_data->elem_node = 4; - problem_data->geo_data_size = 1; - problem_data->quadrature_mode = CEED_GAUSS; - problem_data->residual = PoissonQuadF; - problem_data->residual_loc = PoissonQuadF_loc; - problem_data->setup_rhs = SetupRhs; - problem_data->setup_rhs_loc = SetupRhs_loc; - problem_data->setup_ctx = SetupContext_POISSON_QUAD2D; - - // ------------------------------------------------------ - // Create the libCEED context - // ------------------------------------------------------ - CeedScalar kappa = 1.; - - // ------------------------------------------------------ - // Command line Options - // ------------------------------------------------------ - ierr = PetscOptionsBegin(comm, NULL, "Options for DENSITY_CURRENT problem", - NULL); CHKERRQ(ierr); - // -- Physics - ierr = PetscOptionsScalar("-kappa", "hydraulic conductivity", - NULL, kappa, &kappa, NULL); CHKERRQ(ierr); - - ierr = PetscOptionsEnd(); CHKERRQ(ierr); - - // -- QFunction Context - user->phys->pq2d_ctx->kappa = kappa; - - PetscFunctionReturn(0); -} - -PetscErrorCode SetupContext_POISSON_QUAD2D(Ceed ceed, CeedData ceed_data, - Physics phys) { - PetscFunctionBeginUser; - - CeedQFunctionContextCreate(ceed, &ceed_data->pq2d_context); - CeedQFunctionContextSetData(ceed_data->pq2d_context, CEED_MEM_HOST, - CEED_USE_POINTER, sizeof(*phys->pq2d_ctx), phys->pq2d_ctx); - CeedQFunctionSetContext(ceed_data->qf_residual, ceed_data->pq2d_context); - - PetscFunctionReturn(0); -} \ No newline at end of file diff --git a/examples/Hdiv-mixed/problems/register-problem.c b/examples/Hdiv-mixed/problems/register-problem.c new file mode 100644 index 0000000000..28fc386fbe --- /dev/null +++ b/examples/Hdiv-mixed/problems/register-problem.c @@ -0,0 +1,46 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Command line option processing for H(div) example using PETSc + +#include "../include/register-problem.h" + +// Register problems to be available on the command line +PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx) { + app_ctx->problems = NULL; + PetscFunctionBeginUser; + // 1) darcy2d (Hdiv_DARCY2D is created in darcy2d.c) + PetscCall( PetscFunctionListAdd(&app_ctx->problems, "darcy2d", + Hdiv_DARCY2D) ); + // 2) darcy3d (Hdiv_DARCY3D is created in dacry3d.c) + PetscCall( PetscFunctionListAdd(&app_ctx->problems, "darcy3d", + Hdiv_DARCY3D) ); + // 3) darcy3d-prism + + // 4) richard + PetscCall( PetscFunctionListAdd(&app_ctx->problems, "richard2d", + Hdiv_RICHARD2D) ); + PetscFunctionReturn(0); +} + + +// Free a plain data context that was allocated using PETSc; returning libCEED error codes +int FreeContextPetsc(void *data) { + if (PetscFree(data)) return CeedError(NULL, CEED_ERROR_ACCESS, + "PetscFree failed"); + return CEED_ERROR_SUCCESS; +} \ No newline at end of file diff --git a/examples/Hdiv-mixed/problems/richard2d.c b/examples/Hdiv-mixed/problems/richard2d.c new file mode 100644 index 0000000000..61a03e975d --- /dev/null +++ b/examples/Hdiv-mixed/problems/richard2d.c @@ -0,0 +1,95 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Utility functions for setting up Richard problem in 2D + +#include "../include/register-problem.h" +#include "../qfunctions/richard-system2d.h" +#include "../qfunctions/richard-mms2d.h" +#include "../qfunctions/pressure-boundary2d.h" +#include "petscsystypes.h" + +PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx) { + AppCtx app_ctx = *(AppCtx *)ctx; + RICHARDContext richard_ctx; + CeedQFunctionContext richard_context; + + PetscFunctionBeginUser; + + PetscCall( PetscCalloc1(1, &richard_ctx) ); + + // ------------------------------------------------------ + // SET UP POISSON_QUAD2D + // ------------------------------------------------------ + problem_data->dim = 2; + problem_data->elem_node = 4; + problem_data->q_data_size_face = 3; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->ics = RichardICs2D; + problem_data->ics_loc = RichardICs2D_loc; + //problem_data->force = RichardTrue2D; + //problem_data->force_loc = RichardTrue2D_loc; + //problem_data->residual = RichardSystem2D; + //problem_data->residual_loc = RichardSystem2D_loc; + //problem_data->jacobian = JacobianRichardSystem2D; + //problem_data->jacobian_loc = JacobianRichardSystem2D_loc; + //problem_data->error = DarcyError2D; + //problem_data->error_loc = DarcyError2D_loc; + problem_data->bc_pressure = BCPressure2D; + problem_data->bc_pressure_loc = BCPressure2D_loc; + problem_data->has_ts = PETSC_TRUE; + + // ------------------------------------------------------ + // Command line Options + // ------------------------------------------------------ + CeedScalar kappa = 10., alpha_a = 1., b_a = 10., rho_a0 = 998.2, + beta = 0., g = 9.8, p0 = 101325; + + PetscOptionsBegin(app_ctx->comm, NULL, "Options for Hdiv-mixed problem", NULL); + PetscCall( PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, + kappa, &kappa, NULL)); + PetscCall( PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", + NULL, + alpha_a, &alpha_a, NULL)); + PetscCall( PetscOptionsScalar("-b_a", "Parameter for relative permeability", + NULL, + b_a, &b_a, NULL)); + PetscCall( PetscOptionsScalar("-rho_a0", "Density at p0", NULL, + rho_a0, &rho_a0, NULL)); + PetscCall( PetscOptionsScalar("-beta", "Water compressibility", NULL, + beta, &beta, NULL)); + PetscOptionsEnd(); + + richard_ctx->kappa = kappa; + richard_ctx->alpha_a = alpha_a; + richard_ctx->b_a = b_a; + richard_ctx->rho_a0 = rho_a0; + richard_ctx->beta = beta; + richard_ctx->g = g; + richard_ctx->p0 = p0; + richard_ctx->time = 0.; + richard_ctx->gamma = 5.; + CeedQFunctionContextCreate(ceed, &richard_context); + CeedQFunctionContextSetData(richard_context, CEED_MEM_HOST, CEED_COPY_VALUES, + sizeof(*richard_ctx), richard_ctx); + problem_data->qfunction_context = richard_context; + CeedQFunctionContextSetDataDestroy(richard_context, CEED_MEM_HOST, + FreeContextPetsc); + + PetscCall( PetscFree(richard_ctx) ); + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mass/qfunctions/poisson-true2d.h b/examples/Hdiv-mixed/qfunctions/darcy-error2d.h similarity index 52% rename from examples/Hdiv-mass/qfunctions/poisson-true2d.h rename to examples/Hdiv-mixed/qfunctions/darcy-error2d.h index 42cfb86802..1420ebc571 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-true2d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-error2d.h @@ -15,66 +15,64 @@ // testbed platforms, in support of the nation's exascale computing imperative. /// @file -/// Compute true solution of the H(div) example using PETSc +/// Compute pointwise error of the H(div) example using PETSc -#ifndef TRUE_H -#define TRUE_H +#ifndef DARCY_ERROR2D_H +#define DARCY_ERROR2D_H #include +#include "utils.h" // ----------------------------------------------------------------------------- -// Compuet true solution +// Compuet error // ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupTrueSoln2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { +#ifndef DARCY_CTX +#define DARCY_CTX +typedef struct DARCYContext_ *DARCYContext; +struct DARCYContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; +}; +#endif +CEED_QFUNCTION(DarcyError2D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*coords) = in[0], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1]; + const CeedScalar (*w) = in[0], + (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*p) = (const CeedScalar(*))in[3], + (*true_soln) = in[4]; // Outputs - CeedScalar (*true_soln_Hdiv) = out[0]; + CeedScalar (*error) = out[0]; + // Context + DARCYContext context = (DARCYContext)ctx; + //const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; // Quadrature Point Loop - printf("True solution projected into H(div) space;Qfunction poisson-true2d.h\n"); CeedPragmaSIMD for (CeedInt i=0; i +#include "utils.h" + + +// ----------------------------------------------------------------------------- +// Compuet error +// ----------------------------------------------------------------------------- +#ifndef DARCY_CTX +#define DARCY_CTX +typedef struct DARCYContext_ *DARCYContext; +struct DARCYContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; +}; +#endif +CEED_QFUNCTION(DarcyError3D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*p) = (const CeedScalar(*))in[3], + (*true_soln) = in[4]; + // Outputs + CeedScalar (*error) = out[0]; + // Context + DARCYContext context = (DARCYContext)ctx; + //const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#include "ceed/ceed-f64.h" +#include "utils.h" + +// ----------------------------------------------------------------------------- +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega +// -\div(u) = -f in \Omega +// p = p_b on \Gamma_D +// u.n = u_b on \Gamma_N +// +// Where rho = rho_a/rho_a0, rho_a = rho_a0*exp(\beta * (p - p0)), p0 = 101325 Pa is atmospheric pressure +// rho_a0 is the density at p_0, g_u = g/norm(g) where g is gravity. +// k_r = b_a + alpha_a * (\psi - x2), where \psi = p / (rho_a0 * norm(g)) and x2 is vertical axis +// +// Weak form: Find (u, \psi) \in VxQ (V=H(div), Q=L^2) on \Omega +// (v, K^{-1}/rho*k_r * u) -(v, rho*g_u) -(\div(v), \psi) = -_{\Gamma_D} +// -(q, \div(u)) + (q, f) = 0 +// +// We solve MMS for K = kappa*I and beta=0 ==> rho=1 and \theta = alpha_a*\psi, so +// +// This QFunction setup the mixed form of the above equation +// Inputs: +// w : weight of quadrature +// J : dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// u : basis_u at quadrature points +// div(u) : divergence of basis_u at quadrature points +// p : basis_p at quadrature points +// f : force vector created in true qfunction +// +// Output: +// v : (v, K^{-1}/rho*k_r u) = \int (v^T * K^{-1}/rho*k_r*u detJ*w)dX ==> \int (v^T J^T * K^{-1}/rho*k_r *J*u*w/detJ)dX +// -(v, rho*g_u) = \int (v^T * rho*g_u detJ*w)dX ==> \int (v^T J^T * rho*g_u*w) dX +// div(v) : -(\div(v), \psi) = -\int (div(v)^T * \psi *w) dX +// q : -(q, \div(u)) = -\int (q^T * div(u) * w) dX +// (q, f) = \int( q^T * f * w*detJ )dX +// ----------------------------------------------------------------------------- +#ifndef DARCY_CTX +#define DARCY_CTX +typedef struct DARCYContext_ *DARCYContext; +struct DARCYContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; +}; +#endif +// ----------------------------------------------------------------------------- +// Residual evaluation for Darcy problem +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(DarcySystem2D)(void *ctx, CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*div_u) = (const CeedScalar(*))in[3], + (*p) = (const CeedScalar(*))in[4], + (*f) = in[5], + (*coords) = in[6]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], + (*div_v) = (CeedScalar(*))out[1], + (*q) = (CeedScalar(*))out[2]; + // Context + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; ikappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#include "utils.h" + +// ----------------------------------------------------------------------------- +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega +// -\div(u) = -f in \Omega +// p = p_b on \Gamma_D +// u.n = u_b on \Gamma_N +// +// Where rho = rho_a/rho_a0, rho_a = rho_a0*exp(\beta * (p - p0)), p0 = 101325 Pa is atmospheric pressure +// rho_a0 is the density at p_0, g_u = g/norm(g) where g is gravity. +// k_r = b_a + alpha_a * (\psi - x2), where \psi = p / (rho_a0 * norm(g)) and x2 is vertical axis +// +// Weak form: Find (u, \psi) \in VxQ (V=H(div), Q=L^2) on \Omega +// (v, K^{-1}/rho*k_r * u) -(v, rho*g_u) -(\div(v), \psi) = -_{\Gamma_D} +// -(q, \div(u)) + (q, f) = 0 +// +// We solve MMS for K = kappa*I and beta=0 ==> rho=1 +// +// This QFunction setup the mixed form of the above equation +// Inputs: +// w : weight of quadrature +// J : dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// u : basis_u at quadrature points +// div(u) : divergence of basis_u at quadrature points +// p : basis_p at quadrature points +// f : force vector created in true qfunction +// +// Output: +// v : (v, K^{-1}/rho*k_r u) = \int (v^T * K^{-1}/rho*k_r*u detJ*w)dX ==> \int (v^T J^T * K^{-1}/rho*k_r *J*u*w/detJ)dX +// -(v, rho*g_u) = \int (v^T * rho*g_u detJ*w)dX ==> \int (v^T J^T * rho*g_u*w) dX +// div(v) : -(\div(v), \psi) = -\int (div(v)^T * \psi *w) dX +// q : -(q, \div(u)) = -\int (q^T * div(u) * w) dX +// (q, f) = \int( q^T * f * w*detJ )dX +// ----------------------------------------------------------------------------- +#ifndef DARCY_CTX +#define DARCY_CTX +typedef struct DARCYContext_ *DARCYContext; +struct DARCYContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; +}; +#endif +// ----------------------------------------------------------------------------- +// Residual evaluation for Darcy problem +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(DarcySystem3D)(void *ctx, CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*div_u) = (const CeedScalar(*))in[3], + (*p) = (const CeedScalar(*))in[4], + (*f) = in[5], + (*coords) = in[6]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], + (*div_v) = (CeedScalar(*))out[1], + (*q) = (CeedScalar(*))out[2]; + // Context + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; ikappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + // *INDENT-ON* + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#include "utils.h" + +// ----------------------------------------------------------------------------- +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega +// -\div(u) = -f in \Omega +// p = p_b on \Gamma_D +// u.n = u_b on \Gamma_N +// +// Where rho = rho_a/rho_a0, rho_a = rho_a0*exp(\beta * (p - p0)), p0 = 101325 Pa is atmospheric pressure +// rho_a0 is the density at p_0, g_u = g/norm(g) where g is gravity. +// k_r = b_a + alpha_a * (\psi - x2), where \psi = p / (rho_a0 * norm(g)) and x2 is vertical axis +// +// We solve MMS for K = kappa*I and beta=0 ==> rho=1 and \theta = alpha_a*\psi, so +// +// This QFunction setup the true solution and forcing f of the above equation +// Inputs: +// coords: physical coordinate +// +// Output: +// true_force : = div(u) +// true_solution : = [\psi, u] where \psi, u are the exact solution solution +// ----------------------------------------------------------------------------- +#ifndef DARCY_CTX +#define DARCY_CTX +typedef struct DARCYContext_ *DARCYContext; +struct DARCYContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; +}; +#endif +CEED_QFUNCTION(DarcyTrue2D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*coords) = in[0]; + // Outputs + CeedScalar (*true_force) = out[0], (*true_solution) = out[1]; + // Context + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#include "utils.h" + +// ----------------------------------------------------------------------------- +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega +// -\div(u) = -f in \Omega +// p = p_b on \Gamma_D +// u.n = u_b on \Gamma_N +// +// Where rho = rho_a/rho_a0, rho_a = rho_a0*exp(\beta * (p - p0)), p0 = 101325 Pa is atmospheric pressure +// rho_a0 is the density at p_0, g_u = g/norm(g) where g is gravity. +// k_r = b_a + alpha_a * (\psi - x2), where \psi = p / (rho_a0 * norm(g)) and x2 is vertical axis +// +// We solve MMS for K = kappa*I and beta=0 ==> rho=1 +// +// This QFunction setup the true solution and forcing f of the above equation +// Inputs: +// coords: physical coordinate +// +// Output: +// true_force : = div(u) +// true_solution : = [\psi, u] where \psi, u are the exact solution solution +// ----------------------------------------------------------------------------- +#ifndef DARCY_CTX +#define DARCY_CTX +typedef struct DARCYContext_ *DARCYContext; +struct DARCYContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; +}; +#endif +CEED_QFUNCTION(DarcyTrue3D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*coords) = in[0]; + // Outputs + CeedScalar (*true_force) = out[0], (*true_soln) = out[1]; + // Context + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i - -#ifndef PHYSICS_POISSONQUAD2D_STRUCT -#define PHYSICS_POISSONQUAD2D_STRUCT -typedef struct PQ2DContext_ *PQ2DContext; -struct PQ2DContext_ { - CeedScalar kappa; -}; -#endif - -// ----------------------------------------------------------------------------- -// This QFunction applies the mass operator for a vector field of 2 components. -// -// Inputs: -// w - weight of quadrature -// J - dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim -// u - Input basis at quadrature points -// -// Output: -// v - Output vector (test functions) at quadrature points -// Note we need to apply Piola map on the basis, which is J*u/detJ -// So (v,u) = \int (v^T * u detJ*w) ==> \int (v^T J^T*J*u*w/detJ) -// ----------------------------------------------------------------------------- -CEED_QFUNCTION(PoissonQuadF)(void *ctx, CeedInt Q, const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* - // Inputs - const CeedScalar (*w) = in[0], - (*J)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; - - // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; - // *INDENT-ON* - - // Context - const PQ2DContext context = (PQ2DContext)ctx; - const CeedScalar kappa = context->kappa; - // for simplicity we considered kappa as scalar (it should be tensor) - - // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i \int (v^T J^T* f * w), f=true_soln -// ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupRhs)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* - // Inputs - const CeedScalar (*x) = in[0], - (*w) = in[1], - (*J)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[2]; - // Outputs - CeedScalar (*true_soln) = out[0], - (*rhs) = out[1]; - - // *INDENT-ON* - - // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i_{\Gamma_D} +// -(q, \div(u)) = -(q, f) +// This QFunction sets up the pressure boundary conditions : -_{\Gamma_D} +// Inputs: +// w : weight of quadrature +// p0 : pressure value on the boundary +// +// Output: +// v : p0 * N * w +// Note that the Piola map of the H(div) basis and physical normal "n" got canceled +// and we need to multiply by the reference normal "N" on each face +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(BCPressure2D)(void *ctx, CeedInt Q, + const CeedScalar *const *in, CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0]; + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + // *INDENT-ON* + + // User context + CeedPragmaSIMD + // Quadrature Point Loop + for (CeedInt i=0; i_{\Gamma_D} +// -(q, \div(u)) = -(q, f) +// This QFunction sets up the pressure boundary conditions : -_{\Gamma_D} +// Inputs: +// w : weight of quadrature +// p0 : pressure value on the boundary +// +// Output: +// v : p0 * N * w +// Note that the Piola map of the H(div) basis and physical normal "n" got canceled +// and we need to multiply by the reference normal "N" on each face +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(BCPressure3D)(void *ctx, CeedInt Q, + const CeedScalar *const *in, CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0]; + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + // *INDENT-OFF* + + // User context + CeedPragmaSIMD + // Quadrature Point Loop + for (CeedInt i=0; i +#include "ceed/ceed-f64.h" +#include "utils.h" + +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega x [0,T] +// -\div(u) = -f + d (rho*\theta)/dt in \Omega x [0,T] +// p = p_b on \Gamma_D x [0,T] +// u.n = u_b on \Gamma_N x [0,T] +// p = p_0 in \Omega, t = 0 +// +// Where rho = rho_a/rho_a0, rho_a = rho_a0*exp(\beta * (p - p0)), p0 = 101325 Pa is atmospheric pressure +// rho_a0 is the density at p_0, g_u = g/norm(g) where g is gravity. +// k_r = b_a + alpha_a * (\psi - x2), where \psi = p / (rho_a0 * norm(g)) and x2 is vertical axis +// +// Weak form: Find (u, \psi) \in VxQ (V=H(div), Q=L^2) on \Omega +// (v, K^{-1}/rho*k_r * u) -(v, rho*g_u) -(\div(v), \psi) = -_{\Gamma_D} +// -(q, \div(u)) + (q, f) -(q, d (rho*\theta)/dt ) = 0 +// +// We solve MMS for K = kappa*I and beta=0 ==> rho=1 and \theta = alpha_a*\psi, so +// -(q, d (rho*\theta)/dt ) = -alpha_a*(q, d(\psi)/dt ) +// +// This QFunction sets up the force +// Inputs: +// x : interpolation of the physical coordinate +// w : weight of quadrature +// J : dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// +// Output: +// force_u : which is 0.0 for this problem (- is in pressure-boundary qfunction) +// force_p : 0.0 +// ----------------------------------------------------------------------------- +// We have 3 experiment parameters as described in Table 1:P1, P2, P3 +// Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +#ifndef RICHARD_CTX +#define RICHARD_CTX +typedef struct RICHARDContext_ *RICHARDContext; +struct RICHARDContext_ { + CeedScalar kappa; + CeedScalar alpha_a; + CeedScalar b_a; + CeedScalar rho_a0; + CeedScalar beta; + CeedScalar g; + CeedScalar p0; + CeedScalar time; + CeedScalar gamma; +}; +#endif +// ----------------------------------------------------------------------------- +// Initial conditions for Richard problem +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(RichardICs2D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*coords) = in[0]; + // Outputs + CeedScalar (*u_0) = out[0], (*psi_0) = out[1]; + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; igamma; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + CeedScalar t = context->time; + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#include "utils.h" + +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega x [0,T] +// -\div(u) = -f + d (rho*\theta)/dt in \Omega x [0,T] +// p = p_b on \Gamma_D x [0,T] +// u.n = u_b on \Gamma_N x [0,T] +// p = p_0 in \Omega, t = 0 +// +// Where rho = rho_a/rho_a0, rho_a = rho_a0*exp(\beta * (p - p0)), p0 = 101325 Pa is atmospheric pressure +// rho_a0 is the density at p_0, g_u = g/norm(g) where g is gravity. +// k_r = b_a + alpha_a * (\psi - x2), where \psi = p / (rho_a0 * norm(g)) and x2 is vertical axis +// +// Weak form: Find (u, \psi) \in VxQ (V=H(div), Q=L^2) on \Omega +// (v, K^{-1}/rho*k_r * u) -(v, rho*g_u) -(\div(v), \psi) = -_{\Gamma_D} +// -(q, \div(u)) + (q, f) -(q, d (rho*\theta)/dt ) = 0 +// +// We solve MMS for K = kappa*I and beta=0 ==> rho=1 and \theta = alpha_a*\psi, so +// -(q, d (rho*\theta)/dt ) = -alpha_a*(q, d(\psi)/dt ) +// +// This QFunction setup the mixed form of the above equation +// Inputs: +// w : weight of quadrature +// J : dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// u : basis_u at quadrature points +// div(u) : divergence of basis_u at quadrature points +// p : basis_p at quadrature points +// U_t : time derivative of U = [p, u] +// +// Output: +// v : (v, K^{-1}/rho*k_r u) = \int (v^T * K^{-1}/rho*k_r*u detJ*w)dX ==> \int (v^T J^T * K^{-1}/rho*k_r *J*u*w/detJ)dX +// -(v, rho*g_u) = \int (v^T * rho*g_u detJ*w)dX ==> \int (v^T J^T * rho*g_u*w) dX +// div(v) : -(\div(v), \psi) = -\int (div(v)^T * \psi *w) dX +// q : -(q, \div(u)) = -\int (q^T * div(u) * w) dX +// (q, f) = \int( q^T * f * w*detJ )dX +// -alpha_a*(q, d\psi/dt) = -alpha_a \int (q^T * \psi_t*w*detJ)dX +// +// ----------------------------------------------------------------------------- +#ifndef RICHARD_CTX +#define RICHARD_CTX +typedef struct RICHARDContext_ *RICHARDContext; +struct RICHARDContext_ { + CeedScalar kappa; + CeedScalar alpha_a; + CeedScalar b_a; + CeedScalar rho_a0; + CeedScalar beta; + CeedScalar g; + CeedScalar p0; + CeedScalar time; + CeedScalar gamma; +}; +#endif +// ----------------------------------------------------------------------------- +// Residual evaluation for Richard problem +// ----------------------------------------------------------------------------- +/* +CEED_QFUNCTION(RichardSystem2D)(void *ctx, CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*div_u) = (const CeedScalar(*))in[3], + (*p) = (const CeedScalar(*))in[4], + (*p_t) = (const CeedScalar(*))in[5], + (*coords) = in[6]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], + (*div_v) = (CeedScalar(*))out[1], + (*q) = (CeedScalar(*))out[2]; + // Context + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar beta = context->beta; + const CeedScalar g = context->g; + const CeedScalar p0 = context->p0; // atmospheric pressure + CeedScalar t = context->time; + // *INDENT-ON* + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; ikappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar beta = context->beta; + const CeedScalar g = context->g; + const CeedScalar p0 = context->p0;// atmospheric pressure + // *INDENT-ON* + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i + +#define PI_DOUBLE 3.14159265358979323846 + +// ----------------------------------------------------------------------------- +// Compute alpha * A * B = C +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatMatMult3x3(const CeedScalar alpha, + const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { + for (CeedInt j = 0; j < 3; j++) { + for (CeedInt k = 0; k < 3; k++) { + C[j][k] = 0; + for (CeedInt m = 0; m < 3; m++) { + C[j][k] += alpha * A[j][m] * B[m][k]; + } + } + } + + return 0; +} + +// ----------------------------------------------------------------------------- +// Compute alpha * A^T * B = C +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult3x3(const CeedScalar alpha, + const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { + for (CeedInt j = 0; j < 3; j++) { + for (CeedInt k = 0; k < 3; k++) { + C[j][k] = 0; + for (CeedInt m = 0; m < 3; m++) { + C[j][k] += alpha * A[m][j] * B[m][k]; + } + } + } + + return 0; +} + +// ----------------------------------------------------------------------------- +// Compute determinant of 3x3 matrix +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER CeedScalar MatDet3x3(const CeedScalar A[3][3]) { + // Compute det(A) + const CeedScalar B11 = A[1][1]*A[2][2] - A[1][2]*A[2][1]; + const CeedScalar B12 = A[0][2]*A[2][1] - A[0][1]*A[2][2]; + const CeedScalar B13 = A[0][1]*A[1][2] - A[0][2]*A[1][1]; + return A[0][0]*B11 + A[1][0]*B12 + A[2][0]*B13; + +}; + +// ----------------------------------------------------------------------------- +// Compute inverse of 3x3 symmetric matrix +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int MatInverse3x3(const CeedScalar A[3][3], + const CeedScalar det_A, CeedScalar A_inv[3][3]) { + // Compute A^(-1) : A-Inverse + CeedScalar B[6] = { + A[1][1] * A[2][2] - A[1][2] * A[2][1], /* *NOPAD* */ + A[0][0] * A[2][2] - A[0][2] * A[2][0], /* *NOPAD* */ + A[0][0] * A[1][1] - A[0][1] * A[1][0], /* *NOPAD* */ + A[0][2] * A[1][0] - A[0][0] * A[1][2], /* *NOPAD* */ + A[0][1] * A[1][2] - A[0][2] * A[1][1], /* *NOPAD* */ + A[0][2] * A[2][1] - A[0][1] * A[2][2] /* *NOPAD* */ + }; + CeedScalar A_inv1[6]; + for (CeedInt m = 0; m < 6; m++) { + A_inv1[m] = B[m] / (det_A); + } + A_inv[0][0] = A_inv1[0]; + A_inv[0][1] = A_inv1[5]; + A_inv[0][2] = A_inv1[4]; + A_inv[1][0] = A_inv1[5]; + A_inv[1][1] = A_inv1[1]; + A_inv[1][2] = A_inv1[3]; + A_inv[2][0] = A_inv1[4]; + A_inv[2][1] = A_inv1[3]; + A_inv[2][2] = A_inv1[2]; + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute matrix-vector product: alpha*A*u +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatVecMult3x3(const CeedScalar alpha, + const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { + // Compute v = alpha*A*u + for (CeedInt k = 0; k < 3; k++) { + v[k] = 0; + for (CeedInt m = 0; m < 3; m++) + v[k] += A[k][m] * u[m] * alpha; + } + + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute matrix-vector product: alpha*A^T*u +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult3x3(const CeedScalar alpha, + const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { + // Compute v = alpha*A^T*u + for (CeedInt k = 0; k < 3; k++) { + v[k] = 0; + for (CeedInt m = 0; m < 3; m++) + v[k] += A[m][k] * u[m] * alpha; + } + + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute alpha * A * B = C +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatMatMult2x2(const CeedScalar alpha, + const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { + for (CeedInt j = 0; j < 2; j++) { + for (CeedInt k = 0; k < 2; k++) { + C[j][k] = 0; + for (CeedInt m = 0; m < 2; m++) { + C[j][k] += alpha * A[j][m] * B[m][k]; + } + } + } + + return 0; +} + +// ----------------------------------------------------------------------------- +// Compute alpha * A^T * B = C +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult2x2(const CeedScalar alpha, + const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { + for (CeedInt j = 0; j < 2; j++) { + for (CeedInt k = 0; k < 2; k++) { + C[j][k] = 0; + for (CeedInt m = 0; m < 2; m++) { + C[j][k] += alpha * A[m][j] * B[m][k]; + } + } + } + + return 0; +} + +// ----------------------------------------------------------------------------- +// Compute determinant of 2x2 matrix +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER CeedScalar MatDet2x2(const CeedScalar A[2][2]) { + // Compute det(A) + return A[0][0]*A[1][1] - A[1][0]*A[0][1]; + +}; + +// ----------------------------------------------------------------------------- +// Compute inverse of 2x2 symmetric matrix +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int MatInverse2x2(const CeedScalar A[2][2], + const CeedScalar det_A, CeedScalar A_inv[2][2]) { + // Compute A^(-1) : A-Inverse + A_inv[0][0] = A[1][1]/ det_A; + A_inv[0][1] = -A[0][1]/ det_A; + A_inv[1][0] = -A[1][0]/ det_A; + A_inv[1][1] = A[0][0]/ det_A; + + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute matrix-vector product: alpha*A*u +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatVecMult2x2(const CeedScalar alpha, + const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { + // Compute v = alpha*A*u + for (CeedInt k = 0; k < 2; k++) { + v[k] = 0; + for (CeedInt m = 0; m < 2; m++) + v[k] += A[k][m] * u[m] * alpha; + } + + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute matrix-vector product: alpha*A^T*u +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult2x2(const CeedScalar alpha, + const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { + // Compute v = alpha*A^T*u + for (CeedInt k = 0; k < 2; k++) { + v[k] = 0; + for (CeedInt m = 0; m < 2; m++) + v[k] += A[m][k] * u[m] * alpha; + } + + return 0; +}; + +#endif // utils_qf_h diff --git a/examples/Hdiv-mixed/src/cl-options.c b/examples/Hdiv-mixed/src/cl-options.c index adb719174d..73258ac393 100644 --- a/examples/Hdiv-mixed/src/cl-options.c +++ b/examples/Hdiv-mixed/src/cl-options.c @@ -18,56 +18,39 @@ /// Command line option processing for H(div) example using PETSc #include "../include/cl-options.h" -#include "../include/problems.h" - -// Register problems to be available on the command line -PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx) { - app_ctx->problems = NULL; - PetscErrorCode ierr; - PetscFunctionBeginUser; - // 1) poisson-quad2d (Hdiv_POISSON_QUAD2D is created in poisson-quad2d.c) - ierr = PetscFunctionListAdd(&app_ctx->problems, "poisson_quad2d", - Hdiv_POISSON_QUAD2D); CHKERRQ(ierr); - // 2) poisson-hex3d - - // 3) poisson-prism3d - - // 4) richard - - PetscFunctionReturn(0); -} // Process general command line options -PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx) { +PetscErrorCode ProcessCommandLineOptions(AppCtx app_ctx) { PetscBool problem_flag = PETSC_FALSE; - PetscErrorCode ierr; PetscFunctionBeginUser; - ierr = PetscOptionsBegin(comm, NULL, - "H(div) examples in PETSc with libCEED", - NULL); CHKERRQ(ierr); - - ierr = PetscOptionsFList("-problem", "Problem to solve", NULL, - app_ctx->problems, - app_ctx->problem_name, app_ctx->problem_name, sizeof(app_ctx->problem_name), - &problem_flag); CHKERRQ(ierr); - - app_ctx->degree = 1; - ierr = PetscOptionsInt("-degree", "Polynomial degree of finite elements", - NULL, app_ctx->degree, &app_ctx->degree, NULL); CHKERRQ(ierr); - - app_ctx->q_extra = 0; - ierr = PetscOptionsInt("-q_extra", "Number of extra quadrature points", - NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL); CHKERRQ(ierr); + PetscOptionsBegin(app_ctx->comm, NULL, "H(div) examples in PETSc with libCEED", + NULL); + PetscCall( PetscOptionsFList("-problem", "Problem to solve", NULL, + app_ctx->problems, + app_ctx->problem_name, app_ctx->problem_name, sizeof(app_ctx->problem_name), + &problem_flag) ); // Provide default problem if not specified if (!problem_flag) { - const char *problem_name = "poisson_quad2d"; + const char *problem_name = "darcy2d"; strncpy(app_ctx->problem_name, problem_name, 16); } + app_ctx->degree = 1; + PetscCall( PetscOptionsInt("-degree", "Polynomial degree of finite elements", + NULL, app_ctx->degree, &app_ctx->degree, NULL) ); - ierr = PetscOptionsEnd(); CHKERRQ(ierr); + app_ctx->q_extra = 0; + PetscCall( PetscOptionsInt("-q_extra", "Number of extra quadrature points", + NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL) ); + app_ctx->bc_pressure_count = 16; + // we can set one face by: -bc_faces 1 OR multiple faces by :-bc_faces 1,2,3 + PetscCall( PetscOptionsIntArray("-bc_faces", + "Face IDs to apply pressure BC", + NULL, app_ctx->bc_faces, &app_ctx->bc_pressure_count, NULL) ); + + PetscOptionsEnd(); PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mixed/src/matops.c b/examples/Hdiv-mixed/src/matops.c deleted file mode 100644 index ce3451341c..0000000000 --- a/examples/Hdiv-mixed/src/matops.c +++ /dev/null @@ -1,98 +0,0 @@ -#include "../include/matops.h" -#include "../include/setup-libceed.h" - -// ----------------------------------------------------------------------------- -// This function returns the computed diagonal of the operator -// ----------------------------------------------------------------------------- -PetscErrorCode MatGetDiag(Mat A, Vec D) { - PetscErrorCode ierr; - User user; - - PetscFunctionBeginUser; - - ierr = MatShellGetContext(A, &user); CHKERRQ(ierr); - - // Compute Diagonal via libCEED - PetscScalar *x; - PetscMemType mem_type; - - // -- Place PETSc vector in libCEED vector - ierr = VecGetArrayAndMemType(user->X_loc, &x, &mem_type); CHKERRQ(ierr); - CeedVectorSetArray(user->x_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, x); - - // -- Compute Diagonal - CeedOperatorLinearAssembleDiagonal(user->op, user->x_ceed, - CEED_REQUEST_IMMEDIATE); - - // -- Local-to-Global - CeedVectorTakeArray(user->x_ceed, MemTypeP2C(mem_type), NULL); - ierr = VecRestoreArrayAndMemType(user->X_loc, &x); CHKERRQ(ierr); - ierr = VecZeroEntries(D); CHKERRQ(ierr); - ierr = DMLocalToGlobal(user->dm, user->X_loc, ADD_VALUES, D); CHKERRQ(ierr); - - // Cleanup - ierr = VecZeroEntries(user->X_loc); CHKERRQ(ierr); - - PetscFunctionReturn(0); -}; - -// ----------------------------------------------------------------------------- -// This function uses libCEED to compute the action of the Laplacian with -// Dirichlet boundary conditions -// ----------------------------------------------------------------------------- -PetscErrorCode ApplyLocal_Ceed(Vec X, Vec Y, User user) { - PetscErrorCode ierr; - PetscScalar *x, *y; - PetscMemType x_mem_type, y_mem_type; - - PetscFunctionBeginUser; - - // Global-to-local - ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); CHKERRQ(ierr); - - // Setup libCEED vectors - ierr = VecGetArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x, - &x_mem_type); CHKERRQ(ierr); - ierr = VecGetArrayAndMemType(user->Y_loc, &y, &y_mem_type); CHKERRQ(ierr); - CeedVectorSetArray(user->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, x); - CeedVectorSetArray(user->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); - - // Apply libCEED operator - printf("==== x"); - CeedVectorView(user->x_ceed,"%12.8f", stdout); - CeedOperatorApply(user->op, user->x_ceed, user->y_ceed, CEED_REQUEST_IMMEDIATE); - printf("==== y"); - CeedVectorView(user->y_ceed,"%12.8f", stdout); - - // Restore PETSc vectors - CeedVectorTakeArray(user->x_ceed, MemTypeP2C(x_mem_type), NULL); - CeedVectorTakeArray(user->y_ceed, MemTypeP2C(y_mem_type), NULL); - ierr = VecRestoreArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x); - CHKERRQ(ierr); - ierr = VecRestoreArrayAndMemType(user->Y_loc, &y); CHKERRQ(ierr); - - // Local-to-global - ierr = VecZeroEntries(Y); CHKERRQ(ierr); - ierr = DMLocalToGlobal(user->dm, user->Y_loc, ADD_VALUES, Y); CHKERRQ(ierr); - - PetscFunctionReturn(0); -}; - -// ----------------------------------------------------------------------------- -// This function wraps the libCEED operator for a MatShell -// ----------------------------------------------------------------------------- -PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y) { - PetscErrorCode ierr; - User user; - - PetscFunctionBeginUser; - - ierr = MatShellGetContext(A, &user); CHKERRQ(ierr); - - // libCEED for local action of residual evaluator - ierr = ApplyLocal_Ceed(X, Y, user); CHKERRQ(ierr); - - PetscFunctionReturn(0); -}; - -// ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/src/setup-boundary.c b/examples/Hdiv-mixed/src/setup-boundary.c new file mode 100644 index 0000000000..e3a3a859ae --- /dev/null +++ b/examples/Hdiv-mixed/src/setup-boundary.c @@ -0,0 +1,114 @@ +#include "../include/setup-boundary.h" + +// --------------------------------------------------------------------------- +// Create boundary label +// --------------------------------------------------------------------------- +PetscErrorCode CreateBCLabel(DM dm, const char name[]) { + DMLabel label; + + PetscFunctionBeginUser; + + PetscCall( DMCreateLabel(dm, name) ); + PetscCall( DMGetLabel(dm, name, &label) ); + PetscCall( DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, label) ); + PetscCall( DMPlexLabelComplete(dm, label) ); + + PetscFunctionReturn(0); +}; + +// --------------------------------------------------------------------------- +// Add Dirichlet boundaries to DM +// --------------------------------------------------------------------------- +PetscErrorCode DMAddBoundariesDirichlet(DM dm) { + + PetscFunctionBeginUser; + + // BCs given by manufactured solution + PetscBool has_label; + const char *name = "MMS Face Sets"; + PetscInt face_ids[1] = {1}; + PetscCall( DMHasLabel(dm, name, &has_label) ); + if (!has_label) { + PetscCall( CreateBCLabel(dm, name) ); + } + DMLabel label; + PetscCall( DMGetLabel(dm, name, &label) ); + PetscCall( DMAddBoundary(dm, DM_BC_ESSENTIAL, "mms", label, 1, face_ids, 0, 0, + NULL, + (void(*)(void))BoundaryDirichletMMS, NULL, NULL, NULL) ); + + + PetscFunctionReturn(0); +} + +// --------------------------------------------------------------------------- +// Add Neumann boundaries to DM +// --------------------------------------------------------------------------- +PetscErrorCode DMAddBoundariesPressure(Ceed ceed, CeedData ceed_data, + AppCtx app_ctx, ProblemData problem_data, DM dm, + CeedVector bc_pressure) { + PetscInt dim; + CeedQFunction qf_pressure; + CeedOperator op_pressure; + + PetscFunctionBeginUser; + + PetscCall( DMGetDimension(dm, &dim) ); + + if (app_ctx->bc_pressure_count > 0) { + DMLabel domain_label; + PetscCall(DMGetLabel(dm, "Face Sets", &domain_label)); + // Compute contribution on each boundary face + for (CeedInt i = 0; i < app_ctx->bc_pressure_count; i++) { + + CeedQFunctionCreateInterior(ceed, 1, problem_data->bc_pressure, + problem_data->bc_pressure_loc, &qf_pressure); + + CeedQFunctionAddInput(qf_pressure, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddOutput(qf_pressure, "v", dim, CEED_EVAL_INTERP); + // -- Apply operator + CeedOperatorCreate(ceed, qf_pressure, NULL, NULL, + &op_pressure); + CeedOperatorSetField(op_pressure, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_pressure, "v", ceed_data->elem_restr_u, + ceed_data->basis_u_face, CEED_VECTOR_ACTIVE); + // ---- Compute pressure on face + CeedOperatorApplyAdd(op_pressure, ceed_data->x_coord, bc_pressure, + CEED_REQUEST_IMMEDIATE); + + // -- Cleanup + CeedQFunctionDestroy(&qf_pressure); + CeedOperatorDestroy(&op_pressure); + } + } + + PetscFunctionReturn(0); +} + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif +// --------------------------------------------------------------------------- +// Boundary function for manufactured solution +// --------------------------------------------------------------------------- +PetscErrorCode BoundaryDirichletMMS(PetscInt dim, PetscReal t, + const PetscReal coords[], + PetscInt num_comp_u, PetscScalar *u, void *ctx) { + PetscScalar x = coords[0]; + PetscScalar y = coords[1]; + PetscScalar z = coords[1]; + + PetscFunctionBeginUser; + + if (dim == 2) { + u[0] = -M_PI*cos(M_PI*x) *sin(M_PI*y) - M_PI*y; + u[1] = -M_PI*sin(M_PI*x) *cos(M_PI*y) - M_PI*x; + } else { + u[0] = -M_PI*cos(M_PI*x) *sin(M_PI*y) *sin(M_PI*z) - M_PI*y*z; + u[1] = -M_PI*sin(M_PI*x) *cos(M_PI*y) *sin(M_PI*z) - M_PI*x*z; + u[2] = -M_PI*sin(M_PI*x) *sin(M_PI*y) *cos(M_PI*z) - M_PI*x*y; + } + + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mixed/src/setup-dm.c b/examples/Hdiv-mixed/src/setup-dm.c index 01e7cb312a..91b0758219 100644 --- a/examples/Hdiv-mixed/src/setup-dm.c +++ b/examples/Hdiv-mixed/src/setup-dm.c @@ -1,44 +1,59 @@ #include "../include/setup-dm.h" +#include "petscerror.h" // --------------------------------------------------------------------------- -// Set-up DM +// Setup DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDistributedDM(MPI_Comm comm, DM *dm) { - PetscErrorCode ierr; - PetscSection sec; - PetscBool interpolate = PETSC_TRUE; - PetscInt nx = 2, ny = 1; - PetscInt faces[2] = {nx, ny}; - PetscInt dim = 2, num_comp_u = dim; - PetscInt p_start, p_end; - PetscInt c_start, c_end; // cells - PetscInt e_start, e_end, e; // edges - PetscInt v_start, v_end; // vertices +PetscErrorCode CreateDM(MPI_Comm comm, VecType vec_type, DM *dm) { PetscFunctionBeginUser; - ierr = DMPlexCreateBoxMesh(comm, dim, PETSC_FALSE, faces, NULL, - NULL, NULL, interpolate, dm); CHKERRQ(ierr); - // Get plex limits - ierr = DMPlexGetChart(*dm, &p_start, &p_end); CHKERRQ(ierr); - ierr = DMPlexGetHeightStratum(*dm, 0, &c_start, &c_end); CHKERRQ(ierr); - ierr = DMPlexGetDepthStratum(*dm, 1, &e_start, &e_end); CHKERRQ(ierr); - ierr = DMPlexGetDepthStratum(*dm, 0, &v_start, &v_end); CHKERRQ(ierr); - // Create section - ierr = PetscSectionCreate(comm, &sec); CHKERRQ(ierr); - ierr = PetscSectionSetNumFields(sec,1); CHKERRQ(ierr); - ierr = PetscSectionSetFieldName(sec,0,"Velocity"); CHKERRQ(ierr); - ierr = PetscSectionSetFieldComponents(sec,0,1); CHKERRQ(ierr); - ierr = PetscSectionSetChart(sec,p_start,p_end); CHKERRQ(ierr); - // Setup dofs - for (e = e_start; e < e_end; e++) { - ierr = PetscSectionSetFieldDof(sec, e, 0, num_comp_u); CHKERRQ(ierr); - ierr = PetscSectionSetDof (sec, e, num_comp_u); CHKERRQ(ierr); - } - ierr = PetscSectionSetUp(sec); CHKERRQ(ierr); - ierr = DMSetSection(*dm,sec); CHKERRQ(ierr); - ierr = DMViewFromOptions(*dm, NULL, "-dm_view"); CHKERRQ(ierr); - ierr = PetscSectionDestroy(&sec); CHKERRQ(ierr); + // Create DMPLEX + PetscCall( DMCreate(comm, dm) ); + PetscCall( DMSetType(*dm, DMPLEX) ); + PetscCall( DMSetVecType(*dm, vec_type) ); + // Set Tensor elements + PetscCall( PetscOptionsSetValue(NULL, "-dm_plex_simplex", "0") ); + // Set CL options + PetscCall( DMSetFromOptions(*dm) ); + PetscCall( DMViewFromOptions(*dm, NULL, "-dm_view") ); + + PetscFunctionReturn(0); +}; + +PetscErrorCode PerturbVerticesSmooth(DM dm) { + Vec coordinates; + PetscSection coordSection; + PetscScalar *coords; + PetscInt v,vStart,vEnd,offset,dim; + PetscReal x,y,z; + + PetscFunctionBeginUser; + + PetscCall( DMGetDimension(dm, &dim) ); + PetscCall( DMGetCoordinateSection(dm, &coordSection) ); + PetscCall( DMGetCoordinatesLocal(dm, &coordinates) ); + PetscCall( DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd) ); + PetscCall( VecGetArray(coordinates,&coords) ); + for(v=vStart; vx_ceed); CeedVectorDestroy(&ceed_data->y_ceed); - CeedVectorDestroy(&ceed_data->geo_data); + CeedVectorDestroy(&ceed_data->x_coord); // Restrictions CeedElemRestrictionDestroy(&ceed_data->elem_restr_x); CeedElemRestrictionDestroy(&ceed_data->elem_restr_u); - CeedElemRestrictionDestroy(&ceed_data->elem_restr_geo_data_i); - CeedElemRestrictionDestroy(&ceed_data->elem_restr_u_i); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_U_i); // U = [p,u] + CeedElemRestrictionDestroy(&ceed_data->elem_restr_p); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_p_i); // Bases CeedBasisDestroy(&ceed_data->basis_x); CeedBasisDestroy(&ceed_data->basis_u); + CeedBasisDestroy(&ceed_data->basis_p); + CeedBasisDestroy(&ceed_data->basis_u_face); // QFunctions CeedQFunctionDestroy(&ceed_data->qf_residual); + CeedQFunctionDestroy(&ceed_data->qf_jacobian); + CeedQFunctionDestroy(&ceed_data->qf_error); // Operators CeedOperatorDestroy(&ceed_data->op_residual); - - ierr = PetscFree(ceed_data); CHKERRQ(ierr); + CeedOperatorDestroy(&ceed_data->op_jacobian); + CeedOperatorDestroy(&ceed_data->op_error); + PetscCall( PetscFree(ceed_data) ); PetscFunctionReturn(0); }; @@ -44,151 +53,99 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data) { PetscInt Involute(PetscInt i) { return i >= 0 ? i : -(i + 1); }; + // ----------------------------------------------------------------------------- // Get CEED restriction data from DMPlex // ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt P, - CeedInt topo_dim, CeedElemRestriction *elem_restr) { - PetscSection section; - PetscInt p, num_elem, num_dof, *elem_restr_offsets, e_offset, num_fields, - c_start, c_end; - Vec U_loc; - PetscErrorCode ierr; +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, + DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { + PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; PetscFunctionBeginUser; - ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); - ierr = PetscSectionGetNumFields(section, &num_fields); CHKERRQ(ierr); - PetscInt num_comp[num_fields], field_off[num_fields+1]; - field_off[0] = 0; - for (PetscInt f = 0; f < num_fields; f++) { - ierr = PetscSectionGetFieldComponents(section, f, &num_comp[f]); CHKERRQ(ierr); - field_off[f+1] = field_off[f] + num_comp[f]; - } - ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); - num_elem = c_end - c_start; - ierr = PetscMalloc1(num_elem*PetscPowInt(P, topo_dim), &elem_restr_offsets); - CHKERRQ(ierr); - CHKERRQ(ierr); - for (p = 0, e_offset = 0; p < num_elem; p++) { - PetscInt num_indices, *indices, num_nodes; - ierr = DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, - &num_indices, &indices, NULL, NULL); - CHKERRQ(ierr); - num_nodes = num_indices / field_off[num_fields]; - for (PetscInt i = 0; i < num_nodes; i++) { - PetscInt ii = i; - // Check that indices are blocked by node and thus can be coalesced as a single field with - // field_off[num_fields] = sum(num_comp) components. - for (PetscInt f = 0; f < num_fields; f++) { - for (PetscInt j = 0; j < num_comp[f]; j++) { - if (Involute(indices[field_off[f]*num_nodes + ii*num_comp[f] + j]) - != Involute(indices[ii*num_comp[0]]) + field_off[f] + j) - SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, - "Cell %D closure indices not interlaced for node %D field %D component %D", - p, ii, f, j); - } - } - // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. - PetscInt loc = Involute(indices[ii*num_comp[0]]); - elem_restr_offsets[e_offset++] = loc; - } - ierr = DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, - &num_indices, &indices, NULL, NULL); - CHKERRQ(ierr); - } - if (e_offset != num_elem*PetscPowInt(P, topo_dim)) - SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, - "ElemRestriction of size (%D,%D) initialized %D nodes", num_elem, - PetscPowInt(P, topo_dim),e_offset); - ierr = DMGetLocalVector(dm, &U_loc); CHKERRQ(ierr); - ierr = VecGetLocalSize(U_loc, &num_dof); CHKERRQ(ierr); - ierr = DMRestoreLocalVector(dm, &U_loc); CHKERRQ(ierr); - CeedElemRestrictionCreate(ceed, num_elem, PetscPowInt(P, topo_dim), - field_off[num_fields], 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, + PetscCall( DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, + &elem_size, &num_comp, &num_dof, &elem_restr_offsets) ); + + CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, + 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets, elem_restr); - ierr = PetscFree(elem_restr_offsets); CHKERRQ(ierr); + PetscCall( PetscFree(elem_restr_offsets) ); + PetscFunctionReturn(0); }; // ----------------------------------------------------------------------------- // Get Oriented CEED restriction data from DMPlex // ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, - CeedInt topo_dim, CeedElemRestriction *elem_restr_oriented) { +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, + CeedInt P, CeedElemRestriction *elem_restr_oriented, + CeedElemRestriction *elem_restr) { PetscSection section; - PetscInt p, num_elem, num_dof, *elem_restr_offsets, e_offset, num_fields, - c_start, c_end; + PetscInt p, num_elem, num_dof, *restr_indices_u, *restr_indices_p, + elem_offset, num_fields, dim, c_start, c_end; Vec U_loc; - PetscErrorCode ierr; - const PetscInt *ornt; - + const PetscInt *ornt; // this is for orientation of dof PetscFunctionBeginUser; - ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); - ierr = PetscSectionGetNumFields(section, &num_fields); CHKERRQ(ierr); - PetscInt num_comp[num_fields], field_off[num_fields+1]; - field_off[0] = 0; + PetscCall( DMGetDimension(dm, &dim) ); + PetscCall( DMGetLocalSection(dm, §ion) ); + PetscCall( PetscSectionGetNumFields(section, &num_fields) ); + PetscInt num_comp[num_fields], field_offsets[num_fields+1]; + field_offsets[0] = 0; for (PetscInt f = 0; f < num_fields; f++) { - ierr = PetscSectionGetFieldComponents(section, f, &num_comp[f]); CHKERRQ(ierr); - field_off[f+1] = field_off[f] + num_comp[f]; + PetscCall( PetscSectionGetFieldComponents(section, f, &num_comp[f]) ); + field_offsets[f+1] = field_offsets[f] + num_comp[f]; } - ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); + PetscCall( DMPlexGetHeightStratum(dm, 0, &c_start, &c_end) ); num_elem = c_end - c_start; - ierr = PetscMalloc1(num_elem*topo_dim*PetscPowInt(P, topo_dim), - &elem_restr_offsets); // doesn't alocate as many entries - CHKERRQ(ierr); - bool *orient; - ierr = PetscMalloc1(num_elem*PetscPowInt(P, topo_dim), &orient); - CHKERRQ(ierr); - for (p = 0, e_offset = 0; p < num_elem; p++) { - PetscInt num_indices, *indices, num_nodes; - ierr = DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, - &num_indices, &indices, NULL, NULL); - CHKERRQ(ierr); - num_nodes = num_indices / - field_off[num_fields]; // 8 / 2, but I think there are really 8 nodes - for (PetscInt i = 0; i < num_nodes; i++) { - PetscInt ii = i; - // Check that indices are blocked by node and thus can be coalesced as a single field with - // field_off[num_fields] = sum(num_comp) components. - for (PetscInt f = 0; f < num_fields; f++) { - for (PetscInt j = 0; j < num_comp[f]; j++) { - if (Involute(indices[field_off[f]*num_nodes + ii*num_comp[f] + j]) - != Involute(indices[ii*num_comp[0]]) + field_off[f] + j) - SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, - "Cell %D closure indices not interlaced for node %D field %D component %D", - p, ii, f, j); - } + PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), + &restr_indices_u) ); + PetscCall( PetscMalloc1(num_elem,&restr_indices_p) ); + bool *orient_indices_u; // to flip the dof + PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), &orient_indices_u) ); + for (p = 0, elem_offset = 0; p < num_elem; p++) { + PetscInt num_indices, *indices, faces_per_elem, dofs_per_face; + PetscCall( DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, + &num_indices, &indices, NULL, NULL) ); + + restr_indices_p[p] = indices[num_indices - 1]; + PetscCall( DMPlexGetConeOrientation(dm, p, &ornt) ); + // Get number of faces per element + PetscCall( DMPlexGetConeSize(dm, p, &faces_per_elem) ); + dofs_per_face = faces_per_elem - 2; + for (PetscInt f = 0; f < faces_per_elem; f++) { + for (PetscInt i = 0; i < dofs_per_face; i++) { + PetscInt ii = dofs_per_face*f + i; + // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. + PetscInt loc = Involute(indices[ii*num_comp[0]]); + restr_indices_u[elem_offset] = loc; + // Set orientation + orient_indices_u[elem_offset] = ornt[f] < 0; + elem_offset++; } - // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. - PetscInt loc = Involute(indices[ii*num_comp[0]]); - elem_restr_offsets[e_offset] = loc; // Are we getting two nodes per edge? yes, - // Set orientation - ierr = DMPlexGetConeOrientation(dm, p, &ornt); CHKERRQ(ierr); - orient[e_offset] = ornt[i] < 0; - e_offset++; } - ierr = DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, - &num_indices, &indices, NULL, NULL); - CHKERRQ(ierr); + PetscCall( DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, + &num_indices, &indices, NULL, NULL) ); } - if (e_offset != num_elem*topo_dim*PetscPowInt(P, - topo_dim)) // this probably needs to be like this - SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, - "ElemRestriction of size (%D,%D) initialized %D nodes", num_elem, - PetscPowInt(P, topo_dim),e_offset); + //if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) + // SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, + // "ElemRestriction of size (%" PetscInt_FMT ", %" PetscInt_FMT" ) + // initialized %" PetscInt_FMT " nodes", num_elem, + // dim*PetscPowInt(P, dim),elem_offset); - ierr = DMGetLocalVector(dm, &U_loc); CHKERRQ(ierr); - ierr = VecGetLocalSize(U_loc, &num_dof); CHKERRQ(ierr); - ierr = DMRestoreLocalVector(dm, &U_loc); CHKERRQ(ierr); + PetscCall( DMGetLocalVector(dm, &U_loc) ); + PetscCall( VecGetLocalSize(U_loc, &num_dof) ); + PetscCall( DMRestoreLocalVector(dm, &U_loc) ); // dof per element in Hdiv is dim*P^dim, for linear element P=2 - CeedElemRestrictionCreateOriented(ceed, num_elem, topo_dim*PetscPowInt(P, - topo_dim), // as we're using here - field_off[num_fields], - 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, - elem_restr_offsets, orient, elem_restr_oriented); - ierr = PetscFree(elem_restr_offsets); CHKERRQ(ierr); - ierr = PetscFree(orient); CHKERRQ(ierr); + CeedElemRestrictionCreateOriented(ceed, num_elem, dim*PetscPowInt(P, dim), + 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, + restr_indices_u, orient_indices_u, + elem_restr_oriented); + CeedElemRestrictionCreate(ceed, num_elem, 1, + 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, + restr_indices_p, elem_restr); + PetscCall( PetscFree(restr_indices_p) ); + PetscCall( PetscFree(restr_indices_u) ); + PetscCall( PetscFree(orient_indices_u) ); PetscFunctionReturn(0); }; @@ -196,124 +153,158 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, // Set up libCEED on the fine grid for a given degree // ----------------------------------------------------------------------------- PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, - ProblemData *problem_data, - PetscInt U_g_size, PetscInt U_loc_size, - CeedData ceed_data, CeedVector rhs_ceed, - CeedVector *target) { - int ierr; + ProblemData problem_data, + PetscInt U_loc_size, CeedData ceed_data) { CeedInt P = app_ctx->degree + 1; - CeedInt Q = P + 1 + app_ctx->q_extra; // Number of quadratures in 1D - CeedInt num_qpts = Q*Q; // Number of quadratures per element - CeedInt dim, num_comp_x, num_comp_u; - CeedInt geo_data_size = problem_data->geo_data_size; - CeedInt elem_node = problem_data->elem_node; + // Number of quadratures in 1D, q_extra is set in cl-options.c + CeedInt Q = P + 1 + app_ctx->q_extra; + CeedInt dim, num_comp_x, num_comp_u, num_comp_p; DM dm_coord; Vec coords; PetscInt c_start, c_end, num_elem; const PetscScalar *coordArray; - CeedVector x_coord; - CeedQFunction qf_residual; - CeedOperator op_residual; + CeedQFunction qf_true, qf_residual, qf_jacobian, qf_error; + CeedOperator op_true, op_residual, op_jacobian, op_error; PetscFunctionBeginUser; // --------------------------------------------------------------------------- // libCEED bases:Hdiv basis_u and Lagrange basis_x // --------------------------------------------------------------------------- - ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr); + dim = problem_data->dim; num_comp_x = dim; - num_comp_u = 1; - CeedInt elem_dof = dim*elem_node; // dof per element + num_comp_u = 1; // one vector dof + num_comp_p = 1; // one scalar dof + // Number of quadratures per element + CeedInt num_qpts = PetscPowInt(Q, dim); + // Pressure and velocity dof per element + CeedInt P_p = 1, P_u = dim*PetscPowInt(P, dim); CeedScalar q_ref[dim*num_qpts], q_weights[num_qpts]; - CeedScalar div[elem_dof*num_qpts], interp[dim*elem_dof*num_qpts]; - QuadBasis(Q, q_ref, q_weights, interp, div); - CeedBasisCreateHdiv(ceed, CEED_QUAD, num_comp_u, elem_node, num_qpts, - interp, div, q_ref, q_weights, &ceed_data->basis_u); + CeedScalar div[P_u*num_qpts], interp_u[dim*P_u*num_qpts], + interp_p[P_p*num_qpts], *grad=NULL; + if (dim == 2) { + HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, + problem_data->quadrature_mode); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, + interp_u, div, q_ref, q_weights, &ceed_data->basis_u); + L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); + CeedBasisCreateH1(ceed, CEED_TOPOLOGY_QUAD, num_comp_p, 1, num_qpts, interp_p, + grad, q_ref,q_weights, &ceed_data->basis_p); + HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, + CEED_GAUSS_LOBATTO); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, + interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); + } else { + HdivBasisHex(Q, q_ref, q_weights, interp_u, div, problem_data->quadrature_mode); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, + interp_u, div, q_ref, q_weights, &ceed_data->basis_u); + L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); + CeedBasisCreateH1(ceed, CEED_TOPOLOGY_HEX, num_comp_p, 1, num_qpts, interp_p, + grad, q_ref,q_weights, &ceed_data->basis_p); + HdivBasisHex(Q, q_ref, q_weights, interp_u, div, CEED_GAUSS_LOBATTO); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, + interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); + } + CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, problem_data->quadrature_mode, &ceed_data->basis_x); + // --------------------------------------------------------------------------- // libCEED restrictions // --------------------------------------------------------------------------- - ierr = DMGetCoordinateDM(dm, &dm_coord); CHKERRQ(ierr); - ierr = DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL); - CHKERRQ(ierr); + PetscCall( DMGetCoordinateDM(dm, &dm_coord) ); + PetscCall( DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL) ); + CeedInt height = 0; // 0 means no boundary conditions + DMLabel domain_label = 0; + PetscInt value = 0; // -- Coordinate restriction - ierr = CreateRestrictionFromPlex(ceed, dm_coord, 2, dim, - &ceed_data->elem_restr_x); - CHKERRQ(ierr); + PetscCall( CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, + value, &ceed_data->elem_restr_x) ); // -- Solution restriction - ierr = CreateRestrictionFromPlexOriented(ceed, dm, P, dim, - &ceed_data->elem_restr_u); - // ---- Geometric ceed_data restriction - ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); + PetscCall( CreateRestrictionFromPlexOriented(ceed, dm, P, + &ceed_data->elem_restr_u, &ceed_data->elem_restr_p) ); + // -- Geometric ceed_data restriction + PetscCall( DMPlexGetHeightStratum(dm, 0, &c_start, &c_end) ); num_elem = c_end - c_start; - CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, geo_data_size, - num_elem*num_qpts*geo_data_size, - CEED_STRIDES_BACKEND, &ceed_data->elem_restr_geo_data_i); - CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, num_comp_u, - num_comp_u*num_elem*num_qpts, - CEED_STRIDES_BACKEND, &ceed_data->elem_restr_u_i); - printf("----elem_restr_x:\n"); - CeedElemRestrictionView(ceed_data->elem_restr_x, stdout); - printf("----elem_restr_u:\n"); - CeedElemRestrictionView(ceed_data->elem_restr_u, stdout); - printf("----elem_restr_geo_data_i:\n"); - CeedElemRestrictionView(ceed_data->elem_restr_geo_data_i, stdout); - printf("----elem_restr_u_i:\n"); - CeedElemRestrictionView(ceed_data->elem_restr_u_i, stdout); - + CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, (dim+1), + (dim+1)*num_elem*num_qpts, + CEED_STRIDES_BACKEND, &ceed_data->elem_restr_U_i); + CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, 1, + 1*num_elem*num_qpts, + CEED_STRIDES_BACKEND, &ceed_data->elem_restr_p_i); // --------------------------------------------------------------------------- // Element coordinates // --------------------------------------------------------------------------- - ierr = DMGetCoordinatesLocal(dm, &coords); CHKERRQ(ierr); - ierr = VecGetArrayRead(coords, &coordArray); CHKERRQ(ierr); + PetscCall( DMGetCoordinatesLocal(dm, &coords) ); + PetscCall( VecGetArrayRead(coords, &coordArray) ); + CeedVectorCreate(ceed, U_loc_size, &ceed_data->x_coord); - CeedElemRestrictionCreateVector(ceed_data->elem_restr_x, &x_coord, NULL); - CeedVectorSetArray(x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, + CeedElemRestrictionCreateVector(ceed_data->elem_restr_x, &ceed_data->x_coord, + NULL); + CeedVectorSetArray(ceed_data->x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, (PetscScalar *)coordArray); - ierr = VecRestoreArrayRead(coords, &coordArray); CHKERRQ(ierr); + PetscCall( VecRestoreArrayRead(coords, &coordArray) ); // --------------------------------------------------------------------------- - // Persistent libCEED vectors - // --------------------------------------------------------------------------- - // -- Operator action variables - CeedVectorCreate(ceed, U_loc_size, &ceed_data->x_ceed); - CeedVectorCreate(ceed, U_loc_size, &ceed_data->y_ceed); - /* - // -- Geometric data vector - CeedVectorCreate(ceed, num_elem*num_qpts*geo_data_size, - &ceed_data->geo_data); - + // Setup true solution and force // --------------------------------------------------------------------------- - // Geometric factor computation - // --------------------------------------------------------------------------- - // Create the QFunction and Operator that computes the quadrature data - // geo_data returns dXdx_i,j and w * det. - // --------------------------------------------------------------------------- - // -- QFunction - CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_geo, - problem_data->setup_geo_loc, &qf_setup_geo); - CeedQFunctionAddInput(qf_setup_geo, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_setup_geo, "dx", dim*dim, CEED_EVAL_GRAD); - CeedQFunctionAddOutput(qf_setup_geo, "geo_data", geo_data_size, CEED_EVAL_NONE); - // -- Operator - CeedOperatorCreate(ceed, qf_setup_geo, CEED_QFUNCTION_NONE, - CEED_QFUNCTION_NONE, &op_setup_geo); - CeedOperatorSetField(op_setup_geo, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_setup_geo, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_setup_geo, "geo_data", - ceed_data->elem_restr_geo_data_i, + CeedVector true_vec, true_force; + CeedVectorCreate(ceed, num_elem*num_qpts*(dim+1), &true_vec); + CeedVectorCreate(ceed, num_elem*num_qpts*1, &true_force); + // Create the q-function that sets up the RHS and true solution + CeedQFunctionCreateInterior(ceed, 1, problem_data->true_solution, + problem_data->true_solution_loc, &qf_true); + CeedQFunctionSetContext(qf_true, problem_data->qfunction_context); + //CeedQFunctionContextDestroy(&problem_data->qfunction_context); + CeedQFunctionAddInput(qf_true, "x", num_comp_x, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_true, "true force", 1, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_true, "true solution", dim+1, CEED_EVAL_NONE); + // Create the operator that builds the RHS and true solution + CeedOperatorCreate(ceed, qf_true, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_true); + CeedOperatorSetField(op_true, "x", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_true, "true force", ceed_data->elem_restr_p_i, + CEED_BASIS_COLLOCATED, true_force); + CeedOperatorSetField(op_true, "true solution", ceed_data->elem_restr_U_i, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); - // -- Compute the quadrature data - CeedOperatorApply(op_setup_geo, x_coord, ceed_data->geo_data, + // Setup RHS and true solution + CeedOperatorApply(op_true, ceed_data->x_coord, true_vec, CEED_REQUEST_IMMEDIATE); - // -- Cleanup - CeedQFunctionDestroy(&qf_setup_geo); - CeedOperatorDestroy(&op_setup_geo); - */ + + if (problem_data->has_ts) { + // --------------------------------------------------------------------------- + // Setup qfunction for initial conditions + // --------------------------------------------------------------------------- + CeedQFunction qf_ics; + CeedOperator op_ics; + CeedQFunctionCreateInterior(ceed, 1, problem_data->ics, + problem_data->ics_loc, &qf_ics); + CeedQFunctionSetContext(qf_ics, problem_data->qfunction_context); + CeedQFunctionAddInput(qf_ics, "x", num_comp_x, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_ics, "u_0", dim, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_ics, "p_0", 1, CEED_EVAL_INTERP); + // Create the operator that builds the initial conditions + CeedOperatorCreate(ceed, qf_ics, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_ics); + CeedOperatorSetField(op_ics, "x", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_ics, "u_0", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_ics, "p_0", ceed_data->elem_restr_p, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + // Create local initial conditions vector + CeedVectorCreate(ceed, U_loc_size, &ceed_data->U0_ceed); + // -- Save libCEED data to apply operator in setup-ts.c + ceed_data->qf_ics = qf_ics; + ceed_data->op_ics = op_ics; + } // --------------------------------------------------------------------------- + // Persistent libCEED vectors + // --------------------------------------------------------------------------- + // -- Operator action variables: we use them in setup-solvers.c + CeedVectorCreate(ceed, U_loc_size, &ceed_data->x_ceed); + CeedVectorCreate(ceed, U_loc_size, &ceed_data->y_ceed); // Local residual evaluator // --------------------------------------------------------------------------- // Create the QFunction and Operator that computes the residual of the PDE. @@ -321,10 +312,18 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // -- QFunction CeedQFunctionCreateInterior(ceed, 1, problem_data->residual, problem_data->residual_loc, &qf_residual); + CeedQFunctionSetContext(qf_residual, problem_data->qfunction_context); + //CeedQFunctionContextDestroy(&problem_data->qfunction_context); CeedQFunctionAddInput(qf_residual, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddInput(qf_residual, "dx", dim*dim, CEED_EVAL_GRAD); - CeedQFunctionAddInput(qf_residual, "u", num_comp_u, CEED_EVAL_INTERP); - CeedQFunctionAddOutput(qf_residual, "v", num_comp_u, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_residual, "u", dim, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_residual, "div_u", 1, CEED_EVAL_DIV); + CeedQFunctionAddInput(qf_residual, "p", 1, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_residual, "true force", 1, CEED_EVAL_NONE); + CeedQFunctionAddInput(qf_residual, "x", num_comp_x, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_residual, "v", dim, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_residual, "div_v", 1, CEED_EVAL_DIV); + CeedQFunctionAddOutput(qf_residual, "q", 1, CEED_EVAL_INTERP); // -- Operator CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, @@ -332,50 +331,118 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); CeedOperatorSetField(op_residual, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); + ceed_data->basis_x, ceed_data->x_coord); CeedOperatorSetField(op_residual, "u", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "div_u", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "p", ceed_data->elem_restr_p, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "true force", ceed_data->elem_restr_p_i, + CEED_BASIS_COLLOCATED, true_force); + CeedOperatorSetField(op_residual, "x", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); CeedOperatorSetField(op_residual, "v", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); - - // -- Save libCEED data + CeedOperatorSetField(op_residual, "div_v", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "q", ceed_data->elem_restr_p, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + // -- Save libCEED data to apply operator in matops.c ceed_data->qf_residual = qf_residual; ceed_data->op_residual = op_residual; + // --------------------------------------------------------------------------- - // Setup RHS and true solution + // Add Pressure boundary condition. See setup-boundary.c // --------------------------------------------------------------------------- - CeedQFunction qf_setup_rhs; - CeedOperator op_setup_rhs; - CeedVectorCreate(ceed, num_elem*num_qpts*num_comp_u, target); - // Create the q-function that sets up the RHS and true solution - CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_rhs, - problem_data->setup_rhs_loc, &qf_setup_rhs); - CeedQFunctionAddInput(qf_setup_rhs, "x", num_comp_x, CEED_EVAL_INTERP); - CeedQFunctionAddInput(qf_setup_rhs, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_setup_rhs, "dx", dim*dim, CEED_EVAL_GRAD); - CeedQFunctionAddOutput(qf_setup_rhs, "true_soln", num_comp_u, CEED_EVAL_NONE); - CeedQFunctionAddOutput(qf_setup_rhs, "rhs", num_comp_u, CEED_EVAL_INTERP); - // Create the operator that builds the RHS and true solution - CeedOperatorCreate(ceed, qf_setup_rhs, NULL, NULL, &op_setup_rhs); - CeedOperatorSetField(op_setup_rhs, "x", ceed_data->elem_restr_x, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_setup_rhs, "weight", CEED_ELEMRESTRICTION_NONE, + //DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm); + + // Local jacobian evaluator + // --------------------------------------------------------------------------- + // Create the QFunction and Operator that computes the jacobian of the PDE. + // --------------------------------------------------------------------------- + // -- QFunction + CeedQFunctionCreateInterior(ceed, 1, problem_data->jacobian, + problem_data->jacobian_loc, &qf_jacobian); + CeedQFunctionSetContext(qf_jacobian, problem_data->qfunction_context); + //CeedQFunctionContextDestroy(&problem_data->qfunction_context); + CeedQFunctionAddInput(qf_jacobian, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_jacobian, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_jacobian, "du", dim, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_jacobian, "div_du", 1, CEED_EVAL_DIV); + CeedQFunctionAddInput(qf_jacobian, "dp", 1, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_jacobian, "x", num_comp_x, CEED_EVAL_INTERP); + //CeedQFunctionAddInput(qf_jacobian, "u", dim, CEED_EVAL_INTERP); + //CeedQFunctionAddInput(qf_jacobian, "p", 1, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_jacobian, "dv", dim, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_jacobian, "div_dv", 1, CEED_EVAL_DIV); + CeedQFunctionAddOutput(qf_jacobian, "dq", 1, CEED_EVAL_INTERP); + // -- Operator + CeedOperatorCreate(ceed, qf_jacobian, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_jacobian); + CeedOperatorSetField(op_jacobian, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_setup_rhs, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_setup_rhs, "true_soln", ceed_data->elem_restr_u_i, - CEED_BASIS_COLLOCATED, *target); - CeedOperatorSetField(op_setup_rhs, "rhs", ceed_data->elem_restr_u, + CeedOperatorSetField(op_jacobian, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_jacobian, "du", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "div_du", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dp", ceed_data->elem_restr_p, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "x", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + //CeedOperatorSetField(op_jacobian, "u", ceed_data->elem_restr_u, + // ceed_data->basis_u, CEED_VECTOR_ACTIVE); + //CeedOperatorSetField(op_jacobian, "p", ceed_data->elem_restr_p, + // ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dv", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "div_dv", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dq", ceed_data->elem_restr_p, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + // -- Save libCEED data to apply operator in matops.c + ceed_data->qf_jacobian = qf_jacobian; + ceed_data->op_jacobian = op_jacobian; - // Setup RHS and target - CeedOperatorApply(op_setup_rhs, x_coord, rhs_ceed, CEED_REQUEST_IMMEDIATE); - - // Cleanup - CeedQFunctionDestroy(&qf_setup_rhs); - CeedOperatorDestroy(&op_setup_rhs); - CeedVectorDestroy(&x_coord); + // --------------------------------------------------------------------------- + // Setup Error Qfunction + // --------------------------------------------------------------------------- + // Create the q-function that sets up the error + CeedQFunctionCreateInterior(ceed, 1, problem_data->error, + problem_data->error_loc, &qf_error); + CeedQFunctionSetContext(qf_error, problem_data->qfunction_context); + CeedQFunctionAddInput(qf_error, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_error, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_error, "u", dim, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_error, "p", 1, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_error, "true solution", dim+1, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_error, "error", dim+1, CEED_EVAL_NONE); + // Create the operator that builds the error + CeedOperatorCreate(ceed, qf_error, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_error); + CeedOperatorSetField(op_error, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_error, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_error, "u", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_error, "p", ceed_data->elem_restr_p, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_error, "true solution", ceed_data->elem_restr_U_i, + CEED_BASIS_COLLOCATED, true_vec); + CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_U_i, + CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + // -- Save libCEED data to apply operator in matops.c + ceed_data->qf_error = qf_error; + ceed_data->op_error = op_error; + // -- Cleanup + CeedVectorDestroy(&true_vec); + CeedVectorDestroy(&true_force); + CeedQFunctionDestroy(&qf_true); + CeedOperatorDestroy(&op_true); PetscFunctionReturn(0); }; // ----------------------------------------------------------------------------- \ No newline at end of file diff --git a/examples/Hdiv-mixed/src/setup-matops.c b/examples/Hdiv-mixed/src/setup-matops.c new file mode 100644 index 0000000000..619b189f20 --- /dev/null +++ b/examples/Hdiv-mixed/src/setup-matops.c @@ -0,0 +1,61 @@ +#include "../include/setup-matops.h" +#include "../include/setup-libceed.h" + +// ----------------------------------------------------------------------------- +// Apply the local action of a libCEED operator and store result in PETSc vector +// i.e. compute A X = Y +// ----------------------------------------------------------------------------- +PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, + OperatorApplyContext op_apply_ctx) { + PetscFunctionBeginUser; + + // Zero target vector + PetscCall( VecZeroEntries(Y) ); + + // Sum into target vector + PetscCall( ApplyAddLocalCeedOp(X, Y, op_apply_ctx) ); + + PetscFunctionReturn(0); +} + +PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, + OperatorApplyContext op_apply_ctx) { + + PetscScalar *x, *y; + PetscMemType x_mem_type, y_mem_type; + + PetscFunctionBeginUser; + + // Global-to-local + PetscCall( DMGlobalToLocal(op_apply_ctx->dm, X, INSERT_VALUES, + op_apply_ctx->X_loc) ); + + // Setup libCEED vectors + PetscCall( VecGetArrayReadAndMemType(op_apply_ctx->X_loc, + (const PetscScalar **)&x, + &x_mem_type) ); + PetscCall( VecGetArrayAndMemType(op_apply_ctx->Y_loc, &y, &y_mem_type) ); + CeedVectorSetArray(op_apply_ctx->x_ceed, MemTypeP2C(x_mem_type), + CEED_USE_POINTER, x); + CeedVectorSetArray(op_apply_ctx->y_ceed, MemTypeP2C(y_mem_type), + CEED_USE_POINTER, y); + + // Apply libCEED operator + CeedOperatorApply(op_apply_ctx->op_apply, op_apply_ctx->x_ceed, + op_apply_ctx->y_ceed, CEED_REQUEST_IMMEDIATE); + + // Restore PETSc vectors + CeedVectorTakeArray(op_apply_ctx->x_ceed, MemTypeP2C(x_mem_type), NULL); + CeedVectorTakeArray(op_apply_ctx->y_ceed, MemTypeP2C(y_mem_type), NULL); + PetscCall( VecRestoreArrayReadAndMemType(op_apply_ctx->X_loc, + (const PetscScalar **)&x) ); + PetscCall( VecRestoreArrayAndMemType(op_apply_ctx->Y_loc, &y) ); + + // Local-to-global + PetscCall( DMLocalToGlobal(op_apply_ctx->dm, op_apply_ctx->Y_loc, ADD_VALUES, + Y) ); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/src/setup-solvers.c b/examples/Hdiv-mixed/src/setup-solvers.c new file mode 100644 index 0000000000..dde338a8cc --- /dev/null +++ b/examples/Hdiv-mixed/src/setup-solvers.c @@ -0,0 +1,354 @@ +#include "../include/setup-solvers.h" +#include "../include/setup-matops.h" +#include "../include/setup-libceed.h" + +// ----------------------------------------------------------------------------- +// Setup operator context data +// ----------------------------------------------------------------------------- +PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, + OperatorApplyContext ctx_jacobian) { + PetscFunctionBeginUser; + + ctx_jacobian->dm = dm; + PetscCall( DMCreateLocalVector(dm, &ctx_jacobian->X_loc) ); + PetscCall( VecDuplicate(ctx_jacobian->X_loc, &ctx_jacobian->Y_loc) ); + ctx_jacobian->x_ceed = ceed_data->x_ceed; + ctx_jacobian->y_ceed = ceed_data->y_ceed; + ctx_jacobian->ceed = ceed; + ctx_jacobian->op_apply = ceed_data->op_jacobian; + + PetscFunctionReturn(0); +} + +PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, + OperatorApplyContext ctx_residual) { + PetscFunctionBeginUser; + + ctx_residual->dm = dm; + PetscCall( DMCreateLocalVector(dm, &ctx_residual->X_loc) ); + PetscCall( VecDuplicate(ctx_residual->X_loc, &ctx_residual->Y_loc) ); + ctx_residual->x_ceed = ceed_data->x_ceed; + ctx_residual->y_ceed = ceed_data->y_ceed; + ctx_residual->ceed = ceed; + ctx_residual->op_apply = ceed_data->op_residual; + + PetscFunctionReturn(0); +} + +PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, + OperatorApplyContext ctx_error) { + PetscFunctionBeginUser; + + ctx_error->dm = dm; + PetscCall( DMCreateLocalVector(dm, &ctx_error->X_loc) ); + PetscCall( VecDuplicate(ctx_error->X_loc, &ctx_error->Y_loc) ); + ctx_error->x_ceed = ceed_data->x_ceed; + ctx_error->y_ceed = ceed_data->y_ceed; + ctx_error->ceed = ceed; + ctx_error->op_apply = ceed_data->op_error; + + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// This function wraps the libCEED operator for a MatShell +// ----------------------------------------------------------------------------- +PetscErrorCode ApplyJacobian(Mat A, Vec X, Vec Y) { + OperatorApplyContext op_apply_ctx; + + PetscFunctionBeginUser; + + PetscCall( MatShellGetContext(A, &op_apply_ctx) ); + + // libCEED for local action of residual evaluator + PetscCall( ApplyLocalCeedOp(X, Y, op_apply_ctx) ); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// This function uses libCEED to compute the non-linear residual +// ----------------------------------------------------------------------------- +PetscErrorCode SNESFormResidual(SNES snes, Vec X, Vec Y, void *ctx_residual) { + OperatorApplyContext ctx = (OperatorApplyContext)ctx_residual; + + PetscFunctionBeginUser; + + // Use computed BCs + //PetscCall( DMPlexInsertBoundaryValues(ctx->dm, PETSC_TRUE, + // ctx->X_loc, + // 1.0, NULL, NULL, NULL) ); + + // libCEED for local action of residual evaluator + PetscCall( ApplyLocalCeedOp(X, Y, ctx) ); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// Jacobian setup +// ----------------------------------------------------------------------------- +PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, + void *ctx_jacobian) { + + PetscFunctionBeginUser; + + // J_pre might be AIJ (e.g., when using coloring), so we need to assemble it + PetscCall( MatAssemblyBegin(J_pre, MAT_FINAL_ASSEMBLY) ); + PetscCall( MatAssemblyEnd(J_pre, MAT_FINAL_ASSEMBLY) ); + if (J != J_pre) { + PetscCall( MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY) ); + PetscCall( MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY) ); + } + PetscFunctionReturn(0); +}; + +// --------------------------------------------------------------------------- +// Setup Solver +// --------------------------------------------------------------------------- +PetscErrorCode PDESolver(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, + VecType vec_type, SNES snes, KSP ksp, Vec *U_g) { + + PetscInt U_l_size, U_g_size; + + PetscFunctionBeginUser; + + // Create global unknown solution U_g + PetscCall( DMCreateGlobalVector(dm, U_g) ); + PetscCall( VecGetSize(*U_g, &U_g_size) ); + // Local size for matShell + PetscCall( VecGetLocalSize(*U_g, &U_l_size) ); + Vec R; + PetscCall( VecDuplicate(*U_g, &R) ); + + // --------------------------------------------------------------------------- + // Setup SNES + // --------------------------------------------------------------------------- + // Operator + Mat mat_jacobian; + PetscCall( PetscCalloc1(1, &ceed_data->ctx_jacobian) ); + SetupJacobianOperatorCtx(dm, ceed, ceed_data, ceed_data->ctx_jacobian); + PetscCall( SNESSetDM(snes, ceed_data->ctx_jacobian->dm) ); + // -- Form Action of Jacobian on delta_u + PetscCall( MatCreateShell(comm, U_l_size, U_l_size, U_g_size, + U_g_size, ceed_data->ctx_jacobian, &mat_jacobian) ); + PetscCall( MatShellSetOperation(mat_jacobian, MATOP_MULT, + (void (*)(void))ApplyJacobian) ); + PetscCall( MatShellSetVecType(mat_jacobian, vec_type) ); + + // Set SNES residual evaluation function + PetscCall( PetscCalloc1(1, &ceed_data->ctx_residual) ); + SetupResidualOperatorCtx(dm, ceed, ceed_data, ceed_data->ctx_residual); + PetscCall( SNESSetFunction(snes, R, SNESFormResidual, + ceed_data->ctx_residual) ); + // -- SNES Jacobian + PetscCall( SNESSetJacobian(snes, mat_jacobian, mat_jacobian, + SNESFormJacobian, ceed_data->ctx_jacobian) ); + + // Setup KSP + PetscCall( KSPSetFromOptions(ksp) ); + + // Default to critical-point (CP) line search (related to Wolfe's curvature condition) + SNESLineSearch line_search; + + PetscCall( SNESGetLineSearch(snes, &line_search) ); + PetscCall( SNESLineSearchSetType(line_search, SNESLINESEARCHCP) ); + PetscCall( SNESSetFromOptions(snes) ); + + // Solve + PetscCall( VecSet(*U_g, 0.0)); + PetscCall( SNESSolve(snes, NULL, *U_g)); + + // Free PETSc objects + PetscCall( MatDestroy(&mat_jacobian) ); + PetscCall( VecDestroy(&R) ); + PetscCall( VecDestroy(&ceed_data->ctx_jacobian->Y_loc) ); + PetscCall( VecDestroy(&ceed_data->ctx_jacobian->X_loc) ); + PetscCall( VecDestroy(&ceed_data->ctx_residual->Y_loc) ); + PetscCall( VecDestroy(&ceed_data->ctx_residual->X_loc) ); + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// This function calculates the L2 error in the final solution +// ----------------------------------------------------------------------------- +PetscErrorCode ComputeL2Error(DM dm, Ceed ceed, CeedData ceed_data, Vec U, + CeedScalar *l2_error_u, + CeedScalar *l2_error_p) { + PetscScalar *x; + PetscMemType mem_type; + CeedVector collocated_error; + + PetscFunctionBeginUser; + + PetscCall( PetscCalloc1(1, &ceed_data->ctx_error) ); + SetupErrorOperatorCtx(dm, ceed, ceed_data, ceed_data->ctx_error); + CeedInt c_start, c_end, dim, num_elem, num_qpts; + PetscCall( DMGetDimension(ceed_data->ctx_error->dm, &dim) ); + CeedBasisGetNumQuadraturePoints(ceed_data->basis_u, &num_qpts); + PetscCall( DMPlexGetHeightStratum(ceed_data->ctx_error->dm, 0, &c_start, + &c_end) ); + num_elem = c_end -c_start; + CeedVectorCreate(ceed, num_elem*num_qpts*(dim+1), &collocated_error); + + // Global-to-local + PetscCall( DMGlobalToLocal(ceed_data->ctx_error->dm, U, INSERT_VALUES, + ceed_data->ctx_error->X_loc) ); + + // Setup CEED vector + PetscCall( VecGetArrayAndMemType(ceed_data->ctx_error->X_loc, &x, &mem_type) ); + CeedVectorSetArray(ceed_data->ctx_error->x_ceed, MemTypeP2C(mem_type), + CEED_USE_POINTER, + x); + + // Apply CEED operator + CeedOperatorApply(ceed_data->ctx_error->op_apply, ceed_data->ctx_error->x_ceed, + collocated_error, + CEED_REQUEST_IMMEDIATE); + // Restore PETSc vector + CeedVectorTakeArray(ceed_data->ctx_error->x_ceed, MemTypeP2C(mem_type), NULL); + PetscCall( VecRestoreArrayReadAndMemType(ceed_data->ctx_error->X_loc, + (const PetscScalar **)&x) ); + // Compute L2 error for each field + CeedInt cent_qpts = num_qpts / 2; + CeedVector collocated_error_u, collocated_error_p; + const CeedScalar *E_U; // to store total error + CeedInt length_u, length_p; + length_p = num_elem; + length_u = num_elem*num_qpts*dim; + CeedScalar e_u[length_u], e_p[length_p]; + CeedVectorCreate(ceed_data->ctx_error->ceed, length_p, &collocated_error_p); + CeedVectorCreate(ceed_data->ctx_error->ceed, length_u, &collocated_error_u); + // E_U is ordered as [p_0,u_0/.../p_n,u_n] for 0 to n elements + // For each element p_0 size is num_qpts, and u_0 is dim*num_qpts + CeedVectorGetArrayRead(collocated_error, CEED_MEM_HOST, &E_U); + for (CeedInt n=0; n < num_elem; n++) { + for (CeedInt i=0; i < 1; i++) { + CeedInt j = i + n*1; + CeedInt k = cent_qpts + n*num_qpts*(dim+1); + e_p[j] = E_U[k]; + } + } + + for (CeedInt n=0; n < num_elem; n++) { + for (CeedInt i=0; i < dim*num_qpts; i++) { + CeedInt j = i + n*num_qpts*dim; + CeedInt k = num_qpts + i + n*num_qpts*(dim+1); + e_u[j] = E_U[k]; + } + } + + CeedVectorSetArray(collocated_error_p, CEED_MEM_HOST, CEED_USE_POINTER, e_p); + CeedVectorSetArray(collocated_error_u, CEED_MEM_HOST, CEED_USE_POINTER, e_u); + CeedVectorRestoreArrayRead(collocated_error, &E_U); + + CeedScalar error_u, error_p; + CeedVectorNorm(collocated_error_u, CEED_NORM_1, &error_u); + CeedVectorNorm(collocated_error_p, CEED_NORM_1, &error_p); + *l2_error_u = sqrt(error_u); + *l2_error_p = sqrt(error_p); + // Cleanup + CeedVectorDestroy(&collocated_error); + CeedVectorDestroy(&collocated_error_u); + CeedVectorDestroy(&collocated_error_p); + PetscCall( VecDestroy(&ceed_data->ctx_error->Y_loc) ); + PetscCall( VecDestroy(&ceed_data->ctx_error->X_loc) ); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// This function print the output +// ----------------------------------------------------------------------------- +PetscErrorCode PrintOutput(Ceed ceed, + CeedMemType mem_type_backend, + SNES snes, KSP ksp, + Vec U, CeedScalar l2_error_u, + CeedScalar l2_error_p, AppCtx app_ctx) { + + PetscFunctionBeginUser; + + const char *used_resource; + CeedGetResource(ceed, &used_resource); + char hostname[PETSC_MAX_PATH_LEN]; + PetscCall( PetscGetHostName(hostname, sizeof hostname) ); + PetscInt comm_size; + PetscCall( MPI_Comm_size(app_ctx->comm, &comm_size) ); + PetscCall( PetscPrintf(app_ctx->comm, + "\n-- Mixed H(div) Example - libCEED + PETSc --\n" + " MPI:\n" + " Hostname : %s\n" + " Total ranks : %d\n" + " libCEED:\n" + " libCEED Backend : %s\n" + " libCEED Backend MemType : %s\n", + hostname, comm_size, used_resource, CeedMemTypes[mem_type_backend]) ); + + VecType vecType; + PetscCall( VecGetType(U, &vecType) ); + PetscCall( PetscPrintf(app_ctx->comm, + " PETSc:\n" + " PETSc Vec Type : %s\n", + vecType) ); + + PetscInt U_l_size, U_g_size; + PetscCall( VecGetSize(U, &U_g_size) ); + PetscCall( VecGetLocalSize(U, &U_l_size) ); + PetscCall( PetscPrintf(app_ctx->comm, + " Problem:\n" + " Problem Name : %s\n" + " Global nodes (u + p) : %" PetscInt_FMT "\n" + " Owned nodes (u + p) : %" PetscInt_FMT "\n", + app_ctx->problem_name, U_g_size, U_l_size + ) ); + // -- SNES + PetscInt its, snes_its = 0; + PetscCall( SNESGetIterationNumber(snes, &its) ); + snes_its += its; + SNESType snes_type; + SNESConvergedReason snes_reason; + PetscReal snes_rnorm; + PetscCall( SNESGetType(snes, &snes_type) ); + PetscCall( SNESGetConvergedReason(snes, &snes_reason) ); + PetscCall( SNESGetFunctionNorm(snes, &snes_rnorm) ); + PetscCall( PetscPrintf(app_ctx->comm, + " SNES:\n" + " SNES Type : %s\n" + " SNES Convergence : %s\n" + " Total SNES Iterations : %" PetscInt_FMT "\n" + " Final rnorm : %e\n", + snes_type, SNESConvergedReasons[snes_reason], + snes_its, (double)snes_rnorm) ); + + PetscInt ksp_its = 0; + PetscCall( SNESGetLinearSolveIterations(snes, &its) ); + ksp_its += its; + KSPType ksp_type; + KSPConvergedReason ksp_reason; + PetscReal ksp_rnorm; + PC pc; + PCType pc_type; + PetscCall( KSPGetPC(ksp, &pc) ); + PetscCall( PCGetType(pc, &pc_type) ); + PetscCall( KSPGetType(ksp, &ksp_type) ); + PetscCall( KSPGetConvergedReason(ksp, &ksp_reason) ); + PetscCall( KSPGetIterationNumber(ksp, &ksp_its) ); + PetscCall( KSPGetResidualNorm(ksp, &ksp_rnorm) ); + PetscCall( PetscPrintf(app_ctx->comm, + " KSP:\n" + " KSP Type : %s\n" + " PC Type : %s\n" + " KSP Convergence : %s\n" + " Total KSP Iterations : %" PetscInt_FMT "\n" + " Final rnorm : %e\n", + ksp_type, pc_type, KSPConvergedReasons[ksp_reason], ksp_its, + (double)ksp_rnorm ) ); + + PetscCall( PetscPrintf(app_ctx->comm, + " L2 Error (MMS):\n" + " L2 error of u and p : %e, %e\n", + (double)l2_error_u, + (double)l2_error_p) ); + PetscFunctionReturn(0); +}; +// ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/src/setup-ts.c b/examples/Hdiv-mixed/src/setup-ts.c new file mode 100644 index 0000000000..1983993943 --- /dev/null +++ b/examples/Hdiv-mixed/src/setup-ts.c @@ -0,0 +1,152 @@ +#include "../include/setup-ts.h" +#include "../include/setup-matops.h" +#include "../include/setup-libceed.h" +#include "petscerror.h" + + +// ----------------------------------------------------------------------------- +// Create global initial conditions vector +// ----------------------------------------------------------------------------- +PetscErrorCode CreateInitialConditions(DM dm, CeedData ceed_data, Vec *U0) { + + PetscScalar *u0; + PetscMemType u0_mem_type; + Vec U0_loc; + + PetscFunctionBeginUser; + + PetscCall( DMCreateLocalVector(dm, &U0_loc) ); + PetscCall( VecZeroEntries(U0_loc) ); + + PetscCall( VecGetArrayAndMemType(U0_loc, &u0, &u0_mem_type) ); + CeedVectorSetArray(ceed_data->U0_ceed, MemTypeP2C(u0_mem_type), + CEED_USE_POINTER, u0); + // Apply libCEED operator + CeedOperatorApply(ceed_data->op_ics, ceed_data->x_coord, ceed_data->U0_ceed, + CEED_REQUEST_IMMEDIATE); + // Restore PETSc vectors + CeedVectorTakeArray(ceed_data->U0_ceed, MemTypeP2C(u0_mem_type), NULL); + PetscCall( VecRestoreArrayAndMemType(U0_loc, &u0) ); + + // Create global initial conditions + PetscCall( DMCreateGlobalVector(dm, U0) ); + PetscCall( VecZeroEntries(*U0) ); + // Local-to-global + PetscCall( DMLocalToGlobal(dm, U0_loc, ADD_VALUES, *U0) ); + + // -- Cleanup + CeedVectorDestroy(&ceed_data->U0_ceed); + CeedQFunctionDestroy(&ceed_data->qf_ics); + CeedOperatorDestroy(&ceed_data->op_ics); + // Free PETSc objects + PetscCall( VecDestroy(&U0_loc) ); + PetscFunctionReturn(0); + +} +// ----------------------------------------------------------------------------- +// Setup operator context data +// ----------------------------------------------------------------------------- +PetscErrorCode SetupResidualOperatorCtx_Ut(DM dm, Ceed ceed, CeedData ceed_data, + OperatorApplyContext ctx_residual_ut) { + PetscFunctionBeginUser; + + ctx_residual_ut->dm = dm; + PetscCall( DMCreateLocalVector(dm, &ctx_residual_ut->X_loc) ); + PetscCall( VecDuplicate(ctx_residual_ut->X_loc, &ctx_residual_ut->Y_loc) ); + PetscCall( VecDuplicate(ctx_residual_ut->X_loc, &ctx_residual_ut->X_t_loc) ); + ctx_residual_ut->x_ceed = ceed_data->x_ceed; + ctx_residual_ut->x_t_ceed = ceed_data->x_t_ceed; + ctx_residual_ut->y_ceed = ceed_data->y_ceed; + ctx_residual_ut->ceed = ceed; + ctx_residual_ut->op_apply = ceed_data->op_residual; + + PetscFunctionReturn(0); +} + +PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, + void *ctx_residual_ut) { + OperatorApplyContext ctx = (OperatorApplyContext)ctx_residual_ut; + const PetscScalar *x, *x_t; + PetscScalar *y; + Vec X_loc = ctx->X_loc, X_t_loc = ctx->X_t_loc, + Y_loc = ctx->Y_loc; + PetscMemType x_mem_type, x_t_mem_type, y_mem_type; + PetscFunctionBeginUser; + + // Update time dependent data + + + // Global-to-local + PetscCall( DMGlobalToLocal(ctx->dm, X, INSERT_VALUES, X_loc) ); + PetscCall( DMGlobalToLocal(ctx->dm, X_t, INSERT_VALUES, X_t_loc) ); + + // Place PETSc vectors in CEED vectors + PetscCall( VecGetArrayReadAndMemType(X_loc, &x, &x_mem_type) ); + PetscCall( VecGetArrayReadAndMemType(X_t_loc, &x_t, &x_t_mem_type) ); + PetscCall( VecGetArrayAndMemType(Y_loc, &y, &y_mem_type) ); + CeedVectorSetArray(ctx->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, + (PetscScalar *)x); + CeedVectorSetArray(ctx->x_t_ceed, MemTypeP2C(x_t_mem_type), + CEED_USE_POINTER, (PetscScalar *)x_t); + CeedVectorSetArray(ctx->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); + + // Apply CEED operator + CeedOperatorApply(ctx->op_apply, ctx->x_ceed, ctx->y_ceed, + CEED_REQUEST_IMMEDIATE); + + // Restore vectors + CeedVectorTakeArray(ctx->x_ceed, MemTypeP2C(x_mem_type), NULL); + CeedVectorTakeArray(ctx->x_t_ceed, MemTypeP2C(x_t_mem_type), NULL); + CeedVectorTakeArray(ctx->y_ceed, MemTypeP2C(y_mem_type), NULL); + PetscCall( VecRestoreArrayReadAndMemType(X_loc, &x) ); + PetscCall( VecRestoreArrayReadAndMemType(X_t_loc, &x_t) ); + PetscCall( VecRestoreArrayAndMemType(Y_loc, &y) ); + + // Local-to-Global + PetscCall( VecZeroEntries(Y) ); + PetscCall( DMLocalToGlobal(ctx->dm, Y_loc, ADD_VALUES, Y) ); + + // Restore vectors + PetscCall( DMRestoreLocalVector(ctx->dm, &Y_loc) ); + + PetscFunctionReturn(0); +} + +// TS: Create, setup, and solve +PetscErrorCode TSSolveRichard(DM dm, CeedData ceed_data, AppCtx app_ctx, + Vec *U, PetscScalar *f_time, TS *ts) { + MPI_Comm comm = app_ctx->comm; + TSAdapt adapt; + PetscFunctionBeginUser; + + PetscCall( TSCreate(comm, ts) ); + PetscCall( TSSetDM(*ts, dm) ); + PetscCall( TSSetIFunction(*ts, NULL, TSFormIResidual, + ceed_data->ctx_residual_ut) ); + + PetscCall( TSSetMaxTime(*ts, 10) ); + PetscCall( TSSetExactFinalTime(*ts, TS_EXACTFINALTIME_STEPOVER) ); + PetscCall( TSSetTimeStep(*ts, 1.e-2) ); + PetscCall( TSGetAdapt(*ts, &adapt) ); + PetscCall( TSAdaptSetStepLimits(adapt, 1.e-12, 1.e2) ); + PetscCall( TSSetFromOptions(*ts) ); + + // Solve + PetscScalar start_time; + PetscCall( TSGetTime(*ts, &start_time) ); + + PetscCall(TSSetTime(*ts, start_time)); + PetscCall(TSSetStepNumber(*ts, 0)); + + PetscCall( PetscBarrier((PetscObject) *ts) ); + PetscCall( TSSolve(*ts, *U) ); + + PetscScalar final_time; + PetscCall( TSGetSolveTime(*ts, &final_time) ); + *f_time = final_time; + + PetscFunctionReturn(0); +} + + +// ----------------------------------------------------------------------------- diff --git a/interface/ceed-basis.c b/interface/ceed-basis.c index 5a9358e072..1338d84c42 100644 --- a/interface/ceed-basis.c +++ b/interface/ceed-basis.c @@ -1091,78 +1091,6 @@ int CeedBasisCreateH1(Ceed ceed, CeedElemTopology topo, CeedInt num_comp, CeedIn CeedCall(ceed->BasisCreateH1(topo, dim, P, Q, interp, grad, q_ref, q_weight, *basis)); return CEED_ERROR_SUCCESS; } -/** - @brief Create a basis for H(div) discretizations - - @param ceed A Ceed object where the CeedBasis will be created - @param topo Topology of element, e.g. hypercube, simplex, ect - @param num_comp Number of componenet, we have 1 vector componenet in H(div) - @param num_nodes Total number of nodes - @param num_qpts Total number of quadrature points - @param basis_space 2 for H(div) discretization (1 for H^1, 3 for H(curl)) - @param interp Row-major (dim*num_qpts * num_nodes*dim) matrix expressing the values of - nodal basis functions at quadrature points - @param div Row-major (num_qpts * num_nodes*dim) matrix expressing - divergence of nodal basis functions at quadrature points - @param q_ref Array of length num_qpts holding the locations of quadrature - points on the reference element [-1, 1] - @param q_weight Array of length num_qpts holding the quadrature weights on the - reference element - @param[out] basis Address of the variable where the newly created - CeedBasis will be stored. - - @return An error code: 0 - success, otherwise - failure - - @ref User -**/ -int CeedBasisCreateHdiv(Ceed ceed, CeedElemTopology topo, CeedInt num_comp, - CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, - const CeedScalar *div, const CeedScalar *q_ref, - const CeedScalar *q_weight, CeedBasis *basis) { - int ierr; - CeedInt Q = num_qpts, dim = 0; - ierr = CeedBasisGetTopologyDimension(topo, &dim); CeedChk(ierr); - CeedInt P = dim*num_nodes; // dof per element! - if (!ceed->BasisCreateHdiv) { - Ceed delegate; - ierr = CeedGetObjectDelegate(ceed, &delegate, "Basis"); CeedChk(ierr); - - if (!delegate) - // LCOV_EXCL_START - return CeedError(ceed, CEED_ERROR_UNSUPPORTED, - "Backend does not support BasisCreateHdiv"); - // LCOV_EXCL_STOP - - ierr = CeedBasisCreateHdiv(delegate, topo, num_comp, num_nodes, - num_qpts, interp, div, q_ref, - q_weight, basis); CeedChk(ierr); - return CEED_ERROR_SUCCESS; - } - - ierr = CeedCalloc(1,basis); CeedChk(ierr); - - (*basis)->ceed = ceed; - ierr = CeedReference(ceed); CeedChk(ierr); - (*basis)->ref_count = 1; - (*basis)->tensor_basis = 0; - (*basis)->dim = dim; - (*basis)->topo = topo; - (*basis)->num_comp = num_comp; - (*basis)->P = P; - (*basis)->Q = Q; - (*basis)->basis_space = 2; // 2 for H(div) space - ierr = CeedMalloc(Q*dim,&(*basis)->q_ref_1d); CeedChk(ierr); - ierr = CeedMalloc(Q,&(*basis)->q_weight_1d); CeedChk(ierr); - memcpy((*basis)->q_ref_1d, q_ref, Q*dim*sizeof(q_ref[0])); - memcpy((*basis)->q_weight_1d, q_weight, Q*sizeof(q_weight[0])); - ierr = CeedMalloc(dim*Q*P, &(*basis)->interp); CeedChk(ierr); - ierr = CeedMalloc(Q*P, &(*basis)->div); CeedChk(ierr); - memcpy((*basis)->interp, interp, dim*Q*P*sizeof(interp[0])); - memcpy((*basis)->div, div, Q*P*sizeof(div[0])); - ierr = ceed->BasisCreateHdiv(topo, dim, P, Q, interp, div, q_ref, - q_weight, *basis); CeedChk(ierr); - return CEED_ERROR_SUCCESS; -} /** @brief Create a non tensor-product basis for \f$H(\mathrm{div})\f$ discretizations diff --git a/tests/t330-basis.h b/tests/t330-basis.h index 260cfb6ee8..435f8ef8a3 100644 --- a/tests/t330-basis.h +++ b/tests/t330-basis.h @@ -58,7 +58,7 @@ static void BuildHdivQuadrilateral(CeedInt q, CeedScalar *q_ref, CeedScalar *q_w CeedScalar D[8] = {0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25}; // Loop over quadrature points CeedScalar Bx[8], By[8]; - CeedScalar X[2]; + CeedScalar x[2]; for (CeedInt i = 0; i < q; i++) { for (CeedInt j = 0; j < q; j++) { diff --git a/tests/t999-Hdiv2D.c b/tests/t999-Hdiv2D.c deleted file mode 100644 index aab44aa578..0000000000 --- a/tests/t999-Hdiv2D.c +++ /dev/null @@ -1,136 +0,0 @@ -/// @file -/// test H(div)-mixed fem for 2D quadrilateral element -#include -#include -#include -#include - -// ----------------------------------------------------------------------------- -// Nodal Basis (B=[Bx;By]), xhat is in reference element [-1,1]^2 -// ----------------------------------------------------------------------------- -// B = [b1,b2,b3,b4,b5,b6,b7,b8], size(2x8), -// local numbering is as follow (each edge has 2 dof) -// b6 b8 -// 3---------4 -// b5| |b7 -// | | -// b1| |b3 -// 1---------2 -// b2 b4 -// Sience nodal basis are vector, we have 16 componenets -// For example B[0] = b1_x, B[1] = b1_y, and so on -/* -static int HdivBasisQuad(CeedScalar *xhat, CeedScalar *B) { - B[ 0] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; - B[ 1] = (xhat[1]*xhat[1] - 1)*0.125; - B[ 2] = (xhat[0]*xhat[0] - 1)*0.125; - B[ 3] = (-xhat[0]*xhat[1] + xhat[0] + xhat[1] - 1)*0.25; - B[ 4] = (-xhat[0]*xhat[1] + xhat[0] - xhat[1] + 1)*0.25; - B[ 5] = (xhat[1]*xhat[1] - 1)*0.125; - B[ 6] = (-xhat[0]*xhat[0] + 1)*0.125; - B[ 7] = (xhat[0]*xhat[1] - xhat[0] + xhat[1] - 1)*0.25; - B[ 8] = (xhat[0]*xhat[1] + xhat[0] - xhat[1] - 1)*0.25; - B[ 9] = (-xhat[1]*xhat[1] + 1)*0.125; - B[10] = (xhat[0]*xhat[0] - 1)*0.125; - B[11] = (-xhat[0]*xhat[1] - xhat[0] + xhat[1] + 1)*0.25; - B[12] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; - B[13] = (-xhat[1]*xhat[1] + 1)*0.125; - B[14] = (-xhat[0]*xhat[0] + 1)*0.125; - B[15] = (xhat[0]*xhat[1] + xhat[0] + xhat[1] + 1)*0.25; - return 0; -} -*/ - -// ----------------------------------------------------------------------------- -// Divergence operator; Divergence of nodal basis -// CeedScalar Dhat[8] ={0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25} -// ----------------------------------------------------------------------------- - -#include "t999-Hdiv2D.h" -int main(int argc, char **argv) { - Ceed ceed; - CeedInt P = 2, Q = 2, dim = 2; - CeedInt nx = 1, ny = 1, num_elem = nx * ny; - CeedInt num_nodes = (nx+1)*(ny+1), num_qpts = num_elem*Q*Q; - CeedInt ind_x[num_elem*P*P]; - CeedScalar x[dim*num_nodes]; - CeedVector X, u, U; - CeedElemRestriction elem_restr_x, elem_restr_mass; - CeedBasis basis_x, basis_xc; - CeedQFunction qf_setup; - CeedOperator op_setup; - - CeedInit(argv[1], &ceed); - - //============= Node Coordinates, mesh [0,1]^2 square ============== - for (CeedInt i=0; i Date: Thu, 21 Jul 2022 16:32:11 -0600 Subject: [PATCH 07/15] Setup Richard problem, converged and checked with MMS --- examples/Hdiv-mixed/conv_test.sh | 6 +- examples/Hdiv-mixed/conv_test_result.csv | 16 +- examples/Hdiv-mixed/convrate_mixed.png | Bin 28667 -> 27357 bytes examples/Hdiv-mixed/include/setup-fe.h | 2 +- examples/Hdiv-mixed/include/setup-libceed.h | 13 +- examples/Hdiv-mixed/include/setup-solvers.h | 8 +- examples/Hdiv-mixed/include/setup-ts.h | 16 +- examples/Hdiv-mixed/include/structs.h | 74 +-- examples/Hdiv-mixed/main.c | 94 ++-- examples/Hdiv-mixed/problems/darcy2d.c | 22 +- examples/Hdiv-mixed/problems/darcy3d.c | 20 +- examples/Hdiv-mixed/problems/richard2d.c | 58 ++- .../Hdiv-mixed/qfunctions/darcy-error2d.h | 1 + .../Hdiv-mixed/qfunctions/darcy-error3d.h | 1 + .../Hdiv-mixed/qfunctions/darcy-system2d.h | 4 +- .../Hdiv-mixed/qfunctions/darcy-system3d.h | 1 + examples/Hdiv-mixed/qfunctions/darcy-true2d.h | 14 +- examples/Hdiv-mixed/qfunctions/darcy-true3d.h | 19 +- .../qfunctions/pressure-boundary2d.h | 3 + .../qfunctions/pressure-boundary3d.h | 3 + .../Hdiv-mixed/qfunctions/richard-ics2d.h | 228 +++++++++ .../Hdiv-mixed/qfunctions/richard-system2d.h | 56 +-- .../{richard-mms2d.h => richard-true2d.h} | 127 ++--- examples/Hdiv-mixed/src/setup-fe.c | 33 +- examples/Hdiv-mixed/src/setup-libceed.c | 433 +++++++++++++----- examples/Hdiv-mixed/src/setup-solvers.c | 83 ++-- examples/Hdiv-mixed/src/setup-ts.c | 291 +++++++++--- interface/ceed-operator.c | 4 + 28 files changed, 1170 insertions(+), 460 deletions(-) create mode 100644 examples/Hdiv-mixed/qfunctions/richard-ics2d.h rename examples/Hdiv-mixed/qfunctions/{richard-mms2d.h => richard-true2d.h} (54%) diff --git a/examples/Hdiv-mixed/conv_test.sh b/examples/Hdiv-mixed/conv_test.sh index 1cc1488235..798f2b70eb 100755 --- a/examples/Hdiv-mixed/conv_test.sh +++ b/examples/Hdiv-mixed/conv_test.sh @@ -32,7 +32,7 @@ declare -A run_flags run_flags[pc_type]=svd if [[ $dim -eq 2 ]]; then - run_flags[problem]=darcy2d + run_flags[problem]=richard2d run_flags[dm_plex_dim]=$dim run_flags[dm_plex_box_faces]=2,2 else @@ -43,8 +43,8 @@ declare -A run_flags declare -A test_flags test_flags[res_start]=2 - test_flags[res_stride]=1 - test_flags[res_end]=12 + test_flags[res_stride]=2 + test_flags[res_end]=10 file_name=conv_test_result.csv diff --git a/examples/Hdiv-mixed/conv_test_result.csv b/examples/Hdiv-mixed/conv_test_result.csv index 9997c7c1cc..1c19c3f9ff 100644 --- a/examples/Hdiv-mixed/conv_test_result.csv +++ b/examples/Hdiv-mixed/conv_test_result.csv @@ -1,12 +1,6 @@ run,mesh_res,error_u,error_p -0,2,9.13818,0.09469 -1,3,4.60103,0.05186 -2,4,2.75883,0.03199 -3,5,1.81813,0.02132 -4,6,1.27780,0.01505 -5,7,0.93759,0.01108 -6,8,0.71073,0.00842 -7,9,0.55232,0.00655 -8,10,0.43767,0.00520 -9,11,0.35229,0.00418 -10,12,0.28723,0.00340 +0,2,8.22730,0.00852 +1,4,2.50219,0.00291 +2,6,1.60805,0.00181 +3,8,1.02228,0.00114 +4,10,0.70467,0.00078 diff --git a/examples/Hdiv-mixed/convrate_mixed.png b/examples/Hdiv-mixed/convrate_mixed.png index cd9db19a6311e31055a55496e4c11b31e601b850..e98533fd9d60016ba9282386cd730da8b936864e 100644 GIT binary patch literal 27357 zcmb5W2RxVk-#7e2A(_dDjHrZCk;p8RQZll#X$T30L`HUHq|#7k_9i4EBPtD%S=p8B zlr22(Z|DF2zpwj#uIqk1&-pr6{Kj#7$LI52-{Y$GQFSH;9tH}9!gT1Mnl6Px?M9(c zwbIk#C-dT=pBe9CJMFa?$GiDT=0*i_=+0m$SCkf^MhIU$AvNCnX^*v2&~7 z85b9)3v%1HJN)Mb5{~C>wzpgTD#wScayoeY0)@h2MgE~mQ%<#|Q0z1gsqNEqe>mEC z$yD!KAMH<@c8=qExA*V9puTeLzQ%)x>I;wTZz@>RmUZ3iT~lUfQ`X!12V95GA3F9s zBQmJ?p2Y+#}@mp)36P z@Y~2p%RFKHR&oua8-A;Hzz=UuyF<&!?R&1Xba0BtFxNa#C85%1>9;_+d1{KJq$DdR zXV9H(CMpTqf|8PJoSmJux2z4heqG9CDuibnzKiDR@8$`rS_u-`mXX&|Sf|{n=0It=qQkbQ>{p8E$2&t*xE? z-Jrbl!uKN&_gtVAH_X}PBK@qg%atwV6TrzS*z0w8*{o)&%pGiZ#F5+0NFI|Mlz1 zXS03RfByVgDd6*m%ErdV)(pV@rz*d3kxhS62xzG;{6Sw~v1H z>RaX)hniP8KAWJeSQwLk8!eq#H}K}|+ko=&^0N=FUR{-ymBn9_p*BfB`&>eYZ{g4H zhRC?Mh?Y#lH!3j)7N$pcI(*tS=xS>#sx<#YxX`|h`Ocj?KPM+&l#N$hPfQf*aE^_Q zbsu{=@N*!}M@UeRf>C!L$$$B?^J4d*Q{nuI41p?FT!&k2%`RemFFMeB zc34rb=i{@owY{2SR@V2~@p%+q>ZRtH#hLE@@|iBx2fH0U6xts9`t2JPWw&D&_ zjgB8rFnMyWz0_5_E{vB>OG~TclAaickg#xLk-Z^)Th-Xeb}Z#cRgC;ayUTGrd-m*c zAI^T#GdQ^F@ZrNHrKQ#*?fE6H!*}#j($eUEe}AWn8^i^yl97?Y$Q&MC^t$)rh1%rk zXD-d?UHd*4JLF%UJ8x^@?X8rRoqZ}U9p5TrX=isWa5a~~upDOkb_L6y#YN|@uOjW* zCNSXpD9vWn^_fFb7G-YR)-tRNU=6u;?I!;Oby{9TOo~8X$736|_3J}ZG#?~tKG;2r z(JmU&>ukV+wj63sZ<;Z*xK>QO3+L6*^#>Cw+M_4N!iw2YOQlcF5gA0IX- z{C+Ey_xQ2u#}v)R=RQ7vFro#L0+)YM;S0%XaPF_G>gwWmb93WX_Eun6z53LY=$0*& zrLMzCGbg^3IP3e+tIZ3Bq#Z4?{dhoOy8T3pgaALk1SKRdZwDI($ELb)K54J`_ z>DB<&zVGiIRJXL~mppxXl|}iF_EwYPBMQrZepCC-j{7W&^F6sZZceq={nDjX78Vxy z?i1GePQ5BR+S-ENi&KUP+FJsiJb7aD%strNQs#*Bo=?ZZU>_SBvmR_pu7308@a2hGu0Cuhc1})CAz4`?x65bGW}!x~ zUb%9m@A~!%*5$=spoDi#vR<=#m_GI?FC{<>ch_p>8%6mz2m- z?d2Tl>+fGl$+xPZ?U<0iIM)2hymR59{KXuKptZI23x7SaC{~q4nhxyo7cXDFnci+u zVTvnUoVweZ|7~i#+k(Psad{%}@1@Mn&tGfJHaVPti8Z74ZO=auJmT)?=!hM+9Bi)2 zK4M&aR&?R5_xzv;)!si?AuNjKzvnJ5j6M+(7eAbooO~5U=o+@~YU~we0i~@`*<&^m z0dl7M)ztLdv+4Z#C=;lWue*drL_~I;|9a5YL{n)srMjs}ccX~N;m_Fo8z}VnUF6-n zA`Wpksk!nr5!Bs;uSYyjJY+`C4kMhsLLrRc_yw&FDA02w@Xe zdvj;osm~}{``z731SNMVUE7qQ8vXLt?%lgv#FkNOsVJx4-rMkb3DZF1Yha;(>N?2A zODVK#VTh4)t&S2mOmcXn9&zB%p;Z{VB!{w#;|Go&Wj+wW&qQHW>3J#O^WsB->V|+y z%urJ9K7ZN`GBR8*!})HR$tx(VNA-*}Pfbql#lGT}ciS2*ZNJ%Jy!*Lw>BVsgc{glm zEYq4}T8f*>?!?6j;MYOro-?hpN?0xGv$M07xCi6$%zO}eh&vLt1F_%QUs->*sor_O7XpMgSDij#1d^)_vyz}t^TjCZ@GVq zGR$~J8^81=TEesYk8ot^y|mbFpw z*>&h3jp)Q@+w-VMckd14A_MBx5Pf6DQs71X+ZcI*HC{C|yu^l?K>vP2`=t51RIY@f zp`qxJySdA+Z;2$gNiS~YzpOkmGD4xCUU8gjd#1&;-F!v-(*7jLr%#_2urB7!nE!T+ zPOAVc%up>ZE+(@x_%YSq<%_@aTJaNhcE-JK8v;hAel$MHx2&T6QtG-I>v?u=Aa0`R z@b;>=M;0yoQfm9649(Lg>Ecn-MqKAwOQUtu^++Yp+MoH1&ZpgQMk(_`Cu+utiN0En zF`pe96|UuceQEEg*7cAz_zX< zPX}UaFiVj)ZunMKRtDdh`uWpWBe}ZvIu}{oww?zL(4IekzNfcWCsQtzUy}U2U-_UG6kd%}#Xsq@wXEhtW?(HRI2h!@U>h6?OMs{`pXU z{+nc-jK-!c_00IgnGBUD2GTI=_CeEwa!1YH{@@j1U@E~3yeE36*=0yk@9C0k!#pYe-FLPFw4Yc?-|t6v8O4u75*{Vbw+IANz< z^Oh}98kZ*<)r|*4(T9#6J<2q8)B)H_*12y5t~sMf>tmB7UY3-S()YlzqEYO^c$ebZ zwQKb*#h|Qa8Re^Zn3m42%yTyIE}`}60|ha5BcKLg6kkzN*d&C^QR+1d6}UK&aEXY++K zPLRm}NZjc$dB$b1X@Bvr7guQL@-F>8ZOgn>KtKQx=ES%9C~Vq&z!=W6KdrJ%o-o%0 zv#2{cIb{!%w~^h1U3O2_IkEL*HJOd+vCivV<(?;icz*u+rQ+tcJDg8GsM}|mucMwX}s$m$MxOD+#gLeB>TQSy}m{jZHrgeH|)BZ-2kPp@kQR@}HkH=01zA zBb{Yji}O?dy6G?Geh*C4uqa!iMgC~XY*Wqc@9XnxP+DLFb=!|_z{bZHmXx$H%k=4* z@87={6r89Ft-`9wa~(E#Hc`iGyfFOf?R_co5roeX_=1uC)KqJu!?$_!W}_lI4q6ue zP|ySGKeH2OKBh3~9y^u~=-B@0Q{P<+@AQdFi+}$7#G)pf8RddN_T($PYOeKrA30y; zDSy7aWP<&A+}*^Y7%K3CYv>kgdJTZ%R-bK_8t;O79itFEuta0hw^VmI)3iygVb*RJ70%KfpEsC32)lo0e5 z#mU(Wr8p`YT6%P($XmCB1Jb%|a-O=mrpga~D#~nit*NOg`YxF@W~0+sB<>j}R<_*# zGUED&5890drw)^gP*6~Kk*4!V4WLr*fK=xp1A{vw?b7zGuTr$xR6;tu+}!QPMRtLTp3~f=Vy@@V?qywv^a8zDu7>b)EVQ#lcHO>x zJHD_`uXvk;1hdG|hn2omG>peYX*9I7UIJ823}xo>l!~dHOlr4QSM#_qHKdOk-GkD9 z<@)v2@8gssP>gEczh_R>iaP)-#AH@07y0mE=<}ud)tLs_)!5rkbH7f>?%A`h)L89g zfULV;$YctJ1Vy1HLqGFIRMbj)dwYe2k;k*xGrXEd{))f1s$$qWn{Tj6SlokI_y4;Nim`2UU8A z@=+9d^Cq>SVO#JWtGrCVurMad)B{_5ea|QI))Tkx-lYRW6}_dv=x@fcE5gFT!C@Ww z%2$>dA2)^ii~R53{x4Bw{VMEwP$K*! zA78P@btic;xR!lHCVJ#bm~pp#qLZp@2@I?Me&0S;H3DXvO|f7@dnk!7ij} z3&=RkUU^tUgVB^7`cLPHypL4VZFaJ=$8l{tPy9y|iUN1`yL*=l44cV}1FuZ9!}#EP zSGtwEOq(pB;VDjbW#{aNz=Kle$3; z5M>t=b9#g;t{{V@-P&gF&YgRW?kO}=?X}DB@H`T_l;Z#m0Z<+LttU*TYwy8>^km0} z^2(6f{?c#d>K@cZ@MtQEuCDG5bSMy=NBggHf^Tx?n3s#iY{x36qM`zk4Jt2JM2!jt z3Z$YOeJFdSr^m9ry}h!vl@m2X)?;!DrN^H|$u~DQH|rdFEVP*0=557CaeMPZ0hJ(; z6QF{aQ^~XWQmo0kow_0c(@GmCfz)3`A zZ7n9rGam$3(9#MiDe;40&(8KoOuUt9=mCfZM6LSSTeI`huajp#6|TaXlD#~ew~Kn} z+g%G#0>!+u9g_E=qlH98SE0^JXIN+szu;695^72Y6z&IiI(YEljS9dJ3dPpecItGW zC#^iq)rhup>Ec@Sf}8f6ovWlQwy4ryX>T_g`Bb#pv0`p>K|ujQ7iiRndVlyVF9<;C zF~~OF6j+4%QVD$nY8sjj*ZB(nQ_yQCiY%B3zlsoH-J>#(wqo?a4g*k*1HXCOIiYhuCG` z%D8FMChc@RZmH9ZO0zvQ802`*nXv-STy~M8J?OG|`T3DC{b;f!5FTT_=WWTH`_nR2 zgWCjxjfC<@Ys_zs+j)W(L^=SIBfN3r3ba|jwUo@vl)oz~2P)cnWpC3~9wL1`vUu)I zX?~;4$;I^uw@%NmdS|eYcvNtw1x{sI7XvL-wE}f>B-QI)6}; z6KK`eVA&68MpYog@xe5qK z#8&|Ps5`>|p&pYzuI#z+U2w+^R$iHNYH>-cuOfwY1)ia+ z3W|%bj`LaCBVkbynwGW&+GBs@FqTl*F%J(pu<9LvT?$LH{Y1-y(6iHP-UYkskXYMe z8!ZBljGmn1TUuIhJbCI=FVIdEkz7#?uq@yDUexi#{*M2kZTaTTwSoNWq0Qq8ZqkIN zjdVX3Se|ct&dS4c4XmB$i~uKsXB`}zfrcL{`|uC9X4hcWdq+k>0b)R;S0*PXgTU^F z@OM2llu=ezR&dv@wdme-@%$$zKw|%5OmzXT0QAkin2TM6atfql_)q@1cKv!}ObipM zA7$_9)0;sh3+ru-`|F=ngPryc=RVJQT-wSJ=$_}#{u*4S3OyNY&-h9&HVqkNAU3O0 zr}T&Iw?tvt3=R${x$fIH^(gs0_u!C}uux*@-Pl+O-kuxx?u8I6JMcb6_dXFtKYily zT9}UF&n(8m>Y54jycc{bk}khChku83%lPfh7-y!A+oH$v?As)4`xbslm_OSxoa#09 z&xbXV*>w?9@ceY;f_ot!sVBfyP@?d3Ut@Liz|HsuRuYdbiO+|$o0Cub9 z?&m<-mw&&L(Orj6$CqeqbN(ihEKs?m(Ng#Kv*nH%8()L!+go{sM%Vre^cM;xm_^{u zffQy43T?R->Xw#*^V6f(eSLjtr3_8I{&`!Gfp<@J5EJi?Q}y6EYtKae217V^E@u*( zs*zjK!zHHWasHXL>}w;it)c9x8yVe8$W+0;v5dNNhX|rXR3vCB5u@4u?9$`H?v@sg zl`B^gfd-8v44B9YgI;oBfEmcxA2p4pwZi$#+HG#=_B8>k*z?YH%BdO#qDt&Lboj8U zp59tqu`jywnW8#AStrV)17XyZ$+1pe0>$o1m|0pkZTS>F6mx)WgjhWg?otXZ?BXK3i!K&w;tqbp(rqle=xBR`?2=?Ju%jQ~XEALt_00>ia zjC^uy!5ZN%fWkj8xR1>S#190_09C0DLiTV)VpvFOstBBd;n6vKncWk|?mB`ayFACO zdMdBvJ$VugOxoq%&qQdV*ueMiLAD=LtU%TPOxG4XeoRQ>+c;%`28G{@fZpdS%i`%; z7C6fw`4GXXI$#xjw&g@KiW}4*8tA#|xLCGz>#jl@>#Yu4O(X~^JJSmq$!kb`(b5Wj z{8-A*&(AU=Rk}8aneW=on`=6LiSyu>?tNE9ocdm@a`*Hc{x;9C=loZCD0#_k41v_( z?X_sc!&%H~YIR4_^>9O~#_C0YV98B}vB;7zwW!-7HDSl%*iMe4}Cds}!?^GF6MZkP5?hXK?b-nJ-pY6Hn3h4?#n+eSI4{n-k~;q`TG~H5d)t99o{25|_-mFE@%ljCMSx8)%4oiFWq$lYMRi zx(B1{f%tuNWtiA~>3Y?TjXK4hFtJFJGYdpP(nk21f~&CM{7~pv4hZ4j#*n za+Ce@GB^U@f+}PM?1x}L!5$R%f~QweSHsza4_Lmsu|d}Rk1U`f8!M}J?_Bu$J=A+U zJ?_QEhU4=pF*+MIY%qF!IxK61bm?hI_GLOl1G z5U~@?Jx8UF&&|%dTUlERb*92h`fEi6uv(*?S2s87K?O4t^FfKH+PhxqvJw3THA~BQ z_qlI(#j~~Mey>Vc+_QRvnu|-}nXYF~OMI3qSXDw`B*3gvgLW4-wy2{dB_yO;a{k-3 z+3yb=U!XLqq1+XEA0_Bc%oE1l1VBbeR@T-U{Sr1PSR9ys4)mV2g35vG@HFe8L^X-E-`m9395aBmNyIKo{EAmP`P|rf$T{jQcK8B zXIiuFPf7#HgMntRjHc@O^7Mk!g$uV73#Nbl@&mcR`{^hLH8iTxX#~>WgP;VbrxVGK zfno&H5=%c>{Rygzpoj=vncHYJY>XraxKS^`117+YG?sI*H28BX=EbniH#3D$(2^($ z*n-fo(tel9?B2O^Cs>IZ5GwGc@Zymsb(xIlv>AU9{<`XFYOt7)u&}5D+8KIcFY4+H z=6~dsCMG7ba&RO=WaD}N{=M8b!(+Ic7m)C-qLdR}h8t6bVn(cK=>0u$%YRZ^ccSDK zl@iC4zo-=URV8>v{^@r-0YO35{dHkR1t(t%$K4Hir48Z$MPNIV6Ie*)(+dNF0hQ1j zw=l&I!;wG_5@d4v@iCQ!E@s`ml!#~U@$ykA5r1%RlaiDIxgoE-H&C`Aqa?LqA~0|& z*EzTKjE#k_lR7O3ucO>!N@ed}3Mw)2FT-wTX9H9LeF$#fuC@L88oHG$tI$zlcymkx z5U)keBo5m9Tqe>Ps|#&E<~jEtxIEK&(Y-%R2F&#X(BDHvPfj4T+}2A3V`A4NB_$oq zcC%@Sh3d)t?AbF4rK_t8!V= z&(B8;K=tU+aD%7Ng;L3TO2;((Q6;-60Xvl4 zL|_K8p&-LDYzTlj8*rv62{a;v6#@zusKgV}rt#++;o{vQN0W7l35+_uaSc0rZGviW zKwMlLuJO5^H#T)O_82r#Lkx+*LxoGamtoy<#A?xR0I?G!{W=o(5C!Cv>lg-%xR@9M zkL|~G&a&l&otaT4IMK*#TuotU=%2gQ%iB`HraT@ zh{nWNSNW}MTB7rB2PJrc9<&|pJ1{IPj94cx(3?&T{F!(u5C{rSt{3jG(5_vDO}`fr z2Ou!48X^n&%B%N}aoHV}q0!cmM7&*UqPvaTAlTFabbpNVK5E>kuqA_GTuaq z@RUX<&C@@SajZjwuy=iygJNd)EP!5I6_OSI^l8bB%`qUSrsWngvFLUfpBJu|;T$jHc93iOTb;R4DMQ7AtWL@KPT+zXsl3D-rt zWwWA=SQJW3Rzbw&^XCiR^Vr(085vsaStUe)UoW=X;Zp!m#|dDvpd4nlb)p+qKzjI) zqRDG(0Jmo2~WxbU2HPQH4lR?X`d|v#po)iWP-!zVoLLKqw{myJhac8*#anU zFh2iEs$E~D`!_OFk>1}12Kou3wEqI6CayU`Y(A#y^A|7PEvlQFpRe~@uga%ihuvmF zv(Y7x#IHhCKrg+Q7cYvwiou3y<+=E?XCHyclpUwva)TGpKprmG^fZJ=j>NO^fiA7+ifKrE6rR~+3OX+$xf%9y(dqfwu`y|(5NZhn0`!z zK)mCMSBHcVAB(TJ_Fu(5|1zguYUHGxX2#4YLu}TNCs`CvPmgrCgG3F-yQwwb0Lg@X z<(qyiwYs5!1&-~_=_d}KIRWg00s~i~p!q-P@>oqw3utaPZr=0*m|d4Qtv1DCG9uQ< zeV{faDao?y*(Cx}!1q=nPsJ#pv>vwTe!!CC^B2CqtwIM{#moD^E^dL4-V+guLF6+d z6sPYgWoW4}=|KUnnC)9Vm_Ov7?R@(5DdlMFbZ3ku{ObFewXFIy{$|3i-|9Iat0h8Z zX)m;8|5WU79bniJ*#|T%IYDoKdz5d>xtUp6MXv9L?#9G~LA0|%u{2)3FSln;FHsX=ZdWYL+asUx z2qu#807%UXk^m(BJaj2y=l^^!KmOO_zYpMnJQEEG3Z|yoyc0X5Hf6luK7)Ck_^S9_gRCI0YT368OG_?OTQ{DIy~%iOZ7Ee&;b{opY?`YStP5!!9@ z&b~kS8#4sQm+1@E104%LDfZ3-@A=XN&I2M45aO9MG%)b8RW?!Xg{9%ux;%rdv;#uV zU}**N-9h3_EKrW(^QBsmxal_lUj5hBd?(44b zqM`_R&u&m!oIGZmu|_BZa(tBgZJqqNxu|hZCx;$}l+!owDePGHt|e1C<3RGKCyp$T zAF#r$NWNmBAx1tJ@SNz3fNqtjw1xHe9z1wsF20r_JN_P29ZJ)uP&GZ}&gVV?B->%! zuk`1qs;z#|vu6r`$lR2kA3xGsc-ny72NjSDq)k_5i?Yi5=pQ|K>7!vsux&@drdW?5 z(aVcdZDe(vj5u_z>)BN%-rbfE!?0wC2MsOXBy;$ok{1u8V-Wkyb7Du;?`9haE!pNjcleiCYIJ%s_REfo?-xNL3!=8+b(XP zt?f%pyF0W(iR!oR+}Ug&_8Yds_iz_13+x^+CTc{VIs}6OAaQwVkv&>w88`3WU$=zx zSlbWo;%2tNx`I=0@*KOBP}NB>EARqZ*3FDjN!WY#>{e3rPP05Xaw{a1$KPxdPYrsa z9yVL5@P-Xk*uJ#@7fGW;m<0Z#P_WT|fX^k{XSe|Z9vwcSHWi_wXLIM%fg@58W~g+K zgibAtbw7_l;I0rowEEEH^rc>#15^7DSc1`gGD1HWQr=#^F4zOf22ZfWkx?Lx@8`@+ zuG>X=AxVCgH$p7d*ZJ5YV>_Q+VuxlN^m@5LP6-hk)EaYxx)(22AV{bR-3<*g$x_C) zDM>DOJDapAi7U(LVgKI}(dzq0uR|`-8O+hP$-)PLOh{^&WdMVWKbj9i(!~D0ArAr) zS(NJG>#HD3om^PU5V*Bn6Y49n(7LPn*&=gDYFhson#$2w1<|4xNKvgq@bkv>C?s79 zqCvSWJci!yY=Om8_2NQ!sL;L>HIR|Os$aU{_XVm9EV?bDGe3Xc@H0bsAh4Ho9uWPr zVZuWGM49v}tsO-wYloE7trVuG&i%B29Gk__{6vcCnAS(KwlVNxUs$Sp zkbXe?e;9grR}BDR1edvc)rS_Ln~2~X-rWg!qK)RG8WJ?BH0gHmE`5+ zJN^V=A)w)i=;q>jQBsx6N5uK2pfXYWr0fVD}Kp??k;2sK54Y@%$XGf2s zrtO8QfsQD@?^R)fnVx5~adZ{jBZ z0d=26&NoTB4fu?BS>gs+d#$Y#&t8j!%d;y(n~R;D2D$4MbaW@6bJ8<22SF5rm%(Tz zE!4EL7$p)zM#wnK?&O!cpy;H=lQ<15yl#>v^IjO0F)}tz)QVFw9)P!YPuiXnC3huO z^|Ph<;aRwwNsB1X+)9^aPdCJdRd_AbhokOpA?_4-^^Yl~bgN^~ian>Z&fcg&6gz2g zd&R5}_#lPiKA3#>140&L)#D4!G$sZnWSHf?c=1AT>sCeu(bo`jCTW+1gaq0Q zKDI#~lM$W-3=%OOmuFupCuyhle*GE%$7LNXGgcKrG!2Lr9I-Yi2Vi)0C{aHVIwRil z3xuI56cu_hR9Z@N8_h|I>I4lnHCdf&IXDKFXw$1oFARi_cv=HKAgo(QJPEf^W6@?; z>=IzYsK6)$DoCt}Tk<5mYUeWUjih}I(r5%79w>RqEv#W*$5z}vh;^_pc>%J}sYhe{ z2Cd-5oX^C>O+H4ZU?l{wQk#{!q&xYLodLu_JR^)cT>138hfki!pwH7$0OV-yNSIxN zCaEyhd~6o(3@aZW6G}S{IWST3yca!ey?1bNae?3U;~h9^kfyWvT9!yjttB9z#C{>T zlrj~a(c0PyMeYWzEHM*-S7~#+YFGsXA|Pn#m0HkWfocH|cbI(}yG~8rU&tk$>C%lL zN}kt(+aMNV_9>~3yKM-tzo@M}l&h$yNVZn?d5*>1=6o%CKnXny1dU@!j1TUel%$jt z6$N!pR!>Ynd7x!QYyDVEs3D}hF0UE0^>Qxs#8p~oh*cC{S%%0NC$j%Lr22jIi=YqW zd_1qFw>;tV=0FC@q z`IOG2A3Q(FvJEN*0`-h2vHMWT+Idpdu{4OVWM^j=xNQV6Hfrfk{g|~7AS-mkr1M2S95NqP#Af3(G%_iIOrm$s0MYB z_}Vcs5kujKEMmLi)>zQxC_|Pq?msP$yPx?~D2j*%MCL6ES(=)f6KH>Jm&$pCamog! z!3u~+81jM!^#pASk;(SN@kO#(<=H1;$NC~;Yz-2|Ep10k7P9XMbH`7b(yB4Ik05!? z_67?Oe=Ejx0a^_SX}XPlNx$_Q+!)zNNXdZsDCWp7`6C0_+n~I>fXV#%>xpM*Bt2I}Vk;raB;66n=q~xSLZ@ab zZ|g6?B{@xf4@AnGQTm~bX}s}MWIzMBi~yos+B(IIe=xO?U05073&@Hf=y3X7^gjA4 z&}r&h^`@q$zgGs)g6ZEr7R@S6eKtRn+GHejwX&2Ypg9>8(pKgtZLw2P`!z1CHZOHq zg}|s27?qm-)e!o#`5%d#s^8q)zJZv!WqB_kuC9`M?XY8ATB2Vl=*yRqN3L~eB zAThrm)glvtmoOTejz_Pq!0yP+>4 zB*X{V%Fg%K@86IJlSG|Jd_bnlT{ZJzS(!W_{He@@GvBu!MxF~P!p01JakoT~f6jtU zJ!GZ6yWQeWQ62P)HRxL}(8#FL+VtRa5p8e;bbI&@@R^?MD3NPqM^~~%vXRtqDpUzS zhz><{@^$gxkUU*PCk`9aQz<(snzl-G?Re9Kr4`7#)~kE*ZapXJ6Sr{ z=yWTI|08b{y+D=H^hmM;A|l=p1)-t@0>N(hr>7693Z#lfX&ia5JG**;OJ(nWL{x5) zc#^h6GHUN{n!J({qv9^H77HY#D2z`^o@pdQuYWd@Z#UOkf{4JusvPip5_Bwe8@D&$%qgat*57Fx#qYIGGExESb*^Lc>8u1RMY*y$1s}{ zW?tXKW^*+r@{mZ5f==Mj+vzObbZrCzUP?cTmXwicGdXPL?k>Cg+?NofakK5A=CA{_ zG&OGwiq{hZqr;3eGc0t7u4N#bbI%?k;f4&Y!-u^*a^wil=r6snT7sOjouNRKIQ9B9 zKWVC~qlMg!HWvvIA2M$FY!|@N|1RRWS_umauN%TaMcLwZ{e;b;_e0-U_$&s3J5wPx z0v1?eU~h_PX}&myo{3^b3==`w$ywLat$qJVTOW@`-j!rdSDz_I#l7jeasH06EKa$Kve7qp|(%-|8c6z%$ zdfbjX_m%KmQBf)i3B~{utfY`M<2#w|+kOv&gVoE=g=I>hnjW=PLE@2n1il`-~^Eqc;I?+azZe%n z3kb`Zqorm^7l0Se*awB8_N+n7G@$R2ZHXWwT zSr>Kp&Yd8{%V?1T&2Y*0fR_uKT^QLTtYsW%9paL~!5HX72ws-!jT>fnKlf3FJ>_vB*)8&#VZ zEN6BgE{F~D0tuNU2M{_;;y!F1iXcvll1vDkeB4IfiI>dJmX{WyVlN;rK|9)QIW{-GzjG*!Yr#Lr5f?gS$hZi)Ag`NtgwkKXV(7XLE&eJjHNas>Pp5Xkiuk{!YE zHM^+CPoAhFPDf5G*gilm8ZB67*#1FuwDs$on_vq1t0deAl-JxF8X6jqo^G3+yAQ!2 z?&bxZe3lbdR#Z3!0E1TJGyGHH{&REB9vle&k4i646g{PjwnI+9k@W+eC>ZuKf|sJB zGq9SAilj+T1NpU@8fpPs+j_3MB8+$oups2Jwf;e4LJy*8DW+y-uuV8&+zw58U?0?_ zb>%7$uNTq*6@}yjv2cZDWx2N*7orEr<0KRtKYuvRaga5HaUc-U2@{-Mak0qih`1JP zKXMcTwWJanZPE11;74^H7itQ+FxscBnSoC6uYiB@H|@G8lVQf11qTt8foKd$+xiL0 zTR8e4HHEMxEO2rRjc69&grdWF%UgJ?^WN0g-<+>S_^=5-FxEQJT?fpK76?T-u?8lV@t@H z6OkUW5s`W_O*UY<6?={INL{6 zQ(%a@ZAn75d4Nh1_FH#Y~j zbe+e{*tVj&DbPpEYg!EtJ6^3?Woh^_qpT|(vI0pzBP(~llm#P^!TH`nXO)+xwrS_o zQF(BEuvXGWgfezd%E zH`+>99l%+zqAkS?Ojh4k;pdX(569H#Cd+@Vi1*$qZ&^txkBNgj&BFz>urKbPkiUK} z+(jhN4}=f{?7w@$f#E8zZK4M-NJ-n*bC|3QzqSdlSM5tP`oF&otM9))7{P3lYy`Oh z>0J+B<}f^bNxqEnDzO6bGZi8v{_8;r|LryWA0PJbrz~WyFpL$%#Rmn6Yw2AP*TS?! zZgBp4@BiZs={)*tagdW_V8!zpEaVDWbi@2&o<7|JOzQ3+j;m6!KA?5am7c#ux+j7D6&+ISqO6EB&!TR#HKO3uZ2H6uS5 z62iYP%)b*MqDdVx<3b*4(#&o=h>-&8QB&ffVj|=f^L)& z9b-TyIV2;u?vXWsD#_fFk`E9-@kI=#5-~=&08N)xk_bhbhuxQ#%-YJ~oiIXRD6vq; z$vSc(ogA_m;MlcmI-fjLLp9?NXS&XHGqi1zNCVvEfLiUHq=9n87J05EEdulSG|Q1J z5w;cqX?_L#_X;_>1vovQ_V3xI(^|jh{0gA-_Hs`doZn+clDlK+-aYOZ{eNF{)q7;| zkpyd2HzT+4gAJz}kS)bwb8bZL$-^!t2^++S2J%eh!yBmq<_QP9@M;MrZ?H?R^A|zU zbNVyu{J+-0Li=LBz;Y!}CL#^zDjOQK?0=Gn0=yR^Yh10xus=%V$dN+cxxWut0Az;? zEthHPRT)K2!@RmNS^R4S1?OAZO0CK3la#TPP}S$t=*SH#Cfc#YLh4((f}S}z1r?Q? zk1Pjut%5-1Vj|DCZm0DM#5}{H6{?bOYsf*z489Snxfe+5ki#l$kI7lHNe;YXMh`J7T{!To|fl`TcfBDeq$a(gAik`bEf}*1725mV9b#x-nzAl7T5feia zJO|@&o3ZYFs}Sqk=h6AZ(ZclU`Td!p5+s$sb0-^wq{!m?_wEToApv%mE5B0*32CR# z(&h4p|9O93U*Dur*fm`CY8Z9cHU*>j7a-T=q6YRgYy7YY;(>~xX(6C21?fP1>wm>; zI2-lkQzr$}s>7Etv7(l#5jjlr(E=Efd4+||Cme8{A(J&dB!5;_MTJvP1#`bBg0O>N z5_pum0nLMg^o)<^whfb#HmFg~?1slBpoOX8Tw9@LvhIf6yNjFD(@U{=t~|DB=!4_o zQO=ClM&@w;(K(CfgFqeMSdPy-fHH zLZ~QdR{a9xW(Kv3Oh%wm1rb07nH`C|YJl~~sHi{?V~xX3VD#nsx5+f{=X|P?00O{P zY_dmbB|d41=Zr)U<2v2pB5ShP56WOked+9UhSu7#o0pe&q^q14K4lFaYv5wON40N7 zEg6judh+dx06Zjl+ByxTj6A;w(cLE7TE?)yN8-{F(x-c_LR=Ewwk^Bb3sPA7V;dF- zb19Fnk;~S3r&m^+gK|vLd#|V)U+&3$;6HV_(}<;l&Frt+i#4{3tERg>xj7 z2`BgoPN}gpT*dsauKF)Xpy#P~_RN_dsxt#$zbe@Ms~X}m$uZrhK5)DvOi5=E%80R} zE4#K?WHbIhao+zwrw7sdI|)XU-i6rQ4wp?skFn42Zt`daaA}fd!Y?JIUkl-^G1=Jv z^}$L1Egmpjb@U>_{%TB2Ak);T^5pAP`&>luBegKr|M@9Q|LuqU8(fH0{jITS3BC2~ z`aSjR_^gu$O)vS8Az~v@wf`?y_U{iP1N(n_9@IZqqAw&S#()SC%v%{3ee#}+{Tl*U zgL2fo?%;4Fyh1$Ypc;R-IP-tPYybbQfGn8*uaC1jATUCX`Jw0& zMDh7EFZS|omnd@CFhDF;4N9%2tm*+*JsYNn;%3F)9o}hSE6?#=e6;t1bH3TJ?+Y1jZrs~@M;F0#z$^VxFRrJQ>A6g$FO|CddVl(Z%-ApCUkIp{G6wf=k zij~K%#G_@HQyx=C4o2dbMzP{Q9fvl9gKXo||HmOxtD%+6zb$S4JKCMrX0MLfee-5D zPAB}+ccj;AB$p=VBqpRrJA6##9m~(E@;5^pLiFLr?b|CTiXYT^6-RT%cfSZ1&RRu(j!1X zM;HkUveY*I4_|pYDCaM-QH|H5mT{yO6|D^8J>?@|idY|ashb@FUr-8FaBFy~)!&0% zY91-89HJ|6u-NikPS;FtaAk)6UgPM}FaBMZ*?&%EeH=2lR+U3o0IvPU->FFqI+VUb zQj(&sUfHZy?(lK5<>%r8!5PalhmT6Ewx1c#d_x)P|6E9AYDOg^GZi0) zuk=nmzSEDI3U|d9t}Qvg^2}!gv*}6Oo?wBmFCRoxOGpe=x-ENmOkZVh+2)>;<9qAa zaoMbhzxSHfRsWt=!lISgK3ciIuim(|*nviAE~)SLy?bdUqr0lcI&7{OWPjb}7!_FT zHri~#MD@2{wd{#vc=fiK*-qvh2JDP-4#RIPayGxF_uf#r8<86*N5q*!@mCh9?OaQE1xej-M+1N`46w-?UnLcT2tKu zjJO)GJo$68PE&>_$RWFC`Ylm=FDu-LWtNqv!O_IBIMx;mF%F~z<+5CdR_wA*>8iLS z^|QrrNbqhn!&@X&+Bi@dz9X;M3Ne3X^iOTUkqyM6ikdT4tFKUpTiT;vpb z%F>n0-XVsq$EgE?cyi43mecfxlr@)p=KNHHeX~uzZFBslYGS(n!}I6!;~gDC7W})` zGF&?IxLD$|j7MQ<#DiPMI&KRk@e)N|E91or0SAJSYI&fmfx z62A(MZ@L>;l(ezuaYJE4f_9qf@qPO`if^#?ER4VLd-(Xmw}k9V7pAxd`H{_%-M5oX z+J%&tu_|r4v?ro#D<=nsY}d~k6reQ46w(hcQaKxT1A$lVGN2&ME!!w-C(d zFtyxatsMP)D%&o)a(XBe`dR7Q`l!m=qET@t7CrDxhi0GUeiqB=k!y59B~Q}_ocsG5 z=ac`{;SY!@nmze;aAsG6CVQ*Lq&D5wy`2TqttLZW-$dTD0NHJAw+(l8(F=Y|M`QBb z`_xOtxs6VGY)h@M{?Pg-12YwiFwto5IJ3abl8f{ElPw)AiVxbzc5hQhS|Kwb{g>ss!y{nOZQ_@+JIHk%8uTv4~+qQ}4(g?)d zq{Z@%jFgnt*Roy1nZoS+tp?e-B`L|9G$LZUM)ao7j3)L5@XGkIDC=o;P*H3esF!Ee zzYXA^)W~>e8O>_0dV#27+hfjW?7NVqe{1{Gr?%T!ExbEgw4YM({@Im-Z_#muc)-@a1|cO0NK zzea6N!`je%P#2oxSfpcJ);|-XbljkvtI*o|Ms9Zo=7a{6vD+GT!$1;X^ufWptYN zbvZ|Ny)o+eZ20VZT7*`I^X}c1SJ~&E2@6qkkBD*nhwwq3IA~*SEf(8w_Vvv=nYjBq z-!#x&yWXvx{H$Ldw~0|Ag#6uZ&GhT=zC_y}Ms6 zD3vuaG4S+3;_2eCk^ed=l8~vYXTNjj5C=W~AFkZvS1jLf29*UR<*|l@=eQ?!elfND zb#m&J?q0qMvH9^gUFLl4DymoJe!nSxS(l+6URo+y$ksv!wt|P9V4Ivk2aApU1!%F$ zK4`PZ@BFL{qESQpyTy4!@mL}h@w7JvoUOa?5duD2Jr`e#pjmWK|6+7dV^Y^FpcklJ zBTYRq5fv=AxAWTFn~4%_H$?KY8yNnd(#|}Z>a>C5SCX=fOpz-~GAZP(muMlkLTFOx z+Pj9i++>L<7a>b3+U8CRHDoE-F0w=m*`gFvOm(lNlv30cVo>(?`=s78=ghqCdCz(O z>Ksnj{cX?hdA{Gz_xp%xBneWTR>Lgixr>QMK1|f4)kX`j<9iQgQu_{F&RE7U|loSKpJdP2iK3CMdSkJ=4wFRnOVq z`D>v|(`9pqip>^&n#!$p90iex?xvv-nz7q6y!hlJQx>sf%q)0H20~|t3dF^@V+p_OVzSHXXbrbH(9mVZQ9#MHLTEsS4Chnk~-744x#% zvGUJj1&i+OcDY{?=tRu%Gb}aFq&{;K^2h0W3mi+W}H^VDlr37>w30})h`=0k^zSvNq zZS0V*X2l5^%GN}#VP{osT0lVSMXlvq@dy|XX2N-u!2*_=Fj_uPGchJGz56C(6gt0m z1bl7pthPLhR4=>s!FPpC)zR6emkn?9W_C7oJ{SttH#X*+c$_%bDs8KU8KP++5Na~tO( zs?+ru;ZwO~+M8HvjcMY3!`==pFTx%>e`3sQUFZLc?^Nf)%172^c_zoBqxHPJ+S;U0 zQsat0xU=G$ou#(@C8>Nq8Ohzd8k)ZGlE3a-|3k%4T(HuO0nSzoD^lB}u>#kvbuo3T zId(ja`^q#i0grOG*yc{RA_nzJ6BwXcoXZA zpBkK2&S}osPF`@$Vqy{IXX=J&L-Xe@bm~7;qo+6E(o|j+Y&9ad3tjk!YZWVpa={kj0QP`uii>rJ*{W6709cX0vNN&Ag&o5-T zr#<&|wff0rN7j=w12ZZ+AFu;`iExTfT2WJzxis`ZK3iSartCnyY{#6yH|GjwL`+R= zFMCZNnG@*yoL;ej&lkk4vam1p+sLprn%BC@HTSY{nIxrrQr^By-h+26)fA7AcxrOW z8p!X+48tujf&CBh^$4P1c4I-?EOWxQENOqI?u%NOt3U7FySIA(@{m_E{U$?kUS4-) zpUZu8D7PzFG;G9VOrf(=FlNZ+de8URd12JvqB|kV$RT?nR_Ph>Zqlhz0gw{0!X&f| z@a;ylp&_$9U(4xQ4!>U_V06I((V3!lj6HqM)fPG86g zz7tjK#Z&oE6)Wvn{G$hCjVlH!C?PzCFFb<+C~wiBh*P^-W-TQX5YN;i`oyQJq1}$b z>%fM(yZXE~Ywt?dq?zvdhl_9CWJp7jAg=gWT>f6RmZm7Bngd^Gc{$+Wp(*)1PJhM5oTL8dv3B911$b*Q zw1e_Bmo1CnaHP=v7SSjcT#MR4q#mgD<)Led5RC{NfdXM~Zosz_qF~iboSt<7Q00gO z0oRvUb;!`e!Cedi5bAAk#cFO6M!GdyEOLW9zp+p*OT#JOl{A(=9e5Pqj z2Nn+OwI`omukUoicj$h)iv=NaRb*~xvs01TdXw48oNuMA4%p<4HbX7kOKD#JsCdUp&T@Df2p*rtnJImKLHHwot<|L1A^13GUE5D6V-CG z>{FbNamQ1wA9Pr!Su63=w~1W3^tx-cu9mn@f}T)sub?M??Lt#pr@UeUMd;cFpQUzo4w zzbt>3lwY5-1J%xMk@mNrI;48?(_KBeVNbYo0#r@8O)6SbH=g`n@T!qtkowT~IP@l4|o9p7kcn150^&-s!VrSqYP>_6xn?c948*1A_ce7qy1$YZ3kvxgzu z>>TDWK{V&s?53)R(^~tzd-s0vX#hKhlDtSy_gi$0MLIhan@a-iR+x)Wy7@xcwpm6y zQKZF=8b}J7fqCUrnS%~lI0>Ojrn+^C6DUNvHF0eery|Bb)mc;@pg1~?Aj#XxTVAxz zA`T-29Kld4BrVbyRygjrMKQZxq)#_hUsrYA$3D@p37h8wwuVKDG-p}9wsr(A6mQ(p%&=^`Kbm=_CC-}HxFUT*>>FFmt!QzKqPH(yd%BW& zTAmTXK94xGE1@e79v3GMAH?X*I-Eq6ci9|9=yx!1YCW1ysHI$pTC3p5fM)>VKx8CPjrqUtHojGBQum}>T&{YC}Q0rOJzFwGk zk2VX=>Etovak0Eo49zZD(pb43RUt%Ei$>QQGd&%v7fO#`#(qWW{X1{r4`g;?_A4zT zBZ9_a1bDm0pM758V9M_s$d294CUQ4@o`Qju2tz7q89HdEHTt7bOXX+p*ntTSes5e- zIqk`Q-g4lJ#JnkaT0k0%j9SV~X{aK)wy{l?xvVQPLQrh}ZnuIj|MlQ!1qgz8X0rwn zqG5&bAdMgtj7qHbcp^nH$w;aZrRemY3&{Kz z1nFRVfHq4^{^88+mc?e5Lm>RE+50-?V)jOMO!Cdi{`LIyzK_PUBT?{l`_3H^H8rU| zlUkhk^XJdQJI2zipt#sQU-i*H0p3lPdb)l3wOjcG(}&%kMCVy&RN>&?klDXDQJ)f4g1d^#JMuBM$l<<={6%fL|{`)7%i!QFZ z`NRgKWH9`)-f}G!Uv*sx59KIJVnJo41x{A-#et8BjSztwWA5MhV4im;Gqp7{ixf9(+WOE6G7udnh<+YMX>IzrCaLZId zDGp{(i~SIq#%o0s1#qM-9^DR>t|O#A5l19Iz&;fH&NP|>oR=xr4W2-(Yf8aF>`TF)n$905w z*UWtP8*(r;`x@3S1?+%|bIfUTwMjA+P8z|>0PV$~8JzcN9Y_lqV#jYhc~nAEM`s6m z*N1x^w`R~a4s$phP+#-l@IM+xnnN7#l73L~Hv1>*CKYGl6Toq_6vL6O!CCjBhfg}? zl;H+Aj`ZAapRO+F5S&&SL~-Uq(U-;rSO5|!J73`iv*WI;z9Ls}or+8L-MNGWaSBP^ z@C`v%5;~RSEYU3{4qg<+NF6A3sv(g{1dJUSS3yeRGyn=b#n7B~iHl36xq65|!Nyer z;X#n))`o_%VO*6D@2@u0^%g7+<(y2wh8!+iNk1&$f4h=!G{BxW*spyT9i0?1b0PgT z2z<%HuboBtZpjj!iOo?7luft--+SAXrwp#o)|CpZlXNj4EO8qzPEzoO?jeLLa`;VU ze*p+bJd+Vxp9wH&!^Vw}Lm35d55=OyvQU{WhHN2PVGMzR-|2WA^k@f`l#$PZTM>nne9Wblkqcz*@v7$;J+3YX(Ac z&KEOLwE>Ja~t2XkzpKjXyuw{!FnNK533T%Yi zTHC)~Z(HbmCsU>2gx?Uj%RZ2V!@Kq2wm1oWp=PVj(oFIZ;(+~NGJZtV5+RZU9xa&3 z-5PH7B6tF7b4dx``bWiz3JTczg3H@zFzsV-{P%D#AZbrh#UKTj@4PV={t=o0%+ubC z;Hw~-10(5ow3#3lP=(wdZo_H-baP;SEsL!amqWVMLA7n%x>bSziKsEc9s?PCHu7c>FwyX{G#~g*z*WXS z(xZVbVwSG)thS+yHR~G%vZscMy1MzO0ERpPW^hkb0e9_|E3dtKobzM%Y>Y+Lw6L}t zKdBO?5x|C7E)Qx5DKE*%LG}eCF<#7(@Q%n4!H&UWP=)s8qd<`9&9{+@A;6+}9%eeH z&mSFym zMjHd9A`TPMN)DaI;C=$h^`w^p?gM^rwwlbVe(ORl<8O9(_*U7`*vNSAYXn!4?!=AxHtBo=TizkL32(H4eN`WGt3o zy}_u}CFmUUt1~ifp8Ows3QfPI!6{UsM0h1y773ywPyN%o>#G4XXgo~?WAqXMxby+Q zwp_^;W4G7Du^4pw`~05-Lkl|Vv}F%7`vLw4@(j|-$}4=tXV0!AHwec1ClR`;i^al}#=O{{`^oSxo={ literal 28667 zcma&O2V9T;`!;+<+9;txLs2B9NNFgfj7US0c4ilS*`pu<;YACzCkzjj|dpnFlv-r}N*$%Ql2QIm`3&)Q!+Yi-K!eCC3qwf(sr zVp3ufqWo4DFP?Xl6&JVruOASzzhEieZ1S@Z7hybq@RTD(tu!J3&?G57v8E_{t3%3r zwO#KIw_eiM?pdH8tJ!|`s?Ylseh=2!&h69DGl7ZL*V@+Z7_`wd z5Ow&Wqi1-e{<=h!{&~Sn{X^SaG}!iP(9*6pkttTJwxdHf7M z6mFKZPWl|bkPw}inAr7$8-uQ0d)-#(z9HgZy344cqf2;FooTwD$}8beyT|?{#csb*POe1Jo9=TtgW~7*2IPT_)yJXzLd7N^DWLc z6(85ve>x@a?d{FNBfGkIVRq}jefx~C-XYxDg&~ z=joiYb!R_RR+c$l% z4Yh99S*40F9ur)@l%M|C~7O8`YnxeYt4^%3C|NI~> zwYb7flFBJ8I$@=;(iY_q#``|8J2W)cT3%IrRguMIOfUM)$Y9* zij8_`{rU0O_;_i;fom6MwLjPF6NoTw$+C|2K7BNL=iazIp67plKSY(hkK9_9V|Tjw z>sLW}`So|ijT(11@_GNcTFeqz3@$m5Q$9~o4&i@+k zUPkRxQww3`;}b`8AEWp;uVzYf_VV)DH#a@JY~{+8okokZchWEZ(2Q3N=k-+FC_%zxj^V$jGSEZ{;4=+u}xIEPQ+P zDgHlwii>sf_BF8zOQ?9Wim-jhk6$1AVj0KCyjGNziJ3WI+vyyJRl&;rE!nAmzdl+t z_l#UQ-m$ORH!d!&aa>qER#f-&X=bg*YHT`BbvG1DHJorS!YO`%mmNqq=)XhB~pMJh`U+I@Gr>q)MBLD6f z?zHSloR>(!M75l6T3T9dvA1_xpRlknb52gqJ_MY&no-0l>)N$C!xHtS)oDkB7?zOwulr5KQ>_93P$z9J_qqev0C9vAog%O1<>^)x<@?M|JF8f!KS zXN-u5i1sF@@4$fr`vx1+0`~gRS?47C8N3-~-plIGCaA)I5~}xg{rdI(O!x-|}+hsvI6UXMSmE4zsr*G!%oc4-ypH>%isM{9LEO+r3*QC0AX#bVMPF}ds(P%4Pqhc=}oZ<3Zhwj>^r*>3Lr>Aef_~X-o!B@!kKAD+2=$3Ooa(dd) zVZ76}k;Q#_NaB~g_n&dqi-V2J1yw?7FBvu5Ss{J4*{7sL>0*EF-of0ZbSxAV(>-8j zXBUbhV8i>hrGrmcR3=Mf3pPuqV;-E=twKUWMMEESLe){xje}8|g zsn*&W(~gqMdUqz=XGWG&oVzbPYMi;_mOrbToSZD_d0`(r1Kj8s#ZQVRhDq4LFrlNo&4~ZT$i^JN?CreTcH` z9CtUjgi8~>xfgn_HcsoL>HR2B!A)n4XknYz$zJUD@%N`K_|) z>(`1KhCF(-QGRYH^Vh=MsKmwo<44C_@2V>Fi5cYa3kWQs9vt6%TwB|Ji`L_gcfxW0 z&!5|;?U{&9In=&I%;h&sYFd6%@O+oHeRYm z)pK?-IVUYHj_>{Z_vi8av~+YfIqkU)OJBWuH8wGkdV=q52$vt?ptFL{+yBZHdO)Oc zAVaI^e1vRW9h*qxVRkx}#F4K+0vD%Gw5?@j^)bkIj<;_w3SZ5-wYc~7l8Z`8)WV+$ zbwp$O+s;=nkd_{%rysYiANlsmk$T{ioQ^t++9ijx!W^X*n0oSg*ER8?qy~vllg_e$ z<}7RWJ+u9SPjxfOHpl&bxc#lA#^-Kq)Ty`f>RQuLm8=C7VM$?yzkBbUKYzXiHN&C% z?eds;4spZs8=H<$)c!yg8mj!w8|!S1a2xsg$TZr43j6%-U)x;Q`T?bPxu-(@8d{7Qvm zK)5>3^ph|B=_p+a9I)!#xpNFFRy5kol`)GO1AL<6@Vj=6IUyl|MPb`RMh1owq~;8_ zDI4=MXSjNHc&~o@OVma{MuwAyhUVAzo9bB$1}P2G@;vL;+f0q#s%(}q1&Cj=WXTv{ zn%T#@5)3RX^?H}OUbcH7Rj%79m5x&ph=knzLA+QQz_qNb?0kRi29LSnm&Z4+?r$&l zPMedk_{dp+!j=B=Wd+#|Ej3rN8=00X-DXBY5qA$zRvfortzf!;M|pU5?L#%>r-E^Ek9MSm7bAN`g@`; z!{jxc=&2XVsk!Hl7L31L*PQDhh}I|I&`qHYcuJhU=7LK*fV?iBYt8En;4mziof!m7$A^q-I`1QB7L(j8+-@YZ9T3X{hcZ|y3 zzEw#bcC1mjBHix6PO^mJ;&1hsd8c90SmDxa92{2=(A+&co^LbrSSw|6=1jozqFE*l z4GjYRg^t~4;N#;nH#axNC!N1=p$u@wd~A@8p56=x{nt=tO~>$XD7p(

wPaTXs~v z-DER~K0#P5O7g(i%nZj_JG*77s;cCelyMk&Z1`5}ZS?usqesejrR`&D^PFr;af|2_ z*7Saidf1@CS}w_tkI1Taz)nrxu1BUXpp*b@X9u6V&AB4pH|*M_Lw^43*|W6j>gpya z5@mOcyppZ8PQ>iYMiyq|k-f+|G&JOo+hD#YZNC@th zK#F0oFZ%uXJPw6kOdrZc0P-y(nk0hd0J2Ffw~d@|9qYI}HeSKU93$(rid-6)lcBV< zw5;ILgg-!8VW0N+UGlxK^}7X--c=Dw9Dn}&@lQN*N9Xtb4f_KbC>#AgHG>iJ@vb+r za})y-LE7~1m2)0tuU{`iVcH-qeUjBb+Ov1StSRf;A*A1nVRBx7Tx0YCrRGH^dGb?c zl2axWz5e8~^6+fS&CMmig+TW0Zj)9UcJDs*S;b&vqnzC7!B?p|X`e*m1sHruoWuhV z?7Jy+l$H|p+-D)kM%y-jNZARU7XwaW^DDcs zb>qgWTeol9*nQ1+5hpnfDe>MDEsI-GQNhUnU5L!>E+@e*IFyG4srCNs`2ZMDf`qWr z-roNHZbv$Fcm!^sm08BY!QtqVaqLlIVrR+arN7X@pZ_&xqF><31}N4$vX<=>|FKwp zft%KK3CqxHj5nM#+8`&lZpDffuCqU$WSs9TQ(+&V9E8TogW2m+PHZKIm&HlhTylNJ zKf(iP@xPc3LX)(aoeX{PLQ)@CC@d^YUn~&c{kV^n_FJJlM_XH)?3?df&C*lfP7OAN z<}t{CJJGO9eDY+!(9IKH152NA<9DcX=}+(8z57_WK|mnk?Pf(qzM`U{Xnjo{x_p}? z@nYHCyCn!E6LG5Hd(L)p^ZyIpWsFx&@OZK|mHZCfSN_PuRjteMvo}rB-D!RK z`L}LmMmp^sVQM%&(OcE|nc$$xuJb>Ywg%Zb_VDH(vYhRCAk9Cw*e*9RD)F^>QrC@PkPPf1{<5)iZ zO5VFUg4xIU=5P<%eD!n2FTEGNSk~=g!49GwJb3u-zLg+K(8oV?8EumRc4&3U zTDyw<@?Fl37OCN3F0L+Ttw3-ga;|P}W#D)g(IR8*E@LIl%KRBo0)F*B3@0gUdid*7 z>eJ`X5;#w!$w#Z~uP;{7^Wl!v^OqpQ07^?rOKI?ns>s3)J@400CFl`@K{~t!IyiRh z80y;F2tief8ZBPj0IcAklvG@qX=^?i`9UTF80<+IW46Skio1zT(s z-MyR3vL^Nol3d1_4{LATy2X!%0HJ&E(4o@HOX&&vXW^CST*)i1gd)6}X&EVf0E7e# zi5E`Zu%H_N%-nGE9XX8a72H;5TGj-HhKBB)-;t$`HGh0PCMNVms?Of{<0TCZYv$)> zR6#tv%1_i7WnQg%;J^kEk(BrI0CQZA)uMdC#Vw&o*&@jSZCUBim(1m|It~sJ;Dn^Y zCnhFrn|^Q~JXFV%oO+PeXYyi(fotx&Jes|%*YDpCLE`O>JZZSs*?AWls4*mfQbZFq zR=haB^30htBxUS0v1>>2$irwtDO@<(lb4rQ_tk|>SQ}PuZky?@ znJKUGXR=td5&#-RO8ghFc>zH|3|O|>j;7P;#x(b1Vx)|a4&DwuyZuqEJ7-5ds0Om! zgiaC>VM4!FIyYLplIs7Icuh!&rYo_YnM14uL4povf2KuM?Ta%psW#>Sp|QPgxXSyOA9bqffMM~@$O0sn#Q&UAX~Z?7pXkK|OAWB2thT~b7zXR67ForFA$eS|7mx+!0dyD<0j{4>&a)pO-p4w zex>*F79c^JTUwgcB^<~_pNfQ(fkdu4!j8h|gdL$26&2-YzB%d^dnt^x6*7y9i>t-R zu0e~xBj@awgqcfQwrpuwxX(sQ^W(=4q#ubBO{onJ{9@A5wxOIy+`s?yi;j*?;P>z5 zC?gdi3aL^3R>s{bB^9df`D#Mv!ozl{7mJ`oZJ8cz4@U#OK~|Rg^5x5}*nNk^1y7mX zyGa);I#@M5<53Cc4NXeg{PCXjN*|eb zrl8h>!oyWxo@*tbzsmk#_$O?WfD((I@UifqkJG*Aa9rGokybieW5iwjm?UTo6Kwh2JM zhwv%#Vr`WV9%Sh1>Pknl0Txo}Ys-(miu>^Jkk=MjgN%Izm@y{(JRT09C?1#j9$qiNsMmEeDpKMqs13>xIu#lS~)TO9l0q3bx zr$D@wk%NWcQ2d#Z>MqE|$}Qw|PFeZ(FtV?>MNMqz{rl^ycDcK|vvBWZxbSMdgoMO- zXXj^AgQ?mg&;Zc&?H~1Yh6Vv8z}nhcCBLHjy79?TVFDQkJ-bDs5p$1^-qTT4z|tqo zcDxN;_v^=#xZ3ANiX=by$kc(Ep$B`$iFO@ziZE1K2JtzMG`HLqJ>`4%F6Y6GtI|Re zVre-!I4pDAoR`l6p$Z7t=#!N^aBVH=EyjNSEW5QuE8TfmuWF&P^5D?dmn-pfx`l31 ztC<4o_QmL3yts4Mxo-?{ho#?rFqb_0Etjp%qTo}piKe?Spba}9ZKdEK!~x<*Ql}|8`YDwZL%0wy)83-qKe=(H8Ycv-nt4v&bsIO zcVm!E$3-H4p=ayp?F~F~M+`h>2`<9t^=pH^zCP=kHE-go8vaRqnc%9JZrr%>KwBgO zpeSwY83Idmbk-Iuc-vJqH|sh&W$ra0YG8tFeC*zoScN>RIswtsIck~3XgrW`0DMRr zp7k)=?5}8e6GyldP)6#FZ`0FL52L=J%Lvtaa%`{Io&92W1o`<(J{H>j;vAan_#Qm) zdu|Rjjtd1xrO*SLD)}6uKo@$e%bLmMXaytg-An!Q^3sIG(e!mP=lSVYthah>s3$}x zr!e656ZtPC^T#K2=lc&GV)P=su_r#fzv($2X){2QH*BnE>cgVxNd-dXLXgu2?9;IH z1jg=`aT!@p*bI~ys~jL_lV4+Yndm+;Lb-P z^YrvIviut$@||-~<`7K9Cjz>E`Rj54cdcQQ>c01ZaT`>Tr~!9i+vZ3UO2zXt62{hj^3qBk!TmY}Ow z2{WakrPP^DV%- zg0TV5K0i?(EO556N*V(QVkSz+*ry}n*4gTZ4zXcnclHhR^$8PIjSaGv^HB2!h?JdB zAfu_T#fqqX)F8BB!(OYpgaDG~A?kNR110Jn z@;8g@jOzTipdfC)p`o)TQv(fpizkbqv_ZB514-!LfaU9UG5r4h+hXWUw}t_4a>#xj ziyKi<*7eST{k8E+V_mx#JtnH8DIY*f{!N>f;ob$v`620@5a}KVR8$gx43Us!pF7P? zTK}4zEd%|VnFcP4py*?9dzMoNj~?~>eoCDwU@rij)(ntl&%`NUX-$fP&yU)o{`&_A|b&$CHZQHN3^38QKC$xo)=2nuLcB z#|fzimqGZSJZ0t3FQhDA=sCP`)27b)WG%fpPAjp>o0sss=Z`Q zF-R*XL*eb)SqY#~S8qByhwmV*T?3k0LghIRUj+xqJyQkh%04L;NFAUg;LL~H@mX2* z8#4oggG9eT;Q$-o3dVtCa6*FssX$sZ#`7FJa%4XaAC8j6+cr(R?afia+9C}hGACBJ z3{*gN7rJ+-9{ofJ*A6(d7C_@o@N9z*Kp^uw^-q3{jhlhtZI)mg6T_+vwp*%n`Fmz*9E-pqeL^$s) zu#lV|`L>%V7&xV_qlGg&=eFKFxDjO#r)C)t{uT6HS=OKFfYYOQ!GwSgY3VNewvY!8 z_*po&Q|R+jUKquB&2DiX`Njwd_u^=q!IK)&yuk$#G6wU^smHcW&xpbr6cXZp@7}st z&*@X(!TCYxKr(C6%XIqjDbaP|cV8WPvK=$gA$tWF+y?AeDJz{=qTk%Sd9$$y!jj;& zZOhTw1{N013-UY>I~i-$PmCKlq#ekA*=|J=iqgO~A{#d<4>~7c1C&rWj&x2>~%WJGe`XD@7{<99y4$V>b2b~t?8oWpta8pn&m*rIM8PV_A%Xq3 zhKPy2nx>@It82Dyzx30beI^j(7Va{F(V)<6&EdpV_>1sLt8>g1+S;)8@3$fha?jHs zFb-&Ht^!e>J$<6!$}`De}(Cp%a$!8Qh}js=d#@Vd^5N%qCLqL?Tar#{zNAlfUIqXt1vJyY@fr&eR18A69KHW z3TViW^h{Uyyp(W^$8qgWuq3sjM?WLx^|46s4wZtMH>n-W&&v}4i-ab-0)C037uN#X zZUsq+Qy!?9tbE~uBw=&r5K4ZhGA(E*0Mm4|W+FzyIP@1@-5>P7Strr_qmGklb;aAa zW#DiSj7mVo6qTlzy>Diul?@dD;>X{V)Q+C^!Mq51<)(t)8Fhsr~mY#{ni zLwvi=a<+NR*@2*Z!SLqI8$4ZY-uD*B{8esG)#b-%8SK(&YJAW1Yw_a=yVLaauV?Q7mXLNFDkf-CDaZ#QJaTK1UUYNLa>1ie z3(z0aM5tD{QG8(%DS7{%30k&1;aZ*=4tBJe9&p@QCd_@!?NK1qnd#3*?LAp8e)-B^VSJz-6 zC<@-0<$NAvmvz>Wlv)CV3><+;hKN{RlGbnC@<(!Hl(uiHAIe80plmFqNDf6h`PgtY z;fiUxZMn^hXwc#_lVXJpS=L%dla^9w@6#3;XslG0UQ0Rr49Bm8^oQt;t>%q-hK53* zW+;IZeEoWVW=wq8fE=A1`{vC_Zbm(Grd!W!*rpTND{54vKPYBx{WWJB5Im4ZTBiPT z>i!;2?rnNmK1jnLIvo0{d17`&E;=IP%kGsx2S}muvA7?`&Zdw++Mi@?0g!1ZH5B#h z*RKajNet*OfmG0eLap(GJ`?4B#9RiSzWsZwv{rUn7<5GRj{c~_o!L!>pYv+O5V|<# zZ(xStk2qLc=Fsw6_NANsQ_n+7J7=5n0LAP6ru76~b;gB0PH72GC`lLW7o z7&jeMm~J+cfz=P&=10_3etrtP?W)^2y|HV0gPPN!xWmVN~0EjoJ$?29gyz&ztck~X5Mlc@okN&o|{b2Hk znD0G3rSIMyvTetMMclp}h<0u;M&I}lA%)PM0NW=bi9?-F>^6N<>&t*7fP{gmFT#>XlRV-Xv$J?t*vrx)uKpTB> z7uHqEQ0FP?7P_*Olks6e+(kPqt2>Aak1E#r`bzfnmoHz4AOv$M<@4^Q@V$rElr7%G z`*aa+NcWwf zcSudG97GoBC98G;;A}c_n_pJ8guV+SA#mGTD!AjrcWH21ITg+z1vI$ntZZ=xMNR=**OowaePtc!|l+A|H*X zyn=$1$uk^?r(0Dg7iG5IhCU;-X;b3+Y+11VUSR+AotkhAaSxlKnBUM8Ekb(zeGJ7 z7d@%GT;(4=q^bX4o?6DNJ8@_mX?-XysM-PiGA7pI-om_JwG9ryrvYK zQRFLZpBDz=)@%l9!I_N9g{leQ!>On}0;voEA;bT`AkOKn*E&(8jI+ zObGyq#lzkLxl*mHrG*wzX16CVxxliaO&0lT}yGv2% z2Z%6Ez){QG?5u#8SapZ2v$Fa6KT=nD2k_jWk^f3XUm@qhCKDnO*WJ?-0GvvJ#1%%m zLqoija_x<>B>i`EdWG`jX0+gA?TZT>q>QJgZpDF0{KCP>8AOUddNEk_RxmL!k#p*? zI6p(|2WaRNEl@a_T4Lc15hv(w$D+)Iyq?|8*_7_#SO%DZQPr8x8O5h(Ltk*_i;fLyRd!p;I6A}c7kCSj8wodm! z9}@ZicbaH%8iW9W0K_JuvYSIdQopBVk`!3NGa{=&q@5i4!v42qij@yWubQE|U1Mn} z(&3x`0VvRo622A5Z>S}knTv~yBmo+V5YiCm$I#mh7O22|)aEg9Gz(Z3GP5g|Fnb!o z090-a{6fsgS<{FZ7GR5v=n-MP<^R0G-cY{#Xt2sERp-FK)%g8a zx>BDPl{KWE1Vz8=Xg*vWS`&-DZV9w(Qt|~A)wFQ=rhXHSJe=LQIqg{*ESNfDDJg03 zJC4@GDX!eh%F4-^)WeupoWt2RsjxOpO0EC29Epcm7 zz(;TlZ$*eu6mDkk9Weun!f(M9L4zRf=7XN6NLv#O$k{$H_%6~cGys;5~-JlpjaVe{>V!8c&>J|;rk^Z#?MR% zYjAfb1~U-j=h_MwDc7Ixgf*Zj^bDpfwLQ^yOj$~pff!y0;K7k291}(S%1PQHus$vW zV8ETAuy!P#5MX|5+{BO#e>y0%C+cyHf2gE>By-Ymg1GCz8lw~L_vBf(Ze3c%s^CX0 z+fPQgT?o8)@80D?VF8Ma9iY7-p9Jos8&nWgZp7Qr-UN6{c13E(r{Kz;63tOcd-v{* zoU3OxOifnJolMqGD>UVKr3ymP7sgT2kPz zNalZ4CVS+Fg#U(_f~b_?*P;e_tAUzn1q21h(GVm+dH4l#a_sFo`x0yzpXVeSMPy{K z4y6ckVgz`|qll^i5%pG`VR+^JRqX5%p0gMA^PP`o9u>N2Lf9Gor!#=^GOlC%&}SHx z)^ioRO^M*PI#D-FUI#)zHipYcn!3Dr;(9HZRG8PRj|`j4%Aq zrjz}3uvuM3#zZb7*$vecbPEVyUIjN}pgwr2QZ&Lm!KV=t2#mAD`$4uSEmKngQ&ZE6 zBd4nV>VYia`K^xUveVL#9bV16=VCuA7FTk|4ipO->Q{%~N@HYm)aQHV+Sh})#e$E2 zl^b10D5O^I_0GpNp=#=h zni$|PvVHVS;BO-tn3a`vg)W__p6IkO@3IQ2(Sz(!!mprLtwrPWa4IHoXo+ zaf8>z0mejltvGwUNC0{uf2#CzMO;dJ^hlcv4w%VpD0m>tWj~x-vu4d+d?G|;itxW4 zI1&^E?=0G7P!DE6`RJb{zwBm>IAfCjkUL7kEU?bd7Fl1#eVc&e`;Q-YCK(|AL%&;+ zs2L2WvGKy6-=t_#;~YuA`@A?c#PeLDs%%cijuP}zejL<8I>WR<7{pksNZMIoX(?|P zFx05$A#JJmI>uw*75)?2al0(WS&|#>p zR)-_B9~ri8KwAZA79|UR_4kDbsa*xQNLmXfmyiPcleH7jXUcx)erGNTgCjzU^USW@ zY#K)i2f&5Xd%(3X#s$vlgfG45*@-;>jT3k=P6!aTAK5fw%+lSJMVB_Gk4*^dhnbpa z8|X05w&K}o`JcVihwGZ~Hu1-?eu6*?f{x1O&MeN^=4NI~m7X4WbH;;v#Z-^vRG+H$ zDHtj+9718`U|ofTXPr}w_Mh^>fJQfT1S_CqF89OAkFJ$IyA2sDTG~DgE%Xvrjt-r6 z(W6u0|7GW?gALzMK#=F9ef!&|r!v#wTeuaSc{*`RC;=_MnOpy#8`x~j!a6N zx#tod&%S&6Hc-$h6#9K1Fdz8DaWn{}u*x9I6C@oPy8O170TXa8_`0%|=tYH98n0h3 zz6wziiIm*V+TeRQaUa5h94;UpptgtJ;yj98>*0O90{jlJm_7X}H<$mP#7FtSL&-hW z3|=7=8~jl~LV^vHBvD*Op~70cq;fvwQ3SSZYngaSpA*e06K zJM(boRcK9FfGQha!7UF7rvFylyZ`6;&v)!wk#nw<3A+S}xD!FGQtUNPQ4du@%RmCe zAKwg7Z5dGK!&qiE4Nn0!-R9K8pxl9rh&LU$ak}>#Mq<`!)t;wk#8ZC zx(E07m%4_F=fdzAq>}W0#TR_^h3C&y0$nF*;El-Q9jzy1af_jfWThZ-oRF0vss`_vy1$MaUx^ zg)?ohvG5TvQ6acZ>ytEKSyUD}{#)2=AOa7K{@e$2b$7R;5b;i-A_u(B^Q0@wU6 zW#vbyhV=OW$Hc$zYB95B&ub{-+a0=;sS;G4bjWZ-I=ZXizN?_HFk7WxQc{w2)hf-N z!8IHl4>5}~A)~0+&S1G$+vPviJ}r{x-`O;+ZAqQI6&YFjKE>nuBH!XKdX(-ZO80-X zK+K0miZQhH5GKPfilW1gar4}g^S`lDu~PU%UnJoT@Qf2~6N~~tNvcN`H24w>SWWZ{ z4%+^l;FrAppTjNM6d4h58D;>?7IFWH#yJJI3Ll{YJ-TzS7S_`Wu&5AId(qs8+&9c( zqQiU(uuM{?xyqCqY+a;L0zYu&%9TcE%-s>2>fgiyTheOe8%&RomZ8eVvb;KK2R&Gb zdim;_8q3^=bf~ptKFp1IAw@j6`mV&%3$Omf-#N2GQZgM~GGPx8$N-0RLtit@=*cWA z_17@eV*8wHMmZ>`KVoOI@kuRoyTlG)Ti;k+y#fYap%K_PQQiAP>Q8KBSOWGl;>Hc% zv9U3cuk&NCR}x26R!+8kJMTXzjX>O`a|gaoTJE#yjxYdeix4U&>+c)`njDLPH4&H0 zEiAa}iIRZPv2}!)hd5caSs$iF(!zoWS&!0=zes`7r;2g0TwE*@CK#MSfPSP!DoEM| zmSi&!PrbwIHFzd|{!G!xCo{lZ<-x?dL39PG1jl!0=`%P4b1?~Y&6S-^>iCpwGj#6` z5Z}=iXu`pV9{{UDJi8z7?`H3=2q%7ZHtc+Et_&n_8;f_1I{#2AZ0)>LtAhpz{eoCS zNKUp{(o;a`gudu?z(me=*gIpSPIC=FHw9#CP`~VE3{@8%{^IIs2+C&AQ7In?DyHyQ zSH6Fb=||Q*i+>y+w$k57H&HY#)a36xb8t#xwL0Ws2Ju9L)MX9-NR8$k1Sva_8XSHK0!}bTXtH0=f9HUJD_= zeE&}dZv$bQ2ZS*s0|GGfXEuI^0AdSc!x{|Nv0$wA%!^22nB5kTifu?EwiIUsvRakqjxDQ*XM3;{!#jlU0dt1;T=3^Dl!26HovszTPvm?^p}l~ z^_sON#w-L2qSsPm$?kYLJW|KUi!9+AV$ZAU$*bj zL>wfUbbeI@cc~ERaA4Li-@+ z&_O=13*O#~B6r6Ai-k)4`Qj`jE{uP#K>da60Y9i}=4g~7`BRym{F$@;te z((zPRfIVe6?mu!M4+wJ!G_N$)e$Z*7W|yMwo%ufgz%kyl9|9VTOPg zEBD0d$1bp={|*TTYomuruGKybgU)#!(GyP|557nly}8r}`FKAFhJl77?7i@NRkC7? zUgeGZP)eyg9s(>=L)r%>3IQiZ`g5uTcX}PX4If`z-hdD>mhb-=bb@CpxYCpiE1!Dt zsO9(!)Czbzv?p>QBa=@xzks`wf^|L+{s$TgDt5&6>kk~aG@vAdUDyw-1ftkH?A$6g z4Z*kWLqRE+We($0e70q+2Y>-ePB+F#pjUG$xb37IlEpDR;p*=0XrKe54X8_-bhYgJ zO)&VTpKFyp8oFBw`V4$Zm>=K(xlREa!uFH0l2~<~HqX}F{cx$LukUqf>0WkHkKO-* zDMAGk_Xe^6ab!Zp32_0MQdW6;Dx-n?+B?nhZISWdn7K= z(OJQWY<IDm8IzN>~FK1=;u=xf<~% zW%z<2;oFf6vRn4RR|C)wvJl1wzvMhM?3^X%P}1Jy9~TUXm6{CUKwjab2!T(A$Kegc zFL3TUs~dXqI&48UoB#f}H7AUKd{7%gy%*=&|3g*+)*9PYdH=-5ywZJem;2T~OMp$2 zZsg%a*Ph6eWEs<(|L)w6UpK?J{mEHJktKpCnN919bzeaM!3#qaYBGh$<#q@cGhCnX zS#ZxDUUU!2fPx=eBULcQO75V7OxVC}0mhLQh!E~lcATZ8%;^+Ru!JHBl*KJ)>5e?& z7Io2de5=2#>v{1)8L`b1mj$tXQ@DWmcv7?yR^ugF~?^6_|Hj5vJ_Dk;8V%;Mi0fXa-}LB#`w$|IXdj>c-MP&l``X#3j0r9Mxu$c=(&=W zwKL5bJl$LHH>okhhmn{#$2CP881W;np!^)ri&8#?g$hIr0nP$!!Z1SNv%I5NvkYug zqfH)@e_ksYPX%-%kO8JbbbgPAcyXr>UmL4t>wG@JUq5Vksb`IrGaLX(b=?L(v~BU? z1=8ka+o3u@U>!^%ufPah=f~4fiDZ`nP*X7Z>4@&jb39bWw@Tk-%#%5HK#8OMgs0l} z16sibHp8n8WY5OO$1#`DJMv!8{gyxx8M#7xj};+iG*qP}6siw-DFj8qV0sn2tO*E1 zS*Ia5I&HN7j+%t{^&Cs7tNjpDpL0GP4G{JUw&t`o=R7?;wtM_KAF_V;Dm2OUmsXs- zi~yC~b6YN6gqhyb+#K)PP|=dnWu+yaYOs2QZKTFm*}@A4ng?&_VELyQc64-z41RQ2 z!vLe>M0b#Z6!$<->uVlZyYLpYVVfpFm$hPj$dHM6k4yl9GYbJ$T|KJAw=Z zXVS|`DPrYEVMJm1#}j?;>$CMWKy#_qrj8Gg6r+=4*8W^p3VwuOQGAKKsDXXvDjDIZ zdT>E_)22;O981v69c|!q!{|x}PUc(0s%^a&Dj3cp=YAr(2W+=0uZC-JX0%=Ugi+F3 zJkttZd4@zyMpOZu2F1zUtAhOl{E!Mnub19|SI2A|j7T?dns2t$^Y78%wW7Zyfq_=K@}xrUUxS&O{kN# z{K1ku?>yMJ?QCn_8bD@*9G%zPFf3DkeoJ9>SdOBF2t`+c;gls*tlPkrLZ=~J8H}w| zN^xIj2LCxGzdB+?U6Y1>Spxnl|_(I5h;LG^WKYvCTlODQB}CKrrsOxwcMm)YBWz4{ ztO7Fy@<0Jpq|Kb>RM?iLt0$7R$-6?1B`tkF8gma4`JVYnE#e~y2*7nnGYpbn+RYm~ zRe6C^-t;71NI{GQ?Vd9+y`NA=iTX+Wj}Jhg2>nRfeb==7+&|c~xc{pisA~w3WDFtn zf<7_T-eq|wNY{|D)Q5CusQ08AUx)MQ+c$l{24bs)T`|S!$+~ONYgd92*hTgo!cNDL z48?_Cf+W>tcg$Viu%ARw2(JP+lzwT8MbR+4T1De4ev~bl3wH_H_mp~;fQyR@hMc() zwQh_4WsRP`4_*n(hv^DYUK9dlx*amjQ$Ie_gPLy8yZqO|j*<@~ZdPA?O5h)bMEM-c zH5)g^&AY9pQ;#L-jcz4xNLQCB8EU7-(D)M>j7&>{SXKL^;g~ci>-=9Ip{Hh8@eJxR z5sLB56ovA_Dd+t3%{C$X_r~<;uG?|X3isdn8sLnQad1oqVcujcLQbzeN|u<1*%U(F zy^m>bZnkPYO|p{m)Roek!o-J$(0&L@*y?$Vk)iLE%nfUWph=vMs2dNUa$}CSwk7WF zU8}5Wav_Z>uJuENEy5g8bb>g!)+7T9lXGHXB4Dl}iBSwPMW>MY9A#~0x>hqw@FL!S zy7@MXiS0T;f*>gAHcPBL!PtP?_x|e|MHM3;2MEV>9=at6p+U7+@*0pj*}r!zWjXeq zT}LO+&jYLP%TFe{5;eKf9D7&s@r9!k2FGz<&LwQwRWiqPRxR&QGkF_F>nN!I$`1{; z`j>q1$sen(%bSiR>K$*$CGXxbV!)rgX^0|DaZDxW1i25@yKaG6`S%kvyV>&J|E?)HJwcwf zvYh3a*}p}Hs|ia;QjpG&`)J`6c+Qtl7+>%S-!sdAceYSI(51R($f#hD%~RYPv^u;> zhWD>uKKxtqtnW?0@}SCZYMK?~goL+j zlpO(q&q)sHAF8VZF(=xa0gaK^6wwx&qN&g)@^4V(N8BON(m=x5+BsglSWf0eM%xr1 zqLlT=07?oZM5j8WyALEkNw;Ybi6PMBH3Uz*G|0X_jNDFyiH-avQ#f;Tz2@`UJYitM zSg9~BW0Cb=-#2Z7l7ROSCBL7?D=Ww=Lo{U1oW=Vll+f3)va_3wJR@XnR7jmpY%X$2 z(ZZ~)S>b(+qj7t9NFu|lq7YE#nxUqLGX86tki?kKnd$$(bAh+WhQy|he`nO%A}Xp3 zE?_&}M*&B~;Lv}j?=m1_PL8y$gEkDiU^(%Bjf||I7@!kdTPIgLx(qeb;IVJUEycY` z?GagX{l*e=izSAJN>-csBcorcORaF8-FNZ;jOp0}O27mlrBc2vVoJp1{jDuTuAzu^ zhoFt3-@WUImyEg|9j%vYc2WoLd^&YtDQi>3CMHJv_P2X&(2~FfP7CexqcR zcBuFpSvk3rbHMXgVDIG0cKjb>Y8q`OTgkL$M@Oljio#k7Z&Fekybr=K)Qbfdv}yi3 z6FGdGp2A4vbh+}xB=dIT@?%C0rds(@KRneXn_CVg$_8e{&3O5 z1VLzo7-u33kslxfeOx10(S!RFSE&xr?sbXdr&cj;H>2afbFc5k(=VeLsm%Y=I2LZN z{&?b}{xHIID7ob|i^&&M=Ovm$?Uy>PQnRTL~-r8B71+(y_rG>l*0Hr?K4zqjk zA2+&hi6lXMtDqp{YjyKSuPr%v7Y2uzey#kme->N4oU_50C^CUDj#JS~o~YLFoI^o~ z$9oxe9BEt)8wMMONMQzfVTKUYc+Ji&Qtee$S?e7aqbxA8a(UWZ5si;B>!efa2zk*M zux8VjJ^$x9ntKKI|BXgg^O7f%oSN;7<3I3ma!`qZ}@!hzq}GU@YYLx_zcXtM5P zMRgNL(6t?2JaXtWzlgsfnL%o2U7hKnn{@dreyAJL)TwZi6}lHYMtf358n5` zc#Yh@chD*PwT=XF3shfCqx<{Q5wEyzs6HJ`B6Ugo^5JEV9#K(IK6f}+xtR_o_KvKP zy7>2EPXBXUE34jnZd5-;CeAUca_cTbdAZdUS*o&<^`)>-^b;dKtF$8VG;Rror>>fb$_e+bxdTF6}+s?gx6KsRQ5`0@y`q|%^qI!nER|D&?cC!$`gI02`+Y$}W zGV<=f>>GC()Z$j?=j#+56=LUj)#vs^xwjNA98xw&Qd!k1dsEU~N(GQz)h@9tU{Itz`Zz*GCL`k7dJ++o^QC2(PGXyYmeZ!4?Tnb05> z45IRWtBzf^FeP5{?$EcHcfutcM*VwK1!(^K@mVL$IW&+BEQVvh|BYO1a1ys>vhJsi zLMne&7MbSWJXo&S-KAR-mb3?+7~SF`4_Yy~)9&6Ece1V$SWwu^fO*&cS{kc%!#|n( z={qtie1wh;6bv@pVDK$!-@!-$rgXyGiNQpfWHUK7jZL!y&A(@_#)ml#1r1=7MrJ@~ z{L_!99VIjEOI!1T?QV;*S*D%bcr)_KOY6_&#ZC0cW?@)Kg@k@9C9yt)wNvs_wSt+n zGar5)dCbUIV*C7rg2|83;tzfXx}0Mp`8^Nq@zSHJ=!T>1M}v_9D;XYX^2>kwR>=QC zerCivJTJMpI%ereJBRp_Fz!t`PUknTbc&75yK@ZEhCdj4M@x&xjn(S@J%bT-3`?#@CZ3+KtKV$=X=6rJ zj9kgLS1mP}Idn^wl-D*sKKgsQM5&{IJB~jVaH<&XswQZ$yxB{KpUH-sbQwR0$?eYl?9hs<4^dtIgP^IW(Btyiz@>R5= zZ2Pno8GKz=p6;o8xx>=u$Q_?#t;=UVl;?W(AyD^|bk>!BGLIiWcEv*%~Av@PA) zHqo$xY&+)WFHu<^w3n^j{EUuyH9J?_;Z1F0`VKHOOiAinTNj8PF=sk+rUdU$3okLq zrEcB6Iv3~0)Lc|pP_W7Wxyxj#3QIoi%4^=)1~S9Nv#x#|AOnkP>9Kev-hc$<=| zJ9nY)(V>#g*CoFvx{9v0Q#t zY+fV0EopJ&JMH~F25r;zTgI7Lmm1|e_uNt28eX>Tl%`9clG9(6Q`;)Zb)|ty;K+Bz z2Y-0;YisjH*G0Tqz4q!6bA#d)^79vePZxP!Zuf3lynj#n&7qY8t2Xd?@O!B{^;20k zZ``%l5l3l?mS(by9xJZrXnr+>p6T++kfRJu{fQePlzU&KQrR+sZag` zFX{iU&dxNP%C*tsON6q@P{=&((xzxqN~S2HZOTj%N{C3N5R#B0yAXvUGLIQ6Lo5v{ z4WbM!X^TvS%;Eg+YQNWe&b80=p6h(*ix%s7o^?OZy6^w+yV>jckI$@;y}n(d#f`E4 zVfA-|`Fo$9mGI`2mC4d4`?I@wqkkv&pMI*>KyaSvzc_!nGZZn(u9wl+~-V9jTQ;)&`lpMI>?t|K{H8@ZD8% zj#nz~?zY1Mj(n4PM~jA~F57%0tXR*-ESCSzyIv(4^yUxF!$H=M<19aP^^YQa$bTH*LO%8)K#LPRjheMVZ~Bcf~gP^yx8LAahtTZqw&=YLl-V z^z^0&PMtd6v^wP!#~=~=(GC5G+9DyeU^xxj=1v>Fer$gkY*<4PWl8iF&+DkaL#7@v zn&NCD3)Od5A6vEXzMEK1o{UsV>liB^|D<;-js;W-UFIn*<_<%~F z{7g$}?l7z6KucTWk-cduU*3D1l2YfnX_d<@9j$qC&mgfMz$iR~q8KF*<^ZJqZm>!S z;Kb=$C6vfjTaI!N1PZT}_sup>{2<{WrvIoL_Lvtoi)Pr<7PWoQIglPyZ1Y&}jOg!Hm!7|NSu1F(bA1gDpP7dbr+5}$gqidNA`0@SzOBW|CH3DJ zBn5l&#aI`i4j70ujG;h3#k%&)V%T8sCC1|)S*E1YzNY6#911vH&mNYn7aXW4DD$2L(PhrVaUC7Pyk~BrtbbRQ$-t%W<7O{h}D0Tk1~CBABdeF z*O5KI)fhPVetl_iA*a25@2B$;RX~It+=?9MtcUV!&U4+6uq?g&;KYqPr~kT)lz6tj z#oycx4fXWZF-5ob+8UcnF_+*@pPi(=S<$@1i^r+z^X2H9b5j8ejOL{)*(QxGYm&nH z%V5*~B4?z&T2TK1&N$U`SbTGLWsaSTz}B$EUT-8^d1b=l3>e(PbI#YwY|x`R#I)`l zdZFsoaMIMYGm(*#PxtzrSm<-Hm+XyRFHRxf30E#PTSLR&BJU(BN;7Zp zD9>Nj@@>194%L<7+ILAlxI~R*L}~l{8I|U8vwcDeDKlXMMSr$EzFd=E9UE1ZxSi4R zzIoNctJQ^utfD-FC8a!d7Cn#Iz-3x*enmN-M|uqxCj(~Xn^v_RcHW7Jp?$pZJ-^DM z`qTE;E>ZlxMGuc*RH#|dc2ym}C-z?9c2$R#y4_M@0*^8og$@*%^QdxEH(y>>dW<1f zmRGH#rx8;J?u#M^g>jQJpIiQBy-;_5maiZS%jG0yPEHCiV(X#(8GWeCS*E*piR(zp z#NqwpYdg~a=$TZ@Ghe(f!Ek=3z5S5(?Np5j?IT!4aO356HkNTE^OgPoP-1+O2t{IQQE*C6vdN1%v=(y-92Ml+N zastgjIXysJN#DUoHfgsEg^+BmmsC7(?4d7xpGMjuoy1q`1x`%ll;3I@Yl@Ri?irTJ z3(1=I?FnZ~REVq2i{ai7aJn;@oijSkx*(M~q|xV%kSe99=t}+>+`Fq@8Hmc8 zQoUo$EFz@08v6P5T`K?NKs#KFMKBl^8LA&nSfxv_);tJ-q1AI}c(`+PE=t)-7~O%u zBt=?dXN;um9hcm@$Bgy#5)+v*#UWR4DNc>NPjQLl`s<3)kq>eYNS8)y2Fg~ijC*NN zT*Sqj|K)`$^#u3ayl{PP`L0tG2~+ zr&-U9Ihh)C8A}e?h^&hOCc-U?7-{DIF_JPT`)=SOMG1TO3PVUxqI!PQEMKbrEW@zo zO?F$HPE#Udsk3x-=z1{*(-N&&pNWZ{+96zQ>#}3<@`XW=TEv+ZND)gg^R?H;gB;eZED}OXqIswYCs8mD@VqbO{OCB@?%A&st3a0&O#( z0{9bKnSVfv)|}Ic3|9T$tJo#RkfU zp;K(t()#pJ~{=wzoZMszVIGONSsijMBC#$N~yw)9v ztf{E%=lRm3Ma4ND-1yc^en0QX#Yi6QguZNB`z#|-Ev@<+#v?_UGfe>t0#|pG-0MUQ zwaNX&tL4tRN@i90+Pv0{JRc_^s%FUG*9$781OoTwMVuzkAfG9_D2|sO`7jpS+ZXm*;#$vx(L076prr zg$99|l-=F*;~gA~=C9Lxf-p67TYU~weoBT+w>C!667Q%35+)*SEU>6UOLYOgy z_k9_2%^mV9U*sj0-|*kf6<*Fybou7;R)gKXrHw1*>mS7(-m`EU5{o=P5rgE>zhO8E3uFem8K*cMR^sf zQ~zvdN5>uQ(35?lI@TFSXHLFOlpx1Pq*&7jIi@NB<+CFXN6iX{+QJ8}8ABQCyc@@& zphER?alup-FOB}}YJ80G(39?3WXtDf7^>&i@bu?qm~FPM4;w{5B%-8SDypW$Ar&|2 z>_o$75&RAxf#fuodKZXN%3K`cmT&hp#*)I?=kBfFtm##w54plu2?w94e_?EHF5K3V zv++|rd7}y-DpZYV!J9YL4+(5rRCf;ZAc}K)>=*bmQ}l(v>wULi@E(e&mvzwl2uFWtSGFQfwQ-0%Bv&hxiK#HH!596_$FO+o>v3Hrd$H?a-j< z>m(Mu+Z$!7r^nzST5!;)haMhJCT4nZ0`VIey+5y(xrVz}eFosm!UUeh^WAJnJeR~h zJ2klRHQ~pFtz$H4&#{$TFWgbFxpfz@QtCGBXx`{C(se$23wj1@MhA8hEM-PIw6v>+ zQn6bYPSz#PxLIBw>3@f}*8f;*ZO=>;caSBAhBn2Q->)Q+vAVhu!f}fIHo0$!y0xwI zr)oRpn)}vq%U>l#Lw~dhqUL|pd-2}?alaPVy@ie$yglg0Y;)Ho2U}>+dnsWZPlL6| zMVNPsD6n;&b*#Y2ISvvWo4tLnV>>w#1am*2hyxPKSuh9sD(;^ee*nArkLKc;ccEZ` zkx-JNVxrT4L8_5R)nw+x)U;wk!_0jA%)P~`1dRS+XlR*qcK??YGARN71^`^gaZ6yU zCR}avlu#~+pX&&Jnx^FUW&hB~nD3U0<8#>&W2fJ0Ch|+_Ue!eVzT&Q)?2JnD+pn{E zhd%aB9_1gHHA#McEzaig(SHey_x4N+2#`UAF6M;>1Yu$Bhaj=^!^3D^i=X18AMN@f z#{b&B=9hM!!faC?#!0$bNTfgQ zptb%xZ?s(o=^^BsL2J{%B-;%|z0`?1gY~Ebfe`p6#}P*$a;&}dS;oVo>yp8cW`n=H z!M1^#TIFKvv%tTISz!~~e9LZ+!w6P@S3pCECK5fXlv}r$(6CyGVEA~cvTGLw&DE2o zXe@$H{_))VUWAfTP>VwgF~3OUS6-adqhS8rBtM5)i8zp)GoYg{z@JXcNBKg)yCrrv z3R*Y{vWz4ZtDpUek4T%nYEMeOc{-F`q=D>&f_%DT`^cPA7gLHBtX)a$J7b9U5Eq3N zC8t*o#el`35TaX9T)YBX(VRuhqps^C$#ar89OtQhhVEE@uB5Pn;qdz1ZG+6SfPgFk z5{S5_M`n?!U?Q8GV5^SPeFG-riYvId#OoUx9EBR&+bv8BtbwAuuH3@DgBuAzYL}_$)eY#J-5;8om-7SjSBQP~SwyvmzFKTt918{Xy|m{_NI?>i zB?x!&NDYKyoo)%fFv-I0x>*ArsadwIlJ{2%AU}A5oNXz@-Qx=MY+ItHM4CY+2?3Ie zjAFx%*=fFhUIBp!Y%*bu?{kNfZUVOwY?G0!a;P%~vfHT>;fbbdd#6daACpXIX z*V^0L`>6Be*RLd6?f88M{+l4y^-WD#QPx0LwQt$B*gnvE2%vJ9UL8fEX@J0>|D5{1 z^gFc#gqr2ZRo3CAcWO@hawQd&gRn1hGs z9Dr3_NHm>O7tc(!GIj<~;M4G#cXtpyM;e2`5O7sDV!Dq2Q2rCwhfR7{&}cy%%oZOI zZioDlF?=|d_f~2%XAYO;;n~4ot)s5aM9}7-R4~%*F zD>`c=SQkOpI)_b84xws_7?(%M(DBa75aeq~feg_L*vxy-`%A&kS{dJs_prX^V}C0b6$|a?FGtQ&J*F(3BWLHXsOKZ=;tAC-n97 z2>0u#@a?oTF*!MIB$7x%jzalIWg)@WDASQ}q>xA;Q4b8lZuGn#qHR*>KkJ!DM$Yyf zJNi3x`@6a*NPFL(j31pVrhH0@)gBl4w$Fln$r4UnrF5ShgG@rW2)>=O;>}{pscJau zX=QVWdYlDe=kqaTY(;V@DJgGqJ6QEEeJ>DpddVy(3}^AeC1gJwP0cVuHB#>XEVmTr z$^}F^6H1G<&z}8(qTUTOcp|0R$2R9!dGzQo1${ZFCJFFbMx*H7@q;w_J%VT@_a5=! zzkiAWbB?6`d!EE3&?{T=4@Tka`#OAocs!xCkthGJc73axF7d#z7QsUSX1N(`)c>)4 z+CO@?|LME@5z|i?0M*?qIEofg@m yU-_8`Fk>bD-v8@Tkni&!X5Ig~hh6;wgTL0!+^+A*Z?F*`W$$hs^}JoyXZ{PTqh1F9 diff --git a/examples/Hdiv-mixed/include/setup-fe.h b/examples/Hdiv-mixed/include/setup-fe.h index bb35ff51a0..ca26e65df3 100644 --- a/examples/Hdiv-mixed/include/setup-fe.h +++ b/examples/Hdiv-mixed/include/setup-fe.h @@ -10,6 +10,6 @@ // --------------------------------------------------------------------------- // Setup FE // --------------------------------------------------------------------------- -PetscErrorCode SetupFE(MPI_Comm comm, DM dm); +PetscErrorCode SetupFE(MPI_Comm comm, DM dm, DM dm_u0, DM dm_p0); #endif // setupfe_h diff --git a/examples/Hdiv-mixed/include/setup-libceed.h b/examples/Hdiv-mixed/include/setup-libceed.h index 2bc8901c48..03ecefcaaa 100644 --- a/examples/Hdiv-mixed/include/setup-libceed.h +++ b/examples/Hdiv-mixed/include/setup-libceed.h @@ -6,7 +6,7 @@ // Convert PETSc MemType to libCEED MemType CeedMemType MemTypeP2C(PetscMemType mtype); // Destroy libCEED objects -PetscErrorCode CeedDataDestroy(CeedData ceed_data); +PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data); // Utility function - essential BC dofs are encoded in closure indices as -(i+1) PetscInt Involute(PetscInt i); // Utility function to create local CEED restriction from DMPlex @@ -14,10 +14,13 @@ PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr); // Utility function to create local CEED Oriented restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, - CeedElemRestriction *elem_restr_oriented, CeedElemRestriction *elem_restr); +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, + DM dm_p0, CeedInt P, + CeedElemRestriction *elem_restr_u, CeedElemRestriction *elem_restr_p, + CeedElemRestriction *elem_restr_u0, CeedElemRestriction *elem_restr_p0); // Set up libCEED for a given degree -PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, +PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, Ceed ceed, + AppCtx app_ctx, ProblemData problem_data, - PetscInt U_loc_size, CeedData ceed_data); + CeedData ceed_data); #endif // setuplibceed_h diff --git a/examples/Hdiv-mixed/include/setup-solvers.h b/examples/Hdiv-mixed/include/setup-solvers.h index 3c9490df07..f615d287e9 100644 --- a/examples/Hdiv-mixed/include/setup-solvers.h +++ b/examples/Hdiv-mixed/include/setup-solvers.h @@ -12,17 +12,17 @@ PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual); PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_error); -PetscErrorCode ApplyJacobian(Mat A, Vec X, Vec Y); +PetscErrorCode ApplyMatOp(Mat A, Vec X, Vec Y); PetscErrorCode SNESFormResidual(SNES snes, Vec X, Vec Y, void *ctx); PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, void *ctx); PetscErrorCode PDESolver(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, VecType vec_type, SNES snes, KSP ksp, Vec *U); PetscErrorCode ComputeL2Error(DM dm, Ceed ceed, CeedData ceed_data, Vec U, CeedScalar *l2_error_u, CeedScalar *l2_error_p); -PetscErrorCode PrintOutput(Ceed ceed, +PetscErrorCode PrintOutput(Ceed ceed, AppCtx app_ctx, PetscBool has_ts, CeedMemType mem_type_backend, - SNES snes, KSP ksp, + TS ts, SNES snes, KSP ksp, Vec U, CeedScalar l2_error_u, - CeedScalar l2_error_p, AppCtx app_ctx); + CeedScalar l2_error_p); #endif // setup_solvers_h diff --git a/examples/Hdiv-mixed/include/setup-ts.h b/examples/Hdiv-mixed/include/setup-ts.h index 4d7d7546a3..4da1ea4d18 100644 --- a/examples/Hdiv-mixed/include/setup-ts.h +++ b/examples/Hdiv-mixed/include/setup-ts.h @@ -6,12 +6,20 @@ #include "structs.h" -PetscErrorCode CreateInitialConditions(DM dm, CeedData ceed_data, Vec *U0); -PetscErrorCode SetupResidualOperatorCtx_Ut(DM dm, Ceed ceed, CeedData ceed_data, - OperatorApplyContext ctx_residual_ut); +PetscErrorCode CreateInitialConditions(CeedData ceed_data, + Vec U, VecType vec_type, + OperatorApplyContext ctx_initial_u0, + OperatorApplyContext ctx_initial_p0, + OperatorApplyContext ctx_residual_ut); +PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, + CeedData ceed_data, OperatorApplyContext ctx_residual_ut); +PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm, Ceed ceed, + CeedData ceed_data, OperatorApplyContext ctx_initial_u0); +PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm, Ceed ceed, + CeedData ceed_data, OperatorApplyContext ctx_initial_p0); PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, void *ctx_residual_ut); PetscErrorCode TSSolveRichard(DM dm, CeedData ceed_data, AppCtx app_ctx, - Vec *U, PetscScalar *f_time, TS *ts); + Vec *U, TS *ts); #endif // setup_ts_h diff --git a/examples/Hdiv-mixed/include/structs.h b/examples/Hdiv-mixed/include/structs.h index 8912159106..9713e485eb 100644 --- a/examples/Hdiv-mixed/include/structs.h +++ b/examples/Hdiv-mixed/include/structs.h @@ -18,49 +18,7 @@ struct AppCtx_ { // Problem type arguments PetscFunctionList problems; char problem_name[PETSC_MAX_PATH_LEN]; -}; - -// 2) richard -// We have 3 experiment parameters as described in Table 1:P1, P2, P3 -// Matthew Farthing, Christopher Kees, Cass Miller (2003) -// https://www.sciencedirect.com/science/article/pii/S0309170802001872 - -#ifndef PHYSICS_RICHARDP2_STRUCT -#define PHYSICS_RICHARDP2_STRUCT -typedef struct RICHARDP2Context_ *RICHARDP2Context; -struct RICHARDP2Context_ { - CeedScalar K_star; - CeedScalar theta_s; - CeedScalar theta_r; - CeedScalar alpha_v; - CeedScalar n_v; - CeedScalar m_v; - CeedScalar m_r; - CeedScalar rho_0; - CeedScalar beta; -}; -#endif - -#ifndef PHYSICS_RICHARDP3_STRUCT -#define PHYSICS_RICHARDP3_STRUCT -typedef struct RICHARDP3Context_ *RICHARDP3Context; -struct RICHARDP3Context_ { - CeedScalar K_star; - CeedScalar theta_s; - CeedScalar theta_r; - CeedScalar alpha_star_v; - CeedScalar n_v; - CeedScalar m_v; - CeedScalar rho_0; - CeedScalar beta; -}; -#endif - -// Struct that contains all enums and structs used for the physics of all problems -typedef struct Physics_ *Physics; -struct Physics_ { - RICHARDP2Context richard_p2_ctx; - RICHARDP3Context richard_p3_ctx; + CeedScalar t_final, t; }; // PETSc operator contexts @@ -72,6 +30,9 @@ struct OperatorApplyContext_ { CeedOperator op_apply; DM dm; Ceed ceed; + CeedScalar t, dt; + CeedContextFieldLabel solution_time_label, final_time_label, + timestep_label; ; }; // libCEED data struct @@ -79,23 +40,32 @@ typedef struct CeedData_ *CeedData; struct CeedData_ { CeedBasis basis_x, basis_u, basis_p, basis_u_face; CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_U_i, - elem_restr_p, elem_restr_p_i; - CeedQFunction qf_residual, qf_jacobian, qf_error, qf_ics; - CeedOperator op_residual, op_jacobian, op_error, op_ics; - CeedVector x_ceed, y_ceed, x_coord, U0_ceed, x_t_ceed; - OperatorApplyContext ctx_residual, ctx_jacobian, ctx_error, ctx_residual_ut; + elem_restr_p, elem_restr_p_i, elem_restr_u0, + elem_restr_p0; + CeedQFunction qf_residual, qf_jacobian, qf_error, qf_ics_u, qf_ics_p, + qf_rhs_u0, qf_rhs_p0; + CeedOperator op_residual, op_jacobian, op_error, op_ics_u, op_ics_p, + op_rhs_u0, op_rhs_p0; + CeedVector x_ceed, y_ceed, x_coord, x_t_ceed, rhs_u0_ceed, + u0_ceed, v0_ceed, rhs_p0_ceed, p0_ceed, q0_ceed; + OperatorApplyContext ctx_residual, ctx_jacobian, ctx_error, ctx_residual_ut, + ctx_initial_u0, ctx_initial_p0; + CeedInt num_elem; }; // Problem specific data typedef struct ProblemData_ *ProblemData; struct ProblemData_ { - CeedQFunctionUser true_solution, residual, jacobian, error, ics, - bc_pressure; + CeedQFunctionUser true_solution, residual, jacobian, error, ics_u, ics_p, + bc_pressure, rhs_u0, rhs_p0; const char *true_solution_loc, *residual_loc, *jacobian_loc, - *error_loc, *bc_pressure_loc, *ics_loc; + *error_loc, *bc_pressure_loc, *ics_u_loc, *ics_p_loc, *rhs_u0_loc, + *rhs_p0_loc; CeedQuadMode quadrature_mode; CeedInt elem_node, dim, q_data_size_face; - CeedQFunctionContext qfunction_context; + CeedQFunctionContext true_qfunction_ctx, error_qfunction_ctx, + residual_qfunction_ctx, jacobian_qfunction_ctx, + rhs_u0_qfunction_ctx ; PetscBool has_ts; }; diff --git a/examples/Hdiv-mixed/main.c b/examples/Hdiv-mixed/main.c index fbd47fc1f1..cf8a851b5e 100644 --- a/examples/Hdiv-mixed/main.c +++ b/examples/Hdiv-mixed/main.c @@ -73,9 +73,13 @@ int main(int argc, char **argv) { CeedData ceed_data; PetscCall( PetscCalloc1(1, &ceed_data) ); - Physics phys_ctx; - PetscCall( PetscCalloc1(1, &phys_ctx) ); - + OperatorApplyContext ctx_residual_ut, ctx_initial_u0, ctx_initial_p0; + PetscCall( PetscCalloc1(1, &ctx_residual_ut) ); + PetscCall( PetscCalloc1(1, &ctx_initial_u0) ); + PetscCall( PetscCalloc1(1, &ctx_initial_p0) ); + ceed_data->ctx_residual_ut = ctx_residual_ut; + ceed_data->ctx_initial_u0 = ctx_initial_u0; + ceed_data->ctx_initial_p0 = ctx_initial_p0; // --------------------------------------------------------------------------- // Process command line options // --------------------------------------------------------------------------- @@ -102,80 +106,93 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // Create DM // --------------------------------------------------------------------------- - DM dm; + DM dm, dm_u0, dm_p0; PetscCall( CreateDM(comm, vec_type, &dm) ); + PetscCall( CreateDM(comm, vec_type, &dm_u0) ); + PetscCall( CreateDM(comm, vec_type, &dm_p0) ); // TODO: add mesh option // perturb to have smooth random mesh - PetscCall( PerturbVerticesSmooth(dm) ); + //PetscCall( PerturbVerticesSmooth(dm) ); // --------------------------------------------------------------------------- // Setup FE // --------------------------------------------------------------------------- - SetupFE(comm, dm); + SetupFE(comm, dm, dm_u0, dm_p0); // --------------------------------------------------------------------------- // Create local Force vector // --------------------------------------------------------------------------- - Vec U_loc; - PetscInt U_loc_size; - //CeedVector bc_pressure; - PetscCall( DMCreateLocalVector(dm, &U_loc) ); - // Local size for libCEED - PetscCall( VecGetSize(U_loc, &U_loc_size) ); + Vec U; // U=[p,u], U0=u0 + PetscCall( DMCreateGlobalVector(dm, &U) ); + PetscCall( VecZeroEntries(U) ); // --------------------------------------------------------------------------- // Setup libCEED // --------------------------------------------------------------------------- // -- Set up libCEED objects - PetscCall( SetupLibceed(dm, ceed, app_ctx, problem_data, - U_loc_size, ceed_data) ); + PetscCall( SetupLibceed(dm, dm_u0, dm_p0, ceed, app_ctx, problem_data, + ceed_data) ); //CeedVectorView(force_ceed, "%12.8f", stdout); //PetscCall( DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, // bc_pressure) ); - // --------------------------------------------------------------------------- // Setup TSSolve for Richard problem // --------------------------------------------------------------------------- + TS ts; if (problem_data->has_ts) { // --------------------------------------------------------------------------- // Create global initial conditions // --------------------------------------------------------------------------- - Vec U0; - CreateInitialConditions(dm, ceed_data, &U0); - PetscCall( VecDestroy(&U0) ); + SetupResidualOperatorCtx_U0(comm, dm_u0, ceed, ceed_data, + ceed_data->ctx_initial_u0); + SetupResidualOperatorCtx_P0(comm, dm_p0, ceed, ceed_data, + ceed_data->ctx_initial_p0); + SetupResidualOperatorCtx_Ut(comm, dm, ceed, ceed_data, + ceed_data->ctx_residual_ut); + CreateInitialConditions(ceed_data, U, vec_type, + ceed_data->ctx_initial_u0, + ceed_data->ctx_initial_p0, + ceed_data->ctx_residual_ut); + //VecView(U, PETSC_VIEWER_STDOUT_WORLD); + // Solve Richards problem + PetscCall( VecZeroEntries(ceed_data->ctx_residual_ut->X_loc) ); + PetscCall( VecZeroEntries(ceed_data->ctx_residual_ut->X_t_loc) ); + PetscCall( TSSolveRichard(dm, ceed_data, app_ctx, + &U, &ts) ); + //VecView(U, PETSC_VIEWER_STDOUT_WORLD); } - // --------------------------------------------------------------------------- - // Solve PDE - // --------------------------------------------------------------------------- - // Create SNES SNES snes; KSP ksp; - Vec U; - PetscCall( SNESCreate(comm, &snes) ); - PetscCall( SNESGetKSP(snes, &ksp) ); - PetscCall( PDESolver(comm, dm, ceed, ceed_data, vec_type, snes, ksp, &U) ); - //VecView(U, PETSC_VIEWER_STDOUT_WORLD); + if (!problem_data->has_ts) { + // --------------------------------------------------------------------------- + // Setup SNES for Darcy problem + // --------------------------------------------------------------------------- + // Create SNES + PetscCall( SNESCreate(comm, &snes) ); + PetscCall( SNESGetKSP(snes, &ksp) ); + PetscCall( PDESolver(comm, dm, ceed, ceed_data, vec_type, snes, ksp, &U) ); + //VecView(U, PETSC_VIEWER_STDOUT_WORLD); + } // --------------------------------------------------------------------------- // Compute L2 error of mms problem // --------------------------------------------------------------------------- CeedScalar l2_error_u, l2_error_p; - PetscCall( ComputeL2Error(dm, ceed,ceed_data, U, &l2_error_u, + PetscCall( ComputeL2Error(dm, ceed, ceed_data, U, &l2_error_u, &l2_error_p) ); // --------------------------------------------------------------------------- // Print output results // --------------------------------------------------------------------------- - PetscCall( PrintOutput(ceed, mem_type_backend, - snes, ksp, U, l2_error_u, l2_error_p, app_ctx) ); + PetscCall( PrintOutput(ceed, app_ctx, problem_data->has_ts, mem_type_backend, + ts, snes, ksp, U, l2_error_u, l2_error_p) ); // --------------------------------------------------------------------------- // Save solution (paraview) // --------------------------------------------------------------------------- PetscViewer viewer; - PetscCall( PetscViewerVTKOpen(comm,"solution.vtu",FILE_MODE_WRITE,&viewer) ); PetscCall( VecView(U, viewer) ); PetscCall( PetscViewerDestroy(&viewer) ); @@ -186,9 +203,15 @@ int main(int argc, char **argv) { // Free PETSc objects PetscCall( DMDestroy(&dm) ); + PetscCall( DMDestroy(&dm_u0) ); + PetscCall( DMDestroy(&dm_p0) ); PetscCall( VecDestroy(&U) ); - PetscCall( VecDestroy(&U_loc) ); - PetscCall( SNESDestroy(&snes) ); + if (problem_data->has_ts) { + PetscCall( TSDestroy(&ts) ); + } else { + PetscCall( SNESDestroy(&snes) ); + } + PetscCall( CeedDataDestroy(ceed_data, problem_data) ); // -- Function list PetscCall( PetscFunctionListDestroy(&app_ctx->problems) ); @@ -196,11 +219,12 @@ int main(int argc, char **argv) { // -- Structs PetscCall( PetscFree(app_ctx) ); PetscCall( PetscFree(problem_data) ); - PetscCall( PetscFree(phys_ctx) ); + PetscCall( PetscFree(ctx_initial_u0) ); + PetscCall( PetscFree(ctx_initial_p0) ); + PetscCall( PetscFree(ctx_residual_ut) ); // Free libCEED objects //CeedVectorDestroy(&bc_pressure); - PetscCall( CeedDataDestroy(ceed_data) ); CeedDestroy(&ceed); return PetscFinalize(); diff --git a/examples/Hdiv-mixed/problems/darcy2d.c b/examples/Hdiv-mixed/problems/darcy2d.c index 2a4c3d98f1..fca8b2bdd3 100644 --- a/examples/Hdiv-mixed/problems/darcy2d.c +++ b/examples/Hdiv-mixed/problems/darcy2d.c @@ -21,7 +21,7 @@ #include "../qfunctions/darcy-true2d.h" #include "../qfunctions/darcy-system2d.h" #include "../qfunctions/darcy-error2d.h" -#include "../qfunctions/pressure-boundary2d.h" +//#include "../qfunctions/pressure-boundary2d.h" PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; @@ -47,14 +47,14 @@ PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, void *ctx) { problem_data->jacobian_loc = JacobianDarcySystem2D_loc; problem_data->error = DarcyError2D; problem_data->error_loc = DarcyError2D_loc; - problem_data->bc_pressure = BCPressure2D; - problem_data->bc_pressure_loc = BCPressure2D_loc; + //problem_data->bc_pressure = BCPressure2D; + //problem_data->bc_pressure_loc = BCPressure2D_loc; problem_data->has_ts = PETSC_FALSE; // ------------------------------------------------------ // Command line Options // ------------------------------------------------------ - CeedScalar kappa = 1., rho_a0 = 998.2, g = 9.8, alpha_a = 1., b_a = 10.; + CeedScalar kappa = 10., rho_a0 = 998.2, g = 9.8, alpha_a = 1., b_a = 10.; PetscOptionsBegin(app_ctx->comm, NULL, "Options for Hdiv-mixed problem", NULL); PetscCall( PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, kappa, &kappa, NULL)); @@ -77,9 +77,17 @@ PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, void *ctx) { CeedQFunctionContextCreate(ceed, &darcy_context); CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, sizeof(*darcy_ctx), darcy_ctx); - problem_data->qfunction_context = darcy_context; - CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, - FreeContextPetsc); + //CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, + // FreeContextPetsc); + problem_data->true_qfunction_ctx = darcy_context; + CeedQFunctionContextReferenceCopy(darcy_context, + &problem_data->residual_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, + &problem_data->jacobian_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, + &problem_data->error_qfunction_ctx); + PetscCall( PetscFree(darcy_ctx) ); + PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mixed/problems/darcy3d.c b/examples/Hdiv-mixed/problems/darcy3d.c index 88614f243b..96fba6b598 100644 --- a/examples/Hdiv-mixed/problems/darcy3d.c +++ b/examples/Hdiv-mixed/problems/darcy3d.c @@ -21,7 +21,7 @@ #include "../qfunctions/darcy-true3d.h" #include "../qfunctions/darcy-system3d.h" #include "../qfunctions/darcy-error3d.h" -#include "../qfunctions/pressure-boundary3d.h" +//#include "../qfunctions/pressure-boundary3d.h" PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; @@ -47,8 +47,8 @@ PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, void *ctx) { problem_data->jacobian_loc = JacobianDarcySystem3D_loc; problem_data->error = DarcyError3D; problem_data->error_loc = DarcyError3D_loc; - problem_data->bc_pressure = BCPressure3D; - problem_data->bc_pressure_loc = BCPressure3D_loc; + //problem_data->bc_pressure = BCPressure3D; + //problem_data->bc_pressure_loc = BCPressure3D_loc; problem_data->has_ts = PETSC_FALSE; // ------------------------------------------------------ @@ -77,9 +77,17 @@ PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, void *ctx) { CeedQFunctionContextCreate(ceed, &darcy_context); CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, sizeof(*darcy_ctx), darcy_ctx); - problem_data->qfunction_context = darcy_context; - CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, - FreeContextPetsc); + //CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, + // FreeContextPetsc); + problem_data->true_qfunction_ctx = darcy_context; + CeedQFunctionContextReferenceCopy(darcy_context, + &problem_data->residual_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, + &problem_data->jacobian_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, + &problem_data->error_qfunction_ctx); + PetscCall( PetscFree(darcy_ctx) ); + PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mixed/problems/richard2d.c b/examples/Hdiv-mixed/problems/richard2d.c index 61a03e975d..7240a5375a 100644 --- a/examples/Hdiv-mixed/problems/richard2d.c +++ b/examples/Hdiv-mixed/problems/richard2d.c @@ -19,8 +19,10 @@ #include "../include/register-problem.h" #include "../qfunctions/richard-system2d.h" -#include "../qfunctions/richard-mms2d.h" -#include "../qfunctions/pressure-boundary2d.h" +#include "../qfunctions/richard-true2d.h" +#include "../qfunctions/richard-ics2d.h" +#include "../qfunctions/darcy-error2d.h" +//#include "../qfunctions/pressure-boundary2d.h" #include "petscsystypes.h" PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx) { @@ -39,18 +41,24 @@ PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx) { problem_data->elem_node = 4; problem_data->q_data_size_face = 3; problem_data->quadrature_mode = CEED_GAUSS; - problem_data->ics = RichardICs2D; - problem_data->ics_loc = RichardICs2D_loc; - //problem_data->force = RichardTrue2D; - //problem_data->force_loc = RichardTrue2D_loc; - //problem_data->residual = RichardSystem2D; - //problem_data->residual_loc = RichardSystem2D_loc; + problem_data->true_solution = RichardTrue2D; + problem_data->true_solution_loc = RichardTrue2D_loc; + problem_data->rhs_u0 = RichardRhsU02D; + problem_data->rhs_u0_loc = RichardRhsU02D_loc; + problem_data->ics_u = RichardICsU2D; + problem_data->ics_u_loc = RichardICsU2D_loc; + problem_data->rhs_p0 = RichardRhsP02D; + problem_data->rhs_p0_loc = RichardRhsP02D_loc; + problem_data->ics_p = RichardICsP2D; + problem_data->ics_p_loc = RichardICsP2D_loc; + problem_data->residual = RichardSystem2D; + problem_data->residual_loc = RichardSystem2D_loc; //problem_data->jacobian = JacobianRichardSystem2D; //problem_data->jacobian_loc = JacobianRichardSystem2D_loc; - //problem_data->error = DarcyError2D; - //problem_data->error_loc = DarcyError2D_loc; - problem_data->bc_pressure = BCPressure2D; - problem_data->bc_pressure_loc = BCPressure2D_loc; + problem_data->error = DarcyError2D; + problem_data->error_loc = DarcyError2D_loc; + //problem_data->bc_pressure = BCPressure2D; + //problem_data->bc_pressure_loc = BCPressure2D_loc; problem_data->has_ts = PETSC_TRUE; // ------------------------------------------------------ @@ -72,6 +80,9 @@ PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx) { rho_a0, &rho_a0, NULL)); PetscCall( PetscOptionsScalar("-beta", "Water compressibility", NULL, beta, &beta, NULL)); + app_ctx->t_final = 0.5; + PetscCall( PetscOptionsScalar("-t_final", "End time", NULL, + app_ctx->t_final, &app_ctx->t_final, NULL)); PetscOptionsEnd(); richard_ctx->kappa = kappa; @@ -81,15 +92,28 @@ PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx) { richard_ctx->beta = beta; richard_ctx->g = g; richard_ctx->p0 = p0; - richard_ctx->time = 0.; richard_ctx->gamma = 5.; + richard_ctx->t = 0.; + richard_ctx->t_final = app_ctx->t_final; CeedQFunctionContextCreate(ceed, &richard_context); CeedQFunctionContextSetData(richard_context, CEED_MEM_HOST, CEED_COPY_VALUES, sizeof(*richard_ctx), richard_ctx); - problem_data->qfunction_context = richard_context; - CeedQFunctionContextSetDataDestroy(richard_context, CEED_MEM_HOST, - FreeContextPetsc); - + //CeedQFunctionContextSetDataDestroy(richard_context, CEED_MEM_HOST, + // FreeContextPetsc); + CeedQFunctionContextRegisterDouble(richard_context, "time", + offsetof(struct RICHARDContext_, t), 1, "current solver time"); + CeedQFunctionContextRegisterDouble(richard_context, "final_time", + offsetof(struct RICHARDContext_, t_final), 1, "final time"); + CeedQFunctionContextRegisterDouble(richard_context, "time_step", + offsetof(struct RICHARDContext_, dt), 1, "time step"); + problem_data->true_qfunction_ctx = richard_context; + CeedQFunctionContextReferenceCopy(richard_context, + &problem_data->rhs_u0_qfunction_ctx); + CeedQFunctionContextReferenceCopy(richard_context, + &problem_data->residual_qfunction_ctx); + CeedQFunctionContextReferenceCopy(richard_context, + &problem_data->error_qfunction_ctx); PetscCall( PetscFree(richard_ctx) ); + PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mixed/qfunctions/darcy-error2d.h b/examples/Hdiv-mixed/qfunctions/darcy-error2d.h index 1420ebc571..e5ccbc5719 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-error2d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-error2d.h @@ -21,6 +21,7 @@ #define DARCY_ERROR2D_H #include +#include #include "utils.h" // ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/qfunctions/darcy-error3d.h b/examples/Hdiv-mixed/qfunctions/darcy-error3d.h index 56a24d3bd0..63af1b1e48 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-error3d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-error3d.h @@ -21,6 +21,7 @@ #define DARCY_ERROR3D_H #include +#include #include "utils.h" diff --git a/examples/Hdiv-mixed/qfunctions/darcy-system2d.h b/examples/Hdiv-mixed/qfunctions/darcy-system2d.h index 776f2a15f6..07edd820e9 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-system2d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-system2d.h @@ -21,7 +21,7 @@ #define DARCY_SYSTEM2D_H #include -#include "ceed/ceed-f64.h" +#include #include "utils.h" // ----------------------------------------------------------------------------- @@ -172,7 +172,7 @@ CEED_QFUNCTION(JacobianDarcySystem2D)(void *ctx, CeedInt Q, // *INDENT-ON* DARCYContext context = (DARCYContext)ctx; - const CeedScalar kappa = context->kappa; + const CeedScalar kappa = context->kappa; const CeedScalar rho_a0 = context->rho_a0; const CeedScalar g = context->g; const CeedScalar alpha_a = context->alpha_a; diff --git a/examples/Hdiv-mixed/qfunctions/darcy-system3d.h b/examples/Hdiv-mixed/qfunctions/darcy-system3d.h index bcbb463585..3d8af38444 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-system3d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-system3d.h @@ -21,6 +21,7 @@ #define DARCY_SYSTEM3D_H #include +#include #include "utils.h" // ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/qfunctions/darcy-true2d.h b/examples/Hdiv-mixed/qfunctions/darcy-true2d.h index e91d733cee..3eaef40f0f 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-true2d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-true2d.h @@ -21,6 +21,7 @@ #define DARCY_TRUE2D_H #include +#include #include "utils.h" // ----------------------------------------------------------------------------- @@ -76,17 +77,20 @@ CEED_QFUNCTION(DarcyTrue2D)(void *ctx, const CeedInt Q, CeedScalar x = coords[i+0*Q], y = coords[i+1*Q]; CeedScalar psi = sin(PI_DOUBLE*x)*sin(PI_DOUBLE*y); CeedScalar psi_x = PI_DOUBLE*cos(PI_DOUBLE*x)*sin(PI_DOUBLE*y); + CeedScalar psi_xx = -PI_DOUBLE*PI_DOUBLE*psi; CeedScalar psi_y = PI_DOUBLE*sin(PI_DOUBLE*x)*cos(PI_DOUBLE*y); - + CeedScalar psi_yy = -PI_DOUBLE*PI_DOUBLE*psi; // k_r = b_a + alpha_a * (1 - x*y) CeedScalar k_r = b_a + alpha_a*(1-x*y); + CeedScalar k_rx = -alpha_a*y; + CeedScalar k_ry = -alpha_a*x; // rho = rho_a/rho_a0 CeedScalar rho = 1.; // u = -rho*k_r*K *[grad(\psi) - rho*g_u] - CeedScalar u[2] = {-rho*k_r*kappa*psi_x, -rho*k_r*kappa*(psi_y-1)}; - CeedScalar div_u = -rho*kappa*(-alpha_a*y*psi_x - k_r*PI_DOUBLE*PI_DOUBLE*psi - -alpha_a*x*psi_y - k_r*PI_DOUBLE*PI_DOUBLE*psi); - + CeedScalar u[2] = {-rho*kappa*k_r*psi_x, + -rho*kappa*k_r*(psi_y-1)}; + CeedScalar div_u = -rho*kappa*(k_rx*psi_x + k_r*psi_xx + + k_ry*(psi_y-1) + k_r*psi_yy); // True Force: f = \div(u) true_force[i+0*Q] = div_u; // True Solution diff --git a/examples/Hdiv-mixed/qfunctions/darcy-true3d.h b/examples/Hdiv-mixed/qfunctions/darcy-true3d.h index 6ebc17b1d1..dd18bdd998 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-true3d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-true3d.h @@ -21,6 +21,7 @@ #define DARCY_TRUE3D_H #include +#include #include "utils.h" // ----------------------------------------------------------------------------- @@ -76,20 +77,26 @@ CEED_QFUNCTION(DarcyTrue3D)(void *ctx, const CeedInt Q, CeedScalar x = coords[i+0*Q], y = coords[i+1*Q], z = coords[i+2*Q]; CeedScalar psi = sin(PI_DOUBLE*x) * sin(PI_DOUBLE*y) * sin(PI_DOUBLE*z); CeedScalar psi_x = PI_DOUBLE*cos(PI_DOUBLE*x) *sin(PI_DOUBLE*y) *sin(PI_DOUBLE*z); + CeedScalar psi_xx = -PI_DOUBLE*PI_DOUBLE*psi; CeedScalar psi_y = PI_DOUBLE*sin(PI_DOUBLE*x) *cos(PI_DOUBLE*y) *sin(PI_DOUBLE*z); + CeedScalar psi_yy = -PI_DOUBLE*PI_DOUBLE*psi; CeedScalar psi_z = PI_DOUBLE*sin(PI_DOUBLE*x) *sin(PI_DOUBLE*y) *cos(PI_DOUBLE*z); + CeedScalar psi_zz = -PI_DOUBLE*PI_DOUBLE*psi; // k_r = b_a + alpha_a * (psi - x2) CeedScalar k_r = b_a + alpha_a*(1-x*y*z); + CeedScalar k_rx = -alpha_a*y*z; + CeedScalar k_ry = -alpha_a*x*z; + CeedScalar k_rz = -alpha_a*x*y; // rho = rho_a/rho_a0 CeedScalar rho = 1.; // u = -rho*k_r*K *[grad(\psi) - rho*g_u] - CeedScalar u[3] = {-rho*k_r*kappa*psi_x, - -rho*k_r*kappa*psi_y, - -rho*k_r*kappa*(psi_z-1)}; - CeedScalar div_u = -rho*kappa*(-alpha_a*y*z*psi_x - k_r*PI_DOUBLE*PI_DOUBLE*psi - -alpha_a*x*z*psi_y - k_r*PI_DOUBLE*PI_DOUBLE*psi - -alpha_a*x*y*psi_z - k_r*PI_DOUBLE*PI_DOUBLE*psi); + CeedScalar u[3] = {-rho*kappa*k_r*psi_x, + -rho*kappa*k_r*psi_y, + -rho*kappa*k_r*(psi_z-1)}; + CeedScalar div_u = -rho*kappa*(k_rx*psi_x + k_r*psi_xx + + k_ry*psi_y + k_r*psi_yy + + k_rz*(psi_z-1) + k_r*psi_zz); // True Force: f = \div(u) true_force[i+0*Q] = div_u; diff --git a/examples/Hdiv-mixed/qfunctions/pressure-boundary2d.h b/examples/Hdiv-mixed/qfunctions/pressure-boundary2d.h index a2fc842a33..ee349fbf6b 100644 --- a/examples/Hdiv-mixed/qfunctions/pressure-boundary2d.h +++ b/examples/Hdiv-mixed/qfunctions/pressure-boundary2d.h @@ -11,6 +11,9 @@ #ifndef pressure_bc_2d_h #define pressure_bc_2d_h +#include +#include +#include "utils.h" // ----------------------------------------------------------------------------- // Strong form: // u = -\grad(p) on \Omega diff --git a/examples/Hdiv-mixed/qfunctions/pressure-boundary3d.h b/examples/Hdiv-mixed/qfunctions/pressure-boundary3d.h index 75cc9f6d6f..208ff0aea0 100644 --- a/examples/Hdiv-mixed/qfunctions/pressure-boundary3d.h +++ b/examples/Hdiv-mixed/qfunctions/pressure-boundary3d.h @@ -11,6 +11,9 @@ #ifndef pressure_bc_3d_h #define pressure_bc_3d_h +#include +#include +#include "utils.h" // ----------------------------------------------------------------------------- // Strong form: // u = -\grad(p) on \Omega diff --git a/examples/Hdiv-mixed/qfunctions/richard-ics2d.h b/examples/Hdiv-mixed/qfunctions/richard-ics2d.h new file mode 100644 index 0000000000..49664a40ae --- /dev/null +++ b/examples/Hdiv-mixed/qfunctions/richard-ics2d.h @@ -0,0 +1,228 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Force of Richard problem 2D (quad element) using PETSc + +#ifndef RICHARD_ICS2D_H +#define RICHARD_ICS2D_H + +#include +#include +#include "ceed/ceed-f64.h" +#include "utils.h" + +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega x [0,T] +// -\div(u) = -f + d (rho*theta)/dt in \Omega x [0,T] +// p = p_b on \Gamma_D x [0,T] +// u.n = u_b on \Gamma_N x [0,T] +// p = p_0 in \Omega, t = 0 +// +// Where rho = rho_a/rho_a0, rho_a = rho_a0*exp(\beta * (p - p0)), p0 = 101325 Pa is atmospheric pressure +// rho_a0 is the density at p_0, g_u = g/norm(g) where g is gravity. +// k_r = b_a + alpha_a * (\psi - x2), where \psi = p / (rho_a0 * norm(g)) and x2 is vertical axis +// +// Weak form: Find (u, \psi) \in VxQ (V=H(div), Q=L^2) on \Omega +// (v, K^{-1}/rho*k_r * u) -(v, rho*g_u) -(\div(v), \psi) = -_{\Gamma_D} +// -(q, \div(u)) + (q, f) -(q, d (rho*\theta)/dt ) = 0 +// +// We solve MMS for K = kappa*I and beta=0 ==> rho=1 and \theta = alpha_a*\psi, so +// -(q, d (rho*\theta)/dt ) = -alpha_a*(q, d(\psi)/dt ) +// +// This QFunction setup the true solution and forcing f of the above equation +// Inputs: +// coords: physical coordinate +// +// Output: +// true_force : = div(u) + d (rho*theta)/dt +// true_solution : = [\psi, u] where \psi, u are the exact solution solution +// ----------------------------------------------------------------------------- +// We have 3 experiment parameters as described in Table 1:P1, P2, P3 +// Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +#ifndef RICHARD_CTX +#define RICHARD_CTX +typedef struct RICHARDContext_ *RICHARDContext; +struct RICHARDContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; + CeedScalar beta, p0; + CeedScalar t, t_final, dt; + CeedScalar gamma; +}; +#endif + +// ----------------------------------------------------------------------------- +// We solve (v, u) = (v, ue) at t=0, to project ue to Hdiv space +// This QFunction create rhs_u0 = (v, ue) +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(RichardRhsU02D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*coords) = in[1], + (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[2]; + // Outputs + CeedScalar (*rhs_u0) = out[0]; + // Context + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#include +#include "ceed/ceed-f64.h" #include "utils.h" // See Matthew Farthing, Christopher Kees, Cass Miller (2003) @@ -67,20 +69,17 @@ typedef struct RICHARDContext_ *RICHARDContext; struct RICHARDContext_ { CeedScalar kappa; - CeedScalar alpha_a; - CeedScalar b_a; - CeedScalar rho_a0; - CeedScalar beta; CeedScalar g; - CeedScalar p0; - CeedScalar time; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; + CeedScalar beta, p0; + CeedScalar t, t_final, dt; CeedScalar gamma; }; #endif // ----------------------------------------------------------------------------- // Residual evaluation for Richard problem // ----------------------------------------------------------------------------- -/* CEED_QFUNCTION(RichardSystem2D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { @@ -91,8 +90,9 @@ CEED_QFUNCTION(RichardSystem2D)(void *ctx, CeedInt Q, (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], (*div_u) = (const CeedScalar(*))in[3], (*p) = (const CeedScalar(*))in[4], - (*p_t) = (const CeedScalar(*))in[5], - (*coords) = in[6]; + (*f) = in[5], + (*coords) = in[6], + (*p_t) = (const CeedScalar(*))in[7]; // Outputs CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], @@ -100,16 +100,18 @@ CEED_QFUNCTION(RichardSystem2D)(void *ctx, CeedInt Q, (*q) = (CeedScalar(*))out[2]; // Context RICHARDContext context = (RICHARDContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; + const CeedScalar kappa = context->kappa; const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar beta = context->beta; - const CeedScalar g = context->g; - const CeedScalar p0 = context->p0; // atmospheric pressure - CeedScalar t = context->time; - // *INDENT-ON* + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + //const CeedScalar beta = context->beta; + //const CeedScalar p0 = context->p0; // atmospheric pressure + const CeedScalar gamma = context->gamma; + CeedScalar t = context->t; + //CeedScalar dt = context->dt; + // *INDENT-ON* // Quadrature Point Loop CeedPragmaSIMD for (CeedInt i=0; i -#include "ceed/ceed-f64.h" +#include #include "utils.h" // See Matthew Farthing, Christopher Kees, Cass Miller (2003) @@ -29,7 +29,7 @@ // ----------------------------------------------------------------------------- // Strong form: // u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega x [0,T] -// -\div(u) = -f + d (rho*\theta)/dt in \Omega x [0,T] +// -\div(u) = -f + d (rho*theta)/dt in \Omega x [0,T] // p = p_b on \Gamma_D x [0,T] // u.n = u_b on \Gamma_N x [0,T] // p = p_0 in \Omega, t = 0 @@ -45,15 +45,13 @@ // We solve MMS for K = kappa*I and beta=0 ==> rho=1 and \theta = alpha_a*\psi, so // -(q, d (rho*\theta)/dt ) = -alpha_a*(q, d(\psi)/dt ) // -// This QFunction sets up the force +// This QFunction setup the true solution and forcing f of the above equation // Inputs: -// x : interpolation of the physical coordinate -// w : weight of quadrature -// J : dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// coords: physical coordinate // // Output: -// force_u : which is 0.0 for this problem (- is in pressure-boundary qfunction) -// force_p : 0.0 +// true_force : = div(u) + d (rho*theta)/dt +// true_solution : = [\psi, u] where \psi, u are the exact solution solution // ----------------------------------------------------------------------------- // We have 3 experiment parameters as described in Table 1:P1, P2, P3 // Matthew Farthing, Christopher Kees, Cass Miller (2003) @@ -63,43 +61,15 @@ typedef struct RICHARDContext_ *RICHARDContext; struct RICHARDContext_ { CeedScalar kappa; - CeedScalar alpha_a; - CeedScalar b_a; - CeedScalar rho_a0; - CeedScalar beta; CeedScalar g; - CeedScalar p0; - CeedScalar time; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; + CeedScalar beta, p0; + CeedScalar t, t_final, dt; CeedScalar gamma; }; #endif -// ----------------------------------------------------------------------------- -// Initial conditions for Richard problem -// ----------------------------------------------------------------------------- -CEED_QFUNCTION(RichardICs2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* - // Inputs - const CeedScalar (*coords) = in[0]; - // Outputs - CeedScalar (*u_0) = out[0], (*psi_0) = out[1]; - // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; igamma; - const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar g = context->g; - CeedScalar t = context->time; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + const CeedScalar gamma = context->gamma; + CeedScalar t_final = context->t_final; + // Quadrature Point Loop CeedPragmaSIMD for (CeedInt i=0; i // ----------------------------------------------------------------------------- // Convert PETSc MemType to libCEED MemType @@ -15,13 +16,14 @@ CeedMemType MemTypeP2C(PetscMemType mem_type) { // ----------------------------------------------------------------------------- // Destroy libCEED objects // ----------------------------------------------------------------------------- -PetscErrorCode CeedDataDestroy(CeedData ceed_data) { +PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data) { PetscFunctionBegin; // Vectors CeedVectorDestroy(&ceed_data->x_ceed); CeedVectorDestroy(&ceed_data->y_ceed); + CeedVectorDestroy(&ceed_data->x_t_ceed); CeedVectorDestroy(&ceed_data->x_coord); // Restrictions CeedElemRestrictionDestroy(&ceed_data->elem_restr_x); @@ -29,19 +31,41 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data) { CeedElemRestrictionDestroy(&ceed_data->elem_restr_U_i); // U = [p,u] CeedElemRestrictionDestroy(&ceed_data->elem_restr_p); CeedElemRestrictionDestroy(&ceed_data->elem_restr_p_i); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_u0); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_p0); // Bases CeedBasisDestroy(&ceed_data->basis_x); CeedBasisDestroy(&ceed_data->basis_u); CeedBasisDestroy(&ceed_data->basis_p); CeedBasisDestroy(&ceed_data->basis_u_face); - // QFunctions + if (problem_data->has_ts) { + CeedVectorDestroy(&ceed_data->u0_ceed); + CeedVectorDestroy(&ceed_data->v0_ceed); + CeedVectorDestroy(&ceed_data->p0_ceed); + CeedVectorDestroy(&ceed_data->q0_ceed); + CeedVectorDestroy(&ceed_data->rhs_u0_ceed); + CeedVectorDestroy(&ceed_data->rhs_p0_ceed); + CeedQFunctionDestroy(&ceed_data->qf_rhs_u0); + CeedOperatorDestroy(&ceed_data->op_rhs_u0); + CeedQFunctionDestroy(&ceed_data->qf_ics_u); + CeedOperatorDestroy(&ceed_data->op_ics_u); + CeedQFunctionDestroy(&ceed_data->qf_rhs_p0); + CeedOperatorDestroy(&ceed_data->op_rhs_p0); + CeedQFunctionDestroy(&ceed_data->qf_ics_p); + CeedOperatorDestroy(&ceed_data->op_ics_p); + } + //QFunctions CeedQFunctionDestroy(&ceed_data->qf_residual); - CeedQFunctionDestroy(&ceed_data->qf_jacobian); CeedQFunctionDestroy(&ceed_data->qf_error); - // Operators + //Operators CeedOperatorDestroy(&ceed_data->op_residual); - CeedOperatorDestroy(&ceed_data->op_jacobian); CeedOperatorDestroy(&ceed_data->op_error); + if (!problem_data->has_ts) { + //QFunctions + CeedQFunctionDestroy(&ceed_data->qf_jacobian); + //Operators + CeedOperatorDestroy(&ceed_data->op_jacobian); + } PetscCall( PetscFree(ceed_data) ); PetscFunctionReturn(0); @@ -77,15 +101,19 @@ PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, // ----------------------------------------------------------------------------- // Get Oriented CEED restriction data from DMPlex // ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, - CeedInt P, CeedElemRestriction *elem_restr_oriented, - CeedElemRestriction *elem_restr) { - PetscSection section; - PetscInt p, num_elem, num_dof, *restr_indices_u, *restr_indices_p, - elem_offset, num_fields, dim, c_start, c_end; +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, + DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u, + CeedElemRestriction *elem_restr_p, + CeedElemRestriction *elem_restr_u0, CeedElemRestriction *elem_restr_p0) { + PetscSection section, section_u0, section_p0; + PetscInt p, num_elem, num_dof, num_dof_u0, num_dof_p0, *restr_indices_u, + *restr_indices_p, *restr_indices_u0, *restr_indices_p0, + elem_offset, num_fields, num_fields_u0, num_fields_p0, + dim, c_start, c_end; Vec U_loc; const PetscInt *ornt; // this is for orientation of dof PetscFunctionBeginUser; + // Section for mixed problem PetscCall( DMGetDimension(dm, &dim) ); PetscCall( DMGetLocalSection(dm, §ion) ); PetscCall( PetscSectionGetNumFields(section, &num_fields) ); @@ -95,19 +123,47 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, PetscCall( PetscSectionGetFieldComponents(section, f, &num_comp[f]) ); field_offsets[f+1] = field_offsets[f] + num_comp[f]; } + // Section for initial conditions u0 + PetscCall( DMGetLocalSection(dm_u0, §ion_u0) ); + PetscCall( PetscSectionGetNumFields(section_u0, &num_fields_u0) ); + PetscInt num_comp_u0[num_fields_u0], field_offsets_u0[num_fields_u0+1]; + field_offsets_u0[0] = 0; + for (PetscInt f = 0; f < num_fields_u0; f++) { + PetscCall( PetscSectionGetFieldComponents(section_u0, f, &num_comp_u0[f]) ); + field_offsets_u0[f+1] = field_offsets_u0[f] + num_comp_u0[f]; + } + // Section for initial conditions p0 + PetscCall( DMGetLocalSection(dm_p0, §ion_p0) ); + PetscCall( PetscSectionGetNumFields(section_p0, &num_fields_p0) ); + PetscInt num_comp_p0[num_fields_p0], field_offsets_p0[num_fields_p0+1]; + field_offsets_p0[0] = 0; + for (PetscInt f = 0; f < num_fields_p0; f++) { + PetscCall( PetscSectionGetFieldComponents(section_p0, f, &num_comp_p0[f]) ); + field_offsets_p0[f+1] = field_offsets_p0[f] + num_comp_p0[f]; + } + PetscCall( DMPlexGetHeightStratum(dm, 0, &c_start, &c_end) ); num_elem = c_end - c_start; PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), &restr_indices_u) ); + PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), + &restr_indices_u0) ); PetscCall( PetscMalloc1(num_elem,&restr_indices_p) ); - bool *orient_indices_u; // to flip the dof + PetscCall( PetscMalloc1(num_elem,&restr_indices_p0) ); + bool *orient_indices_u, *orient_indices_u0; // to flip the dof PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), &orient_indices_u) ); + PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), &orient_indices_u0) ); for (p = 0, elem_offset = 0; p < num_elem; p++) { - PetscInt num_indices, *indices, faces_per_elem, dofs_per_face; + PetscInt num_indices, *indices, faces_per_elem, dofs_per_face, + num_indices_u0, *indices_u0, num_indices_p0, *indices_p0; PetscCall( DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL) ); - + PetscCall( DMPlexGetClosureIndices(dm_u0, section_u0, section_u0, p, PETSC_TRUE, + &num_indices_u0, &indices_u0, NULL, NULL) ); + PetscCall( DMPlexGetClosureIndices(dm_p0, section_p0, section_p0, p, PETSC_TRUE, + &num_indices_p0, &indices_p0, NULL, NULL) ); restr_indices_p[p] = indices[num_indices - 1]; + restr_indices_p0[p] = indices_p0[0]; PetscCall( DMPlexGetConeOrientation(dm, p, &ornt) ); // Get number of faces per element PetscCall( DMPlexGetConeSize(dm, p, &faces_per_elem) ); @@ -120,11 +176,21 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, restr_indices_u[elem_offset] = loc; // Set orientation orient_indices_u[elem_offset] = ornt[f] < 0; + PetscInt loc_u0 = Involute(indices_u0[ii*num_comp_u0[0]]); + restr_indices_u0[elem_offset] = loc_u0; + // Set orientation + orient_indices_u0[elem_offset] = ornt[f] < 0; elem_offset++; } } PetscCall( DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL) ); + PetscCall( DMPlexRestoreClosureIndices(dm_u0, section_u0, section_u0, p, + PETSC_TRUE, + &num_indices_u0, &indices_u0, NULL, NULL) ); + PetscCall( DMPlexRestoreClosureIndices(dm_p0, section_p0, section_p0, p, + PETSC_TRUE, + &num_indices_p0, &indices_p0, NULL, NULL) ); } //if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) // SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, @@ -139,22 +205,40 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedElemRestrictionCreateOriented(ceed, num_elem, dim*PetscPowInt(P, dim), 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_u, orient_indices_u, - elem_restr_oriented); + elem_restr_u); CeedElemRestrictionCreate(ceed, num_elem, 1, 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, - restr_indices_p, elem_restr); + restr_indices_p, elem_restr_p); + PetscCall( DMGetLocalVector(dm_u0, &U_loc) ); + PetscCall( VecGetLocalSize(U_loc, &num_dof_u0) ); + PetscCall( DMRestoreLocalVector(dm_u0, &U_loc) ); + // dof per element in Hdiv is dim*P^dim, for linear element P=2 + CeedElemRestrictionCreateOriented(ceed, num_elem, dim*PetscPowInt(P, dim), + 1, 1, num_dof_u0, CEED_MEM_HOST, CEED_COPY_VALUES, + restr_indices_u0, orient_indices_u0, + elem_restr_u0); + PetscCall( DMGetLocalVector(dm_p0, &U_loc) ); + PetscCall( VecGetLocalSize(U_loc, &num_dof_p0) ); + PetscCall( DMRestoreLocalVector(dm_p0, &U_loc) ); + CeedElemRestrictionCreate(ceed, num_elem, 1, + 1, 1, num_dof_p0, CEED_MEM_HOST, CEED_COPY_VALUES, + restr_indices_p0, elem_restr_p0); PetscCall( PetscFree(restr_indices_p) ); PetscCall( PetscFree(restr_indices_u) ); PetscCall( PetscFree(orient_indices_u) ); + PetscCall( PetscFree(restr_indices_u0) ); + PetscCall( PetscFree(orient_indices_u0) ); + PetscCall( PetscFree(restr_indices_p0) ); PetscFunctionReturn(0); }; // ----------------------------------------------------------------------------- // Set up libCEED on the fine grid for a given degree // ----------------------------------------------------------------------------- -PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, +PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, Ceed ceed, + AppCtx app_ctx, ProblemData problem_data, - PetscInt U_loc_size, CeedData ceed_data) { + CeedData ceed_data) { CeedInt P = app_ctx->degree + 1; // Number of quadratures in 1D, q_extra is set in cl-options.c CeedInt Q = P + 1 + app_ctx->q_extra; @@ -189,10 +273,10 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); CeedBasisCreateH1(ceed, CEED_TOPOLOGY_QUAD, num_comp_p, 1, num_qpts, interp_p, grad, q_ref,q_weights, &ceed_data->basis_p); - HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, - CEED_GAUSS_LOBATTO); - CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, - interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); + //HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, + // CEED_GAUSS_LOBATTO); + //CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, + // interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); } else { HdivBasisHex(Q, q_ref, q_weights, interp_u, div, problem_data->quadrature_mode); CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, @@ -200,9 +284,9 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); CeedBasisCreateH1(ceed, CEED_TOPOLOGY_HEX, num_comp_p, 1, num_qpts, interp_p, grad, q_ref,q_weights, &ceed_data->basis_p); - HdivBasisHex(Q, q_ref, q_weights, interp_u, div, CEED_GAUSS_LOBATTO); - CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, - interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); + //HdivBasisHex(Q, q_ref, q_weights, interp_u, div, CEED_GAUSS_LOBATTO); + //CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, + // interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); } CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, @@ -220,25 +304,25 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, PetscCall( CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, value, &ceed_data->elem_restr_x) ); // -- Solution restriction - PetscCall( CreateRestrictionFromPlexOriented(ceed, dm, P, - &ceed_data->elem_restr_u, &ceed_data->elem_restr_p) ); + PetscCall( CreateRestrictionFromPlexOriented(ceed, dm, dm_u0, dm_p0, P, + &ceed_data->elem_restr_u, &ceed_data->elem_restr_p, + &ceed_data->elem_restr_u0, &ceed_data->elem_restr_p0) ); // -- Geometric ceed_data restriction PetscCall( DMPlexGetHeightStratum(dm, 0, &c_start, &c_end) ); num_elem = c_end - c_start; - + ceed_data->num_elem = num_elem; CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, (dim+1), (dim+1)*num_elem*num_qpts, CEED_STRIDES_BACKEND, &ceed_data->elem_restr_U_i); CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, 1, 1*num_elem*num_qpts, CEED_STRIDES_BACKEND, &ceed_data->elem_restr_p_i); + // --------------------------------------------------------------------------- // Element coordinates // --------------------------------------------------------------------------- PetscCall( DMGetCoordinatesLocal(dm, &coords) ); PetscCall( VecGetArrayRead(coords, &coordArray) ); - CeedVectorCreate(ceed, U_loc_size, &ceed_data->x_coord); - CeedElemRestrictionCreateVector(ceed_data->elem_restr_x, &ceed_data->x_coord, NULL); CeedVectorSetArray(ceed_data->x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, @@ -246,7 +330,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, PetscCall( VecRestoreArrayRead(coords, &coordArray) ); // --------------------------------------------------------------------------- - // Setup true solution and force + // Setup true solution for [p,u] // --------------------------------------------------------------------------- CeedVector true_vec, true_force; CeedVectorCreate(ceed, num_elem*num_qpts*(dim+1), &true_vec); @@ -254,57 +338,164 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // Create the q-function that sets up the RHS and true solution CeedQFunctionCreateInterior(ceed, 1, problem_data->true_solution, problem_data->true_solution_loc, &qf_true); - CeedQFunctionSetContext(qf_true, problem_data->qfunction_context); - //CeedQFunctionContextDestroy(&problem_data->qfunction_context); + CeedQFunctionSetContext(qf_true, problem_data->true_qfunction_ctx); + CeedQFunctionContextDestroy(&problem_data->true_qfunction_ctx); CeedQFunctionAddInput(qf_true, "x", num_comp_x, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_true, "true force", 1, CEED_EVAL_NONE); CeedQFunctionAddOutput(qf_true, "true solution", dim+1, CEED_EVAL_NONE); // Create the operator that builds the RHS and true solution CeedOperatorCreate(ceed, qf_true, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_true); + if (problem_data->has_ts) { + double final_time = app_ctx->t_final; + CeedOperatorContextGetFieldLabel(op_true, "final_time", + &ceed_data->ctx_residual_ut->final_time_label); + CeedOperatorContextSetDouble(op_true, + ceed_data->ctx_residual_ut->final_time_label, &final_time); + } CeedOperatorSetField(op_true, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); CeedOperatorSetField(op_true, "true force", ceed_data->elem_restr_p_i, CEED_BASIS_COLLOCATED, true_force); CeedOperatorSetField(op_true, "true solution", ceed_data->elem_restr_U_i, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); - // Setup RHS and true solution + // Setup true solution CeedOperatorApply(op_true, ceed_data->x_coord, true_vec, CEED_REQUEST_IMMEDIATE); + // --------------------------------------------------------------------------- + // Setup initial conditions + // --------------------------------------------------------------------------- if (problem_data->has_ts) { // --------------------------------------------------------------------------- - // Setup qfunction for initial conditions + // Setup RHS for u field // --------------------------------------------------------------------------- - CeedQFunction qf_ics; - CeedOperator op_ics; - CeedQFunctionCreateInterior(ceed, 1, problem_data->ics, - problem_data->ics_loc, &qf_ics); - CeedQFunctionSetContext(qf_ics, problem_data->qfunction_context); - CeedQFunctionAddInput(qf_ics, "x", num_comp_x, CEED_EVAL_INTERP); - CeedQFunctionAddOutput(qf_ics, "u_0", dim, CEED_EVAL_INTERP); - CeedQFunctionAddOutput(qf_ics, "p_0", 1, CEED_EVAL_INTERP); + CeedQFunction qf_rhs_u0; + CeedOperator op_rhs_u0; + // Create the q-function that sets up the RHS + CeedQFunctionCreateInterior(ceed, 1, problem_data->rhs_u0, + problem_data->rhs_u0_loc, &qf_rhs_u0); + CeedQFunctionSetContext(qf_rhs_u0, problem_data->rhs_u0_qfunction_ctx); + CeedQFunctionContextDestroy(&problem_data->rhs_u0_qfunction_ctx); + CeedQFunctionAddInput(qf_rhs_u0, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_rhs_u0, "x", num_comp_x, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_rhs_u0, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddOutput(qf_rhs_u0, "rhs_u0", dim, CEED_EVAL_INTERP); + // Create the operator that builds the RHS + CeedOperatorCreate(ceed, qf_rhs_u0, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_rhs_u0); + CeedOperatorSetField(op_rhs_u0, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_rhs_u0, "x", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_u0, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_u0, "rhs_u0", ceed_data->elem_restr_u0, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + + // -- Save libCEED data to apply operator in setup-ts.c + ceed_data->qf_rhs_u0 = qf_rhs_u0; + ceed_data->op_rhs_u0 = op_rhs_u0; + // --------------------------------------------------------------------------- + // Setup qfunction for initial conditions u0 + // --------------------------------------------------------------------------- + CeedQFunction qf_ics_u; + CeedOperator op_ics_u; + CeedQFunctionCreateInterior(ceed, 1, problem_data->ics_u, + problem_data->ics_u_loc, &qf_ics_u); + CeedQFunctionAddInput(qf_ics_u, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_ics_u, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_ics_u, "u", dim, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_ics_u, "v", dim, CEED_EVAL_INTERP); // Create the operator that builds the initial conditions - CeedOperatorCreate(ceed, qf_ics, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_ics); - CeedOperatorSetField(op_ics, "x", ceed_data->elem_restr_x, + CeedOperatorCreate(ceed, qf_ics_u, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_ics_u); + CeedOperatorSetField(op_ics_u, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_ics_u, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_ics, "u_0", ceed_data->elem_restr_u, + CeedOperatorSetField(op_ics_u, "u", ceed_data->elem_restr_u0, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_ics_u, "v", ceed_data->elem_restr_u0, ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_ics, "p_0", ceed_data->elem_restr_p, + // -- Save libCEED data to apply operator in setup-ts.c + ceed_data->qf_ics_u = qf_ics_u; + ceed_data->op_ics_u = op_ics_u; + // -- Operator action variables: we use them in setup-ts.c + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, &ceed_data->u0_ceed, + NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, &ceed_data->v0_ceed, + NULL); + // --------------------------------------------------------------------------- + // Setup RHS for p field + // --------------------------------------------------------------------------- + CeedQFunction qf_rhs_p0; + CeedOperator op_rhs_p0; + // Create the q-function that sets up the RHS + CeedQFunctionCreateInterior(ceed, 1, problem_data->rhs_p0, + problem_data->rhs_p0_loc, &qf_rhs_p0); + CeedQFunctionAddInput(qf_rhs_p0, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_rhs_p0, "x", num_comp_x, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_rhs_p0, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddOutput(qf_rhs_p0, "rhs_p0", 1, CEED_EVAL_INTERP); + // Create the operator that builds the RHS + CeedOperatorCreate(ceed, qf_rhs_p0, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_rhs_p0); + CeedOperatorSetField(op_rhs_p0, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_rhs_p0, "x", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_p0, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_p0, "rhs_p0", ceed_data->elem_restr_p0, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + + // -- Save libCEED data to apply operator in setup-ts.c + ceed_data->qf_rhs_p0 = qf_rhs_p0; + ceed_data->op_rhs_p0 = op_rhs_p0; + // --------------------------------------------------------------------------- + // Setup qfunction for initial conditions p0 + // --------------------------------------------------------------------------- + CeedQFunction qf_ics_p; + CeedOperator op_ics_p; + CeedQFunctionCreateInterior(ceed, 1, problem_data->ics_p, + problem_data->ics_p_loc, &qf_ics_p); + CeedQFunctionAddInput(qf_ics_p, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_ics_p, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_ics_p, "p", 1, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_ics_p, "q", 1, CEED_EVAL_INTERP); + // Create the operator that builds the initial conditions + CeedOperatorCreate(ceed, qf_ics_p, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_ics_p); + CeedOperatorSetField(op_ics_p, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_ics_p, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_ics_p, "p", ceed_data->elem_restr_p0, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_ics_p, "q", ceed_data->elem_restr_p0, ceed_data->basis_p, CEED_VECTOR_ACTIVE); - // Create local initial conditions vector - CeedVectorCreate(ceed, U_loc_size, &ceed_data->U0_ceed); // -- Save libCEED data to apply operator in setup-ts.c - ceed_data->qf_ics = qf_ics; - ceed_data->op_ics = op_ics; + ceed_data->qf_ics_p = qf_ics_p; + ceed_data->op_ics_p = op_ics_p; + // -- Operator action variables: we use them in setup-ts.c + CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, &ceed_data->p0_ceed, + NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, &ceed_data->q0_ceed, + NULL); } + // --------------------------------------------------------------------------- // Persistent libCEED vectors // --------------------------------------------------------------------------- - // -- Operator action variables: we use them in setup-solvers.c - CeedVectorCreate(ceed, U_loc_size, &ceed_data->x_ceed); - CeedVectorCreate(ceed, U_loc_size, &ceed_data->y_ceed); + // -- Operator action variables: we use them in setup-solvers.c/setup-ts.c + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->x_ceed, + NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->y_ceed, + NULL); + // -- Operator action variables: we use them in setup-ts.c + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->x_t_ceed, + NULL); // Local residual evaluator // --------------------------------------------------------------------------- // Create the QFunction and Operator that computes the residual of the PDE. @@ -312,8 +503,8 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // -- QFunction CeedQFunctionCreateInterior(ceed, 1, problem_data->residual, problem_data->residual_loc, &qf_residual); - CeedQFunctionSetContext(qf_residual, problem_data->qfunction_context); - //CeedQFunctionContextDestroy(&problem_data->qfunction_context); + CeedQFunctionSetContext(qf_residual, problem_data->residual_qfunction_ctx); + CeedQFunctionContextDestroy(&problem_data->residual_qfunction_ctx); CeedQFunctionAddInput(qf_residual, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddInput(qf_residual, "dx", dim*dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_residual, "u", dim, CEED_EVAL_INTERP); @@ -321,6 +512,9 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, CeedQFunctionAddInput(qf_residual, "p", 1, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_residual, "true force", 1, CEED_EVAL_NONE); CeedQFunctionAddInput(qf_residual, "x", num_comp_x, CEED_EVAL_INTERP); + if (problem_data->has_ts) { + CeedQFunctionAddInput(qf_residual, "p_t", 1, CEED_EVAL_INTERP); + } CeedQFunctionAddOutput(qf_residual, "v", dim, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_residual, "div_v", 1, CEED_EVAL_DIV); CeedQFunctionAddOutput(qf_residual, "q", 1, CEED_EVAL_INTERP); @@ -328,6 +522,15 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // -- Operator CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_residual); + if (problem_data->has_ts) { + //double t = ceed_data->ctx_residual_ut->t; + CeedOperatorContextGetFieldLabel(op_residual, "time", + &ceed_data->ctx_residual_ut->solution_time_label); + //CeedOperatorContextGetFieldLabel(op_residual, "time_step", + // &ceed_data->ctx_residual_ut->timestep_label); + //CeedOperatorContextSetDouble(op_residual, + // ceed_data->ctx_residual_ut->solution_time_label, &t); + } CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); CeedOperatorSetField(op_residual, "dx", ceed_data->elem_restr_x, @@ -342,6 +545,10 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, CEED_BASIS_COLLOCATED, true_force); CeedOperatorSetField(op_residual, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + if (problem_data->has_ts) { + CeedOperatorSetField(op_residual, "p_t", ceed_data->elem_restr_p, + ceed_data->basis_p, ceed_data->x_t_ceed); + } CeedOperatorSetField(op_residual, "v", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_residual, "div_v", ceed_data->elem_restr_u, @@ -351,68 +558,69 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // -- Save libCEED data to apply operator in matops.c ceed_data->qf_residual = qf_residual; ceed_data->op_residual = op_residual; + if (!problem_data->has_ts) { + // --------------------------------------------------------------------------- + // Add Pressure boundary condition. See setup-boundary.c + // --------------------------------------------------------------------------- + //DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm); - // --------------------------------------------------------------------------- - // Add Pressure boundary condition. See setup-boundary.c - // --------------------------------------------------------------------------- - //DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm); - - // Local jacobian evaluator - // --------------------------------------------------------------------------- - // Create the QFunction and Operator that computes the jacobian of the PDE. - // --------------------------------------------------------------------------- - // -- QFunction - CeedQFunctionCreateInterior(ceed, 1, problem_data->jacobian, - problem_data->jacobian_loc, &qf_jacobian); - CeedQFunctionSetContext(qf_jacobian, problem_data->qfunction_context); - //CeedQFunctionContextDestroy(&problem_data->qfunction_context); - CeedQFunctionAddInput(qf_jacobian, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_jacobian, "dx", dim*dim, CEED_EVAL_GRAD); - CeedQFunctionAddInput(qf_jacobian, "du", dim, CEED_EVAL_INTERP); - CeedQFunctionAddInput(qf_jacobian, "div_du", 1, CEED_EVAL_DIV); - CeedQFunctionAddInput(qf_jacobian, "dp", 1, CEED_EVAL_INTERP); - CeedQFunctionAddInput(qf_jacobian, "x", num_comp_x, CEED_EVAL_INTERP); - //CeedQFunctionAddInput(qf_jacobian, "u", dim, CEED_EVAL_INTERP); - //CeedQFunctionAddInput(qf_jacobian, "p", 1, CEED_EVAL_INTERP); - CeedQFunctionAddOutput(qf_jacobian, "dv", dim, CEED_EVAL_INTERP); - CeedQFunctionAddOutput(qf_jacobian, "div_dv", 1, CEED_EVAL_DIV); - CeedQFunctionAddOutput(qf_jacobian, "dq", 1, CEED_EVAL_INTERP); - // -- Operator - CeedOperatorCreate(ceed, qf_jacobian, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_jacobian); - CeedOperatorSetField(op_jacobian, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_jacobian, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_jacobian, "du", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "div_du", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "dp", ceed_data->elem_restr_p, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "x", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - //CeedOperatorSetField(op_jacobian, "u", ceed_data->elem_restr_u, - // ceed_data->basis_u, CEED_VECTOR_ACTIVE); - //CeedOperatorSetField(op_jacobian, "p", ceed_data->elem_restr_p, - // ceed_data->basis_p, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "dv", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "div_dv", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "dq", ceed_data->elem_restr_p, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); - // -- Save libCEED data to apply operator in matops.c - ceed_data->qf_jacobian = qf_jacobian; - ceed_data->op_jacobian = op_jacobian; - + // Local jacobian evaluator + // --------------------------------------------------------------------------- + // Create the QFunction and Operator that computes the jacobian of the PDE. + // --------------------------------------------------------------------------- + // -- QFunction + CeedQFunctionCreateInterior(ceed, 1, problem_data->jacobian, + problem_data->jacobian_loc, &qf_jacobian); + CeedQFunctionSetContext(qf_jacobian, problem_data->jacobian_qfunction_ctx); + CeedQFunctionContextDestroy(&problem_data->jacobian_qfunction_ctx); + CeedQFunctionAddInput(qf_jacobian, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_jacobian, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_jacobian, "du", dim, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_jacobian, "div_du", 1, CEED_EVAL_DIV); + CeedQFunctionAddInput(qf_jacobian, "dp", 1, CEED_EVAL_INTERP); + CeedQFunctionAddInput(qf_jacobian, "x", num_comp_x, CEED_EVAL_INTERP); + //CeedQFunctionAddInput(qf_jacobian, "u", dim, CEED_EVAL_INTERP); + //CeedQFunctionAddInput(qf_jacobian, "p", 1, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_jacobian, "dv", dim, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_jacobian, "div_dv", 1, CEED_EVAL_DIV); + CeedQFunctionAddOutput(qf_jacobian, "dq", 1, CEED_EVAL_INTERP); + // -- Operator + CeedOperatorCreate(ceed, qf_jacobian, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_jacobian); + CeedOperatorSetField(op_jacobian, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_jacobian, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_jacobian, "du", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "div_du", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dp", ceed_data->elem_restr_p, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "x", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + //CeedOperatorSetField(op_jacobian, "u", ceed_data->elem_restr_u, + // ceed_data->basis_u, CEED_VECTOR_ACTIVE); + //CeedOperatorSetField(op_jacobian, "p", ceed_data->elem_restr_p, + // ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dv", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "div_dv", ceed_data->elem_restr_u, + ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dq", ceed_data->elem_restr_p, + ceed_data->basis_p, CEED_VECTOR_ACTIVE); + // -- Save libCEED data to apply operator in matops.c + ceed_data->qf_jacobian = qf_jacobian; + ceed_data->op_jacobian = op_jacobian; + } // --------------------------------------------------------------------------- // Setup Error Qfunction // --------------------------------------------------------------------------- // Create the q-function that sets up the error CeedQFunctionCreateInterior(ceed, 1, problem_data->error, problem_data->error_loc, &qf_error); - CeedQFunctionSetContext(qf_error, problem_data->qfunction_context); + CeedQFunctionSetContext(qf_error, problem_data->error_qfunction_ctx); + CeedQFunctionContextDestroy(&problem_data->error_qfunction_ctx); CeedQFunctionAddInput(qf_error, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddInput(qf_error, "dx", dim*dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_error, "u", dim, CEED_EVAL_INTERP); @@ -443,6 +651,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, CeedVectorDestroy(&true_force); CeedQFunctionDestroy(&qf_true); CeedOperatorDestroy(&op_true); + PetscFunctionReturn(0); }; // ----------------------------------------------------------------------------- \ No newline at end of file diff --git a/examples/Hdiv-mixed/src/setup-solvers.c b/examples/Hdiv-mixed/src/setup-solvers.c index dde338a8cc..4f47c4fcc4 100644 --- a/examples/Hdiv-mixed/src/setup-solvers.c +++ b/examples/Hdiv-mixed/src/setup-solvers.c @@ -53,7 +53,7 @@ PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, // ----------------------------------------------------------------------------- // This function wraps the libCEED operator for a MatShell // ----------------------------------------------------------------------------- -PetscErrorCode ApplyJacobian(Mat A, Vec X, Vec Y) { +PetscErrorCode ApplyMatOp(Mat A, Vec X, Vec Y) { OperatorApplyContext op_apply_ctx; PetscFunctionBeginUser; @@ -133,7 +133,7 @@ PetscErrorCode PDESolver(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, PetscCall( MatCreateShell(comm, U_l_size, U_l_size, U_g_size, U_g_size, ceed_data->ctx_jacobian, &mat_jacobian) ); PetscCall( MatShellSetOperation(mat_jacobian, MATOP_MULT, - (void (*)(void))ApplyJacobian) ); + (void (*)(void))ApplyMatOp) ); PetscCall( MatShellSetVecType(mat_jacobian, vec_type) ); // Set SNES residual evaluation function @@ -166,6 +166,9 @@ PetscErrorCode PDESolver(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, PetscCall( VecDestroy(&ceed_data->ctx_jacobian->X_loc) ); PetscCall( VecDestroy(&ceed_data->ctx_residual->Y_loc) ); PetscCall( VecDestroy(&ceed_data->ctx_residual->X_loc) ); + PetscCall( PetscFree(ceed_data->ctx_jacobian) ); + PetscCall( PetscFree(ceed_data->ctx_residual) ); + PetscFunctionReturn(0); }; @@ -253,6 +256,7 @@ PetscErrorCode ComputeL2Error(DM dm, Ceed ceed, CeedData ceed_data, Vec U, CeedVectorDestroy(&collocated_error_p); PetscCall( VecDestroy(&ceed_data->ctx_error->Y_loc) ); PetscCall( VecDestroy(&ceed_data->ctx_error->X_loc) ); + PetscCall( PetscFree(ceed_data->ctx_error) ); PetscFunctionReturn(0); }; @@ -260,11 +264,10 @@ PetscErrorCode ComputeL2Error(DM dm, Ceed ceed, CeedData ceed_data, Vec U, // ----------------------------------------------------------------------------- // This function print the output // ----------------------------------------------------------------------------- -PetscErrorCode PrintOutput(Ceed ceed, +PetscErrorCode PrintOutput(Ceed ceed, AppCtx app_ctx, PetscBool has_ts, CeedMemType mem_type_backend, - SNES snes, KSP ksp, - Vec U, CeedScalar l2_error_u, - CeedScalar l2_error_p, AppCtx app_ctx) { + TS ts, SNES snes, KSP ksp, + Vec U, CeedScalar l2_error_u, CeedScalar l2_error_p) { PetscFunctionBeginUser; @@ -301,6 +304,25 @@ PetscErrorCode PrintOutput(Ceed ceed, " Owned nodes (u + p) : %" PetscInt_FMT "\n", app_ctx->problem_name, U_g_size, U_l_size ) ); + // --TS + if (has_ts) { + PetscInt ts_steps; + TSType ts_type; + TSConvergedReason ts_reason; + PetscCall( TSGetStepNumber(ts, &ts_steps) ); + PetscCall( TSGetType(ts, &ts_type) ); + PetscCall( TSGetConvergedReason(ts, &ts_reason) ); + PetscCall( PetscPrintf(app_ctx->comm, + " TS:\n" + " TS Type : %s\n" + " TS Convergence : %s\n" + " Number of TS steps : %" PetscInt_FMT "\n" + " Final time : %g\n", + ts_type, TSConvergedReasons[ts_reason], + ts_steps, (double)app_ctx->t_final) ); + + PetscCall( TSGetSNES(ts, &snes) ); + } // -- SNES PetscInt its, snes_its = 0; PetscCall( SNESGetIterationNumber(snes, &its) ); @@ -319,30 +341,31 @@ PetscErrorCode PrintOutput(Ceed ceed, " Final rnorm : %e\n", snes_type, SNESConvergedReasons[snes_reason], snes_its, (double)snes_rnorm) ); - - PetscInt ksp_its = 0; - PetscCall( SNESGetLinearSolveIterations(snes, &its) ); - ksp_its += its; - KSPType ksp_type; - KSPConvergedReason ksp_reason; - PetscReal ksp_rnorm; - PC pc; - PCType pc_type; - PetscCall( KSPGetPC(ksp, &pc) ); - PetscCall( PCGetType(pc, &pc_type) ); - PetscCall( KSPGetType(ksp, &ksp_type) ); - PetscCall( KSPGetConvergedReason(ksp, &ksp_reason) ); - PetscCall( KSPGetIterationNumber(ksp, &ksp_its) ); - PetscCall( KSPGetResidualNorm(ksp, &ksp_rnorm) ); - PetscCall( PetscPrintf(app_ctx->comm, - " KSP:\n" - " KSP Type : %s\n" - " PC Type : %s\n" - " KSP Convergence : %s\n" - " Total KSP Iterations : %" PetscInt_FMT "\n" - " Final rnorm : %e\n", - ksp_type, pc_type, KSPConvergedReasons[ksp_reason], ksp_its, - (double)ksp_rnorm ) ); + if (!has_ts) { + PetscInt ksp_its = 0; + PetscCall( SNESGetLinearSolveIterations(snes, &its) ); + ksp_its += its; + KSPType ksp_type; + KSPConvergedReason ksp_reason; + PetscReal ksp_rnorm; + PC pc; + PCType pc_type; + PetscCall( KSPGetPC(ksp, &pc) ); + PetscCall( PCGetType(pc, &pc_type) ); + PetscCall( KSPGetType(ksp, &ksp_type) ); + PetscCall( KSPGetConvergedReason(ksp, &ksp_reason) ); + PetscCall( KSPGetIterationNumber(ksp, &ksp_its) ); + PetscCall( KSPGetResidualNorm(ksp, &ksp_rnorm) ); + PetscCall( PetscPrintf(app_ctx->comm, + " KSP:\n" + " KSP Type : %s\n" + " PC Type : %s\n" + " KSP Convergence : %s\n" + " Total KSP Iterations : %" PetscInt_FMT "\n" + " Final rnorm : %e\n", + ksp_type, pc_type, KSPConvergedReasons[ksp_reason], ksp_its, + (double)ksp_rnorm ) ); + } PetscCall( PetscPrintf(app_ctx->comm, " L2 Error (MMS):\n" diff --git a/examples/Hdiv-mixed/src/setup-ts.c b/examples/Hdiv-mixed/src/setup-ts.c index 1983993943..dd0728951b 100644 --- a/examples/Hdiv-mixed/src/setup-ts.c +++ b/examples/Hdiv-mixed/src/setup-ts.c @@ -1,55 +1,61 @@ #include "../include/setup-ts.h" #include "../include/setup-matops.h" #include "../include/setup-libceed.h" +#include "../include/setup-solvers.h" +#include "ceed/ceed.h" #include "petscerror.h" +#include "petscsystypes.h" +#include // ----------------------------------------------------------------------------- -// Create global initial conditions vector +// Setup operator context data for initial condition, u field // ----------------------------------------------------------------------------- -PetscErrorCode CreateInitialConditions(DM dm, CeedData ceed_data, Vec *U0) { +PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm_u0, Ceed ceed, + CeedData ceed_data, + OperatorApplyContext ctx_initial_u0) { + PetscFunctionBeginUser; + + ctx_initial_u0->comm = comm; + ctx_initial_u0->dm = dm_u0; + PetscCall( DMCreateLocalVector(dm_u0, &ctx_initial_u0->X_loc) ); + PetscCall( VecDuplicate(ctx_initial_u0->X_loc, &ctx_initial_u0->Y_loc) ); + ctx_initial_u0->x_ceed = ceed_data->u0_ceed; + ctx_initial_u0->y_ceed = ceed_data->v0_ceed; + ctx_initial_u0->ceed = ceed; + ctx_initial_u0->op_apply = ceed_data->op_ics_u; - PetscScalar *u0; - PetscMemType u0_mem_type; - Vec U0_loc; + PetscFunctionReturn(0); +} +// ----------------------------------------------------------------------------- +// Setup operator context data for initial condition, p field +// ----------------------------------------------------------------------------- +PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm_p0, Ceed ceed, + CeedData ceed_data, + OperatorApplyContext ctx_initial_p0) { PetscFunctionBeginUser; - PetscCall( DMCreateLocalVector(dm, &U0_loc) ); - PetscCall( VecZeroEntries(U0_loc) ); + ctx_initial_p0->comm = comm; + ctx_initial_p0->dm = dm_p0; + PetscCall( DMCreateLocalVector(dm_p0, &ctx_initial_p0->X_loc) ); + PetscCall( VecDuplicate(ctx_initial_p0->X_loc, &ctx_initial_p0->Y_loc) ); + ctx_initial_p0->x_ceed = ceed_data->p0_ceed; + ctx_initial_p0->y_ceed = ceed_data->q0_ceed; + ctx_initial_p0->ceed = ceed; + ctx_initial_p0->op_apply = ceed_data->op_ics_p; - PetscCall( VecGetArrayAndMemType(U0_loc, &u0, &u0_mem_type) ); - CeedVectorSetArray(ceed_data->U0_ceed, MemTypeP2C(u0_mem_type), - CEED_USE_POINTER, u0); - // Apply libCEED operator - CeedOperatorApply(ceed_data->op_ics, ceed_data->x_coord, ceed_data->U0_ceed, - CEED_REQUEST_IMMEDIATE); - // Restore PETSc vectors - CeedVectorTakeArray(ceed_data->U0_ceed, MemTypeP2C(u0_mem_type), NULL); - PetscCall( VecRestoreArrayAndMemType(U0_loc, &u0) ); - - // Create global initial conditions - PetscCall( DMCreateGlobalVector(dm, U0) ); - PetscCall( VecZeroEntries(*U0) ); - // Local-to-global - PetscCall( DMLocalToGlobal(dm, U0_loc, ADD_VALUES, *U0) ); - - // -- Cleanup - CeedVectorDestroy(&ceed_data->U0_ceed); - CeedQFunctionDestroy(&ceed_data->qf_ics); - CeedOperatorDestroy(&ceed_data->op_ics); - // Free PETSc objects - PetscCall( VecDestroy(&U0_loc) ); PetscFunctionReturn(0); - } + // ----------------------------------------------------------------------------- -// Setup operator context data +// Setup operator context data for Residual of Richard problem // ----------------------------------------------------------------------------- -PetscErrorCode SetupResidualOperatorCtx_Ut(DM dm, Ceed ceed, CeedData ceed_data, - OperatorApplyContext ctx_residual_ut) { +PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, + CeedData ceed_data, OperatorApplyContext ctx_residual_ut) { PetscFunctionBeginUser; + ctx_residual_ut->comm = comm; ctx_residual_ut->dm = dm; PetscCall( DMCreateLocalVector(dm, &ctx_residual_ut->X_loc) ); PetscCall( VecDuplicate(ctx_residual_ut->X_loc, &ctx_residual_ut->Y_loc) ); @@ -63,27 +69,207 @@ PetscErrorCode SetupResidualOperatorCtx_Ut(DM dm, Ceed ceed, CeedData ceed_data, PetscFunctionReturn(0); } +// ----------------------------------------------------------------------------- +// Create global initial conditions vector +// ----------------------------------------------------------------------------- +PetscErrorCode CreateInitialConditions(CeedData ceed_data, + Vec U, VecType vec_type, + OperatorApplyContext ctx_initial_u0, + OperatorApplyContext ctx_initial_p0, + OperatorApplyContext ctx_residual_ut) { + + PetscFunctionBeginUser; + // ---------------------------------------------- + // Create local rhs for u field + // ---------------------------------------------- + Vec rhs_u_loc; + PetscScalar *ru; + PetscMemType ru_mem_type; + PetscCall( DMCreateLocalVector(ctx_initial_u0->dm, &rhs_u_loc) ); + PetscCall( VecZeroEntries(rhs_u_loc) ); + PetscCall( VecGetArrayAndMemType(rhs_u_loc, &ru, &ru_mem_type) ); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, + &ceed_data->rhs_u0_ceed, + NULL); + CeedVectorSetArray(ceed_data->rhs_u0_ceed, MemTypeP2C(ru_mem_type), + CEED_USE_POINTER, ru); + + // Apply operator to create RHS for u field + CeedOperatorApply(ceed_data->op_rhs_u0, ceed_data->x_coord, + ceed_data->rhs_u0_ceed, CEED_REQUEST_IMMEDIATE); + + // ---------------------------------------------- + // Create global rhs for u field + // ---------------------------------------------- + Vec rhs_u0; + CeedVectorTakeArray(ceed_data->rhs_u0_ceed, MemTypeP2C(ru_mem_type), NULL); + PetscCall( VecRestoreArrayAndMemType(rhs_u_loc, &ru) ); + PetscCall( DMCreateGlobalVector(ctx_initial_u0->dm, &rhs_u0) ); + PetscCall( VecZeroEntries(rhs_u0) ); + PetscCall( DMLocalToGlobal(ctx_initial_u0->dm, rhs_u_loc, ADD_VALUES, rhs_u0) ); + + // ---------------------------------------------- + // Solve for U0, M*U0 = rhs_u0 + // ---------------------------------------------- + Vec U0; + PetscCall( DMCreateGlobalVector(ctx_initial_u0->dm, &U0) ); + PetscCall( VecZeroEntries(U0) ); + PetscInt U0_g_size, U0_l_size; + PetscCall( VecGetSize(U0, &U0_g_size) ); + // Local size for matShell + PetscCall( VecGetLocalSize(U0, &U0_l_size) ); + + // Operator + Mat mat_ksp_u0; + // -- Form Action of residual on u + PetscCall( MatCreateShell(ctx_initial_u0->comm, U0_l_size, U0_l_size, U0_g_size, + U0_g_size, ceed_data->ctx_initial_u0, &mat_ksp_u0) ); + PetscCall( MatShellSetOperation(mat_ksp_u0, MATOP_MULT, + (void (*)(void))ApplyMatOp) ); + PetscCall( MatShellSetVecType(mat_ksp_u0, vec_type) ); + + KSP ksp_u0; + PetscCall( KSPCreate(ctx_initial_u0->comm, &ksp_u0) ); + PetscCall( KSPSetOperators(ksp_u0, mat_ksp_u0, mat_ksp_u0) ); + PetscCall( KSPSetFromOptions(ksp_u0) ); + PetscCall( KSPSetUp(ksp_u0) ); + PetscCall( KSPSolve(ksp_u0, rhs_u0, U0) ); + + // ---------------------------------------------- + // Create local rhs for p field + // ---------------------------------------------- + Vec rhs_p_loc; + PetscScalar *rp; + PetscMemType rp_mem_type; + PetscCall( DMCreateLocalVector(ctx_initial_p0->dm, &rhs_p_loc) ); + PetscCall( VecZeroEntries(rhs_p_loc) ); + PetscCall( VecGetArrayAndMemType(rhs_p_loc, &rp, &rp_mem_type) ); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, + &ceed_data->rhs_p0_ceed, + NULL); + CeedVectorSetArray(ceed_data->rhs_p0_ceed, MemTypeP2C(rp_mem_type), + CEED_USE_POINTER, rp); + + // Apply operator to create RHS for p field + CeedOperatorApply(ceed_data->op_rhs_p0, ceed_data->x_coord, + ceed_data->rhs_p0_ceed, CEED_REQUEST_IMMEDIATE); + + // ---------------------------------------------- + // Create global rhs for p field + // ---------------------------------------------- + Vec rhs_p0; + CeedVectorTakeArray(ceed_data->rhs_p0_ceed, MemTypeP2C(rp_mem_type), NULL); + PetscCall( VecRestoreArrayAndMemType(rhs_p_loc, &rp) ); + PetscCall( DMCreateGlobalVector(ctx_initial_p0->dm, &rhs_p0) ); + PetscCall( VecZeroEntries(rhs_p0) ); + PetscCall( DMLocalToGlobal(ctx_initial_p0->dm, rhs_p_loc, ADD_VALUES, rhs_p0) ); + + // ---------------------------------------------- + // Solve for P0, M*P0 = rhs_p0 + // ---------------------------------------------- + Vec P0; + PetscCall( DMCreateGlobalVector(ctx_initial_p0->dm, &P0) ); + PetscCall( VecZeroEntries(P0) ); + PetscInt P0_g_size, P0_l_size; + PetscCall( VecGetSize(P0, &P0_g_size) ); + // Local size for matShell + PetscCall( VecGetLocalSize(P0, &P0_l_size) ); + + // Operator + Mat mat_ksp_p0; + // -- Form Action of residual on u + PetscCall( MatCreateShell(ctx_initial_p0->comm, P0_l_size, P0_l_size, P0_g_size, + P0_g_size, ceed_data->ctx_initial_p0, &mat_ksp_p0) ); + PetscCall( MatShellSetOperation(mat_ksp_p0, MATOP_MULT, + (void (*)(void))ApplyMatOp) ); + PetscCall( MatShellSetVecType(mat_ksp_p0, vec_type) ); + + KSP ksp_p0; + PetscCall( KSPCreate(ctx_initial_p0->comm, &ksp_p0) ); + PetscCall( KSPSetOperators(ksp_p0, mat_ksp_p0, mat_ksp_p0) ); + PetscCall( KSPSetFromOptions(ksp_p0) ); + PetscCall( KSPSetUp(ksp_p0) ); + PetscCall( KSPSolve(ksp_p0, rhs_p0, P0) ); + + // ---------------------------------------------- + // Create final initial conditions U + // ---------------------------------------------- + // Global-to-local for U0, P0 + PetscCall( DMGlobalToLocal(ctx_initial_u0->dm, U0, INSERT_VALUES, + ctx_initial_u0->X_loc) ); + PetscCall( DMGlobalToLocal(ctx_initial_p0->dm, P0, INSERT_VALUES, + ctx_initial_p0->X_loc) ); + // Get array u0,p0 + const PetscScalar *u0, *p0; + PetscCall( VecGetArrayRead(ctx_initial_u0->X_loc, &u0) ); + PetscCall( VecGetArrayRead(ctx_initial_p0->X_loc, &p0) ); + + // Get array of local vector U = [p,u] + PetscScalar *u; + PetscInt U_l_size; + PetscCall( VecGetLocalSize(U, &U_l_size) ); + PetscCall( VecZeroEntries(ctx_residual_ut->X_loc) ); + PetscCall( VecGetArray(ctx_residual_ut->X_loc, &u) ); + for (PetscInt i = 0; inum_elem; i++) { + u[i] = p0[i]; + } + for (PetscInt i = ceed_data->num_elem; inum_elem]; + } + PetscCall( VecRestoreArray(ctx_residual_ut->X_loc, &u) ); + PetscCall( VecRestoreArrayRead(ctx_initial_p0->X_loc, &p0) ); + PetscCall( VecRestoreArrayRead(ctx_initial_u0->X_loc, &u0) ); + PetscCall( DMLocalToGlobal(ctx_residual_ut->dm, ctx_residual_ut->X_loc, + ADD_VALUES, U) ); + + // Clean up + PetscCall( VecDestroy(&rhs_u_loc) ); + PetscCall( VecDestroy(&rhs_u0) ); + PetscCall( VecDestroy(&U0) ); + PetscCall( VecDestroy(&ctx_initial_u0->X_loc) ); + PetscCall( VecDestroy(&ctx_initial_u0->Y_loc) ); + PetscCall( VecDestroy(&rhs_p_loc) ); + PetscCall( VecDestroy(&rhs_p0) ); + PetscCall( VecDestroy(&P0) ); + PetscCall( VecDestroy(&ctx_initial_p0->X_loc) ); + PetscCall( VecDestroy(&ctx_initial_p0->Y_loc) ); + PetscCall( MatDestroy(&mat_ksp_p0) ); + PetscCall( MatDestroy(&mat_ksp_u0) ); + PetscCall( KSPDestroy(&ksp_p0) ); + PetscCall( KSPDestroy(&ksp_u0) ); + PetscFunctionReturn(0); + +} + PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, void *ctx_residual_ut) { OperatorApplyContext ctx = (OperatorApplyContext)ctx_residual_ut; const PetscScalar *x, *x_t; PetscScalar *y; - Vec X_loc = ctx->X_loc, X_t_loc = ctx->X_t_loc, - Y_loc = ctx->Y_loc; PetscMemType x_mem_type, x_t_mem_type, y_mem_type; PetscFunctionBeginUser; // Update time dependent data - - + if(ctx->t != time) { + CeedOperatorContextSetDouble(ctx->op_apply, + ctx->solution_time_label, &time); + ctx->t = time; + } + //PetscScalar dt; + //PetscCall( TSGetTimeStep(ts, &dt) ); + //if (ctx->dt != dt) { + // CeedOperatorContextSetDouble(ctx->op_apply, + // ctx->timestep_label, &dt); + // ctx->dt = dt; + //} // Global-to-local - PetscCall( DMGlobalToLocal(ctx->dm, X, INSERT_VALUES, X_loc) ); - PetscCall( DMGlobalToLocal(ctx->dm, X_t, INSERT_VALUES, X_t_loc) ); + PetscCall( DMGlobalToLocal(ctx->dm, X, INSERT_VALUES, ctx->X_loc) ); + PetscCall( DMGlobalToLocal(ctx->dm, X_t, INSERT_VALUES, ctx->X_t_loc) ); // Place PETSc vectors in CEED vectors - PetscCall( VecGetArrayReadAndMemType(X_loc, &x, &x_mem_type) ); - PetscCall( VecGetArrayReadAndMemType(X_t_loc, &x_t, &x_t_mem_type) ); - PetscCall( VecGetArrayAndMemType(Y_loc, &y, &y_mem_type) ); + PetscCall( VecGetArrayReadAndMemType(ctx->X_loc, &x, &x_mem_type) ); + PetscCall( VecGetArrayReadAndMemType(ctx->X_t_loc, &x_t, &x_t_mem_type) ); + PetscCall( VecGetArrayAndMemType(ctx->Y_loc, &y, &y_mem_type) ); CeedVectorSetArray(ctx->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, (PetscScalar *)x); CeedVectorSetArray(ctx->x_t_ceed, MemTypeP2C(x_t_mem_type), @@ -98,39 +284,38 @@ PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, CeedVectorTakeArray(ctx->x_ceed, MemTypeP2C(x_mem_type), NULL); CeedVectorTakeArray(ctx->x_t_ceed, MemTypeP2C(x_t_mem_type), NULL); CeedVectorTakeArray(ctx->y_ceed, MemTypeP2C(y_mem_type), NULL); - PetscCall( VecRestoreArrayReadAndMemType(X_loc, &x) ); - PetscCall( VecRestoreArrayReadAndMemType(X_t_loc, &x_t) ); - PetscCall( VecRestoreArrayAndMemType(Y_loc, &y) ); + PetscCall( VecRestoreArrayReadAndMemType(ctx->X_loc, &x) ); + PetscCall( VecRestoreArrayReadAndMemType(ctx->X_t_loc, &x_t) ); + PetscCall( VecRestoreArrayAndMemType(ctx->Y_loc, &y) ); // Local-to-Global PetscCall( VecZeroEntries(Y) ); - PetscCall( DMLocalToGlobal(ctx->dm, Y_loc, ADD_VALUES, Y) ); - - // Restore vectors - PetscCall( DMRestoreLocalVector(ctx->dm, &Y_loc) ); + PetscCall( DMLocalToGlobal(ctx->dm, ctx->Y_loc, ADD_VALUES, Y) ); PetscFunctionReturn(0); } // TS: Create, setup, and solve PetscErrorCode TSSolveRichard(DM dm, CeedData ceed_data, AppCtx app_ctx, - Vec *U, PetscScalar *f_time, TS *ts) { + Vec *U, TS *ts) { MPI_Comm comm = app_ctx->comm; TSAdapt adapt; PetscFunctionBeginUser; PetscCall( TSCreate(comm, ts) ); PetscCall( TSSetDM(*ts, dm) ); + PetscCall( TSSetType(*ts, TSBDF) ); PetscCall( TSSetIFunction(*ts, NULL, TSFormIResidual, ceed_data->ctx_residual_ut) ); - PetscCall( TSSetMaxTime(*ts, 10) ); + PetscCall( TSSetMaxTime(*ts, app_ctx->t_final) ); PetscCall( TSSetExactFinalTime(*ts, TS_EXACTFINALTIME_STEPOVER) ); PetscCall( TSSetTimeStep(*ts, 1.e-2) ); PetscCall( TSGetAdapt(*ts, &adapt) ); PetscCall( TSAdaptSetStepLimits(adapt, 1.e-12, 1.e2) ); PetscCall( TSSetFromOptions(*ts) ); - + ceed_data->ctx_residual_ut->t = -1.0; + //ceed_data->ctx_residual_ut->dt = -1.0; // Solve PetscScalar start_time; PetscCall( TSGetTime(*ts, &start_time) ); @@ -143,7 +328,7 @@ PetscErrorCode TSSolveRichard(DM dm, CeedData ceed_data, AppCtx app_ctx, PetscScalar final_time; PetscCall( TSGetSolveTime(*ts, &final_time) ); - *f_time = final_time; + app_ctx->t_final = final_time; PetscFunctionReturn(0); } diff --git a/interface/ceed-operator.c b/interface/ceed-operator.c index 8f2bf988eb..4d7ed19e6f 100644 --- a/interface/ceed-operator.c +++ b/interface/ceed-operator.c @@ -76,6 +76,10 @@ static int CeedOperatorCheckField(Ceed ceed, CeedQFunctionField qf_field, CeedEl CeedEvalModes[eval_mode]); // LCOV_EXCL_STOP } + bool is_oriented; + ierr = CeedElemRestrictionIsOriented(r, &is_oriented); CeedChk(ierr); + CeedInt scale_r; + scale_r = is_oriented ? qf_field->size : 1; // Field size switch (eval_mode) { case CEED_EVAL_NONE: From eb7ecef2f39d67d8bac1809d0ed444c8b58c32be Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Mon, 15 Aug 2022 14:58:04 -0600 Subject: [PATCH 08/15] Added post-processing.c to solve the projection problem for velocity to visualize it --- examples/Hdiv-mass/conv_test.sh | 4 +- examples/Hdiv-mass/conv_test_result.csv | 13 +- examples/Hdiv-mass/convrate_mass.png | Bin 25544 -> 23385 bytes .../Hdiv-mass/qfunctions/poisson-error2d.h | 16 +- .../Hdiv-mass/qfunctions/poisson-error3d.h | 34 +-- .../Hdiv-mass/qfunctions/poisson-mass2d.h | 22 +- .../Hdiv-mass/qfunctions/poisson-mass3d.h | 37 +-- examples/Hdiv-mass/qfunctions/poisson-rhs2d.h | 8 +- examples/Hdiv-mass/qfunctions/poisson-rhs3d.h | 8 +- examples/Hdiv-mass/qfunctions/utils.h | 205 +++++++++++++ examples/Hdiv-mixed/conv_plot.py | 13 + examples/Hdiv-mixed/conv_test.sh | 12 +- examples/Hdiv-mixed/conv_test_result.csv | 10 +- examples/Hdiv-mixed/convrate_mixed.png | Bin 27357 -> 25042 bytes examples/Hdiv-mixed/include/post-processing.h | 21 ++ .../Hdiv-mixed/include/register-problem.h | 12 +- examples/Hdiv-mixed/include/setup-dm.h | 4 +- examples/Hdiv-mixed/include/setup-fe.h | 5 +- examples/Hdiv-mixed/include/setup-libceed.h | 4 +- examples/Hdiv-mixed/include/setup-solvers.h | 13 +- examples/Hdiv-mixed/include/setup-ts.h | 11 +- examples/Hdiv-mixed/include/structs.h | 65 ++-- examples/Hdiv-mixed/main.c | 234 ++++++++++----- examples/Hdiv-mixed/main.h | 1 + examples/Hdiv-mixed/problems/darcy2d.c | 27 +- examples/Hdiv-mixed/problems/darcy3d.c | 17 +- .../Hdiv-mixed/problems/register-problem.c | 2 + examples/Hdiv-mixed/problems/richard2d.c | 16 +- examples/Hdiv-mixed/problems/richard3d.c | 134 +++++++++ .../Hdiv-mixed/qfunctions/darcy-error2d.h | 1 + .../Hdiv-mixed/qfunctions/darcy-error3d.h | 1 + .../qfunctions/darcy-system-quartic2d.h | 170 +++++++++++ .../Hdiv-mixed/qfunctions/darcy-system2d.h | 2 + .../Hdiv-mixed/qfunctions/darcy-system3d.h | 1 + .../qfunctions/darcy-true-quartic2d.h | 98 ++++++ examples/Hdiv-mixed/qfunctions/darcy-true2d.h | 14 +- examples/Hdiv-mixed/qfunctions/darcy-true3d.h | 18 +- .../Hdiv-mixed/qfunctions/post-processing2d.h | 97 ++++++ .../Hdiv-mixed/qfunctions/post-processing3d.h | 100 +++++++ .../Hdiv-mixed/qfunctions/richard-ics2d.h | 1 + .../Hdiv-mixed/qfunctions/richard-ics3d.h | 204 +++++++++++++ .../Hdiv-mixed/qfunctions/richard-system2d.h | 1 + .../Hdiv-mixed/qfunctions/richard-system3d.h | 279 ++++++++++++++++++ .../Hdiv-mixed/qfunctions/richard-true2d.h | 1 + .../Hdiv-mixed/qfunctions/richard-true3d.h | 137 +++++++++ examples/Hdiv-mixed/src/cl-options.c | 32 ++ examples/Hdiv-mixed/src/post-processing.c | 272 +++++++++++++++++ examples/Hdiv-mixed/src/setup-dm.c | 91 +++++- examples/Hdiv-mixed/src/setup-fe.c | 56 +++- examples/Hdiv-mixed/src/setup-libceed.c | 108 ++++++- examples/Hdiv-mixed/src/setup-solvers.c | 195 +++--------- examples/Hdiv-mixed/src/setup-ts.c | 163 ++++++---- interface/ceed-operator.c | 4 - 53 files changed, 2501 insertions(+), 493 deletions(-) create mode 100644 examples/Hdiv-mass/qfunctions/utils.h create mode 100644 examples/Hdiv-mixed/include/post-processing.h create mode 100644 examples/Hdiv-mixed/problems/richard3d.c create mode 100644 examples/Hdiv-mixed/qfunctions/darcy-system-quartic2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/darcy-true-quartic2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/post-processing2d.h create mode 100644 examples/Hdiv-mixed/qfunctions/post-processing3d.h create mode 100644 examples/Hdiv-mixed/qfunctions/richard-ics3d.h create mode 100644 examples/Hdiv-mixed/qfunctions/richard-system3d.h create mode 100644 examples/Hdiv-mixed/qfunctions/richard-true3d.h create mode 100644 examples/Hdiv-mixed/src/post-processing.c diff --git a/examples/Hdiv-mass/conv_test.sh b/examples/Hdiv-mass/conv_test.sh index d039a9b01a..cdc2fddd74 100755 --- a/examples/Hdiv-mass/conv_test.sh +++ b/examples/Hdiv-mass/conv_test.sh @@ -33,6 +33,8 @@ declare -A run_flags run_flags[problem]=mass2d run_flags[dm_plex_dim]=$dim run_flags[dm_plex_box_faces]=2,2 + run_flags[dm_plex_box_lower]=0,0 + run_flags[dm_plex_box_upper]=1,0.1 else run_flags[problem]=mass3d run_flags[dm_plex_dim]=$dim @@ -42,7 +44,7 @@ declare -A run_flags declare -A test_flags test_flags[res_start]=2 test_flags[res_stride]=1 - test_flags[res_end]=5 + test_flags[res_end]=10 file_name=conv_test_result.csv diff --git a/examples/Hdiv-mass/conv_test_result.csv b/examples/Hdiv-mass/conv_test_result.csv index 8dcc28a8f8..296e9812fd 100644 --- a/examples/Hdiv-mass/conv_test_result.csv +++ b/examples/Hdiv-mass/conv_test_result.csv @@ -1,5 +1,10 @@ run,mesh_res,error_u -0,2,0.55094 -1,3,0.23091 -2,4,0.12473 -3,5,0.07812 +0,2,0.06228 +1,3,0.02836 +2,4,0.01616 +3,5,0.01043 +4,6,0.00728 +5,7,0.00537 +6,8,0.00413 +7,9,0.00327 +8,10,0.00265 diff --git a/examples/Hdiv-mass/convrate_mass.png b/examples/Hdiv-mass/convrate_mass.png index 94490982785c5ba47db7a2aa09bfa34525c1ae02..51859ba6748fdf371533b42c3666dc917583e74f 100644 GIT binary patch literal 23385 zcma*P2{@H)yFUJqOqDD`$`C3gnWAJYEtyJ)NHQy-%#tFBEEGv-FhpcZNE0PwOeqR! zG-XyPN`v8d-g@``zP*pV|NGzZI^Kh|*0Y}H9m-hkpL+fOe zWaOonIs5o{c&o_D9{kTY$hdp$mu&H=Ft= zeR_Yy!nC(%{piufyJ*}3EJ7@IO<1hB`S>aWADhJ9n!C9sQ>#kxlWUYyZ_23^B?VVb ziMu$JxL>_kl6LlzYh*||&$1}?bHqm3j@9!UWcysFj|Lr43w1|4->Ydj*3(BV- zed~Ra8Lb!aacZI?JTXz=+qZ8X)00P}_m(aS=MvV^(qbO@^y$fD5F3|JMa79ZU%SFZ zT@uR1Ul`OsetaS)=YaOs0{(rCm%@UBsrgG3b%$0tc4EY&UY}XIVYGIsd&TS5g7|3g z$&)AhN}8ICtjYrTR1beXFV_`ybn>VCw(Z;P9IH<l* zAoXj8$(DT+26}p%^DS@RN)QwjtbF%wQu``T^SsWEQhL1GhX+UMpFXY5HOsrDzxTwo z?U%)+rMHbKYif$x1b*kcq* zOG-8_*>LFe_u*HrZ5w@u7C9;iw&m|JcfNoB@Rw8DR#wTDPp0qo>whMEdWjMz-W7iF zqM+p+w}{p}bJwyPokkron$wIOnp2AwEK%HaW{J|-*jOg2{if4AtG#>e4jtm!uwjGS z+dB&+ww?}-ii&FcliliqRdChmNdl%?XX^KN`9trNC;xmYcbZO``{Kom(MmSaikh0a z`}XZKTD6Fslk?F@wuOqrAN%^cv9fetUfXWBZy&3qq~x|_mD^1dd(OSg+HrMfmSJIG z;ijuw&lLI$nlyS}ym+zj)~%K|S#u9{JX}y>75%z+f1-=(lSJ89lgB%Id&A-+cO?VJHy)<)IT3WjE)2A~(fBI7uH=Uj|-8!)I!sW|V&z~=xnH<}Z zb}(7llR5CukFYam7Pz!zGuhbK2&wvQb4`|g_>)=T;(YT0>ly}4!YFUo+V=K#+mH3J z_0OKQHN?fohxrb-9e5Z+lh~?uT#|b*WLdg?>y^!On(uk`m=uT#2}x57EIDjRv5=4u z7E0c^X<7dFq~zq9C$i8}%D(GSnEj8D*Wzo|7)*Y9B8$P`Sh#Rukw=%-*Vp?~Iri+?)796v68CcZj%L8P zvz?opbnR(Di3FLwQX0PpB2P}X0?p>&rjDSP!X_`TTGz}{^ ze)QWnp8d~L__Wy5OmgU7K9p%=ZBnpmW zV+OS>;QP=DaR~`+{L*%ZOGM3LK-J63i^tB+t~*>bh#S#KfT~7_$=zKb{f9Ytbaa&4 z(9qESS<*)G^Qg9VtD;-Gk+k&#i+iP|k#pzqlbvLG;nJnZL+?uulm*R9Pd#dBG4(%s zG>W|FUOu(U-`}5o=gys;-PIh80jbBQOZx6lOiT!9PL7^jtR8TST85C3yIWR4!2|)1 zLsV3>apEg>SK>VzhEG^cbT&_BR@OW!Q)+=0J54}9l43Mp)p1=rGg`H9F6G%-!PKa* zfB$}_O+p*J=*7}2R%qiY5s%{}T@x@w+b>+Wz`*@9-hB7Y$`~tF-lb(#(;GWIi@b^<=EM{FIB(m zO|COfUcEA>xaaXpH%UKfY~;(|bC(~}f9b{TqZ5aiHckF`>uy;vfzf{W@ZpL>SFg0U z=IqpQP5snfg!$H-ne>h*kZsq@GcVwu_|hSur>AF+-JgEo%s?e^AqO{oE-k0(fJp3&c%I!be zd7>|CN2-SIjvaLu*X_S`M_5Q`{=9kf8hbJ_mRHx+Ngzimxwf@PPkjF#cK`nUHrKmu zubG!FT{=G0YT&vjB_$<$ckxQH*8859_zv&7<=h; z?;f4_Oc^=f*Op(uz7|1CF=qSr?IhY~WMp)X55A;=Q`C<>YHp^HC2&3Q?AhS4{^t^M za-$`0sg%zf5JVb#LY~ibJhwV$e06J@by)y*IZ5-eDJjwQ^}7B?jxeq2(2&i)q~z)n zXoyX5%e}*T&C%Tx+A$*Px48Nb4>U1Ra*p-f`IA9f-JGjlnU0q$NimN&?`nRVW zFK@7ObPPqJby<7&?p+_K2>3uz&CSi( zd+y%o6K(!8@kL_o+672NvOk92oj<=QzYcLt>b&r#w2`?_ruXpNuqp`|ANOg0{W{)d zPv);Atjj8_x5#7VGlwR+YofA|FLLheS)`+*V_|9Ovi9g;W+J7QeDa1vU9_Bn!lSqD z~LQgLd*Qrkm^qF>n@pTeke9`6e|H!-r2QFBovPzA~FEH?MSw|Bbg zM31+W-pf?`^WfGZ&ai9SW;V?H8tYezc~UByGNs|>=0;J7TiP;x&3Z zcD#~mUJ*Dk_xQDlTuy$~bbZHz2N&SRmM>p^e$e;%^i>O?5-aPEUSgqbJ$6;6?E0Qw z#PVahe*OCB%N?3sjT<^bi@v@r=i%Hc%{*sL)D&A&Rn?Z^ug7(U8}##cRXuxV=t>_h zPnoKFcq_G7@vZOS!*!3s=0zQwetm9K963+!`1CPAJz@8UzgDh4^p3$aht)Z4!p?E0VddlSb#+~L&dpZqn}TcIl=5;nqC1HEY&nn@VjLT$LB%hS^&?{;v25 zfLRDYK^k(7tSNVD;PZaIftA}Yb-pX{9za0Oc`{af?%cVv@$s2TE{A`8bE2A5Ob7b< zPOU%q#dNt%X$l$2Bj1;EtCIDR z-gqx3=jQae@7}+s2@9VY7;sirR=(o;>c*x1UayU(<_m_9pu0Qzb4U5iyMxLqDvuf) z7a&IJ?Av#3->&dLFthQ3@ z-Z}&iZV{1a72lyNHswK1Qx<&J^x{@7R&Zo*yRkP4aAsG%#&H^7k*t2z%a^>{FDX`T z=s0%u`;60xZ$X9z22Z@Ej_aE?E8FXsJ|?pPv~J&CXop7*YfF|4nwdK6_WGvwI<{ST zyYjh(giiN9NhCWL3FRknXwjkfDggWJ?=j(rqsi?59v>utR##Uy?Qv-T(~Bn&6w|ka zb1h!CZ$6-KEOPp~{m*O>zI3;55B$8IR17EPu%%-d7H{-gK0P&I101jz<%#7?_V3@n z?Ey|Ms2^S5-QC@E|FAnUZ)ZdN8Un1b`ZAPC4jtHbQDLsUyuAO^_g5qj`v(MwD=G15 zPL1;dXGQMHx3v2?+SUHx$hr?l`h|tm{JE1=51-P{e3WUBSa@V?-^hourJvs1UgN16 z9UXl^^{~wJuYS=7C~Fwe(Oa#ogc4+JxOjPa^L;x4hSvgY^N|$>Y_(_LSHpUzhD9hn z1iQMrF09*c7&bEYEte5!yac)0qQq;(@9)Fmm^gcYS?8uJOk`#;8js$*SweE}#g12) zkUu{@Rbe-G4Gb{0o>_80=H0SfQ=YwN3>-&gY#z>{MHDKbIXO9nH*fOb&iOlz&y@L3 z{w&&8_TYhUPYG}~EcmIA=&d#hz8o`xo_uN@TF5A|&3-c5yxNaX~e0*G7^X2ioK$r%eLoKVj zK7CsIw6QU)xVYHjfxl-_!C*&uIb}3-*DeYtx-%ylB z2wD7CnS4k_udeTud;fUq&%|x(`}-P~iOx7Qv2Jt2EtK{acp*1&H=7!s#(%MaU6kU_djcZQ>R{T#)i+}Xqxkfpr^-rF} zsjyCkM?`RZ^eSnQ$H%1jqTbKr*6RHH{yuPBoC+sj8g_W$-Mjrlp>(>VoW+Rsrp-B6 zJ6nJ!ESBZ6i?Q4+E{;8Sj+yfC@HnfCZ!_+-Em|WPub;Wv*LSETR)tHw>d_-+>MY)) zs;VlIutHq?WJQ!@Lr!N>=@KOu-K5aSorUiH(-JhAASeHQf$FLPGl7W(_-zUQeQ^vy zMQbacwY9a0ju=X2DJA?d^wgCv8i3 z;enScg;adlrQ-GXzYX}_YTkU;y+b^8r&K&)hK^4^cy`C;JW6!?k@liXbtsbG-gPf% znUAmh9b98*kIK)(({nSb^3k?E?rD4O9wNWGxm`EBBf!HGQg(Er`^TrZVJUlx_botr zV?$}0>~>!n*`r5$6)wVui)i5@?Ck8TOmC9tggQF;{C(huD?phR_xT6Iu4K%xgVvstGT`UwR+#eK0RO4931g{o5Wsf{%IZW5lnTHDE>(8$a z_%sS0ep2<{^Ss(V5!f7vMPPa!>6qQz+`O^8HA+CXZSaTwki_$Dqi{MNd5urmP3I2} zHUeCPg6Nt<-LNj>%ruEWrJJOBmG_zBB=P%0ZZ%U;z)33Pq@$bvP} zsDSIXq#YtlI0OZaalK*sP%@CbPy~k_G}{)uKrIFVV~59(UsD@O!e`@qwl^Qf%PCi-M*C z2s4OHibA3$Y@PrJu$Nkz*REX)`;Y|+S9jYsy1jk+zSnr^Y(fGnMurLw4`=)FrNd^( zZRN_9+S6_8z!8nSy}J-3#++|sV_jdq*hKF@kTvx~Qm8>H@d`!0| z3TP(qMlcFqiaW$JF|sCGT-tkH|*0%Hb~`YH;Da-5=7m=lP7Z(?Ai?J{%isWjF`=_<8xW-PmD^TnM$1;N+}bZ;f_MI$=s4b-wL=tu zn%Q5a)DQdB>ftd#4Gj(Fm)90t+mU+X$n*lhfhbMpar`GgY7TPlX`22nzRww4?no*zRj+9Y3X|Vk*cMiTKMe_E| z#f60xz?h!+AWuOEpVl&Yd76Ia&70{iu^h(L$Ck2C^~h@AF|n~RBl+Lo_0+|X46W$W z!rs@{CwogcUVkpO3DP}^g^Id5o&6HNt?$ik0>f}q=Z8Ko<|~O|FzV&Nk}LyHy{=;z z6_)(Ir;8iMF0ViMXpei@_jePo?|Dkhj^PtiE72nfo?&jW)(nvUsi~H4gA_OHfkhoHV=0i4x9iy;=ecn zo6hvmA(?C2E{1?->jGN=T80)av@v9(cV7j*z;$nqY|t?ge$`xlQcQsDLOdChQ=-w# zvSTH5x-YvwJi)ECF3U9$NZZPMC`vRnyygZwlB|cXZ&+etVv8|J^>v^L2vv?t(yb6 zHGMz3SfSr1%h}xX_;Do&-s>Ax%-c-ZOdF#_|H6tdnHd?j_@eeA2R7;oFel-Zs&k}7 zQ1;;xWj|1IS7ND?g#%mxgme;c>?)?uL?_Qf=r&nCMCrQdgGjO z{AZ79A`p%50-J?6FbV~v`{U!dfW1Q$zv_$F>S{XRXeH2qh=>R(l&F?iOfsc*i;y-?6corVS?|UG1#XAx^PYDmSOU(l%s%ZUm+w7jf?@~PQOKM zsi2@hkO!jbX<=28V}~rx*%z(bgu<=8?C6FMN580BSXel#@5jX4?Fd>5DvVoeQ;zAQ zmoJO!CwAXGBrLslEg`d?Uf!?}H2Vq?q&f%PO&}O;Ld7P3?B}Rf!NHD)>eEnDpaSih zWJ6>v%QVTMD8xZh7(6&KMkqYRgKtc`9nI!Qu4+4R?#A@_C%u-5dlGNmh>eZi^;PQ7 zUex>wz4wtFL%C7qYsWsn>)gx?G$#uRBj)l3cTkZ<_21Rh)J738=lHAaXae6>bbKa= zEXBHL;Vq!^@M>Y;A3v&T{>3@LSS~4QM?yd~JbL}QWZSO@M*XOxlT%mKLQVRvUHokY zHVdh2n{o}r8F9~>xnVViY%XorcaL~u%Ilsbt188O8W}m8oV@pADQE?vi1rl$B!Wjb za#k0kmv|-g`EJ~}vF6#Ip02L6@9mY9OsiL~rh>s_ZOY#z@aN&&u-o^ZlRQZhW5sA! zmFM>%GpYhx^LtQw*SGF8i#aWjqYx3T0JQ4+Wvw5u0~8U~m>^SRu00JUf=7LUpH66 zN?Tsx#pm}k$vU64TXJ@?l?8l%gj629P~+#OVUI5mL5l3ESXt)I-Mlg`4Jv|4Nq~}= z7!$6Co5&xXSSg-9^aVq`SmT&7*wIc3$T3Yv zw&wetf*2zTkRcBZ6l%i~Ft*5?wIE6@hV)Nh=Kw&=$7wb--AHGd2^73g#hinkYkE^~eXD&Ki+8wz2qw5qD! zBFD!(L`21I&I$IZS(gC-Yx*9#6$c=3UNwi<{2*(Uf0Fa|*uamw)#`4rgX&;ku%Pu= z?5WT3=GK*wxVPy4X4S0E+VjFbT~{(e)uf8o|>p$Mo2nEzvtYq7*1wiM+m&AJA1O<%u${d3@I z44FWRlmwxm>IpI5?bs?h>mV?@1a0V+>np=`i9-inBnzB4aRNm& zmGVLMf$c-nTnn{De?pWO*tXULZlNzv#*K%DhT4KyrznI!K#|eUG?iM~eJ=%hd8X0v zMwM7S*d zpWwXbQgVJH77;s}$Nnyd1PPt!WvF{LxPJ&W$*D%LuycFbU?l7z6ptlFQ6(iM7ZjX0 zfd`3vgYwoE^nkQ)EBnMEptvq*0vIBK_o-#em$P&P{aFjyV!7c~r&&{sV`q0yPZxXy zK+&_5L6QAjRxY7czS2@`f4Pxm;^NGpH--q`pwf>7t`Ycq6FdVok$iwJaVk9VaogC% zGPsP*oE#l1u@38ysRR z^O8lNZot0dO{&NYy|_ZJUVUoE^PDJHXb_Gw7i{H=CIhS+vts#jTR@+CfGseE0Dy!7 zF)RTXG0~9_o}Hi<4qRMZaKUWG%@Rf&C{Ry0!e@4Zbe}_G=Y=X3x^3ka1eHSXenZ7W z?-s9Lzn*-)$E*k=6QGs9s|S)5u`Ha<&R*5{4b@g*iy-^#GE(;#Zd*G(@|qXgrxx&f z(A0PH9En(#*>#vJFUFFdo=!qBVcNkVTa@~&h77dISMeM*3wm>}iD(>S9R6^G4Kz6w z94)MB55I|X+x3gpk9DPHA#d(AOeBq$cMd}KF`yeGsd#B$*63N1u(kg9mqlHGn2_Ep zUcC~4^vNT6H8=S0i&%P+or8lB-~tXLKwXH)j8^=6WB0&MZ%fq>25DUj-^thVnW-H^ zx~AG+jZkhG^Z5_dVwpPAX;8w!EOgy4J>RT?X3i&H4_f+oM} zxV0BDAyt{wU6qQQy#cc(PynOA>vO1piO*-iQe549l9eYbJ@@b3n>7Isklal&5sF#+ zoU10YGoQ;09*Ed9#AIa`QnoHGkz}hVDw?h<<7g(u5+YDFqFfY=C*V6dNEDKHV5KVR z89lP5>nAlqjp~Xc%KOhuO*BKASiWLK8sde2Z@hU%w07%a9YS+1_W-3$7A7hrX6nJj zhocuZxQhVmtuWl$DoFBx@rpe#I3b3EFkT4!e%-ok2{LB93jZLJNF>O}Tt^fuPykjF z_ZF1t;uhnuE(Sgs8_gl+0mw)SZff*}fovCO59b#d90mTFoHaBOS%mcs4QtmFJP0vN zR_Ou-e5n_}!zt$~XC3jhWk_H_czAnPq3&lzXtgR2+B>A1(eJ+srD1jm91;$XLOEb6 zW1+}206;pwzL|+d)8R=9p=@S27}()xz&k=?V`&6VPkyZvmG(7Vjl2NKZY za2SJ;dvWQ8L!^`@5)_n?=A509Sj9}S9Acp@V!62p((p?xQ&$&mYil#96BAFD8bTx| zVI0mE6B8c86txHvr~mF8b)y80N8}mPy-adU$)F-Y!eDb6d9BeK&qdTcko_yR>7F_@ zOLE6o0o6C%b~QhSn?x{Zmhzb0FbGDs%8< zRu7CsBo*~NPmMwSmoVh}`0-;E!>!|s2z8J5V0U09h(H3Tfhw1lZiA z)XhGlJ$xuEu+v^)y1J|v3UPL`-yHy- zn+&lTAslrUlLvwhij?X^Wj3*p_Kq(@CbSJr) zLBt^VZYmo7^XHGm%9W6H6q_t>7_H*w<*hn0l5O(!8SjeDWJv~NemEFykak}Ovr@Yu zPyjuPkjw{VFX8uh{Y)M_G5B|KzdA$NTBFZ+NrqLluV$xfYco;AVRhZ2h#B6M%%HNr z97psET0Gan8Xg*r35FiU>-qUb2+`2F%eG}kXXkd39Uzziv7-laFM+re2~Nm!bQJ3u z3$~6kL(?Q_o{T<*6>Gt>w}G$GD?ugqGnwdmTgQF-2(|8yfJZ26Lf=5BidEq^4x0@T zCr7~nT*W%DKwkSN_Cq*MgUbc6J6DbjS;hpxC7=Vbd4ypOXfHpm$uDg|p=ivxN;6wU zBKV7=F1MH9KX5S+igW;>GotAkybq+X&BmiaoPvQMf)ceuXM{LDQKIT4oz}fO`v3vG z@FBt~Bk=pjr@Rt5RI}$Zqvtb|&@l*nL1I5v-yAlsPUuyPsHiyHex^MZU%JZ|kOYg@ zyDdJoP&66{o5TrEPfuqIVM{YnQg8_~SH#2+49^|7TmBXNNpc|X5hg1%-Nk_Y1Yz(n ziG&R%UpY9q+E-fq@hl;dyB1_AY8bm08G6dzee(^31=TKcYv~H7=MwqnKZ&xnzCw^z zzzT(v1d5Ta1Om&9-eS7C;f2m@%*hQS>KEi%<-ZP0+&CAAQeZB#)oP0j=ybD77%kLD z&DXXsg<`;oRM39w0Iz|tF>Q9R%-WD`X1y>?!Vu24oG3&(+em3KE3H8IB~%OryPj@E^EF$%+`Dcbkh>B z)NC+Rf;VGAiJYvT8B!#_P%gGh%0P5BIP1v(%wflpnS-(i-J7#>P|a7`m!4Utxt2~} zo&&=sTt3{hiGgV7&_#K&#B)<-ALLy9^XGN2tDHf$CPE}OPX(ehat=FcKz-quY|C!4 z1I$W6(~-jb_~chMsuLEex^pWGh|M@6j%Maga%sB0dGWsOWN{;MG9n|jP`_iq=FaXQ zvq+GXie6i?vh3NDppwp6OU6ngHe#$B?*bR`kXSRgymw{Cq@U(cR;>>J1Q&z^;(pcOu& z>t58(hf;fRjSXdag&io~_mw1d%5^M$rT@YHp1PsKzW@`G6?*ZBF4ZB3F7G{uJ7*2Yy25m=eU1=56~getwAdeXe!6- zY+g6HcHu0>* z*pSsv$T(Qeu3|0!1f{%th>^<#WttXS3l{*Kc&8Db=1|sw-_1yjhW9y9S&$abN}i92 zdX}mg10tpq9s*d6=}0JmGAPb6qPNmBXEz*0VGDyt1?^!+aOaDR2s}adX|h7*w?nII z$}|ulR$-8##I@me@Zb>*VPmD5D3Vx=_U&5c^)LBY-JmEMDl@l6-EjzEJa%LkN?vH1}!MmBH_1kV8;B2Tj66DQ;1crhEn z@OTxs&;}zE&a_{IzXYhGv_T82n|tZfC?K|l;NME{lYOgb0=*iypoib`Flsetvgk6nJRH91z^)gC_mQKfM+BCu2m5YDNQ6 zZAvp*q#7TwZ6z6aNXb$grQ}Y@`ECE2kFvDBw z{-eu4Jms2)I~_5TrkM#q{M!opw;85at>-nhwWLpAc*p`=HuiZ{M@PAW1CQ~>^O-T{ zOWIqrjpv~D*b2OKD}mTgkvi1@_Zm{}MIytd=MLazYDw`3&j_(94k=+v{WrA3$mK}p zzb~R5a0MHX^fJKQ!Zg`IPbAdvU3Z&bapv*>5~lI&V%#*^2>?WKEFGf6A-y=nOK&)!5EIV45_PCYvP`CTBg#*VwF2uWT6W?W&YO*b>U81#OfynH3J zCf|4<{}g#Rauq=*BIDIzb0NTF)15@>7XH`W!@3sF4d46b%H0aPcI_hSJ-&zu-m8e0 zG-EUKz9>mj0$|ZPz{ulq_;7e;W@g(M3=Dky(Tw`DgvG>?sRPFW!^+VwWo*42;3gil@R*8`jIJOezt|48z>Gpwf>!J-{3=YeQi*NB6XcA*#W9z{l zBHA|MI)pV{$p)(ta8kapCmFh5)Pg1#rw09A%`amGe+_Ay0k`Qh)S@@!OwY*D4>YO6 zZHEJxH1?p}6oaHSrVP&@X0D*!2UM#mY<1{+D739)W*5V1S`0J6tSc$ErREUQFzNhI z6WMS`2ri$jBT1JpM*$9)yQu()x*%-BTNsQEDB>@Xws|OoNOxX^pEg)xA{rU^V@Z^P zwfAPga!wiXIl*XcS_WSLa2P6*l2!V!%578h?sCc`sKI#?*uKSc<3n8M*poxvL9CSE!6 z8AvG$QBV`y0_ccZo+emmvcqVkzu!D}&wH2)~q zAte=g_=A{ua?@j$8Z=%7G+TinB26|Vc%TZ=m>k_gI2yTi>wxuX#;+w;trS=Au|b~` z#FX$a5C3pM0f+~K7RyTVd_Z()D9h7SO`EZMFla)8mo^ZlnQbJ+ON~)ps|6HsHFb4) z-yzeHH@6MOtPVWexKcdVjo@I@P^?;UW#z>%_OVj)7t2u~+hdjaXvU8TN*x!~V!v)v z4z*K68NxFXT9JJXCmDlTF8nKlp>Nt}pS3G5YHtuYH32Tb-b^U1Vz?q&xCUBsc*z}O ziH$X_p_$Fgl<~u#iCGqxGA<7aB=iwL(by!G4zW2Vhb~7Fg0K=&R3uNlBA{%p!0k%- zWvSaHsSXq1a|jCpMuLtkLWJ?8joz$?5rB_#N5{EYX~w?i;P!^|{3?tZR4}Gp80IYq zo3ZN`(-8Rsst-ZMu(;zUh%PiaI5J3`f%I6HSMV?rSc`g~f0~h-h~Rb=HO~M{LS&Wh zeulNvYL*j4k}8zb>qo5a9@eKe zw$}OI-4O)d!q!*c4}!Pw@E2>m_u8Wn0Ep$N?tmo8rY9oPP-`A+ilBJK(>cwC?DxC6 z!eI2)K6M8gDWD#84iE1Us{}eW$TZoUV=4~j3@*FPh}dF@Pe^AJ;sggo&CVKYdPbsS zdPv`@jW9^QxpOd2>F!?NM6}IeKV@WPRlRv*IiyTCzsGcVvV~g9qpzJA*oKGS$pWCl* zy*1XIbJfbE)`PHtDTHSyM(Bo#joQ}--aAoSSSv;u63+)T%6}K(6A}`5Bqh=#(|6^` z!-8aze;GKeN0Cl{vHCkB`OKN^5t`4QJwunDly@-JbZ4u%O&DZV?Djj5jGU8En1P$G zK)DLatpdfw?!He{1-K=2g~5(wdgB$ZaVsR8&>b4TGE9g0C)0K488Q0m_U0`gtN9Rt z=YbS?lx=)9Qi$&~WIlpf&_{$=&jkO$tUx&Z#~l!ciL$(V=nUilt%CQ(`(yT{Fbk+C`5M?@pC~vMZq^`^tgEqbCcBJx!!Fx)zuyiyyCfjOq#nw>Sl*rmJ-OEQIdg)MiCu-Q`^5I7=Q<&8gAjfB zu67jOmdzmC6`%hb_Z5(=m1dhGA%4;*GwMAyCX%ZW!c=}QilRE?D1+}J{|MxmwrFHf z5+aB<*y}yaMq=vfd&lA~T=@AY^#sG(3h@cao?s)=_6bjQHGC9u7}Ujr2WFJ|zI=%} zKK+XgPQhx_Kc1tVn<0#z%#3Et*UEWbly{@lXOI~_$z0#Yr%!FsY|sPn&JQ_bk%4DsX*HYPFoaO?+;PTU01j-)S+8aUhzsW}hER#^j+gO62}^Dijgv}GK+wayw$R<(og^rP?%l z=+IIZj>=d6LL?)5C~nfo4OJYxgSkhJ!6rqFfI#og5VeS!(%ESX6_gA6o9K$zYJ|Vx z*vT6!^Rd6*0k+hi-BGel*LSZY01vivXnRSfNsR18%VqtDKn*%gF-CIxB^HXEi>nsJ zEE&>w<;O({+{C1ysl^qUEw5=VAKkQUFC={MaF^D6*o$t2%a_seH4jt?%}mtHMdW_& zHR-WhZsT7NQIW~ET~g#gxdY~aa88K0*%yx!qzQIGR|EO))G$()|5)ROB8WOvaB8Y3 zraT-$P(SH3BRe4X1?+>xA|y!Lq@cN z84~r+$;+27gF#^As-p@w&fJL93YvqUz($2Itx6;s)6Etmk*F4!P+7xb3w~W!mc(bG zl@oy;dJ!SO0QXGlKtHNJkh;SHwsD{a>=+khVIU}qIxV8U1ThY7tatz;Ms7G_DKVrQ zqD$v#vZ?^G;y*}>6k|X@g)N{Q*-`cnRs(0t!^Q;Z)eh1aR(PFjLD@>=1o5)kJo||3 zN91ktMK#$bNv^MIYd+9_lU4xI`TU72&1vDT1;0g1}g>8 zaSpGfVKsyoWNfBjY~E~6`=T{k&d#gg1rA1p(#O;({0l|-3~&Sb5WmGUJq$+}(aQL~ zbym(fy>x@&y4NU3Hb6CR2CYHXtT7obT?14Nq~e1sr4kehs+tPePayM_%V1VjrnhgW zTV7g!kdruG;My=YHir0~PrPZ_=kfXZr@>u9Ve#@(KuzD-({mDVkL5N)Pwv;DNp5R1*Ejev#KLsEHDq)Qb7Gc+4g%pQg_+YKH zRX_Y};gCWO8U6Y-=VK_|6^jdv8x>`>YO}dkvj*q}YP$!AzY#eVQ~sC8&Ty1Cy5S;tcL^wuA*yjmo7lj0f!$?Lhga zXW%~(15phhgf=`fKx$Da;z{obBNqa7x(f+uD8~u$Zi+%lScB4)^!a#7LEdyBU$S6)_|-yK530~0>j48kxiKjo#ERT2sMg62_y2VJ z6-E8{@q?H#hT8JifRGZ;Z6F`6A-*isJT*9$g19S)#slsbzSVRoE%p;=StEH^IcQQD zDiRPA!E*Fgc5+)x#JUVu&%Yu#jz2)>_w4raB?4o_Czz1Xj@FLzGvJCWLeUBmFBN;4 z(Qq0uoCn@N(nd?jJBV5T6xFDz!K~;qgXFn1^Z8YQDE0xd_s!mzn6#fZwBg{qyfr9; zGorNuR)PQ^!5e;g?|I(d4L^kxxSkhy?>7Y;Rp=S-H#E^*g`CdFz5vX zf?h>eGSO?r)uC0UbB@DqA+c=PChs+JNUhg*7oQ?SB{dgw*|7O=TM9_<+n*Y^QX5Yyp5mZmY6(IP)qj}&`{Vf*MOr(*Vceyb;`*ys$Lr@;6=>^i z+XiFI^$h5d^fw3#`l~fG9-0oXkdz!&=0J#T_Ky~B_69e{#}@_+7w-}Xjb2z1mzP?C zN76?YoAUtfI$(D#m>sb~qV}4vjH`V7m;+|@_zP!|6=vYY;p>6z^CS`44!!3DvHCP= zBQHWRo(cb14)YWKGTcfgQCtST8>~z|C#Y#Z(ZJ6{t+l+h3CaaF9vq=bzR#aIWFgpy zh;Sj;=zSvsDP~>U1wsL3i$aD3GK6cz=fD2^aX@yqJe-2S*QE=(Ue9o){-yIzD*n6I zX3z+l7!NP6QkWr(D8S4_j{(UKbpfCN zUU`$;7&swJf&}1K5Ar1Gg_zxNriauP?O_&)Y_-9h)*>esxp!z#n-KJh0=~Q8P?1gD zj${&`1ZK0q22WH;qOxHX{ik<>sG2e6HRm?dw1{_@fThGl0i?vQIDTU`BxP$slbj!7 zna?`vBi;qnVhgLOvfs#Ja<_wgDGuh$_?dEooN_frLkkiTFq9r0C;CN*9&!6#g}4LlKwo>PJ?Q95{Q|=$XVhK5z}-v#$fyuNGV(& z{6WAn>|^u<@RMf{T=Z9j-18m94*ahUXhONuInZ3XL|4>8kxvBIe_ET6)bu} zJmG2KiNd{U7q%P6qZ#XpsZ?xI(tPZOpFe+Yl1Y>RS3xj5%!rgNN)>YQkr9N)2wq++ z6JcD}sd|=f|GTAku3GeuLjzR2EC?|QL52_|E4Cr$nks$GZRZ)yZExU(27|V=Mc>P> z!a#*yk>3+vVn8jn;BcOhP^_01s|-Z#iq;@Vk{AIVr+G=v(F7Eh2ysm4yN>`aV`P}a zEy5A0=Y^8J+JucFJ&RKD@#pbUb^svbW9)bH0RV{EsIuG~_hORY-`RQP?K1qWRVYeW z?-@?Afk(JTL-s#9fKv6hVNsY)6}HDNhtoYaF_Db|x@r-NNx`9YO{_?_jso(`ZDe$gDY_YYyvhPC)8kLD0fu0vEpRkSC zk2tu%1G*;>#6WpTdv@F5lDTMS39)X+=SE>9x(ZVnvE32dK3o_@s6S|C3eoM&^YnCe ziO(BlCfjV)m#!TJs3dU^1h)7be0#4Sk|n7sQ4oWu=R*I$GgUe&COw$)8uHPPMTk2n zgoQ4xQnQ`3z?bz`irS5HAguuyjemZ>6y^fN=61B7KxoXrdXc3hK8pAy}_m|L!FKXA}#@-WF`t;Q9Cq@V6mN5mdBs~ zqCN}$_k_9T-rYQuqYCTRp$D6^%%Bw6@+bpyW4c5^EcC1j-7NmUF>QQO>(}#xH(TX0 z@x)P~y1M!b@;d1Ug7wnu7Zb*ZebejT>`{9;#&5Uhm`Z_kA2+#X zBLWm{${Y(ZOI3aoHe0(1n_6;#%)iJoT5}ZXi-0i5S1N)xn=O#`W37Mj;-={^u{@ES z3(C=&&ovl4r2esookFUjqN)0b2|(f}B>Y&r%U{*|47V+SxJnrfe5TEfLDT4wE*5MR zr7HXV6p^{Hu>c9pirr99|8EXw|Khzr4zdEX05Dz3`l>ON3UygV97RDme2B zrVE^>Fo&2IQI_EtH#|tj7NMe->p+QD&B|23pKQodb09)SfKQ|uJ9RRWQ;xv?-vpB! zbV(w35z>IpiZ1H^(q5Ia2Eqz5tqu}6Re??ysE@!qa0q{fbx>hBsr=2Ra@+qE@~eT5 zPQlG%0_g|Kp%UI3<&2#R)IlE*e<{)p=^+3ybH%;mA)yb%#Us9UN4WI;W0yucrzkW9 zRN)wjlOUW4*MR_AYj1uA@YfC&cC>A8fzV%st@Yqr!QQW4#^z!J)yC$5Hq^qw_NKH&9Ie&rR z^z(qNjNGLkj-<1c-Wv-=MNJNQ(|un&g3cE7(jyOvy#+u|ABsAhlKtqszEead7X{xk zV(*&OctqOZJpTLo2}wy-X~X1KKHH)^M}XwNo5Yw2FZT8H%n8#n9!#Grm~Vu2<2~Aqq7d{U9gWyu!3jT)43K=bt|Y=gGX~ zuEQo^-n;GaH=F-Kx&|OHz+RSIm2mby8V~Uq-5s;;*#G$mMDp^i_5A-lKM}@3eIt*% zw_*H@h{L_crb!KmWtPm7Mu-9CVWB*{yw1(W1Hn@C4OiCIUaR(XOjrE=d{a)V3*5Gq z4GrkG-6r{#v|}RS1Hh&pb+8eaT8jNYTPZCzjkx#RH>ShWu5{WEhdzxD(1e7A?Ho7ViTmk@1+# z6h3>u?>TH`VfNe5f7LE~$W?_ns39Og<@g+|<#Yj5j1eGEyHEt-gusil{g2s~dWcmE z<27rj`fuabORVc~M2Vwj8d`-L?!XOyjUom_sc&egt4P3PLoeEh9)0-(FP0y}!7Ic*NQgUh2`s~*YsH3$kvKwc*MKCXsne;KATE%O zO;TE5_YHg~Q$@Q0qkQ^zUj6ryjwfHx@V+`22375D=ZzCb ze}S~rg#%(6SXpy;%_5Px#b5OtzTN?!^of*-xATw3iM>cjN#UZhN%vMi=$TRUrF4^+ z%gKpqrj~e@(Zx(4008~kSL;(DC1M<|qSey!?GVlnfyu`D;Xeic1$2_m!`y|X z^ash71Hq9{Y?)fLYBKu!<)Rj*alP`Osip#(a{E-9dmduM!-n=TW{R|8f+e;GZSb>W zW}1zHr8Mt!HvJq9Pr$T`!#O#B!-l)!uRMKxsu3>GH>C#o)lwM#TZ4!m>L6zD2sD*q}Pk;no~bh39@<3v%2571$R5f6!40MlaB+QaX@WI`ciV6zd{9GxAR zLBiz@Q`K9F;xjTj96x_3+cPvVh#@8pe8NSco{|nTU3*4UQ>1+I+dP~C$BxL{T?!ec z3XR8Khw=)b4p?s6Og3%7~QKv|AIloX<{5as>o4>w}!0beHN z7x;PNg=*k$MWhBI*ut4fY;lm^YY%q>vVk|qHq8~csrzD~iI~5R93BL{ITjtW$aUWS zmvZ^USO}*{4l@H82KZ(iF|3K=3JY)(zJNH|@Bs-Gl_jDCd0ni7if55!P~THQ5ONkDUCBv*AJ>C>y505%-6q;#V8cM}5i>Q3dDi1S z#_~`xRwN}SlfM(Y31D4#d-47qMZ@0Z<@e5AQ>BG;6luh9h|!(+^#Xr%bnL^Cy(fXE z($O+~sbe2Thqx`_EII)yNv>=+xp!}?1JcOa! z0Tbh2n$RWTHX0GINa|okSiWs5{Wqao=_=j-)&$IxDJH%JG6Gsz{^{Qp#Ce(Y1F+)! zFCfGpF2MgS75{UJ^ZzjjsEq$Vj({c~p442RQnBJqd2BC^xu*2A4e2+uY(xGZU+^Ey literal 25544 zcmbSz30RJ8yY7={o@r2$MwBK*nl&he25*BjCqn}jB{XW#Kncm5CK^zoXr2v)NGT@BcsEdf(^0@9Vy<^E%J-x}!{X=(Dp3vQQMozSTh2 zjG`9!QWRY?69fLne|>=j{%_L}J*y)-5A8j2)Y038+U|J7)BVs9_XAEVd|kW`A2@Vy zot%=K;+hrvj~wwltRgS(@vk3{JLJ7jzSVKG6mP=pX|U@sMR7Wkf9WzbZy%s2XQi#W zn`wU6zIPn6+3{^obGXGebN}M##=SymXD(Fgwp%^rVsKtP%@_Ja;LEyd_jgOBnyj8! z1PGjYCg2>A7MUj@<{b0f|NfF>vp3$IEkm4|zm8ssHuU-N!>-|JPoUbDvR6JyV;a|s zzxAqoRneCU<60pmCYEAttkb_B68~hlDztLrpG;;3RlFdkROz6vtE-zPB``={Zdw>B zj925YIOK}sRkOuxzIavF^<-r%UNzWGez}?J=+UFC@86pbY&h6awCM7-h(qNw1=f#t zntuECO+e+yD#48h+3>d&68gW4T)7 zB38Z?Yu2y@{2tG&%5qDRG`i?8^s&?|akHy|5x!wxbKuNR%aUIoPrQEfCL||EesFNm z#Ka`H=V@F|YMe$@QqXMF8KI4awo;k#oEnpqoSfXMHEU{WWBBs|XO81>yBpJu_P@Kc zE4=0C=V#n(%apkee|=?MP&)EN$nW>~>WBIH+4sk-9iN|*B;S;QT~NYjb=qDm9?izk zkY&qh0XjD~w`H5W7w+A=x6WmDcJ{GX4+D87W*Ps0h8yak1qB<{ulM8GZ2q{g(6R4z za@{YF_JZ1n2i|fC3Pv9r`(o}F@N1}3zdC|FCjY?x{aTimzkG}&Q-ZqY^rC1VqIqS` zozeSLj5bxv`#-ghy|YbaH2esuWj4*%cZZtwqCvPp3L{@}qucdFu^ZLx!! z8>_9YEy6-{?OGn^r?KH@&TP;7xoQc^LI#Euyu<6}X1%w!wkx{SEl~^n^SEkyN_+2K zsV^@s(+dd+;ZrKAsuq-$l|6g*%w*zuL&MV)y_nZUK5pcd#zvm8FE3?A#sV&fg)KZX z@Q(fGK#P71pKLDntoQ!4Ys>n+edATVCI6u;KUe1cG1Yl zNbkLo&JQ0xP-^88T}gI6FCKZcGhE)rAiD5mXlTN1Ke`LY&ve8LaLL=3v2k*8uHXNf z!V*8lY8y7uK00HO6r7*1^&ZU*ocZ0?mM?Se+&PL8+TgA=@a}GMb~Y;x#D^lE?3#id zNg8J~0)AIKd&a~rxLykz?$uorUkn6FP~$j zURYSj#LR3utM(*Tc)CASi?;k^B+VLMTCe`1Y<)*(LxVBJg+=78tgNhRX|bfZLK)bl z-!-(gSwHk>w^;AdM)MzizEW(}DuW~aO4EL<*iz^ zic6a*{^CWcHEY&{rKhJ)j@HZ97WoX+6c3fjP?1Ar(>taLFI^I#(`s#NV_CU!Wu9xZ zbl#C~X7N|A%1jJ(%I{fw+0)aLE8F5OPg20dWrYnJ_O%S|?WSxVGxy!-q9# zThG`0{y9+lR3vE4`1p8OOiaxEThW7G&#yENyMF!p>!zk9=cTsuVDT2)2mDO%^z!2V z+K|Mr%_N#wua{$2s*7(|Gx+{NZ3Mee&FQ72gq9Qc{oN5?1wH za(DM5zWPLYe)976E9<%W`1ooAH>4l)^7sATW_~ANZgyrt@Z@AqT!KgMrY{>0eO%(` z=(r%*Z~FTqJu9nAOsuRj1$Qr9y2PU?Ze(Pn==N^S2tI+;*w~m(YnigA*83c>rwf9~ zBH^)}-rsj2|5iPFW?;2zSJcg$D;(Y2WGwF4tk#$r-(uB@7xT6BPhii-7qH@6rZ zrRW41%V@F@%F4=29v&WEzsC=_lg_>(K0c`< zM~;MDy?S-@>j;*-Wc)E#0)MD*JWG9D9S>EaCVA3N2z`_BGs)?|2g9 z(1KuhdOG+uDTt4%kuU!<+SpL8tg*YmottyH8rRXGkG$n`v&H*6it^7s-_!l#^18aR zUiB~9YMhqH*XaibIx9JXnb_H7C)T;XyK`A!?ViZ{cgdEbq;RiZy}GV|U1$Rnm9p!> zUNKeG-S5jBDi_dcy|+v@iQM4P7WuLCgiJXj4q0O2O6!OFMLz~kd$_d6IQ(kpWEBk? z9vR_A0g`)n>-O!HfByVgXlmY*X(h%x(LJ6P60*7Qw}`SQMR9R)ncSJF=G0sm%+u2S zzO7B#IFBwkDvG(Sz$2!7_P5dd4gQm!OZfPla31OkkQqeRty^4uij$FI63spF>mZwe z0;SEgXV0Eh_N6C&ZLzT6dv#sa(!D(}FfhZuylix)qxfk2g$oPq?d|&p24vc?3(O}@ zOgklY6NDCM^7^r)OvP*WcYupAf3A z%K}b3bZfcS>Z`755mUMeIc(^=j+7xZJS9&xr@N=gbpte>P2a5f@s5`WiSKRs zI(~l2Qrj+36cZ<>K2qC-AYBnPzu*@ywz+?JocH2=AxgUZL>bPoq${II$^^g192@iV z=g*(j)t$`DM6oizF;jAEOe%tHsagN;*}@J4dqqu6noYaxnZm+N$f42s1cFdEIzxZF z+}Ifw5y7zKj9|FWz&jKWPL!GoEG#Nf43N)ILOW&nV=U(EuOT&9q=kCmP*T&TS;#KFHn>{A5bSWLRMpo9yBWUEX;f@{H z8xb*==XG_`d@1%54}wNix6^D1*twLPEc)W*OH`S3mVh6fK}+gWGBbC%7IUSvPGd!# zN4jfK$%8{e=0LLzx3#xBH)olr%gr4%@Dm0snEZ0}Q0~zoyOO!t>DD}F#*fFo3cC#ZCA4(x zJ8|NK--#3S6tGxWNr~FFE9;q25%};eWPD!KXRDDQxsdHHy0 z{Y&)^c3|5b^wYY@yJ4faR~p~YZMkP_-`3LBCdd-L06Vn5v#j(!f?N3V<;%GR1@~y| zIAu@y?fpY=f>IB>z4h$Xt0$4%;uq?-Z`j@}wz_aswPGx2k9lvP z#lDxcMGJ!ea>F*?c2ArEnT~?_~?+F(lT|w;jV;s9L!VKuM2DI=)|8t zpX#o8d^}^cV2M1N&;~Z|!eyxE=21VF@bV%dJy>w!*9V+Q&rem0bhm7|Xf@%K=F*z$ z`1tTwj=;%Jr+5NxXaqz9mRTOWosq!=JY`ZQziL$ls=kAh(}I2bzB&awJ-sxPq&Ecx zE~L#Zon-+sMZ3*!VF8hsnF>7GBLFz0H+{)Tm5$XrrmU)(ikBvly}KUF1;*I+|28)b$(T>u&M%*C%$l5>-m+llRcb+ZTUvWIM?Xw>rYNiIrcm`^X$b7 zT>{;{yjU6*7UuB!=H}ty;fy1~Wb3QKS;;wLUMzLt@#W$BF7;fC7l$IDIW(qi&CSoR zP1TR{{PAJaG7W!$xaI1{dpBHMv+HCCJ!8s=Uq1vk9$dCuGw?J(0Dv;@bZXm!ea562 zGD?)wb8-?p{MxoQAgOZ1>%zdHepv+@Gm2r{S*36m%Bah-a(ayJH-y zBcl5H_PZqUKTPCd6Ae6|LT|gZQ&^F&9wJ_(4gjn;J&z~1A zUc6XWPj9QO;hC|Kk`|+6AaLtZArRq6q`?^+eU?Kl6)6=;lh$@ z%3ceuZt&3d_g6dku_S8m%PU)L`M8@GTvOYuD_z~%Du9h?eRyE;ay35+IT@Kt1dua( zT{4HC#$rD-SGo7^Ya19a=?HTt2(c zbp3i-|5`$c#CB)4N`LvR1Hq_(1@NQFi3=hYvQ&Dk@Gm1p;b* zo|yye$mUvF)Vmz}IAo|5$e{p(Px9SbP;gngrb-{q2FNf_%18RRP|IbC1{4$&C@SsN zt*Y0rbE}e8;>-U`yndY>U5?8^zopMt8ox-=6hV`xWn=R}W?Fg)3z|)4kG6-EIQYq# znatE<-|vFQCkCT}gQ*u+Hx8u_0ANR+K3#Kjb7*w_(Ql25z_&25u$UdJM^C?Q->VwL zYkjFEVhDdL-SRmKbOORcR7NIdo^UcWk1ts<^*JhFdLkJ-jNee{FFblQQuxK+Ci}Xk z)=hIPQw|>c+ECrn!jFQDk{1SC4Ja^PXJUB~hf+d9qN2UMxV509wA5?#^K-+yV=n0V z(Sxz_^CwE%Zv@-bYR4XF;|&l9I)+tm&1BL2g4+S=NN-wz>qiR;I9A^8ey z@|I9jSO4_=`x$^Q^P-5;r`ewMj`H00!Y@Xn7J279ZEqLCgD%ITQLL;nYju*#%QfZf z%T7;DP8zmsLna?Tbk5%Sw#lm3Kx1om-Csz(&U4ncDLTT8ihc^DO=eOa0@=z+fUG47 zz7v=LX~pbuzzDNMExduxrO%bGUc6u{C@83HYkPDk4qzm4>hk#bk!?5BmeMn@Y2gTg zTM5NpTNFh|ZNH+mbLV2vbCLzAEJytl>wCd^T_|GveG27fbw^uc zzPxT$%F)6YQ>5b6uO+g4~6S+;J8t^a4Jp9qCm zfQgBxrhg65`)TxJThdr0$~%~;!On6KR=zb_-rj4`zJ(x^QKT}P%1lj7Jr5ray?5_m z-UNxbnwo`R4kR@e3kk7ua>ns|$EUUycw8L~FfrjOoBAS@mY!bu{5dlyip_~~wswcU z+I73UP3-mFL>71F&bKN*hsWAJ11*yt>7!o{`;Ci|VkxdW1aC;bqLyQb`<&O3sjRkEfB=bWg1V!{D{;c>vUZ{CFJ(I4aw=fYJPRFx!~ z`43iwF-Jy6OXt~AtwlaEDo1~?F*2TVeeb&qxepjDRIoz|Zz5l46N~hn{7@*SWdBw5 z69p3co(l{%oxNcH_~^%DjA$Kee{eFXm2VwKz$$>RSf&}c3||_3uLhcekxE996&p7` zdf_uW6f{>|5kjAwlf$W_qXPhYr>3V9$b}!>CU{qVkV7GHadF=7HH~ZAT3ewF`1`Lo z{~QYvy8F?=%{n^tU>sx%UI8Yc(CQ?c&}JF6L6)zSle^bMcbY$p%V7Iw!?%klTxGeA4&`}anQdRZ!1NpAY*YD?A zucXAibSX1M4z2fnW^V2`nqUbktEy672sPL5S!-o&UG?(i-3S@y+?_^Q!{JvI92fWw zeLRCM(E*iWe{&Wu$o+;zA4$TKD2+R5L)o$%veUMooJ z5%_1a#ZDe^EP~=sWDl+9x4BCFL<9s@J)apQfh_I+2^|mvJ3ISeYaTtIwt!}!+Mcyb zRebuxz+0xG?nv6TwY5pUdkgvnk_h?DDD-g z$vE)3Ute9Pqrlcz#c9q)f&%$emmp(cXjnDcTVH?Y<&!52Z{EByFfyu!Mw9TJ6q>7t z%A)YI0z>6@@@4S|$r-HkO$EWUYF`v~y6lq%o1|_Lzmd{>C(B?88DzK+$O-DDhoq zco%bX&m`Tsv*r%gU7IP>h_1pxzgYroKceFls4#S0jvgLo@XLg`tNksDBty^05eBs> zz2N-Wv*2GiaPX;$+ev}ZkoCl6WCnbWhD1j8r30Hd9y%0Pc=)Rm5QkYJeQ;%LR;7K= zOgI8o96dMUTKwRL$01OcNJ6zO_BD^a*W26c2>5kL!BIQCV65+rI62SY2f`DkG%2GuMEw4^{wPm*;`y++p$Er4{- z%KnoIKRy(O&i(nSNd@DJXaQOsDx2CHdGhNfJ+(yD*fQx-LbAO6(FL7ojRq%6oQftMkTT6#@Vx6V3I!J zm#p%f7M1-OH9}`dM}=_-sQRt}90mb#i6_a&$EOxJj>wLmen&+{*6}P^5=I~;&LQdh zaGqn4yf&i%_BUP-7<3f{o4eLtCxUJDyOyS=hc5v7CS>2WJb7|UHz^wQV+7X%ebYnHz~v_@4Y0f)f$vbG$$lo5jGl$!^qZ*xuk>HJZAI-$8ku#M|^yY zY`=7MnGBX*zPuFr2q)z?aQ6}D<2sw6nD{D?ccT}LaQIHz$mwfqF96!4Q{(i-r@3(0 zWh`JQ2HCOl_6rx}iQ13lOgiarUa!({m~Pk|OS3>dfTrl#TgQu@;DS{k{sVk6ft0`; zl?ZhsTjSE>Kch<%pYM@pKRh??LqvM#&!w&{`It8o?>T0f`<3sT*mS zcaE;4D%~j20bC<`1=>_mRHQ-aYG}8NjMZ2{RJ<@$Y4qf)eRlOeanLuHTeXU5YHF&l zqj)3UBLbP?&+PONk;vYZ96L-ZKlQr!=%GJ|T9?>1a|;`rW#}}_(+ALL{7k%ZB^nEk zK4Iyz`ufmQrx>B94*w_~GPJcWRBSa(aC>`;{nZWi7}ENHFC>*#%J>FRX0NFoBW?Er z3cO6}&f7Z(IhEz!4n$IocT_ykm?ph@C2~|b;->oJ$K|Mc5pWrx4xRydR}H`RSPIM683s}uwzEcyF;NAV+@c7m9&t45_8mjg!c!jX>l4iFx?Y^^n>*X7gjN9>N#UV2g{3?{RTiU2Y|>c4C4J64QQgUY8#&6g@B^fcmfl z-18+RH$Ji?5Wo}Wf+p)ntI7$whQ@5`pX%cI(!y`6?9r`7=uS)F)^)RPdWr{Z!BcJeKAU1O{nV z$YGS$(jNx-M7BJ38oQtZZUASl8X60FK1QJZJG&k*prV>3TJ+#oE^U(v#UY2|1iaQN zpH1VOKl__|41~GkSpeZ{(f4~z4m(o8iHZCL?(Y}K$jC&=oM%8vy_uQ$oNF^WdU-uP zJ&?o|unm}&2L`OMFI$F$U$-x=MbGDG1Cn8e#oNZls&kS?uOU?dhdQA>_fcu0^S8K@ zCOta15nH)n;ztK7>T1c@E7iIU5E9<)E08|N?T8gjHz$W_VuL>#Y28T@Wzpesu=?62=J^=$8(zyvI^{>8=hSLa!x_ z7KWTt1%t+7scjW8s(DKV1S(ovt-AYN`5B4{A;&>aPoG&215=g~p6Jt;FA>HXQwu@5 zl3s3fbTmya`<+>)!y^P0T2I7Pv0=-J!B$10euC>p0JS0sWzG=72VE@uDM!FT?R#}C zvrz?)*tZEp18OBWvu;#NfHhr^hhpO5n>KeVWo*lhiHmy*Bn~t^+!#n`FK97Q69gcs ztCu!4eE#yK>dhO|O}0hrK~pd>F;&2`Lb!jtUkX%a1ajH^IA9%kkPJ(Hd@SAlsubxj z1&l+gWsV&=me4;~zvI5EI6sW?aQ?!??T23SX- zqjkF+JZKkHDup$@imoXhsTkE+9O)d58wiwyCYQ`bcmf1(UuU@{STp-JV|2QV9Lp-E z#=d?U7+}SsUEO%7>h0UbuxV7`NJ;0<{AJpfw3k7c{J%48@zQm$_i)q^e+`xl;zJmy zG4QZru^Ee=2b}{WP=k_|i(T^mxeHzU>Sy zQUKfuqF4a|*^-rt+5ltB8pSCwsEzpH6L)<8hMV(?s;&HC5W)+#oc zuiL6WX!qje)NN4x$tXs|uwtY>xpe7L=-k=>OnGiCayI3G!a&|IE>u414BVZm0{Bei zBj^Eag6o+o#-aq#d|gs>-je=qE1(R2@0Tx5A4`r+qL|ZB0Pb6GAh6_8UM@bNX_a6) zD42a}+aO54Mv(D*uNcd1esstl^oO{b+A?cvYe-;(r~oEJtr#=YAM*D0_Uw7W-a8E* z3u!OKG<8eeE@e78ItV8^0BxW8`=ddi;4nFXz&9^x!2N=$IriSb4C-}fn)&0VoV_P&e!LqPZMC&8C z5!p!B+FJ11wQIw~fO5$MSzEW3S@c3nQ9IOT5Db7B%`!v2o6U`-?9|SlakcUF-2f{D z90-e0Fy_^RtX#Z}19&Q+68dY`ucyoD%bH8b8w3_FCZ#xlR5^A}i9a*r+Qi`dP+;r@ezOnL(YwMMXV#t7mo)F*%&=kLTz;C6V6aD4f~}c6ZOL0B z4oXTly$DtH&&&)YTl%gk4`Rb$#0iN|;jQH+C_>s1pLA)h3k!@r6a|+Vb(5!I@9q1F z2N8quW+*-W0Rhk~xL_%%sH~)elj~y98tIkuWn%$_ZlDrq3;FT&>(>=07qNOmIz<{Z zv=vNPY+TC&jDcLn*!#=|EIIs!Cqc~_+VGk$ng3QyLnUJzSJodmxpdv$E+_(QKKXxN z=)d{rk3S)W1U9$}P!%9AkvH|wd`aFrZO1tOyZx82BgEu~Fwoz#C++_Aeb932GM`vsk5}zcnNN2B&P%30_A|;>p^ThWd!^Fb)8)^+x#0heE>S_`usTz zWsr39sFkg7!022TL~lzRvg;HTEAbjp0{+>E{&N{u=H0cKOGOtFwgQ4DMg9IY6o>PM zPI_!c z6*u1odja4aiarvRiXNT0@`;HxB==<+r3wO@GBB;edqNTbxIbW;6>Ha?OO7@C84{L zqKQ!_L=X|&q86{r^&4@DP|4!C#w#QG3LBE8y5pKM9jZ{t)R(Iig({8mHL^5k+>rM< z>wJ#M+Jcsi)Cu5aPQvN}X21f^&W^Y7X7e?FhKK|nhp{=MDV>X8-)-9ndza@_%K|on zB;8~a16#gKMZh!2>TA$)7}_#$b8o!^`vweM5iv2W02vFRu?%(;^S2jx^rUa3-Bp$; zU@0sv7AH1v2(AX&=x6%g-sXf^mYsr9sQ`hJ6KZpmOlpzVD0eY90%RaMtOS8%fIA~- zV9md6Ae|;1JwpW;zO|RxKyl0F+Y=hg-TfvU8TjBXux94gjOhO~(Ne8H(72Gq1}iJ@ z=0Y9eW`XCS*TcgX5eE(0sB0(}ICk)ru9!uN`Kzh|TS~UU1$-J6&P?ENVNTF^g0 zA3Z+ujb}^Ta`wyHq&&Q6kC=G(xVx``TacDwE0;(An5JdR>JfPGmBjb{9y}%;V8>`y#b4? zgdNKRT~2Hj$QR;1o6>IGzkk2(I=_TS-tX``Ogf__7i`3=}wT6(^H`I=u1XL80mpAUepAow=${`V+L(s|~rybzH10-8B zM)wR*9bGF^tcYeT03QLCFbi5eI%u2zC@DUk4v%_k`&B--Lv0UNH#Robl`cPY=n%2^ z!x2ou#+)U&1l{F0sCjl!a)a%UI6;nE6}T@&ygZq1jQ>n?vH%qDh`)3x63ilwYeho? zCzvnUbsxadiej9A(A01e#0zXAxs_0Nb!5ZrbxiTkz=C20(pyqQ7R9YOK!TfjI zfpm~L3D<~!CL4lgL%|*@G{qa5MsXJh?Oe6wSwn-o&#o<77(g}=uO*yHtf1*mfvkpk zW%&D}P7Bu)cnZS59yziex_8RmdqT3U;kxQdP%%a` zPz6s@Cf%?ETi)q}Y8{U51r?qT%pPgya`F%5n8kmqM=?8$K1&)J$4Z#jsTIqXEeo@e zO9T)|MR&aiu5;TvciQhict9-q2oMd`vyldR*EEBcgP}70xGD&W10>Nf*e7ceOl@IF zxXE*VCD2%!1zK1jRU*AjqFBR~9F5XI)E+h;nwvr383GifNqyFs9$ZINpav4-i$+g^ ztMEK>qCoXZ5g#~Q{KQgX7B!v9W4?qSCe!QVkZXE6j&|09Q%LY2B?TGerHOt%9|0&C zJGiwS5Y>bjBIgXplTdJ|aWqL%12bL`O$A{%QZHXlQ2_A~Xmz8q#j;f9;g1hs;X;Bk z@UqtK-9kXwPYE-GnTt##@vOD;6(gU2`SMt9@hubuml=%mPxh6ZvvwlrDu3ht*DHwq z10}JRc;e7n`E1o~o*xuy+OcB?2Yg&HXy#JvN{^HNBXj3%6BID=-WCMOq*>s+a{#6P z0SqDH7@B7)I61a|;f@>V`16Mc->)P5xBBt%3UKs`K3Q4=7T8Nc1eK_#0*g(&`ks%?F_K&zaJ@4Ns2Zt+XrL-4s12C zC}La!Rq*ZKcaGLhu7E0@ToQN>!#Tv>Wc}#i%Y7wg*Pjx!oTW57JADYS2jRP18N)31 zOF=@j`RJ0NsT6B1rVs)MfrlYe5vBss3M63AJ4{=H(u9_9Ju#_*24JL)41E+PWpbTe zlwl=#Z?jzpv)l&{$RG~Z^bES6p1wD!cypI70-G}L5Xv6X2W+g`zz~DqzXP-xq$>h( z#T!0{aS^zOjsgv&4+2@isfGzt1xC9COl2#`mrmnBKc?W*PXm8Sx>yQ&cGti_y@@_; z2iX;gLi}bmOwBN3RsflLbyd)18i{2kkdQK1OL$hab;KJ=wvE`Wkao7jE?W)RC!c2? zENvy_kyv(dBJdzj&QxRB8Bvd;WW=*d=ii41_U0NgiiNyRv=sj*GN!!qizg@^W}@-i zjr7VbEl+_C-=u8?E)X~4EO~-W&AP6gto65`(TgsmZUUE*PE1GGly@GLn`c9qcSIi< zdFIR_3QA218iqzby|i2N8{4eS6oSK44R8UPcmL~~bP!Qak@!I*M1F+hAlyzj+_c9Se5$JjWhbu{a@h>q>qf3eJ!I2Qfhkr^DV5B70Ja0o{S&E?G1etQFh9MOfV z24+xIhM{u-AR4}h1>t6LrBPv&ou%HWcKE*94Up0y*km5@La?y^wPGYN`Bcnx*dIi1 z3N+w{LX-;fNSUIb?{R(+Usmz^Z4ba**%o<>f`pAeZaI|M5yJ zgORX4Ujum|EiH|rq;_131nFsRUZ_iBPAH4(zyJ))07{e=PK~3JlbMSi=<8+xFS~?s ztj|)CX#Z-5P}m&tL&Uioy*8>(iKg62*bPuM=oZ4YlPPMyt1E`7_`%idfyv4$F@U%d zuy^m?oQ@nwMYcspDj^`wpQWb=;im&(Q8e9f`C!8I0YQ_zx&lWCV3uggUgLeH{^0AH zO~tvn&FR2iSy@?KI3}_QV`j)*2!w<}4%h*RX%X3?7Ft&>;6{Xs7_Zr*|JbNI zpEMm1LeHK*54mw;IppA|=jT_VXV%Tv3fCVD{(9RP#>?1j%bW!ivAlsuN`@)Q-0r3d z(atl^!Dod$HF#wOQ!Iabccel0+MX01$->GV8MF+M55Dgy;TJO}28+LiU zH7331jUzlU7O88>=j|b`T0r( z2Rm4xs-Y;SNYj$t22qMB!WUgSRUl5yEahNxPGsE%Pc# zlM^1n`LCN5shu1g@nkHh96hV^-(z1LOh=E!kIXW{kw+Ilf{f4&Bn_}Cr+BQYvNE|Y z8uhZS(0^(zj7c##Y(2<54Af}>MeXja$c!R$Ka0Vbpc&rXQDjsT7MRc8#}asN|4^2> z7@2kDZz<52`n(v=OT<2YIh(GYib&Ty*Xz8~PEx$6mKVV~o)%W+gLi|T!ZR>}e{#wY z&(e`zHt6R8ea96j3RE)Av=%T6y1rcRUPCz7E+@xk#IF8zrxln}eC100saZH*bPpc9 zH;N1)1kq@8{^MUZc&Y9`$puca7Dws=;EG2z=!>QOk^PTZI&NOFd)3h)%)r2a329x| z!HN)if-Q)sF*KCp{`G*R*aF&$R$1GkC2;1#{u`W`xta{pASeF6G~A{x>WEs9f?H7J zFkKn}E6pd&Tc0ahBW9+^dxf_c{u{#jQ)Cz$8%?+u3R0Y9<^qyAKt+Nxy$Kc}$$yj? zN^v;gXyW53MT%I$0k1KQXqQK%msFH|w8sac{*hm}iQolXZFH9q&|rSaNJP<*ZzsVi zkN@ZpLh0FQXW0iH4=0l+N&O$M^Mh$!-JByx*(5k%EISFl2qioZoJvy;u_r_t)Lu9- z8dh99+SdswtbePCkIpMRx<^uxjDrHz7G+8DjG+pGA zV1(&D>L${mVJ|=w`KUro zP~rZuWS8XLFkuaUt9~A^n#IT_`0nv%&j#N%PIAIwq$*7Gb$g|z?zHm5rV=tU{YzX* ziUDO%{`=J%H)2s_oGXIq;6$i~C)v)eo!CBc{Qy#*k^5~Nw4D5dVkwsK^}SYw-jc+G z2l5DR4HAiDEAr2NuH3;C0`408B@bq3h|$Zg4Ga{$ot+)#$J7aD4+a>X8fSQY67GT5 z6N2^}85?U+0F#BTzWx|Zn%g{^zl9miEkE&9fE^>Y6p*~2(7Btn&m~3wgan9kO<3V- z=rEAIC`w#Bm1pM^FUz9e6W>`8A;h+W@&sErjI<{q5@JxruD{O~sSU=aWdEdd=T4%l zUhmq(gjJ-1VQvP+O04ysK0bN?*4~E?E8-_zKb#QDst06HIQT*N^XJc6#>Si|rB$e# zwC+Y8v8?NL0mbL}VfbFZ-jHJ)UjpWW-Agaj*=jAFqi_G~G~Jz|4nZ2uTa5n8z6+fF za||5sh2w%7F)|LFMzU?@&!1e6_6@z$XiWXVKBHkJ)-c?@KnAL4`-7xUHbhHiwL=dH z2cAwvO`kVdzi-#ooukEy8~;kp3?W^>lX=>p9x+$h!HraT_GCkwhm7yH$ywB8J-(;k;e4WXY;& zn&)sFx@o`3VNr6u1DqDn6?eV=@#V`C#FhE250wXDa3*aCLDS@6kVr5oN5{Y>5Dfi- z@@G=wHa$S50RRWEL#Y`VIKBQmbQdD^Vy2FI_wHTYFJk<`Nhed&;}9Ym@y*4|EHQXe ziA5(oHtgidXttH%@xrL-3N&14x9Wrkaa7?WN^tT(k|&cKL{b(JNs^Y2h32Vpe0&w? zj@$!prHDX&c^xA&GcyGK2Y@Z{f-*;Q2b#0_0RpU3Z0+qukl@xR8YM(#$f?3a1p#9b zu4bsIsd2z4rsTm-zyr;70V2g1R9OIdTh`|?#ygQ!A$b={%bdo?tzNT67qE$8+L8`Gh2;to zU3Yi)&77P#0O4-@f=dPlnM{6g7gvTdmLEB`ckh>hciOr%nLD<{d~lbLCIaU^4W3Pd z%ny~Xo;~YJs7D)SUkozC`~hywAUXubVkrve6f*|)Ksb?x(noV;zJI9!5jcp=CpNsm z-``eJT0qQQU%p6rnIj?hd}bC)G5hm-Am@^@mlzsSm`ETXmiYSh7aIH|Q>;TkhV;!X)~*1pip_@%atT+d5U@+)-bLVe;N$o-H!uAkE&05FxG5@MvEgC3oCVHBMhXpDFkC$~=J+BS5KBz2 z!{smnAXtUzL8uQp;Kg#WeuEzh7sL8>u1HNRrR53vn8jo~0ti=BPL6{jmL~VMhs@~T zcr3T*(tJqA?{gC7FBE8c?8F~M!G!a;t!mpwexc1fZm=nq#H`J`ACt~k%4gW|; zb_O;}xlO0M5Gu{b;7GuTpzxD8kvRUjK&AET_aF@NiOS|Gi3E?>BnIHPr*R@Zhzq|z zlX+xAaFdX#n3)hd4KgW9c69%N1D}w%pQ6}o!m{_XL9t1(md8*w7V-(GNjeHin>41F z8blm&H?JW}arZVly}yGMq_>3S4=r?xN));;nv8nD?OZwzXdkPnV@c_6VqqoF<~GVrT_=9(bxq0~d&x zrgw*Ne?m8=5o>U<%px@UpaECl&G3n4iCc8j)`H0~NPphzO~?p9`X|JTm|-f81lL2L zGJ1e?xj6iYC#Fy`AHuFhR7fGpHEgYuy|YYG%yZOe8P>5U}`)#7c|FlZZ_?!h}zH z?y_f(1IANeUgoE6qRe51$wAIqK_twr-!bX~nUuVW&WBiRd*D0QLT)0+BjX(rDz{3Z zh{0w{QMeFA?C+@(JI;>hmRJhw4hU-EF$F`ZgjN|;9E@I&pHl?Mpbl9YbQZyAXJK3* zz5^Yjm^m4Ay)1vqS93Q<;#omb5!)VWpANJvN}&mjnkXpH6jyUVWi%8iNG*?#|CA;( z*(d{e{QekHopqRkobmhcM2m#Cgb5B(8g&YO_zZ>7(vr~0zWaGPEa?Raz}tO zNa>eIjAkm4DBvRmG<-mB^4%=S0Z~}DZhr0apCZJ~L6;#L06#9E@aditgS#+^G|bGS ztE&_S2D~r7qwp}}%D+>Xb>K3%w-R=N=16WN!tVlw5DN{dIzfL%7;${# zYslH=wZNBxmhyL7$jq);A{Z3Hk|w-G%D*tD4gm(?C@v}*g0MFTM_9ixBBustjvxU~ zh~F;LxsKH!BxD?2$Xp(_m39jv%uQU1LT2SbdC&$eq`(9oN4#f*fo1ofc)t&>IUyie z1~7cYaReJcA_|Pyab~h{PDBEQ$C)Sy)G#=C;;tf@>+xFr2ys+O??r(m7#BBXorc=^ z36tIP%RXxOa6q zfI~{Ljs(PsEy83ypkNFrpik%naNEOe;+A~l2F^aqIgE=~f5G{a0Y@+y-MI6o8qq=0 zMWo$gq~BEQ0`;B$ec1#BOh$-=5NFL0M6LK*{;Q|+oWv)8p0STf%9DD7$ z3N{BAun1z+nCzmLkdl&41gnW*FY>VqaIXL%3XcvgM;M|mGHl=e(ro(1ro#;2naRv4 zXpEUX68Z@B;#pZ+DqKZv8)8#{pCmnQ0{?Wp`&Y36l~QR=-x=?TU`fN zT@)rU4D^r|UtCej*3F>5>5dV#4VdyG<}J{AWLTFt#z2!*q9D>3aZEEakm5g-1r)#N z)n+2|Jfxl}VEhDEpDn^9sonMj1Gvs2D)G)3^vtoD!N3)EUrKCeMu5ThGunasI)r<$)R!ulZi)PP?)`Uci@k9T>-fN!vsAi_!!*YModA(RSAP& za7aj{OcG|bb$t@IUGTYaD$DjiTBvxvaH|@oF*zVYV#qJMsTU$;&0mwNfgVvyf&R%w zEpxNKG?%IRvVmPC8ACUpiCkzztU6@CLCNhMMcvwQJsNz=O?ZYI`7i~O&w?KbV_{(# zhABJ+LICNaF)BdZ10?f7QAh-M5X*AGQI;47&?a~Rn~A8dfy?FfZDSFVT~n;1A2>z1 zUIL>9Tktk19h9;>p8*SEQA0qG>ZhB(3>uv4YhXj7g`fiMeE$wUIGjYWEFLVL5=0q$G=~bQ3%vzVorcmH*~A5o!2CSXBj5|6sDF2+!#9nibs8z*3RGz^jKsp% zaY-{sM9LlD+-q_5#R`gjaxOvPzAe%R@{qRYC^_Q zMt~*=y^6}ofR{ARR(_+mJpuoNNHlf#Cp?{1_on@Xt|ASiIWVs z2p+@r7K9CfWu1%%V>W?^iNvKxR?n>aGjob1@zPwqDnt|nY_4=WD6vR`X4HU$%}gOU zw#y+>g8z`fpudBQi=+?sM%Se~(<;RbM^T6fXz(#o704S1rSSaxd;-h36V4!C&5a_Y zDse{(yqRcjYjJRhKb-8o*)hPb--&>cD>OAl(9{qX8kc#b0U1PQ*x-9n z6E^Agj_Yvh;cJ`2>fQek5;U4}^7g2Ar|x$&Hz$xaC6aO0gMWsYAmT8Idg5X*h;t|^ zvTVMzujKSV-GEDW3)v3HSV$}A`SihN;oP!@cDRj>jD+OnDI%9i`>dvMfFRidP64-2 zETk@N@DM~tMJ{86-bR!VHs72BIE{cz2}Nwuw-IJ!ppIPpN6NYX+{~}}6NUXsk*>_H zwm%P70&ZJKmkYHM!*>Tzq=L~7T9x*hOI3YXk8rPza;wK|Xho``pllc&mxhPUANH+#wH7lH2oD^(QckUmD zEhicvC8h42z9h3~?!gWff83rz23*i4V;$+CY0ZacmanV2)33F-83K4d_|c}@+SF`O_iGP+1>z+;2RB2`t0cx%wx{#$xS&)_ON zh*=s;?Q`}{g94+-d*C}o;Fm}nXZ6@q6m2#!f0DcQYQk8~fyrC0F+1tpyUuPm4i@I& zNc$4mC$guzc99{i=il8U0)m2uwy}mMNMpR%oTOKDHuG7Jwvr|xpYbzNI5Q5;g&ri0 z){$zU?YQ?x|KYH>Gx`G)pHlXkCHl!4{(N7 zPck`hIh3#w4PTzD>|c4sL30|2IdXLmr4<;cF$?1nnbae#Ij}SeKAuKN8iXlMrGSRN zGW05nLdqbWovt}1GhKBN4#afqH|#+C+n{@Ojvp@~tqXT1Kr}O%O@rG9x`kPynK*bT zBAMW-xc&S0J77K%eph{WapQK-7z<{g_wnKT`ln{^UK@SAl@L#%g&b_AC6Gr(u-jIs$afj zpPOn3G65-u$qh`VvAS+r8b%5i=Hk-HZK{hFk$w{;<0SY_Jbv!JhUJ&8TrspIGiujy ziydt>xr7HE>|u1EXiptsY=gl9@lzu(m|+G+I2L>>Q5n&;oQhyz3_@i}ds6;{CWl+< zh?asm%De7kV`J4IOEIJt>!)<1So&zy7Z{5NY z6IU6O4s!3^ya$83`tLS$#24dzOIt-f8}AmbI#`n-UMbjy#uGrap^MV$#6DExrWae+$}4zk3#F# zjI7Y>Hgc!P?Y!|tLFb!iUYfW!V>NjgAn{+{|AX4f!sqBj$WE4b0 zqs;o}Y01fSxGu=GBLa(r#wy!KNY{rh zC@~{zL^a(mYuc~OHSNcu9XCu4gY1GmHH~kwSl9ea=jxv+(ofI-+xMQIUHBh-f75jM z5HM{gK-`r@OH!=QlpE26Crh)g;=DEfGiQj|yR4u1l;)7m>>`vW>6iZ!lW7AZ_%`(A zmOhI^ZP|ANq2c(D`wt;llhGpRuGx(qq=N%@7L37t988Qi?shHzPcSd*AxBC-uCi89 zR#vXSs4Vn9a>sHYDyVb^Lr`LW0j{3g3?DTTka>|^U`~dLVrz3QD4wd$P7!26GPeab z^G&v;JbJW?ZF4nIi$OH5c|&f;f*m>W_dhpdL12CSLIqR+Hy0O~9sh(fXs@~1 z02dK36PVaf@arf&@#y4isQD=XXiih3y*W0T6s{lHl>&Tk z4%~&?Rv|%hZ1OP0OdGA;UHn%qq}K#Bi1Ie62R4Ib8aO7Zq8#z##Eo8Nm_Fc2fUw z3@|X~e^qvNAx(v09Ji1NCfGz_30*i6hS zENYaV8*ds3hxtW_7ImW%B8rF&onB~nA|s-k4NXLU&$`PTc|X{D-t!)w=lOrVrGvk0 zW9m6*W2eW^9}EW9@E3>dw`wZCZETdF2Nu(vm|ejG=*yKU>l1CS1z}GohecGhIOiB- z=fMCzI9JkP3yKs?`+Zv$kh8g-laVM(z|1Sj@ImZ05+E3yQ zxA$hvJU=)*1O>eo3rul1T*y`vTt1&saw5hii{BN}pi2}7Ll^6vqZMNs<_l5jy2_8V zrhj+raXDvyU@DFvo%8?LWHP<|wDSNu8t6rd+-YTpGRq2Xe4MyY9qwSGN21oQz1OJT z+dNTDiibG8`&LSNZ8(<_J`id6@@O2{YKrK~`S6C}^SgUq=?ju>jf>GBL6oR;sJ&m| zDwlq&a{Rs8_D}UP{}fq#cEw-v<~nuCSZ8Q#mLhnbn-k6`IW`xQvy&*P?yDHDM9KTt z58s=4d5}aecuykd+Q+XVR6{#BirJjT(0Y+JUw~J9`?GN29$mKpW%=!DtW_vdZ6ORe z;x$l|s;ynZC-Ep`H_TMbFS+$xp*eg51eRN(J9zBq0yRC|9C!&>;kF5(8_S`nwxp)i x-BqW-Us&Ji8zR8WVwLxQE6}N@jI)==$;&t0jirU7+1#p;P - +#include "utils.h" // ----------------------------------------------------------------------------- // Compuet error // ----------------------------------------------------------------------------- @@ -41,17 +41,13 @@ CEED_QFUNCTION(SetupError2D)(void *ctx, const CeedInt Q, // Setup, J = dx/dX const CeedScalar J[2][2] = {{dxdX[0][0][i], dxdX[1][0][i]}, {dxdX[0][1][i], dxdX[1][1][i]}}; - const CeedScalar detJ = J[0][0]*J[1][1] - J[0][1]*J[1][0]; + const CeedScalar det_J = MatDet2x2(J); // Compute Piola map:uh = J*u/detJ - CeedScalar uh[2]; - for (CeedInt k = 0; k < 2; k++) { - uh[k] = 0; - for (CeedInt m = 0; m < 2; m++) - uh[k] += J[k][m] * u[m][i]/detJ; - } + CeedScalar u1[2] = {u[0][i], u[1][i]}, uh[2]; + AlphaMatVecMult2x2(1/det_J, J, u1, uh); // Error - error[i+0*Q] = (uh[0] - target[i+0*Q])*(uh[0] - target[i+0*Q])*w[i]*detJ; - error[i+1*Q] = (uh[1] - target[i+1*Q])*(uh[1] - target[i+1*Q])*w[i]*detJ; + error[i+0*Q] = (uh[0] - target[i+0*Q])*(uh[0] - target[i+0*Q])*w[i]*det_J; + error[i+1*Q] = (uh[1] - target[i+1*Q])*(uh[1] - target[i+1*Q])*w[i]*det_J; } // End of Quadrature Point Loop diff --git a/examples/Hdiv-mass/qfunctions/poisson-error3d.h b/examples/Hdiv-mass/qfunctions/poisson-error3d.h index dd8f0fbc66..6413a87b59 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-error3d.h +++ b/examples/Hdiv-mass/qfunctions/poisson-error3d.h @@ -21,23 +21,7 @@ #define ERROR3D_H #include - -// ----------------------------------------------------------------------------- -// Compute determinant of 3x3 matrix -// ----------------------------------------------------------------------------- -#ifndef DetMat -#define DetMat -CEED_QFUNCTION_HELPER CeedScalar ComputeDetMat(const CeedScalar A[3][3]) { - // Compute det(A) - const CeedScalar B11 = A[1][1]*A[2][2] - A[1][2]*A[2][1]; - const CeedScalar B12 = A[0][2]*A[2][1] - A[0][1]*A[2][2]; - const CeedScalar B13 = A[0][1]*A[1][2] - A[0][2]*A[1][1]; - CeedScalar detA = A[0][0]*B11 + A[1][0]*B12 + A[2][0]*B13; - - return detA; -}; -#endif - +#include "utils.h" // ----------------------------------------------------------------------------- // Compuet error // ----------------------------------------------------------------------------- @@ -58,18 +42,14 @@ CEED_QFUNCTION(SetupError3D)(void *ctx, const CeedInt Q, const CeedScalar J[3][3] = {{dxdX[0][0][i], dxdX[1][0][i], dxdX[2][0][i]}, {dxdX[0][1][i], dxdX[1][1][i], dxdX[2][1][i]}, {dxdX[0][2][i], dxdX[1][2][i], dxdX[2][2][i]}}; - const CeedScalar detJ = ComputeDetMat(J); + const CeedScalar det_J = MatDet3x3(J); // Compute Piola map:uh = J*u/detJ - CeedScalar uh[3]; - for (CeedInt k = 0; k < 3; k++) { - uh[k] = 0; - for (CeedInt m = 0; m < 3; m++) - uh[k] += J[k][m] * u[m][i]/detJ; - } + CeedScalar u1[3] = {u[0][i], u[1][i], u[2][i]}, uh[3]; + AlphaMatVecMult3x3(1/det_J, J, u1, uh); // Error - error[i+0*Q] = (uh[0] - target[i+0*Q])*(uh[0] - target[i+0*Q])*w[i]*detJ; - error[i+1*Q] = (uh[1] - target[i+1*Q])*(uh[1] - target[i+1*Q])*w[i]*detJ; - error[i+2*Q] = (uh[2] - target[i+2*Q])*(uh[2] - target[i+2*Q])*w[i]*detJ; + error[i+0*Q] = (uh[0] - target[i+0*Q])*(uh[0] - target[i+0*Q])*w[i]*det_J; + error[i+1*Q] = (uh[1] - target[i+1*Q])*(uh[1] - target[i+1*Q])*w[i]*det_J; + error[i+2*Q] = (uh[2] - target[i+2*Q])*(uh[2] - target[i+2*Q])*w[i]*det_J; } // End of Quadrature Point Loop diff --git a/examples/Hdiv-mass/qfunctions/poisson-mass2d.h b/examples/Hdiv-mass/qfunctions/poisson-mass2d.h index e5c5349d4b..6748c54d1d 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-mass2d.h +++ b/examples/Hdiv-mass/qfunctions/poisson-mass2d.h @@ -21,7 +21,7 @@ #define POISSON_MASS2D_H #include - +#include "utils.h" // ----------------------------------------------------------------------------- // This QFunction applies the mass operator for a vector field of 2 components. // @@ -54,25 +54,19 @@ CEED_QFUNCTION(SetupMass2D)(void *ctx, CeedInt Q, const CeedScalar *const *in, // Setup, J = dx/dX const CeedScalar J[2][2] = {{dxdX[0][0][i], dxdX[1][0][i]}, {dxdX[0][1][i], dxdX[1][1][i]}}; - const CeedScalar detJ = J[0][0]*J[1][1] - J[0][1]*J[1][0]; + const CeedScalar det_J = MatDet2x2(J); - const CeedScalar u1[2] = {u[0][i], u[1][i]}; + CeedScalar u1[2] = {u[0][i], u[1][i]}, v1[2]; // *INDENT-ON* // Piola map: J^T*J*u*w/detJ // 1) Compute J^T * J - CeedScalar JTJ[2][2]; - for (CeedInt j = 0; j < 2; j++) { - for (CeedInt k = 0; k < 2; k++) { - JTJ[j][k] = 0; - for (CeedInt m = 0; m < 2; m++) - JTJ[j][k] += J[m][j] * J[m][k]; - } - } + CeedScalar JT_J[2][2]; + AlphaMatTransposeMatMult2x2(1., J, J, JT_J); + // 2) Compute J^T*J*u * w /detJ + AlphaMatVecMult2x2(w[i]/det_J, JT_J, u1, v1); for (CeedInt k = 0; k < 2; k++) { - v[k][i] = 0; - for (CeedInt m = 0; m < 2; m++) - v[k][i] += JTJ[k][m] * u1[m] * w[i]/detJ; + v[k][i] = v1[k]; } } // End of Quadrature Point Loop diff --git a/examples/Hdiv-mass/qfunctions/poisson-mass3d.h b/examples/Hdiv-mass/qfunctions/poisson-mass3d.h index 4b50b86d0a..b98109eabc 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-mass3d.h +++ b/examples/Hdiv-mass/qfunctions/poisson-mass3d.h @@ -21,23 +21,7 @@ #define POISSON_MASS3D_H #include - -// ----------------------------------------------------------------------------- -// Compute determinant of 3x3 matrix -// ----------------------------------------------------------------------------- -#ifndef DetMat -#define DetMat -CEED_QFUNCTION_HELPER CeedScalar ComputeDetMat(const CeedScalar A[3][3]) { - // Compute det(A) - const CeedScalar B11 = A[1][1]*A[2][2] - A[1][2]*A[2][1]; - const CeedScalar B12 = A[0][2]*A[2][1] - A[0][1]*A[2][2]; - const CeedScalar B13 = A[0][1]*A[1][2] - A[0][2]*A[1][1]; - CeedScalar detA = A[0][0]*B11 + A[1][0]*B12 + A[2][0]*B13; - - return detA; -}; -#endif - +#include "utils.h" // ----------------------------------------------------------------------------- // This QFunction applies the mass operator for a vector field of 2 components. // @@ -71,24 +55,17 @@ CEED_QFUNCTION(SetupMass3D)(void *ctx, CeedInt Q, const CeedScalar *const *in, const CeedScalar J[3][3] = {{dxdX[0][0][i], dxdX[1][0][i], dxdX[2][0][i]}, {dxdX[0][1][i], dxdX[1][1][i], dxdX[2][1][i]}, {dxdX[0][2][i], dxdX[1][2][i], dxdX[2][2][i]}}; - const CeedScalar detJ = ComputeDetMat(J); - const CeedScalar u1[3] = {u[0][i], u[1][i], u[2][i]}; + const CeedScalar det_J = MatDet3x3(J); + CeedScalar u1[3] = {u[0][i], u[1][i], u[2][i]}, v1[3]; // *INDENT-ON* // Piola map: J^T*J*u*w/detJ // 1) Compute J^T * J - CeedScalar JTJ[3][3]; - for (CeedInt j = 0; j < 3; j++) { - for (CeedInt k = 0; k < 3; k++) { - JTJ[j][k] = 0; - for (CeedInt m = 0; m < 3; m++) - JTJ[j][k] += J[m][j] * J[m][k]; - } - } + CeedScalar JT_J[3][3]; + AlphaMatTransposeMatMult3x3(1., J, J, JT_J); // 2) Compute J^T*J*u * w /detJ + AlphaMatVecMult3x3(w[i]/det_J, JT_J, u1, v1); for (CeedInt k = 0; k < 3; k++) { - v[k][i] = 0; - for (CeedInt m = 0; m < 3; m++) - v[k][i] += JTJ[k][m] * u1[m] * w[i]/detJ; + v[k][i] = v1[k]; } } // End of Quadrature Point Loop diff --git a/examples/Hdiv-mass/qfunctions/poisson-rhs2d.h b/examples/Hdiv-mass/qfunctions/poisson-rhs2d.h index a5f528b889..b3fb894fe7 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-rhs2d.h +++ b/examples/Hdiv-mass/qfunctions/poisson-rhs2d.h @@ -21,6 +21,7 @@ #define POISSON_RHS2D_H #include +#include "utils.h" #ifndef M_PI #define M_PI 3.14159265358979323846 @@ -63,11 +64,8 @@ CEED_QFUNCTION(SetupRhs2D)(void *ctx, const CeedInt Q, CeedScalar ue[2] = {-M_PI*cos(M_PI*x) *sin(M_PI*y), -M_PI*sin(M_PI*x) *cos(M_PI*y)}; //CeedScalar ue[2] = {x-y, x+y}; CeedScalar rhs1[2]; - for (CeedInt k = 0; k < 2; k++) { - rhs1[k] = 0; - for (CeedInt m = 0; m < 2; m++) - rhs1[k] += J[m][k] * ue[m]; - } + AlphaMatTransposeVecMult2x2(1, J, ue, rhs1); + // Component 1 true_soln[i+0*Q] = ue[0]; rhs[i+0*Q] = rhs1[0] * w[i]; diff --git a/examples/Hdiv-mass/qfunctions/poisson-rhs3d.h b/examples/Hdiv-mass/qfunctions/poisson-rhs3d.h index 81d42d323a..9b6c8b05c1 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-rhs3d.h +++ b/examples/Hdiv-mass/qfunctions/poisson-rhs3d.h @@ -21,7 +21,7 @@ #define POISSON_RHS3D_H #include - +#include "utils.h" #ifndef M_PI #define M_PI 3.14159265358979323846 #endif @@ -66,11 +66,7 @@ CEED_QFUNCTION(SetupRhs3D)(void *ctx, const CeedInt Q, }; //CeedScalar ue[3] = {x,y,z}; CeedScalar rhs1[3]; - for (CeedInt k = 0; k < 3; k++) { - rhs1[k] = 0; - for (CeedInt m = 0; m < 3; m++) - rhs1[k] += J[m][k] * ue[m]; - } + AlphaMatTransposeVecMult3x3(1, J, ue, rhs1); // Component 1 true_soln[i+0*Q] = ue[0]; rhs[i+0*Q] = rhs1[0] * w[i]; diff --git a/examples/Hdiv-mass/qfunctions/utils.h b/examples/Hdiv-mass/qfunctions/utils.h new file mode 100644 index 0000000000..b0f1d98fda --- /dev/null +++ b/examples/Hdiv-mass/qfunctions/utils.h @@ -0,0 +1,205 @@ +/// @file +/// Utility helpers QFunction source + +#ifndef utils_qf_h +#define utils_qf_h + +#include "ceed/ceed-f64.h" +#include + +#define PI_DOUBLE 3.14159265358979323846 + +// ----------------------------------------------------------------------------- +// Compute alpha * A * B = C +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatMatMult3x3(const CeedScalar alpha, + const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { + for (CeedInt j = 0; j < 3; j++) { + for (CeedInt k = 0; k < 3; k++) { + C[j][k] = 0; + for (CeedInt m = 0; m < 3; m++) { + C[j][k] += alpha * A[j][m] * B[m][k]; + } + } + } + + return 0; +} + +// ----------------------------------------------------------------------------- +// Compute alpha * A^T * B = C +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult3x3(const CeedScalar alpha, + const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { + for (CeedInt j = 0; j < 3; j++) { + for (CeedInt k = 0; k < 3; k++) { + C[j][k] = 0; + for (CeedInt m = 0; m < 3; m++) { + C[j][k] += alpha * A[m][j] * B[m][k]; + } + } + } + + return 0; +} + +// ----------------------------------------------------------------------------- +// Compute determinant of 3x3 matrix +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER CeedScalar MatDet3x3(const CeedScalar A[3][3]) { + // Compute det(A) + const CeedScalar B11 = A[1][1]*A[2][2] - A[1][2]*A[2][1]; + const CeedScalar B12 = A[0][2]*A[2][1] - A[0][1]*A[2][2]; + const CeedScalar B13 = A[0][1]*A[1][2] - A[0][2]*A[1][1]; + return A[0][0]*B11 + A[1][0]*B12 + A[2][0]*B13; + +}; + +// ----------------------------------------------------------------------------- +// Compute inverse of 3x3 symmetric matrix +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int MatInverse3x3(const CeedScalar A[3][3], + const CeedScalar det_A, CeedScalar A_inv[3][3]) { + // Compute A^(-1) : A-Inverse + CeedScalar B[6] = { + A[1][1] * A[2][2] - A[1][2] * A[2][1], /* *NOPAD* */ + A[0][0] * A[2][2] - A[0][2] * A[2][0], /* *NOPAD* */ + A[0][0] * A[1][1] - A[0][1] * A[1][0], /* *NOPAD* */ + A[0][2] * A[1][0] - A[0][0] * A[1][2], /* *NOPAD* */ + A[0][1] * A[1][2] - A[0][2] * A[1][1], /* *NOPAD* */ + A[0][2] * A[2][1] - A[0][1] * A[2][2] /* *NOPAD* */ + }; + CeedScalar A_inv1[6]; + for (CeedInt m = 0; m < 6; m++) { + A_inv1[m] = B[m] / (det_A); + } + A_inv[0][0] = A_inv1[0]; + A_inv[0][1] = A_inv1[5]; + A_inv[0][2] = A_inv1[4]; + A_inv[1][0] = A_inv1[5]; + A_inv[1][1] = A_inv1[1]; + A_inv[1][2] = A_inv1[3]; + A_inv[2][0] = A_inv1[4]; + A_inv[2][1] = A_inv1[3]; + A_inv[2][2] = A_inv1[2]; + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute matrix-vector product: alpha*A*u +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatVecMult3x3(const CeedScalar alpha, + const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { + // Compute v = alpha*A*u + for (CeedInt k = 0; k < 3; k++) { + v[k] = 0; + for (CeedInt m = 0; m < 3; m++) + v[k] += A[k][m] * u[m] * alpha; + } + + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute matrix-vector product: alpha*A^T*u +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult3x3(const CeedScalar alpha, + const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { + // Compute v = alpha*A^T*u + for (CeedInt k = 0; k < 3; k++) { + v[k] = 0; + for (CeedInt m = 0; m < 3; m++) + v[k] += A[m][k] * u[m] * alpha; + } + + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute alpha * A * B = C +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatMatMult2x2(const CeedScalar alpha, + const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { + for (CeedInt j = 0; j < 2; j++) { + for (CeedInt k = 0; k < 2; k++) { + C[j][k] = 0; + for (CeedInt m = 0; m < 2; m++) { + C[j][k] += alpha * A[j][m] * B[m][k]; + } + } + } + + return 0; +} + +// ----------------------------------------------------------------------------- +// Compute alpha * A^T * B = C +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult2x2(const CeedScalar alpha, + const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { + for (CeedInt j = 0; j < 2; j++) { + for (CeedInt k = 0; k < 2; k++) { + C[j][k] = 0; + for (CeedInt m = 0; m < 2; m++) { + C[j][k] += alpha * A[m][j] * B[m][k]; + } + } + } + + return 0; +} + +// ----------------------------------------------------------------------------- +// Compute determinant of 2x2 matrix +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER CeedScalar MatDet2x2(const CeedScalar A[2][2]) { + // Compute det(A) + return A[0][0]*A[1][1] - A[1][0]*A[0][1]; + +}; + +// ----------------------------------------------------------------------------- +// Compute inverse of 2x2 symmetric matrix +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int MatInverse2x2(const CeedScalar A[2][2], + const CeedScalar det_A, CeedScalar A_inv[2][2]) { + // Compute A^(-1) : A-Inverse + A_inv[0][0] = A[1][1]/ det_A; + A_inv[0][1] = -A[0][1]/ det_A; + A_inv[1][0] = -A[1][0]/ det_A; + A_inv[1][1] = A[0][0]/ det_A; + + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute matrix-vector product: alpha*A*u +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatVecMult2x2(const CeedScalar alpha, + const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { + // Compute v = alpha*A*u + for (CeedInt k = 0; k < 2; k++) { + v[k] = 0; + for (CeedInt m = 0; m < 2; m++) + v[k] += A[k][m] * u[m] * alpha; + } + + return 0; +}; + +// ----------------------------------------------------------------------------- +// Compute matrix-vector product: alpha*A^T*u +// ----------------------------------------------------------------------------- +CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult2x2(const CeedScalar alpha, + const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { + // Compute v = alpha*A^T*u + for (CeedInt k = 0; k < 2; k++) { + v[k] = 0; + for (CeedInt m = 0; m < 2; m++) + v[k] += A[m][k] * u[m] * alpha; + } + + return 0; +}; + +#endif // utils_qf_h diff --git a/examples/Hdiv-mixed/conv_plot.py b/examples/Hdiv-mixed/conv_plot.py index d6c7f6653b..d4bfcff84e 100644 --- a/examples/Hdiv-mixed/conv_plot.py +++ b/examples/Hdiv-mixed/conv_plot.py @@ -46,6 +46,7 @@ def plot(): E_p = data['error_p'] #E_hdiv = data['error_hdiv'] h = 1/data['mesh_res'] + N = data['mesh_res'] H1 = amin(E_p)* (h/amin(h)) # H = C h^1 H2 = amin(E_u)* (h/amin(h))**2 # H = C h^2 @@ -62,6 +63,18 @@ def plot(): #xlim(.06, .3) fig.tight_layout() plt.savefig('convrate_mixed.png', bbox_inches='tight') + + conv_u = [] + conv_p = [] + conv_u.append(0) + conv_p.append(0) + for i in range(1,len(E_u)): + conv_u.append(log10(E_u[i]/E_u[i-1])/log10(h[i]/h[i-1])) + conv_p.append(log10(E_p[i]/E_p[i-1])/log10(h[i]/h[i-1])) + + result = {'Number of element':N, 'order of u':conv_u, 'order of p':conv_p} + df = pd.DataFrame(result) + print(df) diff --git a/examples/Hdiv-mixed/conv_test.sh b/examples/Hdiv-mixed/conv_test.sh index 798f2b70eb..33ca308c8a 100755 --- a/examples/Hdiv-mixed/conv_test.sh +++ b/examples/Hdiv-mixed/conv_test.sh @@ -32,19 +32,23 @@ declare -A run_flags run_flags[pc_type]=svd if [[ $dim -eq 2 ]]; then - run_flags[problem]=richard2d + run_flags[problem]=darcy2d run_flags[dm_plex_dim]=$dim run_flags[dm_plex_box_faces]=2,2 + run_flags[dm_plex_box_lower]=0,0 + run_flags[dm_plex_box_upper]=1,1 else run_flags[problem]=darcy3d run_flags[dm_plex_dim]=$dim run_flags[dm_plex_box_faces]=2,2,2 + run_flags[dm_plex_box_lower]=0,0,0 + run_flags[dm_plex_box_upper]=1,1,1 fi declare -A test_flags - test_flags[res_start]=2 + test_flags[res_start]=4 test_flags[res_stride]=2 - test_flags[res_end]=10 + test_flags[res_end]=12 file_name=conv_test_result.csv @@ -64,7 +68,7 @@ for ((res=${test_flags[res_start]}; res<=${test_flags[res_end]}; res+=${test_fla args="$args -$arg ${run_flags[$arg]}" fi done - ./main $args | grep "L2 error of u and p" | awk -v i="$i" -v res="$res" '{ printf "%d,%d,%.5f,%.5f\n", i, res, $8, $9}' >> $file_name + ./main -view_solution $args | grep "L2 error of u and p" | awk -v i="$i" -v res="$res" '{ printf "%d,%d,%.5f,%.5f\n", i, res, $8, $9}' >> $file_name i=$((i+1)) done diff --git a/examples/Hdiv-mixed/conv_test_result.csv b/examples/Hdiv-mixed/conv_test_result.csv index 1c19c3f9ff..074a2faf64 100644 --- a/examples/Hdiv-mixed/conv_test_result.csv +++ b/examples/Hdiv-mixed/conv_test_result.csv @@ -1,6 +1,6 @@ run,mesh_res,error_u,error_p -0,2,8.22730,0.00852 -1,4,2.50219,0.00291 -2,6,1.60805,0.00181 -3,8,1.02228,0.00114 -4,10,0.70467,0.00078 +0,4,26.30005,0.03133 +1,6,11.97420,0.01464 +2,8,6.79226,0.00838 +3,10,4.36393,0.00540 +4,12,3.03689,0.00377 diff --git a/examples/Hdiv-mixed/convrate_mixed.png b/examples/Hdiv-mixed/convrate_mixed.png index e98533fd9d60016ba9282386cd730da8b936864e..fbe94c51338d9f7668e1790525711b8bf4cf4fa1 100644 GIT binary patch literal 25042 zcmbTe2{@MPzc2ohu?V50QmAA|hLp_7RHV$3xgZmqrd?_9sL&)NU8u4`SZmB;%&&pmvXP=v@v74ULDK}3mR~yPr9HCpSks>n$EOuIKEW&Ps|( ziHeJCIql}=d`@Q9t~396fvA(K?XEVf$x?g>qqD}ba})}T75SGcRWZemLa|ajsIp(* z>(P%6Z__(1J#^!?ZERFJC)sIQX>`PBBvs#wMLkRlF?rBqQgAg%_gJUu=Mww7Uk#dS zO`j?=zxxs{eCYc-vr4lAswdfLY4%%6zf%0ZcNddnHQ^T(*Z$!?Kbm9V@Ks(S4k0T4ZUNd zqSl#veWhKTAI@P|x$;tEBxiep(~a;Adk@)Y$H+0>*SgHSVNa@==ko6*OlDiYL)5cf zQc|3HDcZqLpGu1C-1$D=!6?)0g<+pxZT8E}7iOuJ7yA|8Cm-Ih+ubqU#VA$$RGjjIk`}Kn$e)y0AH*6G8X8gTMRodowf5Y`o~oigk* zho#>OV_Xs^8Ag8mc%OMpa(b+5v-%Al1Hmc${ekVbzs}DL-;?u}FnM;G%GcLd>%Ii- z4IX*jv?KfXQ|)nV+v2CV{YcE>H6OeS0jau!ct-{n7lbD5pn0v)<}o0m!({L9PhQe zCu`m3+qiL~ii=Agt8&QnwA_N>mRkdwZc%Z}0Jv?^T!| z;Ai9F+JRlpAH=FG8g;cl*54d+p!GIJ{`mR1z9{SOZ|_&UeXDhiOS~sE_Ar)e@WY1> zon2kmNZQtOEYCIgt8~5eqoqu|+uNnlw_WSrmPgUis&;m1wF`=K-`II?|MePcG`&R4 z)%6Kwi_i1`uZ+jYj=Lhq_!RKtuLA=+zI1dLyxjX>hhi+baHI>{fU>32@0Xcf?4H;; z?3UNbhab{V#7{SF$+P>go%|c=fc>WN!}r_Que_(;J=pegwr7>NbGH(e(#WT$>Ov`X zwp2=PW1VJRUS1(sS6A`xDopwN=Vy-Rx%M6U+S|*ourMMy(cu4E&)C>FglYv{YKfwP zf;su0&Qjm24GIfdWH#b`e~ngDR46ZE3F#Kmu3D?nwUC=DHc_?CB4qGG$|{QW=$9hh zB2-33O-)Up`wF2RtY zh#Iw%Wkd^$G}!qUSUb{V`OBc4mgyn zsinn1d5ubL(D}-#OCf)$y|aIDZi<$|wr-t(kg%{YyYh|;ybtf)3wixoC9;<+Au?H7;6fcM@E-j%=+~z3$Io7p{5`sHPFL{NriuYgs-C}-V{fJ4v zebVIpCG;oifZ4hJ*bKi{lT*iDcx1h)tzF|e*2%f>^9MUNFo(o_t&NHoOocLj*op@6 znX0O(*_RmhS<)*xeks)H^ZnM-!>4wGXXlR(eNQP{P!d(Kj-HLx?Rv#Q3A%dq>YKW{ z+dmr~?ma@eS$j7!GE_TSCbT?|UUPYM5Ni+aJ>KkvhjnAz{w4*R+>F`fU{>X8+){_0 z*|!|Bi46}AN2g^~xm=xX9Pg7j67wuSza@P!M&6e~(b3TnTIli_-bp#?ySAaGMt#V7 zMZm|box3Tdq@MjEO$lJ7R{TB7f01W`v>1;UKvUAyT6g-@ z8yP6}tu3$LzGccO{rOW`_6;8#CtXqfDp_3HObj!+K1I3hZF=hCuP^G)Y*8Xb9Dyn-Tke)ed_ zBnISCXec8}M(UHUsw&!@CeO-HCG>w5ocV0{W8>=8tA!fU=H})eYetENgojr>v-?1y zt~yj$l0kfQ#Lk&%&Z%nwjgbf?i=l(hkxuhi%|<x#O%#4!kHj|W$Cr>1#_w7r2cG^IHE~@nTue5jW zSb~>r>h9XfneW@$T%CCEcCOo??nqnS3bb8`mkXn^CF2#VbjQ7JwRfJw6tQvx(*u%i z<&pOZS}%Dj&Z+C=`1pA0dpzTrq3mH>otAyEZae$@M9qphe*gZ>U1#WM_3`^76=p_) z-+aFY{{e?z+wx8fpu6SU$ji&?7#d#2^Ot}Ae7rq2BI4qsM;pC-d_JbG?yU~5q~Cl# zV6*=sTYY^!3P2^m-_}h|9mP_PXU_0O)g_6=3EOmJTfEBl$eetMcG?|fabZ8Yqdh09 zaO~7Us^SAzR!e)1ZjA9A-IrB)&Ia{Jb^rd0hcY@^GK{E{k}?`wTi40^OqZcnqElc0 zT;Oz~DvVRA_>`G(iI@0ur>^V3v(__1E$)-|Th+|YuWdJ{Ke`xU5cMmqL56+sQbQYC z%hRHB{T0>KbRSZVsH>@6Zl6HGAZVKsU{!gUiV~lYU{Jf`xYX~-s8`LYtI00@n6<(B z#0jO9xUi55f7oVHa5g+3fD-wHT|oQw#TE3dn>K|5E?5mTJ}&Mwj2>)! zyaGizd}g?nUtYdcTVbj_^=gCi1}Z8lT(Lr7X<9GRKkATF>5&Z^9&#_qZcBG?%;QhD zMbj?x?DD(f(pydD)uu1B>%s!_@q)9F-^(VarVgm6(7c=<68`q>TO9g`^o2!f;IaUe z3+u7~Dyi(6LYH1K|KI1yCTo9YXMdYGba3^BH<$JK^XKsxC2Oqp_|#MlAtPN~K84xt z74NgnWUbJ_2x2Yqp3<*nczloU+b;jPtL^Udzs7Q04uxZ@GE&g$L8qiXiu3ayP*GD` zi8@Ap&D7NN^|=Ol0W!B~S_V^zf=lC-Eb%C?U>Les%wf^8o@AY0MSVz(TpG&AQBhI~ z$bJ5NEx9AC4}!9KP_d+n8%9PP#m{}$A|%Ml%8C~lM^!~7sMd|As;bHwLmQ7fAs;I( zEgga-JW)=+IRxc1{Fna313W#az>ydZz?HRNOcftK=w+G~J*oZ0v|*1HrO_uwvd>Kb zw`CwprzB#W8`u6zac~OYiOs{qgHJ#}o#AnZZFb4?=O3coezY6>Cbgkx7%CXew!1kNJXFu{zc+ z&CjrZ{P>ZYlA^#H7{OScq8OTbVQyg|0arcg&4+c~?(EId?ib{T!?MR4* z6OhMI6O;5OyLfr2unZ4qXoOe^*(llB*|nAW?%l5a==4B3YeaEdu)(lcsbj;R0AX$Q;>J+9rq(?7wWxdzXSkYJ7 zQ40waz&C=wuwvayIegqAEWB#Zo;|a57QZh6S?|4nlzxXA^%3x?GoPPVfzxsg;rce~ zzx9mSYyYc|alSg6+s4MG(D#=p%5{+`6A?jzSgBbyD+GaAw4a=e6mK)=#caD5Gf3ly zI){@I6U#a}O4}uvJX&(|^J(K|)!ILOT3@&af=M$+UteE&W#xg|Z7o*08JC|dwA9yM z|0A(59eP+(A?>9cc<&w;O2Za0vBpLNUZ^Qe-2HhOYUbuVV2TkBAJ(lU3XBAU*(Ag9 zf5eyAyo?ptzuUHLyPQoYf3c`&AKELk88fac#kbJ%(lmYe5OMqViGlQ+cL||fw{Bg% zrUulV(DLlHcOKXO^%vv22T~`DPnQL;dQA_o0fSLdva_@4vdI&5oXE}KEix4iE=E;d zUo4YI3sOx{I(2IMqeqY0c532tt^^9$Ys_hx0a!4ZvNr`!PW4DeKp(osHDVvykrTd3 zaTo6S)W2y;c1OE3!~W`1Ql=hX`W3cZ+c#g@`R%XU;9(GiEUm0Ag4p)}gQ$R_UD&0C zPgMElx(|90F7M*UYI5~krf#JUjHZi-h$!@$;M0C2{rHzdRTY)FIjV5k>_phIkGJ5@ z;SV=?58S+amo(|GSp}UH3uCWtBxy#aM+%FIesof`kK_Fq8WQs0ZegJ`WCknzE&+mc zOk9**yLP=ETv#{DF4ngGsOd&lp4DH95{eb24Du{um-hhUf9>g^#?~qMJwMFA%v@C; zElX65Ya92iS+!0Si(~_#-4vxmhgMc!-$Zy(9KZ^ZjLvqJ2u71Fv48)5FyH<7S!2qP z&0mV$S5cftzwBe;5~HA`eVHPMsSMTxQzu;v#-XQ&Zgan+6dmhya#%qGlX5 z?-2h{0j+x!fI_kXiQ3{jR!V>F+_~tX6o+Uqm7!W}FMpwLklXIsNMkjbtycY`D5m> z6{}XQqLdYMUAPJWn0f4}+DZpt{sR~?$c6zRL43gdyGq8XW!wfRl<}WG?`FF!{5(@K zRWA#^v5FE0mLSur*D52wyf`JboXpCmb6XFf4dt3wP>^w*sA(Bc#%fdWQC=ba8SMbK(~ky+%4qHvpwq zU7}ruhNx)SoT?)xE*|k9J92l{N}b!dU!4zkqoc#I>NURp2;bXz)8`I$Dk~>vl5Qu+ z9Y7<*AA`A4NqiWF%-kn-PkH@Ohg*_u>ouIkx<|9Qc03;3CwpJZMNZmSv9$nOqUQRh ze4z#Ka(>*Dwzf9*LdFwQP?*L9b(Xge_p+clRuhnB8{o)pj{kiXN%RwTp zp(|P8E>Gk6Pyl&@ZWik|Kq#_x=i`e{=Et<78bn?)?K6BD7q>;Qjh2>{AW+<7oMfPm*q>xc<>-+a%u9x?K zWJ$}&SicTo)6mjVadnj-wd(srryi6A(zF0boJZQk$sPv-F3Wdl?f&s29GcDBNBh{( zybIlj*Iwt54+E6dSUwRP6hzur?DW$!dM}g7nqIR9Ayv+QjzW#DXhA_4SL9CB~**|^yM08sT35o83 zfe`3Y=f}(GDI9?!L=RhB=vuB;-|QDe2*i(qu4*F-KZ6XcsR$QRr+!b|@ZW+7Yt1%0 zaO~LqS{Fq{#l^l|{#2CNg@qH)1Bq~a-p>zOS;9{d4>Kt6mCw(77IXM05?M`y8B|eM z=YJdUs?0`NPnM60GC4VE>Ev_+P%A(8Cj*2*N8Py-G|^jg0w9kPpt!uKXHympQJ~o!DEto(O4^pK@K>KJ zsTw zJ@FCGs-td6qQ{?W*$1Cg$oZ)siHWqx`5&V?;sFpzs z&@z^{y@>zU3fe&j6!4KRMeI;$2io!y>ND7MTK-r(sfS1zJ3sfN{hae&r%n~{({U6` z>Z3tYHbxmh5mGwJ6zQ3u+IIk1#Sx9K-0K2_Ib4$!$nBgz>d88_N~u( zIiKz(!pBFAt!6(4>1F@t=V##9>~POG3&sQxaCYG%Yp5h?Ie2NB@M+*9TulE_R=PFCOx#W5sw-jMN6Oni2FWrY>5 zdlR0K;XWQVHr{u09}Ao;5|fhBo|U-|HU(#l>=tM8U{>Lj znm;=oQR|D5PAp%=>F=SYqOt|!Xg+*|nFWmn#Kq23#S7Aa&oLD*5L?pPu@_cglSQ5& zJsBl-SgM&y34b1*@ljE^JdK4y+Mtog21~ z*VMbcQrUeeZfN=t|EYmJu0%&~B8cnSHKs@jY*a}YJXSU~wCd{WL{*jf-q#ll6gB8T z;gYiFfX4$nj94@G?%nGK1{LbRcK!NgOmA&%t^IFiUcU4L$_yC!uifQ=vx`#=;%C}- zPY<=MB6YU*y zpe+0VTHc^`@xGUzp4zs~nfjbMb&6C;?CKoX9}o}WawI_UAx;qyF{b((NGp8xdtUfd z{iC#r^B^Y#OOAHEx`FxcB{w(KpePP)H|;H0iMW094)8@L+xkcQ&!3kAVAvfq_BxcE z!jW|a^h?^MhX#*Lzz89FcgzaTZi26rx#!Z=kW>Zz#e~V(iC*1t*K_A8@u@dbQ+L4N zTMK=^2F^gzlTx3Vii+S>6CaK!eg(YqnjPnbC~zHJjZa8uB?R-`uYcUkW7SVWp`}v# z_WgVMCSef~j&-}tIf8?OrDT3RK5&I5K;SG?k@P;Z)E91px7v$POo{BrM~nk=B8E6X2Rj`WiU3@u(8wl03a&c6oKpiYR%hbg ztx;H99EKH=hX1!sjQtD|BOZDOq0CT8Euf&N_StTY$WE{0kDOR9yCO|G!_(}}2UXOn zaZJ1QkM?{9M#ggJP7!b^SQHmoi9AAdY^+YrzF^iJ!omjt5lTu+4eGI*x4{4^12=5% z@9hl)Pv~w+(qv#_dX1F+fM$1175(EdajPL` zcXxMFD5PaVYf0a;#z0U|@Zfwg8bVv4OTz9o2K$P3>G++s3Jc;~kz|R?rT;mS$SP3C95hdx3~e6nnn{ zH36C@CMIfRd0;)MXlq}eK9B=^8gk=C%^g9#XEG_!%ByWxr5uS}4IlA5T(BKGced1} zTJcHn5K8a2uxI6}RY{$fAwEjW%4USC7^Jt!*dl3wVI&j1@tIOM9ZKH42M?@K?-*EE zu4rx7WAv<@nS~xYvnSRJTb}GR0z~0&6T6nuja#mgY>*4Ynz06~2eB)M>_EUY)EnI& za*rQBCfX^y#{jIYJ9qE)IQcJZfG~9fEQXpw-0c1sU(ADB8I}iIBdk(qffsw; zMDSY!&^EmU%d&EE%5kAQz^rn~dF9sbYieo=G8N4kOJ{GnCmJYlQu*`%{Z@+7e+nh5 zWMyR|PhiJmpR0ke*_WDX`en<#Z}XdeuTJz*>_m%%U#)%pYAQzzAI3UirBtI|>c<4W zfZX$kG8X#Gr0YM&hR`^2#K_^|a9bWfq|X^wCzY9`VmaOQFRV9EfX#w}g1CygnN865 zLFz*x*!J|^P~79@9-^_TJu`D^cH)F-kt^-Ng9qQdeXD;=avq6;D!8M~^}~aMq0kZI zGBdRv)J!%$KJe(nEe0l!W(~pzy8M^r)vj+0#g*K}GAh$FwUO>Zo+~?w&e3Cus$pOz zy31F<7COu=VsrNFb##}s;4IuNrkOy=w5m&wuWMX1%WFx^$u>?`1dN^5+}Cdd0s3Zi zbXJQm6s4r@=Ux-m&rZKreigaZ{$v{`;!s&GvV6 zN-FsuwqEK;)kz?{n=DiSC`mkM2zMkpVJefQ#-T$MsC8QoGf%!+USjDwKNYUu2}vTu zJzHK*t`SOTY0JY@LHjl3XTj_5$+!zaHn)WS$0oHB?vD!KU#&)S*zeh17Qr^=O)?zG z$;rB&XrP{0b#Vgb2P}k#S*2_a^R*Fn#LHXW*s_Jso>l0&N><^SFNJJiu(Xtuwzj=x z7gsd-l66HVPTX}Ff}95ZDC5Y{tar276rY?~PX7Wp$P&>ZY4;%xtfySR1rG=}x=HZM znC^<2g#kIv&dpgtVn!rJLMu2lgqxwr^g1>r1XF4X{Nx*`&VYhh12+_Z-=uC2pjt5w zRoO^(O}@}&^J7<11s%qFD_5;t+1=lN1^g|%<^)U=PR?Ms^WD(h$ET($(X%7&-#^wM zwT(}LiI)a8X)vDl1mxj+(k^@mAy94Du;Ki#4o}@8z=f}qeRc3oatjL75oP#e2jKRk z-3D0AFU;AbWrFb~ABnw=vY?i7G=*dB+RG^4Y4Qj9VGZF=E?KsG@^<4pJ>V);VZCL@ zy@c8h7w~|Isp)NvG1N*NT9>y3?#LhAA6C0;u7#Ynfb6*z4MOB=S?YHw`J zrFM<0JkmX6NGFiRqeW0uv>NMN%&Ak(E*2&#q!1Sn<>-cA*gSF<7L9$tWsL-898-05 zb#pDtsYgB+u>2bB2n5L_djzgz0>ok6BJZj1q@e|5Wo3QrC+UXwFa3Pc!Rdc|E@)Ms z3BA@11tzg-M3@wqSY|cbQPK(WL+Yg)gAC9X55#8QfBe~LHeyJWw!Yvcpo~Bh zB2SQQjp4ikp`@Fn%@E<3AX{bU@#n8`lX(uUqS41YH*VS_?lo>nEL+@A%Lg-7sS6lv zB66YxK?G!>KrFa>XiLT$U?ZCz3y8X=h0avbGVVPOQdOD@1EZf4Unz^|a4bj&G!2)O z$N^#uWSG!#u3ygxO=zREGX;7+g~NaD*TDM~q~|F9o?s#0dHlKK#n@OB#N<0H)+vdJ z#3eN@I2&C%O^XInLoNdu-C=k#^~?NSP+~A>N*OFsVhAdUzdDZA_&(i0I5Hv?WQSP( z)5{`pT-UmLdr#c-<%_?ZqC+nWB#zz+iQs^ZznU5yD2JOuEE?NxpUgcgSEehRvR|oh z`d51KlkMA;f@8Z3?&jyc&$5Wy%Ku&$EHT466NhCC7{&^mgQ6_h3L;a(@hPxDGA9Rb9Nz5;t1;9nzQ zw$F7R$byckY35HJtmR1cO(+Oh+ZVC_T8#2T-)QU}vb2Rf4bSFvO^v3aTwQG~>*mcj z(Db=8&EGvV@^fqxv-~wW*{ZEwmZ1Ki?&lbb{-X;Gb*l2_V3*|j%-FjPG*VH-U>}{I{dz^n24M(_epi-CbK8%O7waI!MwxV@ zb!?T-AOe}D;zKNpiMNssN}^F2V23~2Ik6HAWp^Fe#MYvMEr#Gh;f1B*X&LZfs6eVn zWs!)HimEDH25NcVpDGv@02>|1jN!MkLQ$<=zkYo>f2;P1WV@cmT7V@hgS^m*Wtb{j zE8PRE>E>$n-tIWQVzWV{X(%^UpN77TGvK}c`dlfL2-8v@gFbmM>cAT}STLFf#QZHZ zAJS4eY=pakv-KLqGO@MRwi|L*?u8|vYa8~iK6dQbJw<;7$aPrZH&Dydm{S{7PoCV0 zRT2>uMGt{Cy33+tzBN*P&S-B(%)z8%bWWYkMMB)}BW-LF_n82`E6|;quUizmF(Pq6 z#6N5$(kSyKA~!c>%P|VZ4NTI;}a5^z=_L0E2OQj4lOCNeCM`pwAg16*PYs*F#@_qnuDFYjkM{k zJPVrLG*uYu_w!h3YivlW5VJyC#03DzHOP92pdwvEyD_#YzK!Lraqc8Dj~o}A`zxlR zt=_utQ4WBjGngBbP2-{S;FOO@4b(`p$5T9UHx6aEGsrk55X)qXX<=f1!i%X4n6!xCj8Wg~)>~rA^ z#u05Nm>>nz_`P-#lX&V!B>NeWs-nWI1F`MN<*5x(+?gatx^Le;aMQ;h>%urWI4nDg z-I;}&b(U_`^+}oY>+f5WKY7c9H~?QOuRwlep@iv7VS~`daRr-sOLT=bc+w$*D=l4E zKXyALmG`TYnGYZ0Ig7oUmm3G=p;-8&GfUCAxVYSta#)MH!vEu(r)Le?G~3#>$+|xg z@XCGm?9TTy3!^W_h!&grW+AzZlM$S z@(2`zNx(&BN-4I zYY|?*9tsY!EJdOj2US#%43A$4Z3I*}bGa}5 zQ_-3iU^K6X!M}%Q6ky8<(aFYdAHYC1>GP2;i9SQMD&6JuqNPG+%Y#ARp3K&j zC~0UgAYI9}X;apOJf!ZXH|l^PV0a`5N~kZPuw`XRB{Lt7&$!Wt*MPm}f%ZX;J&08c z(G8#}(OJ%K-mPS|=c?{ahY!i&9N(a}+gay2RQz9{9*H%QnkogZx!vo5TiY~qk?M`K?G!5ZnGwR?r`loi^^UNF zzSNpXJh%{($6@HleF8nK%>aK?9pij^&YvTnR*=;Wf6l%i zI#MMH1xZ6d{qN54U&`3MhMj`w>orvGp3+~PdKD<7<5ro<6L9Ib; zY_9YB$O&2nk(Qn8?AvfArUz!5## zCk_H0X}2d`*umO5eoaZbP{y#v2RM8C8K}_V!fQ%P|Ip-)U zAh3pwZSWEv1hgIbqeD~G1L^oE1D*|g9Jz2lWoM~}t7`>_{_eX!x?U;5NK9-sH#aZz z9E&k2dZ4YE`fdS`EaTBw7FwlHj+vPmSR^B500uE6>vXvsiCjbXv0|_BZYu1fqo$@J zKc43123)y9k8sTHm;Im}L?=YDON4o7_UC7(jEy%f&zuQ|UOqUJYgKvk)~yQCOq=7# z0>FqM&R>blgcu~{{9fR!OM!takiBCDj3-sM*Bb!T01)PheWj@?77RV*+1U=0n)VDI z+ye0Rv;jTm|in z7<;5YMzW=mI7EJamO%%?8bjQH(L4~ijJz`^-!9ay6+JAJQ;7k5XVRjX1}aEqlvs;I z%}si-_upU{vCQ3FJHp_tgSM?!SFhoYJ@CohBpSM6et$Q{kl;6Lj>v|}uCC4CUK>LX zsi=gn#n%o@zJ2(zR^hMONZ|=0^^SL!&(8MVFi4g~Ph*B0#z28cB}UF60om(+$~wjl z5!(x)0*7gHtaQ{QqK3P@ve!lM3M~vpxeQJQv<|Vodx^N-GFsEzd`Ht9u6n|$$!~lo zugSO%1>F`n5)V+!CF>zVssT(=&J}b_We5^9KC#iHD1ko^k@N(xy}QQ`z$+4i#R-el zC)Zf1A%&5}Fbabh-rFF8iR*9OzMVAsMPOolTnPYz3wX%e_v33l@UHcqOh{F)7*r) z5w0Y@W`In4F`?8 zbPV--$BrFJR#uO9hdjnH9tsD)i7=fk_#B!!$|`Y=?%uuIqP>mp_Fvs!zdm-+&%A}* zk;2c*7r#C5?+VkZ4*-Scn&uW;3v52rd{F3iR8)$DQ8aie#}^^R{$wWaV(4v3kj~v= zYH(QY4VQS^dv-=%eUs9&T`$97eGvH~D-+tW;Mm9r?i*)=U_>QF>7)>Xf-q+ma1>78 zjc|=<@CB9i*4!{K=|&Mbs;f&4qg!xCD7qP*ovs z9V~x~3O=Nq^@ZAKUzy7tL9DsT*Oo?_2Kml*3!TM2qM5Mov3i<@Rt8wB*!%btS5Ja! z!Ev}t{SfExz^`I~|MN25^&{Tc+$i4Ga#Mm8k>pgh3~Yn(I5qx#Ajr)oU3jR(DZn*V zdmTNGXhYJ64kh-OJUeW&#-fkG%!8MmEkfKHaWm9(f^P5MzfTDO>9*@=ihTGm!)0O~ zHTO%Q%L-V2C1ivP0Yv}&pcy=|#{;M{LdOT>jgpg50J=X}+7T7sZau49>g#OPLFJ7F$ z{~$ymdGsAVH&y&Lqjr@=67(Y+_236)#D;diZ~=u33kOV@JU70|F^!!5O!jDM<)i{9909Hsi#%@nyMWwb>8&W*Otp@dEF+-N>!_ ze?77KKe+)6$N&Dg>wo{tR@3EY(fYlH(o+5aj$==&8(aP)J2yxkFj9&|z>X-w5M;|c zJ9&_SCVn)IB-sq4V|v(H_9dLnlSRCo@CfjIf;YHi+;*(qC~Y}lPp%kL4P5P^etl#4 z_}5AtgQ3GAM6~PFNVJ@bkb^-S8#9b{k%LPzkT1s-@GP_z3F6?uhT zvKE;iM0UfmZ>`Y2DHI$6h+JEYq7EEo;9ZK1!M|(Q5gW2t>_xcUax=8Ck73FNH#Q#q zSMisk1Vsw%88j{o+AOk9>!96cPd-BI?-F3$GjFJwOCX4NfI|t)|46e}o<4nAdjVx0 z&eICY+qZAWaS)6oIzfYKpePvp2x-F*@d#mo4jc`82ZI&Ha8m1EfBi+G{3g#GLlIaawmH7< zpKKpF$Z+WrHO_t}8MXqKsel&zom{f`{22{{FMS=eOk1>^_ZonJo961Os#FyG?f*$j zBfn+866d|G1`h4kuU-2B?FspNq_`u861H?#usyB-1C_sf$NVAr za3w(bw2nHtuIW;JxhN(Lr&X)Lr>+rT_w8FZay=E0I3_+GcN$E%PQERXHd?(3HP@OH zRz^mhPyRUhaBY+9S}30G#}otqPChX~iOx1Jt%VkYqABKMCj1kMl(UD2;pZ_Z?7PkK z{wwEfMLaV$HoL?9maT9WKxmX~Xv2W1Nv@8ZOo}!ng)rn<%2a9_tY$(-&=|1AS5Tmo zk+2)0rT)=IH~(CfFeT{`5HITi0F9iN1f}fr=MC)JwF2_<-J^{fZx&9{km-NWX&8g{ zwjc067d{1`V1i-fRt!XZanc7#4>U=khL5-q2;%WncPOZh$gw9hz)JI}Eab0SZq^41 zAf$Z}`-Hd;pjUwiq}EQ5s87a5a*5p$JW#yqs%t#*CY#;u#Duhu0_UhS8xQOjdJZ5$ zQU@eVOU~avHD{()uR=wNcszKfkCCWyh!s@8_t;hPvSZMgl(d5JYdRg>G(mIJCOI~k)EGL2XlmPHA)JX(N$f(LBMtP42^($_wF(s27 z5i=-v!%UBroqYw4Tq9$|T`WXy!sNu?u{R_kz^?_E{%MVayM;V{FV3C*n7N)32ku1z z>7}Lelx{34cMI|bd`%BN1`6O%I=D`Z@hA+z{A~RL<$t#A$Z| zgGsl@b<-zJ8czIzfUyb!awQT#H*en#gw)LiZ(a?eD2gnFVp8u;7KDmVocq-d1a-%u zNAux@5>UWv{<xH&JiEF9lAeSa?Hlcr!nJI$M&hRavrGUu3}`o2*dVq2C2-6MszIyBREs? zTNqqYrMYaqTpNV_vW5m0Oau3gn(AuONLQ^DDFc)K3g;ctF%qbHLcOp}5eCZtfz3YS zzJZJio$aZCl}yMPpebcd&OWiJBZu?I`xYp~gd?eUh)GrCbOQyQ+R;xRDqTqGeo3#Z zX6_KQ$zkQ7%a;!=z#=3$I!KExb2CJH16+rte?B>SWBbV@_HG0zIofUcos>)eKkE<^%frcc$?w znHyh8HR2AZ+cX0Dg|{Sxp>9MjtzW<1*~KMoYWSFm2@aM$sc%4@Lq&fxx0_KW!dsAl zVN>q$4EQ{_BEP2!TH_d3K`SZK~ z`nsAFDujyAmQK7bXK(o%sYvYp18?~3Fa5Wqgxgu4pdKl;jhx4sum33&2YUY?k@)Q) zfAY=$hrP19+f9HhPcE0@+xE`I?4-)<*7hXT;eT_3q%r*4F8;6g>2Nz=vc?AhUF~;~ zlHM;CWM>6XrqP}O;Z!a$Y_f+7sPzW=)WJ;jbj}>&INrb z(tFv^vB~)5Mut_Arlo%mUSeRO^cFc&65+K^xAr%EWHm(l_uGH$4vQK+YqonJqD>AH zsqQ=Q_w6<5TF3innyD&7#={wivOli|N=$n8taztUD?;7fx2*AOiJ=(|9d|Pv{&qU9 z;r4`X)lu!*Ri(sunDNkd95gtro&6&1NMRxunRcFg4}_9;Q*rQS^Ysl6oUVLlTl4ql z2Ih)ce%=#zctgw5=xEr|$A;Os;`*=E$a98^aPAM zw;ZLXq@{%|xwu?TTNjac0>2k8vn@*4WN7zc|C}#-eV@l~WNX<;z!+BS4L{$=W=C8}*pkk<*3%hU$#aETbY5Qn@;f&6 z>Gzswja(2?C_w=&BgE;s>)LTgR3=9lqkhLul!1_b1aY|p7 z-n1}|yHV(@Z9V1Z&#))47gnBMa&PIceS0!DV=w$H=cVPrcV?!h5 zGje_7zDtnLdvGLMLsPR7AzpHeOV4)~`wqEm4S8hP-FO2zLWEV(3d#tXO+T_{MI~}=Hvy*k{4bk_H@q8@yq^Dc0&wM)Tc$B}H z%j)WaPF=_Fi{*i4cx@0l`RzWWA7pngPXcinM1sh9aG#kW5`$+#gup1<^djUuN*M+W zkllVv9vFzkPAJ6wflD^-?tUf>lcS$9d!46Kzu&wY%kt$Fi?sBP z+Or)iGtJFr<|HNS0?jP9+>Fq1uX~VUJGa2c&-pMfg7LBAo85BvwUR$tw?dX6L3y+X zN&v)2q?5Xl-6}(~O=9%f7y^=rTST!-?7-yqd)ryF1uDwA^YoQzCK$GgkOhFg* zt3_ShSB9|Vsqs@nOSSk|ND7ADiTnM9|7q*U^&V-rwYI(lU1FoGhgQZp23|3o?PS_0 zo!gF$1{tCPFLyYaA%prOwI?{ZZ@c&f77~^S7kL(#faD}5G z%Z~51V69MKN7=4&p`H2z@z6QXvditBjz3stt2UF@^^QMc=&ysv&G!l-k;PoGfK z)vH=-ZU~#a-JSPzg{y0AU`ONQux`VBiTP)Uia~=`IRWR{+#wfsc2&*$w{OR* zYOnuTeVw-0{lk}~w2fclcAvg!olu>ypI#Q>HdvBd6co(4kCK;jfNmmDtLM}j{^x`d zcy>`|_WqNZJv|&|B9T{Yl9GjfI(@EoRi|-z`yikn1%q$$s z4sBUn;JB)XN8kR;)YS9KuS1n$71H--*=WAY`c_^Z7Jlt^xq8+2{{CL51(X12&6b~@ z*dU-!`44din-28@n{^i({7-B;lH|a-dPCoz+1Y~#kMnm*K3wr3v0VO6~gZ!45tv_6#M?zu}@59)HR5oInv7LyC4T_5A z4dHnF1)XGcgy5j&Iz=7-St{ zWluQGxgsDOTe;4fQCtQS%vb{7duXY zBBUd>2pxJc+7Onpo^?G@+96KqOJc@s2XysX=#NnP8rmkxeCl=JMV~0B7ceuNq0su{Gd`~O#>LGSQe`rOFXO<}@Y2taYdms{P9;vLTGB?4@6rYO54&{=%<|S6C2bR{a#d;bOi+hJyd8Wa}rVDPHq- zs&#hcIiAJA@^1@Cb+I;blpMoZ0k}BeQt}f*h4$HR{8!RnibT-TFVDZ@X(s-tMQ>4HvW|m#Ctv5n6 zy0}K(>~OYc?tN_cqan&%h(+`2I$=Y>R8+V-w^y`%7Jm8Ci>>XI#S(XIzT(s*>iIqK zu+fjT`wxF;kKI)g*9P3r4Vok^^xP%%>Ib>?qmzA)mr1TpaUHuMy5(I(GCZ&&YOd{n+A!^n68nW?e@(-s{z?3A5K<@=vt2Fb zZtq=Z;_l$%F>Lr^0a6exovi({rlz6irUzjE%}natzaKUuA8B=F(W{C(#=v`dX)^cJ znJfnPT9g#CN7~xyW4>eZmOmYT6f4TQr&qmO!=@81)~1utH)K-K$HXhUq0qNr=x+XI zf040UkL1>eglEm0#`+~Q*eK1ZJSX+^p#+7x>=JEQ`Kop|f1!2$0?0>)mS5-2>b9(s zbW2Nn@PaL}p^cWl)v0v!3l$>5U()YJCf%bkS|4|+KJ%r`kKeeZ@81&J@xV4&J(D^J zw8x&;_4}1_YjP1}b5dVXQ{tUj$z;BqR)f=Uo&w3fQnWeYL}@ z<<<9=mX{Czf3=-^IMn$b$7glfwkV1%4ijCpGpUeKOJOzIgc>sLm)SuW)lx1MrA0`m z;$%vaYcYllm1vY|OKI$Gt|_gPORha7&+qxo&+qsBeLwHZ zYx-{QI^t0E*V?|ez+gMIamPb>8`nI}Nu$5BzgUx|`eqBWUN69Q4};ua)p}No4oVeL2RdnX+5&pM+)}sl?lS2MeF&RE zfALv^-#Lx%X^{#0J5EH+tl-ID3qEA9I_74*?b`=VPEMnL`(kx+c$+TVSN+Y58*hf8 zGuJBR7-np}D%e&$PulX6*_-6-wbl7rX7SexS(d)ncFPqWI?&Kn9wG|UKAx7wZ82r9 z@tZ%tqwk`zdlA{Xd&&l14%vUJSDj)~^D(F`s?+-!)o3Qw*e;4;qNDMy-k zRuzqR4%0dv^KM5aO8q)~tSL!`_T&i4^JW{?UXYKoyOyt&?(tYU{wjRI*zm=}tg654 zZs%|vKK1|1!COO=fECg=>x0w|*zu7@0N6G<238O7$v`}d=r zx8@22xBUisa;iqc2ju);#~yZAz1B{3ot9Rgesjsr8gcds%j~9W`abb(KbA@~tX_z- zT^_1YcK^Y`)5)}jC^Dl)?_YJzt~!^@T(z9moj)%czreq8u+-hRVy?Q8+nrCla=8u` zmiY$xj9&}7x$NlT)(1)>>ACEy>^y0Y35C+IF0naHq8{8w)#frs`*#{y-$-I_a_b*EL*<`oO3^hd8ngB4?2MBSmU@o*RJ)cb z)*bcXIkx}6!sdLuJ#baAZ9qgD2#wEt3(sQa7YP@O>&wpHe*x`CfR|*g3C?^-`s#f#!NmtVdUWpHT zzM75fT8A>bpu1Q1jvnxl9BQDnbX&I!zwIqEhG;72Z`~9u=irn)+%Yk{fiviXhxF5+ zHMwr}fygPW?|5C>lsIor^{=i_Bq~djmWXAnto#exl*PhSud`1S-=xeY&mBG#^n-)r zot+&=v!V?7=Y*-L?JXT81Rep#^YzCHwJQ=eC48mCznd95gC-?lIU^5mb-c|Fb*o4u z#0Vo#qH=sLO@1M_+X*s7LeZ3~_BA+J1*+OTKytxBjV>T&V8XM~%MOgeqA6WuEvzlz z$;i~RaI$SaJ))VvN|>RlopS!XcS|ynzII*tL!-&k^LLK@DJao06r}xRU5F~h>!lin z&4Dt^tMaFo@h}3kV&Xu);y5xZhSViXV$mI6lD_+6T~Z%y-MR&sNw=Asx89CqdfreO zch z4nR@3FbV(t`xSPYIdihlO#o0->b`CvOKwUFq;OC-BPCdf%*D{CtQLNzuUuxc7?5gb zXJ=^9yx;DhPF^v=-p&m2LM8`>Z4`JoKpbgfOhq-ULrADdev$(>W@apo^I(jv`; z*4|z^B^E`ZJia5&8i_?kT)<@z6$lsBz`YA54;P%1AeIMW!bsT29NQT7p|D8@h$lz{ z%!3C@QIzEmyO)PKZZtGJxzgg?rv5SZxQp!ExrD1gZ^Z1q2(xc(?!?%O=cPpzr&Ur` zjeAM%Za*Q=sQUBIS(F=lUf8@%LW>JA)ksnYsJ(1XLaKyV3GgjpBcWe=wT6K?@QZ#f zbXl#Zr-&gp@5L!rmM1iMdA5FFHw()A4M{ulY!8s7(e{!kQtko4T@*U9k2KNI*Pv1e zreVmMjpPYzr=-oH`Uw8cfXaW0RA%14`aH21*%X3vf>#ucQ6BXs;iJYoA$?T1t)!zc zjM#TO^7>9`t8fkkPzJRG#pEKeO$YHLJQ~XU2=(z)oK#c`)7kvf2YZ?VLlM`4lpRUr zckeBj_gq-R@OF#|HXAka*}50_^Af1lOU0t0*La?A>Qp`&HVGV#T;Fxv5ru`wl_gp+FnbDfSY7p=B=*ON_&zqXI-}x3! z1+aH^G-nLh8CzmNs9S_v7EotW`5|a2z!)n{u`?iV>fd6$GzA78LGy3m9n|0{O9(m1G6%d4hB{#Lq~Q&L%`5qp|l}R5u;0 zu5N%juDQr`&KCQK*vjYVGNVDZTt~;cX)&%f{)OEBwu;&L$_mQsZ<6DfqXEfwD-(-R zf~hkp*2tVBy=>e0CH)&ellPq{TzAKZu@`cv?dlDw=dN$?2KgL_Z2AS MW;>V#rksfX06v7;`~Uy| literal 27357 zcmb5W2RxVk-#7e2A(_dDjHrZCk;p8RQZll#X$T30L`HUHq|#7k_9i4EBPtD%S=p8B zlr22(Z|DF2zpwj#uIqk1&-pr6{Kj#7$LI52-{Y$GQFSH;9tH}9!gT1Mnl6Px?M9(c zwbIk#C-dT=pBe9CJMFa?$GiDT=0*i_=+0m$SCkf^MhIU$AvNCnX^*v2&~7 z85b9)3v%1HJN)Mb5{~C>wzpgTD#wScayoeY0)@h2MgE~mQ%<#|Q0z1gsqNEqe>mEC z$yD!KAMH<@c8=qExA*V9puTeLzQ%)x>I;wTZz@>RmUZ3iT~lUfQ`X!12V95GA3F9s zBQmJ?p2Y+#}@mp)36P z@Y~2p%RFKHR&oua8-A;Hzz=UuyF<&!?R&1Xba0BtFxNa#C85%1>9;_+d1{KJq$DdR zXV9H(CMpTqf|8PJoSmJux2z4heqG9CDuibnzKiDR@8$`rS_u-`mXX&|Sf|{n=0It=qQkbQ>{p8E$2&t*xE? z-Jrbl!uKN&_gtVAH_X}PBK@qg%atwV6TrzS*z0w8*{o)&%pGiZ#F5+0NFI|Mlz1 zXS03RfByVgDd6*m%ErdV)(pV@rz*d3kxhS62xzG;{6Sw~v1H z>RaX)hniP8KAWJeSQwLk8!eq#H}K}|+ko=&^0N=FUR{-ymBn9_p*BfB`&>eYZ{g4H zhRC?Mh?Y#lH!3j)7N$pcI(*tS=xS>#sx<#YxX`|h`Ocj?KPM+&l#N$hPfQf*aE^_Q zbsu{=@N*!}M@UeRf>C!L$$$B?^J4d*Q{nuI41p?FT!&k2%`RemFFMeB zc34rb=i{@owY{2SR@V2~@p%+q>ZRtH#hLE@@|iBx2fH0U6xts9`t2JPWw&D&_ zjgB8rFnMyWz0_5_E{vB>OG~TclAaickg#xLk-Z^)Th-Xeb}Z#cRgC;ayUTGrd-m*c zAI^T#GdQ^F@ZrNHrKQ#*?fE6H!*}#j($eUEe}AWn8^i^yl97?Y$Q&MC^t$)rh1%rk zXD-d?UHd*4JLF%UJ8x^@?X8rRoqZ}U9p5TrX=isWa5a~~upDOkb_L6y#YN|@uOjW* zCNSXpD9vWn^_fFb7G-YR)-tRNU=6u;?I!;Oby{9TOo~8X$736|_3J}ZG#?~tKG;2r z(JmU&>ukV+wj63sZ<;Z*xK>QO3+L6*^#>Cw+M_4N!iw2YOQlcF5gA0IX- z{C+Ey_xQ2u#}v)R=RQ7vFro#L0+)YM;S0%XaPF_G>gwWmb93WX_Eun6z53LY=$0*& zrLMzCGbg^3IP3e+tIZ3Bq#Z4?{dhoOy8T3pgaALk1SKRdZwDI($ELb)K54J`_ z>DB<&zVGiIRJXL~mppxXl|}iF_EwYPBMQrZepCC-j{7W&^F6sZZceq={nDjX78Vxy z?i1GePQ5BR+S-ENi&KUP+FJsiJb7aD%strNQs#*Bo=?ZZU>_SBvmR_pu7308@a2hGu0Cuhc1})CAz4`?x65bGW}!x~ zUb%9m@A~!%*5$=spoDi#vR<=#m_GI?FC{<>ch_p>8%6mz2m- z?d2Tl>+fGl$+xPZ?U<0iIM)2hymR59{KXuKptZI23x7SaC{~q4nhxyo7cXDFnci+u zVTvnUoVweZ|7~i#+k(Psad{%}@1@Mn&tGfJHaVPti8Z74ZO=auJmT)?=!hM+9Bi)2 zK4M&aR&?R5_xzv;)!si?AuNjKzvnJ5j6M+(7eAbooO~5U=o+@~YU~we0i~@`*<&^m z0dl7M)ztLdv+4Z#C=;lWue*drL_~I;|9a5YL{n)srMjs}ccX~N;m_Fo8z}VnUF6-n zA`Wpksk!nr5!Bs;uSYyjJY+`C4kMhsLLrRc_yw&FDA02w@Xe zdvj;osm~}{``z731SNMVUE7qQ8vXLt?%lgv#FkNOsVJx4-rMkb3DZF1Yha;(>N?2A zODVK#VTh4)t&S2mOmcXn9&zB%p;Z{VB!{w#;|Go&Wj+wW&qQHW>3J#O^WsB->V|+y z%urJ9K7ZN`GBR8*!})HR$tx(VNA-*}Pfbql#lGT}ciS2*ZNJ%Jy!*Lw>BVsgc{glm zEYq4}T8f*>?!?6j;MYOro-?hpN?0xGv$M07xCi6$%zO}eh&vLt1F_%QUs->*sor_O7XpMgSDij#1d^)_vyz}t^TjCZ@GVq zGR$~J8^81=TEesYk8ot^y|mbFpw z*>&h3jp)Q@+w-VMckd14A_MBx5Pf6DQs71X+ZcI*HC{C|yu^l?K>vP2`=t51RIY@f zp`qxJySdA+Z;2$gNiS~YzpOkmGD4xCUU8gjd#1&;-F!v-(*7jLr%#_2urB7!nE!T+ zPOAVc%up>ZE+(@x_%YSq<%_@aTJaNhcE-JK8v;hAel$MHx2&T6QtG-I>v?u=Aa0`R z@b;>=M;0yoQfm9649(Lg>Ecn-MqKAwOQUtu^++Yp+MoH1&ZpgQMk(_`Cu+utiN0En zF`pe96|UuceQEEg*7cAz_zX< zPX}UaFiVj)ZunMKRtDdh`uWpWBe}ZvIu}{oww?zL(4IekzNfcWCsQtzUy}U2U-_UG6kd%}#Xsq@wXEhtW?(HRI2h!@U>h6?OMs{`pXU z{+nc-jK-!c_00IgnGBUD2GTI=_CeEwa!1YH{@@j1U@E~3yeE36*=0yk@9C0k!#pYe-FLPFw4Yc?-|t6v8O4u75*{Vbw+IANz< z^Oh}98kZ*<)r|*4(T9#6J<2q8)B)H_*12y5t~sMf>tmB7UY3-S()YlzqEYO^c$ebZ zwQKb*#h|Qa8Re^Zn3m42%yTyIE}`}60|ha5BcKLg6kkzN*d&C^QR+1d6}UK&aEXY++K zPLRm}NZjc$dB$b1X@Bvr7guQL@-F>8ZOgn>KtKQx=ES%9C~Vq&z!=W6KdrJ%o-o%0 zv#2{cIb{!%w~^h1U3O2_IkEL*HJOd+vCivV<(?;icz*u+rQ+tcJDg8GsM}|mucMwX}s$m$MxOD+#gLeB>TQSy}m{jZHrgeH|)BZ-2kPp@kQR@}HkH=01zA zBb{Yji}O?dy6G?Geh*C4uqa!iMgC~XY*Wqc@9XnxP+DLFb=!|_z{bZHmXx$H%k=4* z@87={6r89Ft-`9wa~(E#Hc`iGyfFOf?R_co5roeX_=1uC)KqJu!?$_!W}_lI4q6ue zP|ySGKeH2OKBh3~9y^u~=-B@0Q{P<+@AQdFi+}$7#G)pf8RddN_T($PYOeKrA30y; zDSy7aWP<&A+}*^Y7%K3CYv>kgdJTZ%R-bK_8t;O79itFEuta0hw^VmI)3iygVb*RJ70%KfpEsC32)lo0e5 z#mU(Wr8p`YT6%P($XmCB1Jb%|a-O=mrpga~D#~nit*NOg`YxF@W~0+sB<>j}R<_*# zGUED&5890drw)^gP*6~Kk*4!V4WLr*fK=xp1A{vw?b7zGuTr$xR6;tu+}!QPMRtLTp3~f=Vy@@V?qywv^a8zDu7>b)EVQ#lcHO>x zJHD_`uXvk;1hdG|hn2omG>peYX*9I7UIJ823}xo>l!~dHOlr4QSM#_qHKdOk-GkD9 z<@)v2@8gssP>gEczh_R>iaP)-#AH@07y0mE=<}ud)tLs_)!5rkbH7f>?%A`h)L89g zfULV;$YctJ1Vy1HLqGFIRMbj)dwYe2k;k*xGrXEd{))f1s$$qWn{Tj6SlokI_y4;Nim`2UU8A z@=+9d^Cq>SVO#JWtGrCVurMad)B{_5ea|QI))Tkx-lYRW6}_dv=x@fcE5gFT!C@Ww z%2$>dA2)^ii~R53{x4Bw{VMEwP$K*! zA78P@btic;xR!lHCVJ#bm~pp#qLZp@2@I?Me&0S;H3DXvO|f7@dnk!7ij} z3&=RkUU^tUgVB^7`cLPHypL4VZFaJ=$8l{tPy9y|iUN1`yL*=l44cV}1FuZ9!}#EP zSGtwEOq(pB;VDjbW#{aNz=Kle$3; z5M>t=b9#g;t{{V@-P&gF&YgRW?kO}=?X}DB@H`T_l;Z#m0Z<+LttU*TYwy8>^km0} z^2(6f{?c#d>K@cZ@MtQEuCDG5bSMy=NBggHf^Tx?n3s#iY{x36qM`zk4Jt2JM2!jt z3Z$YOeJFdSr^m9ry}h!vl@m2X)?;!DrN^H|$u~DQH|rdFEVP*0=557CaeMPZ0hJ(; z6QF{aQ^~XWQmo0kow_0c(@GmCfz)3`A zZ7n9rGam$3(9#MiDe;40&(8KoOuUt9=mCfZM6LSSTeI`huajp#6|TaXlD#~ew~Kn} z+g%G#0>!+u9g_E=qlH98SE0^JXIN+szu;695^72Y6z&IiI(YEljS9dJ3dPpecItGW zC#^iq)rhup>Ec@Sf}8f6ovWlQwy4ryX>T_g`Bb#pv0`p>K|ujQ7iiRndVlyVF9<;C zF~~OF6j+4%QVD$nY8sjj*ZB(nQ_yQCiY%B3zlsoH-J>#(wqo?a4g*k*1HXCOIiYhuCG` z%D8FMChc@RZmH9ZO0zvQ802`*nXv-STy~M8J?OG|`T3DC{b;f!5FTT_=WWTH`_nR2 zgWCjxjfC<@Ys_zs+j)W(L^=SIBfN3r3ba|jwUo@vl)oz~2P)cnWpC3~9wL1`vUu)I zX?~;4$;I^uw@%NmdS|eYcvNtw1x{sI7XvL-wE}f>B-QI)6}; z6KK`eVA&68MpYog@xe5qK z#8&|Ps5`>|p&pYzuI#z+U2w+^R$iHNYH>-cuOfwY1)ia+ z3W|%bj`LaCBVkbynwGW&+GBs@FqTl*F%J(pu<9LvT?$LH{Y1-y(6iHP-UYkskXYMe z8!ZBljGmn1TUuIhJbCI=FVIdEkz7#?uq@yDUexi#{*M2kZTaTTwSoNWq0Qq8ZqkIN zjdVX3Se|ct&dS4c4XmB$i~uKsXB`}zfrcL{`|uC9X4hcWdq+k>0b)R;S0*PXgTU^F z@OM2llu=ezR&dv@wdme-@%$$zKw|%5OmzXT0QAkin2TM6atfql_)q@1cKv!}ObipM zA7$_9)0;sh3+ru-`|F=ngPryc=RVJQT-wSJ=$_}#{u*4S3OyNY&-h9&HVqkNAU3O0 zr}T&Iw?tvt3=R${x$fIH^(gs0_u!C}uux*@-Pl+O-kuxx?u8I6JMcb6_dXFtKYily zT9}UF&n(8m>Y54jycc{bk}khChku83%lPfh7-y!A+oH$v?As)4`xbslm_OSxoa#09 z&xbXV*>w?9@ceY;f_ot!sVBfyP@?d3Ut@Liz|HsuRuYdbiO+|$o0Cub9 z?&m<-mw&&L(Orj6$CqeqbN(ihEKs?m(Ng#Kv*nH%8()L!+go{sM%Vre^cM;xm_^{u zffQy43T?R->Xw#*^V6f(eSLjtr3_8I{&`!Gfp<@J5EJi?Q}y6EYtKae217V^E@u*( zs*zjK!zHHWasHXL>}w;it)c9x8yVe8$W+0;v5dNNhX|rXR3vCB5u@4u?9$`H?v@sg zl`B^gfd-8v44B9YgI;oBfEmcxA2p4pwZi$#+HG#=_B8>k*z?YH%BdO#qDt&Lboj8U zp59tqu`jywnW8#AStrV)17XyZ$+1pe0>$o1m|0pkZTS>F6mx)WgjhWg?otXZ?BXK3i!K&w;tqbp(rqle=xBR`?2=?Ju%jQ~XEALt_00>ia zjC^uy!5ZN%fWkj8xR1>S#190_09C0DLiTV)VpvFOstBBd;n6vKncWk|?mB`ayFACO zdMdBvJ$VugOxoq%&qQdV*ueMiLAD=LtU%TPOxG4XeoRQ>+c;%`28G{@fZpdS%i`%; z7C6fw`4GXXI$#xjw&g@KiW}4*8tA#|xLCGz>#jl@>#Yu4O(X~^JJSmq$!kb`(b5Wj z{8-A*&(AU=Rk}8aneW=on`=6LiSyu>?tNE9ocdm@a`*Hc{x;9C=loZCD0#_k41v_( z?X_sc!&%H~YIR4_^>9O~#_C0YV98B}vB;7zwW!-7HDSl%*iMe4}Cds}!?^GF6MZkP5?hXK?b-nJ-pY6Hn3h4?#n+eSI4{n-k~;q`TG~H5d)t99o{25|_-mFE@%ljCMSx8)%4oiFWq$lYMRi zx(B1{f%tuNWtiA~>3Y?TjXK4hFtJFJGYdpP(nk21f~&CM{7~pv4hZ4j#*n za+Ce@GB^U@f+}PM?1x}L!5$R%f~QweSHsza4_Lmsu|d}Rk1U`f8!M}J?_Bu$J=A+U zJ?_QEhU4=pF*+MIY%qF!IxK61bm?hI_GLOl1G z5U~@?Jx8UF&&|%dTUlERb*92h`fEi6uv(*?S2s87K?O4t^FfKH+PhxqvJw3THA~BQ z_qlI(#j~~Mey>Vc+_QRvnu|-}nXYF~OMI3qSXDw`B*3gvgLW4-wy2{dB_yO;a{k-3 z+3yb=U!XLqq1+XEA0_Bc%oE1l1VBbeR@T-U{Sr1PSR9ys4)mV2g35vG@HFe8L^X-E-`m9395aBmNyIKo{EAmP`P|rf$T{jQcK8B zXIiuFPf7#HgMntRjHc@O^7Mk!g$uV73#Nbl@&mcR`{^hLH8iTxX#~>WgP;VbrxVGK zfno&H5=%c>{Rygzpoj=vncHYJY>XraxKS^`117+YG?sI*H28BX=EbniH#3D$(2^($ z*n-fo(tel9?B2O^Cs>IZ5GwGc@Zymsb(xIlv>AU9{<`XFYOt7)u&}5D+8KIcFY4+H z=6~dsCMG7ba&RO=WaD}N{=M8b!(+Ic7m)C-qLdR}h8t6bVn(cK=>0u$%YRZ^ccSDK zl@iC4zo-=URV8>v{^@r-0YO35{dHkR1t(t%$K4Hir48Z$MPNIV6Ie*)(+dNF0hQ1j zw=l&I!;wG_5@d4v@iCQ!E@s`ml!#~U@$ykA5r1%RlaiDIxgoE-H&C`Aqa?LqA~0|& z*EzTKjE#k_lR7O3ucO>!N@ed}3Mw)2FT-wTX9H9LeF$#fuC@L88oHG$tI$zlcymkx z5U)keBo5m9Tqe>Ps|#&E<~jEtxIEK&(Y-%R2F&#X(BDHvPfj4T+}2A3V`A4NB_$oq zcC%@Sh3d)t?AbF4rK_t8!V= z&(B8;K=tU+aD%7Ng;L3TO2;((Q6;-60Xvl4 zL|_K8p&-LDYzTlj8*rv62{a;v6#@zusKgV}rt#++;o{vQN0W7l35+_uaSc0rZGviW zKwMlLuJO5^H#T)O_82r#Lkx+*LxoGamtoy<#A?xR0I?G!{W=o(5C!Cv>lg-%xR@9M zkL|~G&a&l&otaT4IMK*#TuotU=%2gQ%iB`HraT@ zh{nWNSNW}MTB7rB2PJrc9<&|pJ1{IPj94cx(3?&T{F!(u5C{rSt{3jG(5_vDO}`fr z2Ou!48X^n&%B%N}aoHV}q0!cmM7&*UqPvaTAlTFabbpNVK5E>kuqA_GTuaq z@RUX<&C@@SajZjwuy=iygJNd)EP!5I6_OSI^l8bB%`qUSrsWngvFLUfpBJu|;T$jHc93iOTb;R4DMQ7AtWL@KPT+zXsl3D-rt zWwWA=SQJW3Rzbw&^XCiR^Vr(085vsaStUe)UoW=X;Zp!m#|dDvpd4nlb)p+qKzjI) zqRDG(0Jmo2~WxbU2HPQH4lR?X`d|v#po)iWP-!zVoLLKqw{myJhac8*#anU zFh2iEs$E~D`!_OFk>1}12Kou3wEqI6CayU`Y(A#y^A|7PEvlQFpRe~@uga%ihuvmF zv(Y7x#IHhCKrg+Q7cYvwiou3y<+=E?XCHyclpUwva)TGpKprmG^fZJ=j>NO^fiA7+ifKrE6rR~+3OX+$xf%9y(dqfwu`y|(5NZhn0`!z zK)mCMSBHcVAB(TJ_Fu(5|1zguYUHGxX2#4YLu}TNCs`CvPmgrCgG3F-yQwwb0Lg@X z<(qyiwYs5!1&-~_=_d}KIRWg00s~i~p!q-P@>oqw3utaPZr=0*m|d4Qtv1DCG9uQ< zeV{faDao?y*(Cx}!1q=nPsJ#pv>vwTe!!CC^B2CqtwIM{#moD^E^dL4-V+guLF6+d z6sPYgWoW4}=|KUnnC)9Vm_Ov7?R@(5DdlMFbZ3ku{ObFewXFIy{$|3i-|9Iat0h8Z zX)m;8|5WU79bniJ*#|T%IYDoKdz5d>xtUp6MXv9L?#9G~LA0|%u{2)3FSln;FHsX=ZdWYL+asUx z2qu#807%UXk^m(BJaj2y=l^^!KmOO_zYpMnJQEEG3Z|yoyc0X5Hf6luK7)Ck_^S9_gRCI0YT368OG_?OTQ{DIy~%iOZ7Ee&;b{opY?`YStP5!!9@ z&b~kS8#4sQm+1@E104%LDfZ3-@A=XN&I2M45aO9MG%)b8RW?!Xg{9%ux;%rdv;#uV zU}**N-9h3_EKrW(^QBsmxal_lUj5hBd?(44b zqM`_R&u&m!oIGZmu|_BZa(tBgZJqqNxu|hZCx;$}l+!owDePGHt|e1C<3RGKCyp$T zAF#r$NWNmBAx1tJ@SNz3fNqtjw1xHe9z1wsF20r_JN_P29ZJ)uP&GZ}&gVV?B->%! zuk`1qs;z#|vu6r`$lR2kA3xGsc-ny72NjSDq)k_5i?Yi5=pQ|K>7!vsux&@drdW?5 z(aVcdZDe(vj5u_z>)BN%-rbfE!?0wC2MsOXBy;$ok{1u8V-Wkyb7Du;?`9haE!pNjcleiCYIJ%s_REfo?-xNL3!=8+b(XP zt?f%pyF0W(iR!oR+}Ug&_8Yds_iz_13+x^+CTc{VIs}6OAaQwVkv&>w88`3WU$=zx zSlbWo;%2tNx`I=0@*KOBP}NB>EARqZ*3FDjN!WY#>{e3rPP05Xaw{a1$KPxdPYrsa z9yVL5@P-Xk*uJ#@7fGW;m<0Z#P_WT|fX^k{XSe|Z9vwcSHWi_wXLIM%fg@58W~g+K zgibAtbw7_l;I0rowEEEH^rc>#15^7DSc1`gGD1HWQr=#^F4zOf22ZfWkx?Lx@8`@+ zuG>X=AxVCgH$p7d*ZJ5YV>_Q+VuxlN^m@5LP6-hk)EaYxx)(22AV{bR-3<*g$x_C) zDM>DOJDapAi7U(LVgKI}(dzq0uR|`-8O+hP$-)PLOh{^&WdMVWKbj9i(!~D0ArAr) zS(NJG>#HD3om^PU5V*Bn6Y49n(7LPn*&=gDYFhson#$2w1<|4xNKvgq@bkv>C?s79 zqCvSWJci!yY=Om8_2NQ!sL;L>HIR|Os$aU{_XVm9EV?bDGe3Xc@H0bsAh4Ho9uWPr zVZuWGM49v}tsO-wYloE7trVuG&i%B29Gk__{6vcCnAS(KwlVNxUs$Sp zkbXe?e;9grR}BDR1edvc)rS_Ln~2~X-rWg!qK)RG8WJ?BH0gHmE`5+ zJN^V=A)w)i=;q>jQBsx6N5uK2pfXYWr0fVD}Kp??k;2sK54Y@%$XGf2s zrtO8QfsQD@?^R)fnVx5~adZ{jBZ z0d=26&NoTB4fu?BS>gs+d#$Y#&t8j!%d;y(n~R;D2D$4MbaW@6bJ8<22SF5rm%(Tz zE!4EL7$p)zM#wnK?&O!cpy;H=lQ<15yl#>v^IjO0F)}tz)QVFw9)P!YPuiXnC3huO z^|Ph<;aRwwNsB1X+)9^aPdCJdRd_AbhokOpA?_4-^^Yl~bgN^~ian>Z&fcg&6gz2g zd&R5}_#lPiKA3#>140&L)#D4!G$sZnWSHf?c=1AT>sCeu(bo`jCTW+1gaq0Q zKDI#~lM$W-3=%OOmuFupCuyhle*GE%$7LNXGgcKrG!2Lr9I-Yi2Vi)0C{aHVIwRil z3xuI56cu_hR9Z@N8_h|I>I4lnHCdf&IXDKFXw$1oFARi_cv=HKAgo(QJPEf^W6@?; z>=IzYsK6)$DoCt}Tk<5mYUeWUjih}I(r5%79w>RqEv#W*$5z}vh;^_pc>%J}sYhe{ z2Cd-5oX^C>O+H4ZU?l{wQk#{!q&xYLodLu_JR^)cT>138hfki!pwH7$0OV-yNSIxN zCaEyhd~6o(3@aZW6G}S{IWST3yca!ey?1bNae?3U;~h9^kfyWvT9!yjttB9z#C{>T zlrj~a(c0PyMeYWzEHM*-S7~#+YFGsXA|Pn#m0HkWfocH|cbI(}yG~8rU&tk$>C%lL zN}kt(+aMNV_9>~3yKM-tzo@M}l&h$yNVZn?d5*>1=6o%CKnXny1dU@!j1TUel%$jt z6$N!pR!>Ynd7x!QYyDVEs3D}hF0UE0^>Qxs#8p~oh*cC{S%%0NC$j%Lr22jIi=YqW zd_1qFw>;tV=0FC@q z`IOG2A3Q(FvJEN*0`-h2vHMWT+Idpdu{4OVWM^j=xNQV6Hfrfk{g|~7AS-mkr1M2S95NqP#Af3(G%_iIOrm$s0MYB z_}Vcs5kujKEMmLi)>zQxC_|Pq?msP$yPx?~D2j*%MCL6ES(=)f6KH>Jm&$pCamog! z!3u~+81jM!^#pASk;(SN@kO#(<=H1;$NC~;Yz-2|Ep10k7P9XMbH`7b(yB4Ik05!? z_67?Oe=Ejx0a^_SX}XPlNx$_Q+!)zNNXdZsDCWp7`6C0_+n~I>fXV#%>xpM*Bt2I}Vk;raB;66n=q~xSLZ@ab zZ|g6?B{@xf4@AnGQTm~bX}s}MWIzMBi~yos+B(IIe=xO?U05073&@Hf=y3X7^gjA4 z&}r&h^`@q$zgGs)g6ZEr7R@S6eKtRn+GHejwX&2Ypg9>8(pKgtZLw2P`!z1CHZOHq zg}|s27?qm-)e!o#`5%d#s^8q)zJZv!WqB_kuC9`M?XY8ATB2Vl=*yRqN3L~eB zAThrm)glvtmoOTejz_Pq!0yP+>4 zB*X{V%Fg%K@86IJlSG|Jd_bnlT{ZJzS(!W_{He@@GvBu!MxF~P!p01JakoT~f6jtU zJ!GZ6yWQeWQ62P)HRxL}(8#FL+VtRa5p8e;bbI&@@R^?MD3NPqM^~~%vXRtqDpUzS zhz><{@^$gxkUU*PCk`9aQz<(snzl-G?Re9Kr4`7#)~kE*ZapXJ6Sr{ z=yWTI|08b{y+D=H^hmM;A|l=p1)-t@0>N(hr>7693Z#lfX&ia5JG**;OJ(nWL{x5) zc#^h6GHUN{n!J({qv9^H77HY#D2z`^o@pdQuYWd@Z#UOkf{4JusvPip5_Bwe8@D&$%qgat*57Fx#qYIGGExESb*^Lc>8u1RMY*y$1s}{ zW?tXKW^*+r@{mZ5f==Mj+vzObbZrCzUP?cTmXwicGdXPL?k>Cg+?NofakK5A=CA{_ zG&OGwiq{hZqr;3eGc0t7u4N#bbI%?k;f4&Y!-u^*a^wil=r6snT7sOjouNRKIQ9B9 zKWVC~qlMg!HWvvIA2M$FY!|@N|1RRWS_umauN%TaMcLwZ{e;b;_e0-U_$&s3J5wPx z0v1?eU~h_PX}&myo{3^b3==`w$ywLat$qJVTOW@`-j!rdSDz_I#l7jeasH06EKa$Kve7qp|(%-|8c6z%$ zdfbjX_m%KmQBf)i3B~{utfY`M<2#w|+kOv&gVoE=g=I>hnjW=PLE@2n1il`-~^Eqc;I?+azZe%n z3kb`Zqorm^7l0Se*awB8_N+n7G@$R2ZHXWwT zSr>Kp&Yd8{%V?1T&2Y*0fR_uKT^QLTtYsW%9paL~!5HX72ws-!jT>fnKlf3FJ>_vB*)8&#VZ zEN6BgE{F~D0tuNU2M{_;;y!F1iXcvll1vDkeB4IfiI>dJmX{WyVlN;rK|9)QIW{-GzjG*!Yr#Lr5f?gS$hZi)Ag`NtgwkKXV(7XLE&eJjHNas>Pp5Xkiuk{!YE zHM^+CPoAhFPDf5G*gilm8ZB67*#1FuwDs$on_vq1t0deAl-JxF8X6jqo^G3+yAQ!2 z?&bxZe3lbdR#Z3!0E1TJGyGHH{&REB9vle&k4i646g{PjwnI+9k@W+eC>ZuKf|sJB zGq9SAilj+T1NpU@8fpPs+j_3MB8+$oups2Jwf;e4LJy*8DW+y-uuV8&+zw58U?0?_ zb>%7$uNTq*6@}yjv2cZDWx2N*7orEr<0KRtKYuvRaga5HaUc-U2@{-Mak0qih`1JP zKXMcTwWJanZPE11;74^H7itQ+FxscBnSoC6uYiB@H|@G8lVQf11qTt8foKd$+xiL0 zTR8e4HHEMxEO2rRjc69&grdWF%UgJ?^WN0g-<+>S_^=5-FxEQJT?fpK76?T-u?8lV@t@H z6OkUW5s`W_O*UY<6?={INL{6 zQ(%a@ZAn75d4Nh1_FH#Y~j zbe+e{*tVj&DbPpEYg!EtJ6^3?Woh^_qpT|(vI0pzBP(~llm#P^!TH`nXO)+xwrS_o zQF(BEuvXGWgfezd%E zH`+>99l%+zqAkS?Ojh4k;pdX(569H#Cd+@Vi1*$qZ&^txkBNgj&BFz>urKbPkiUK} z+(jhN4}=f{?7w@$f#E8zZK4M-NJ-n*bC|3QzqSdlSM5tP`oF&otM9))7{P3lYy`Oh z>0J+B<}f^bNxqEnDzO6bGZi8v{_8;r|LryWA0PJbrz~WyFpL$%#Rmn6Yw2AP*TS?! zZgBp4@BiZs={)*tagdW_V8!zpEaVDWbi@2&o<7|JOzQ3+j;m6!KA?5am7c#ux+j7D6&+ISqO6EB&!TR#HKO3uZ2H6uS5 z62iYP%)b*MqDdVx<3b*4(#&o=h>-&8QB&ffVj|=f^L)& z9b-TyIV2;u?vXWsD#_fFk`E9-@kI=#5-~=&08N)xk_bhbhuxQ#%-YJ~oiIXRD6vq; z$vSc(ogA_m;MlcmI-fjLLp9?NXS&XHGqi1zNCVvEfLiUHq=9n87J05EEdulSG|Q1J z5w;cqX?_L#_X;_>1vovQ_V3xI(^|jh{0gA-_Hs`doZn+clDlK+-aYOZ{eNF{)q7;| zkpyd2HzT+4gAJz}kS)bwb8bZL$-^!t2^++S2J%eh!yBmq<_QP9@M;MrZ?H?R^A|zU zbNVyu{J+-0Li=LBz;Y!}CL#^zDjOQK?0=Gn0=yR^Yh10xus=%V$dN+cxxWut0Az;? zEthHPRT)K2!@RmNS^R4S1?OAZO0CK3la#TPP}S$t=*SH#Cfc#YLh4((f}S}z1r?Q? zk1Pjut%5-1Vj|DCZm0DM#5}{H6{?bOYsf*z489Snxfe+5ki#l$kI7lHNe;YXMh`J7T{!To|fl`TcfBDeq$a(gAik`bEf}*1725mV9b#x-nzAl7T5feia zJO|@&o3ZYFs}Sqk=h6AZ(ZclU`Td!p5+s$sb0-^wq{!m?_wEToApv%mE5B0*32CR# z(&h4p|9O93U*Dur*fm`CY8Z9cHU*>j7a-T=q6YRgYy7YY;(>~xX(6C21?fP1>wm>; zI2-lkQzr$}s>7Etv7(l#5jjlr(E=Efd4+||Cme8{A(J&dB!5;_MTJvP1#`bBg0O>N z5_pum0nLMg^o)<^whfb#HmFg~?1slBpoOX8Tw9@LvhIf6yNjFD(@U{=t~|DB=!4_o zQO=ClM&@w;(K(CfgFqeMSdPy-fHH zLZ~QdR{a9xW(Kv3Oh%wm1rb07nH`C|YJl~~sHi{?V~xX3VD#nsx5+f{=X|P?00O{P zY_dmbB|d41=Zr)U<2v2pB5ShP56WOked+9UhSu7#o0pe&q^q14K4lFaYv5wON40N7 zEg6judh+dx06Zjl+ByxTj6A;w(cLE7TE?)yN8-{F(x-c_LR=Ewwk^Bb3sPA7V;dF- zb19Fnk;~S3r&m^+gK|vLd#|V)U+&3$;6HV_(}<;l&Frt+i#4{3tERg>xj7 z2`BgoPN}gpT*dsauKF)Xpy#P~_RN_dsxt#$zbe@Ms~X}m$uZrhK5)DvOi5=E%80R} zE4#K?WHbIhao+zwrw7sdI|)XU-i6rQ4wp?skFn42Zt`daaA}fd!Y?JIUkl-^G1=Jv z^}$L1Egmpjb@U>_{%TB2Ak);T^5pAP`&>luBegKr|M@9Q|LuqU8(fH0{jITS3BC2~ z`aSjR_^gu$O)vS8Az~v@wf`?y_U{iP1N(n_9@IZqqAw&S#()SC%v%{3ee#}+{Tl*U zgL2fo?%;4Fyh1$Ypc;R-IP-tPYybbQfGn8*uaC1jATUCX`Jw0& zMDh7EFZS|omnd@CFhDF;4N9%2tm*+*JsYNn;%3F)9o}hSE6?#=e6;t1bH3TJ?+Y1jZrs~@M;F0#z$^VxFRrJQ>A6g$FO|CddVl(Z%-ApCUkIp{G6wf=k zij~K%#G_@HQyx=C4o2dbMzP{Q9fvl9gKXo||HmOxtD%+6zb$S4JKCMrX0MLfee-5D zPAB}+ccj;AB$p=VBqpRrJA6##9m~(E@;5^pLiFLr?b|CTiXYT^6-RT%cfSZ1&RRu(j!1X zM;HkUveY*I4_|pYDCaM-QH|H5mT{yO6|D^8J>?@|idY|ashb@FUr-8FaBFy~)!&0% zY91-89HJ|6u-NikPS;FtaAk)6UgPM}FaBMZ*?&%EeH=2lR+U3o0IvPU->FFqI+VUb zQj(&sUfHZy?(lK5<>%r8!5PalhmT6Ewx1c#d_x)P|6E9AYDOg^GZi0) zuk=nmzSEDI3U|d9t}Qvg^2}!gv*}6Oo?wBmFCRoxOGpe=x-ENmOkZVh+2)>;<9qAa zaoMbhzxSHfRsWt=!lISgK3ciIuim(|*nviAE~)SLy?bdUqr0lcI&7{OWPjb}7!_FT zHri~#MD@2{wd{#vc=fiK*-qvh2JDP-4#RIPayGxF_uf#r8<86*N5q*!@mCh9?OaQE1xej-M+1N`46w-?UnLcT2tKu zjJO)GJo$68PE&>_$RWFC`Ylm=FDu-LWtNqv!O_IBIMx;mF%F~z<+5CdR_wA*>8iLS z^|QrrNbqhn!&@X&+Bi@dz9X;M3Ne3X^iOTUkqyM6ikdT4tFKUpTiT;vpb z%F>n0-XVsq$EgE?cyi43mecfxlr@)p=KNHHeX~uzZFBslYGS(n!}I6!;~gDC7W})` zGF&?IxLD$|j7MQ<#DiPMI&KRk@e)N|E91or0SAJSYI&fmfx z62A(MZ@L>;l(ezuaYJE4f_9qf@qPO`if^#?ER4VLd-(Xmw}k9V7pAxd`H{_%-M5oX z+J%&tu_|r4v?ro#D<=nsY}d~k6reQ46w(hcQaKxT1A$lVGN2&ME!!w-C(d zFtyxatsMP)D%&o)a(XBe`dR7Q`l!m=qET@t7CrDxhi0GUeiqB=k!y59B~Q}_ocsG5 z=ac`{;SY!@nmze;aAsG6CVQ*Lq&D5wy`2TqttLZW-$dTD0NHJAw+(l8(F=Y|M`QBb z`_xOtxs6VGY)h@M{?Pg-12YwiFwto5IJ3abl8f{ElPw)AiVxbzc5hQhS|Kwb{g>ss!y{nOZQ_@+JIHk%8uTv4~+qQ}4(g?)d zq{Z@%jFgnt*Roy1nZoS+tp?e-B`L|9G$LZUM)ao7j3)L5@XGkIDC=o;P*H3esF!Ee zzYXA^)W~>e8O>_0dV#27+hfjW?7NVqe{1{Gr?%T!ExbEgw4YM({@Im-Z_#muc)-@a1|cO0NK zzea6N!`je%P#2oxSfpcJ);|-XbljkvtI*o|Ms9Zo=7a{6vD+GT!$1;X^ufWptYN zbvZ|Ny)o+eZ20VZT7*`I^X}c1SJ~&E2@6qkkBD*nhwwq3IA~*SEf(8w_Vvv=nYjBq z-!#x&yWXvx{H$Ldw~0|Ag#6uZ&GhT=zC_y}Ms6 zD3vuaG4S+3;_2eCk^ed=l8~vYXTNjj5C=W~AFkZvS1jLf29*UR<*|l@=eQ?!elfND zb#m&J?q0qMvH9^gUFLl4DymoJe!nSxS(l+6URo+y$ksv!wt|P9V4Ivk2aApU1!%F$ zK4`PZ@BFL{qESQpyTy4!@mL}h@w7JvoUOa?5duD2Jr`e#pjmWK|6+7dV^Y^FpcklJ zBTYRq5fv=AxAWTFn~4%_H$?KY8yNnd(#|}Z>a>C5SCX=fOpz-~GAZP(muMlkLTFOx z+Pj9i++>L<7a>b3+U8CRHDoE-F0w=m*`gFvOm(lNlv30cVo>(?`=s78=ghqCdCz(O z>Ksnj{cX?hdA{Gz_xp%xBneWTR>Lgixr>QMK1|f4)kX`j<9iQgQu_{F&RE7U|loSKpJdP2iK3CMdSkJ=4wFRnOVq z`D>v|(`9pqip>^&n#!$p90iex?xvv-nz7q6y!hlJQx>sf%q)0H20~|t3dF^@V+p_OVzSHXXbrbH(9mVZQ9#MHLTEsS4Chnk~-744x#% zvGUJj1&i+OcDY{?=tRu%Gb}aFq&{;K^2h0W3mi+W}H^VDlr37>w30})h`=0k^zSvNq zZS0V*X2l5^%GN}#VP{osT0lVSMXlvq@dy|XX2N-u!2*_=Fj_uPGchJGz56C(6gt0m z1bl7pthPLhR4=>s!FPpC)zR6emkn?9W_C7oJ{SttH#X*+c$_%bDs8KU8KP++5Na~tO( zs?+ru;ZwO~+M8HvjcMY3!`==pFTx%>e`3sQUFZLc?^Nf)%172^c_zoBqxHPJ+S;U0 zQsat0xU=G$ou#(@C8>Nq8Ohzd8k)ZGlE3a-|3k%4T(HuO0nSzoD^lB}u>#kvbuo3T zId(ja`^q#i0grOG*yc{RA_nzJ6BwXcoXZA zpBkK2&S}osPF`@$Vqy{IXX=J&L-Xe@bm~7;qo+6E(o|j+Y&9ad3tjk!YZWVpa={kj0QP`uii>rJ*{W6709cX0vNN&Ag&o5-T zr#<&|wff0rN7j=w12ZZ+AFu;`iExTfT2WJzxis`ZK3iSartCnyY{#6yH|GjwL`+R= zFMCZNnG@*yoL;ej&lkk4vam1p+sLprn%BC@HTSY{nIxrrQr^By-h+26)fA7AcxrOW z8p!X+48tujf&CBh^$4P1c4I-?EOWxQENOqI?u%NOt3U7FySIA(@{m_E{U$?kUS4-) zpUZu8D7PzFG;G9VOrf(=FlNZ+de8URd12JvqB|kV$RT?nR_Ph>Zqlhz0gw{0!X&f| z@a;ylp&_$9U(4xQ4!>U_V06I((V3!lj6HqM)fPG86g zz7tjK#Z&oE6)Wvn{G$hCjVlH!C?PzCFFb<+C~wiBh*P^-W-TQX5YN;i`oyQJq1}$b z>%fM(yZXE~Ywt?dq?zvdhl_9CWJp7jAg=gWT>f6RmZm7Bngd^Gc{$+Wp(*)1PJhM5oTL8dv3B911$b*Q zw1e_Bmo1CnaHP=v7SSjcT#MR4q#mgD<)Led5RC{NfdXM~Zosz_qF~iboSt<7Q00gO z0oRvUb;!`e!Cedi5bAAk#cFO6M!GdyEOLW9zp+p*OT#JOl{A(=9e5Pqj z2Nn+OwI`omukUoicj$h)iv=NaRb*~xvs01TdXw48oNuMA4%p<4HbX7kOKD#JsCdUp&T@Df2p*rtnJImKLHHwot<|L1A^13GUE5D6V-CG z>{FbNamQ1wA9Pr!Su63=w~1W3^tx-cu9mn@f}T)sub?M??Lt#pr@UeUMd;cFpQUzo4w zzbt>3lwY5-1J%xMk@mNrI;48?(_KBeVNbYo0#r@8O)6SbH=g`n@T!qtkowT~IP@l4|o9p7kcn150^&-s!VrSqYP>_6xn?c948*1A_ce7qy1$YZ3kvxgzu z>>TDWK{V&s?53)R(^~tzd-s0vX#hKhlDtSy_gi$0MLIhan@a-iR+x)Wy7@xcwpm6y zQKZF=8b}J7fqCUrnS%~lI0>Ojrn+^C6DUNvHF0eery|Bb)mc;@pg1~?Aj#XxTVAxz zA`T-29Kld4BrVbyRygjrMKQZxq)#_hUsrYA$3D@p37h8wwuVKDG-p}9wsr(A6mQ(p%&=^`Kbm=_CC-}HxFUT*>>FFmt!QzKqPH(yd%BW& zTAmTXK94xGE1@e79v3GMAH?X*I-Eq6ci9|9=yx!1YCW1ysHI$pTC3p5fM)>VKx8CPjrqUtHojGBQum}>T&{YC}Q0rOJzFwGk zk2VX=>Etovak0Eo49zZD(pb43RUt%Ei$>QQGd&%v7fO#`#(qWW{X1{r4`g;?_A4zT zBZ9_a1bDm0pM758V9M_s$d294CUQ4@o`Qju2tz7q89HdEHTt7bOXX+p*ntTSes5e- zIqk`Q-g4lJ#JnkaT0k0%j9SV~X{aK)wy{l?xvVQPLQrh}ZnuIj|MlQ!1qgz8X0rwn zqG5&bAdMgtj7qHbcp^nH$w;aZrRemY3&{Kz z1nFRVfHq4^{^88+mc?e5Lm>RE+50-?V)jOMO!Cdi{`LIyzK_PUBT?{l`_3H^H8rU| zlUkhk^XJdQJI2zipt#sQU-i*H0p3lPdb)l3wOjcG(}&%kMCVy&RN>&?klDXDQJ)f4g1d^#JMuBM$l<<={6%fL|{`)7%i!QFZ z`NRgKWH9`)-f}G!Uv*sx59KIJVnJo41x{A-#et8BjSztwWA5MhV4im;Gqp7{ixf9(+WOE6G7udnh<+YMX>IzrCaLZId zDGp{(i~SIq#%o0s1#qM-9^DR>t|O#A5l19Iz&;fH&NP|>oR=xr4W2-(Yf8aF>`TF)n$905w z*UWtP8*(r;`x@3S1?+%|bIfUTwMjA+P8z|>0PV$~8JzcN9Y_lqV#jYhc~nAEM`s6m z*N1x^w`R~a4s$phP+#-l@IM+xnnN7#l73L~Hv1>*CKYGl6Toq_6vL6O!CCjBhfg}? zl;H+Aj`ZAapRO+F5S&&SL~-Uq(U-;rSO5|!J73`iv*WI;z9Ls}or+8L-MNGWaSBP^ z@C`v%5;~RSEYU3{4qg<+NF6A3sv(g{1dJUSS3yeRGyn=b#n7B~iHl36xq65|!Nyer z;X#n))`o_%VO*6D@2@u0^%g7+<(y2wh8!+iNk1&$f4h=!G{BxW*spyT9i0?1b0PgT z2z<%HuboBtZpjj!iOo?7luft--+SAXrwp#o)|CpZlXNj4EO8qzPEzoO?jeLLa`;VU ze*p+bJd+Vxp9wH&!^Vw}Lm35d55=OyvQU{WhHN2PVGMzR-|2WA^k@f`l#$PZTM>nne9Wblkqcz*@v7$;J+3YX(Ac z&KEOLwE>Ja~t2XkzpKjXyuw{!FnNK533T%Yi zTHC)~Z(HbmCsU>2gx?Uj%RZ2V!@Kq2wm1oWp=PVj(oFIZ;(+~NGJZtV5+RZU9xa&3 z-5PH7B6tF7b4dx``bWiz3JTczg3H@zFzsV-{P%D#AZbrh#UKTj@4PV={t=o0%+ubC z;Hw~-10(5ow3#3lP=(wdZo_H-baP;SEsL!amqWVMLA7n%x>bSziKsEc9s?PCHu7c>FwyX{G#~g*z*WXS z(xZVbVwSG)thS+yHR~G%vZscMy1MzO0ERpPW^hkb0e9_|E3dtKobzM%Y>Y+Lw6L}t zKdBO?5x|C7E)Qx5DKE*%LG}eCF<#7(@Q%n4!H&UWP=)s8qd<`9&9{+@A;6+}9%eeH z&mSFym zMjHd9A`TPMN)DaI;C=$h^`w^p?gM^rwwlbVe(ORl<8O9(_*U7`*vNSAYXn!4?!=AxHtBo=TizkL32(H4eN`WGt3o zy}_u}CFmUUt1~ifp8Ows3QfPI!6{UsM0h1y773ywPyN%o>#G4XXgo~?WAqXMxby+Q zwp_^;W4G7Du^4pw`~05-Lkl|Vv}F%7`vLw4@(j|-$}4=tXV0!AHwec1ClR`;i^al}#=O{{`^oSxo={ diff --git a/examples/Hdiv-mixed/include/post-processing.h b/examples/Hdiv-mixed/include/post-processing.h new file mode 100644 index 0000000000..5a121f0352 --- /dev/null +++ b/examples/Hdiv-mixed/include/post-processing.h @@ -0,0 +1,21 @@ +#ifndef post_processing_h +#define post_processing_h + +#include +#include + +#include "structs.h" +#include "../include/setup-libceed.h" +PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, + CeedMemType mem_type_backend, + TS ts, SNES snes, KSP ksp, + Vec U, CeedScalar l2_error_u, + CeedScalar l2_error_p); +PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, + CeedData ceed_data, OperatorApplyContext ctx_Hdiv); +PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, + CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_H1); +PetscErrorCode ProjectVelocity(AppCtx app_ctx, + Vec U, Vec *U_H1); +PetscErrorCode CtxVecDestroy(AppCtx app_ctx); +#endif // post_processing_h diff --git a/examples/Hdiv-mixed/include/register-problem.h b/examples/Hdiv-mixed/include/register-problem.h index ec623dd49b..9d54e26794 100644 --- a/examples/Hdiv-mixed/include/register-problem.h +++ b/examples/Hdiv-mixed/include/register-problem.h @@ -9,15 +9,21 @@ PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx); // Set up problems function prototype // ----------------------------------------------------------------------------- // 1) darcy2d -PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, void *ctx); +PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, DM dm, + void *ctx); // 2) darcy3d -PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, void *ctx); +PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, DM dm, + void *ctx); // 3) darcy3dprism // 4) richard -PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx); +PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, DM dm, + void *ctx); + +PetscErrorCode Hdiv_RICHARD3D(Ceed ceed, ProblemData problem_data, DM dm, + void *ctx); extern int FreeContextPetsc(void *); diff --git a/examples/Hdiv-mixed/include/setup-dm.h b/examples/Hdiv-mixed/include/setup-dm.h index 610d74efd0..b61ac7e17d 100644 --- a/examples/Hdiv-mixed/include/setup-dm.h +++ b/examples/Hdiv-mixed/include/setup-dm.h @@ -10,7 +10,9 @@ // --------------------------------------------------------------------------- // Setup DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDM(MPI_Comm comm, VecType vec_type, DM *dm); +PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, + VecType vec_type, DM *dm); PetscErrorCode PerturbVerticesSmooth(DM dm); +PetscErrorCode PerturbVerticesRandom(DM dm); #endif // setupdm_h diff --git a/examples/Hdiv-mixed/include/setup-fe.h b/examples/Hdiv-mixed/include/setup-fe.h index ca26e65df3..24dcda84ab 100644 --- a/examples/Hdiv-mixed/include/setup-fe.h +++ b/examples/Hdiv-mixed/include/setup-fe.h @@ -10,6 +10,7 @@ // --------------------------------------------------------------------------- // Setup FE // --------------------------------------------------------------------------- -PetscErrorCode SetupFE(MPI_Comm comm, DM dm, DM dm_u0, DM dm_p0); - +PetscErrorCode SetupFEHdiv(MPI_Comm comm, DM dm, DM dm_u0, DM dm_p0); +PetscErrorCode SetupFEH1(ProblemData problem_data, + AppCtx app_ctx, DM dm_H1); #endif // setupfe_h diff --git a/examples/Hdiv-mixed/include/setup-libceed.h b/examples/Hdiv-mixed/include/setup-libceed.h index 03ecefcaaa..aac44d76fd 100644 --- a/examples/Hdiv-mixed/include/setup-libceed.h +++ b/examples/Hdiv-mixed/include/setup-libceed.h @@ -19,8 +19,8 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, CeedElemRestriction *elem_restr_u, CeedElemRestriction *elem_restr_p, CeedElemRestriction *elem_restr_u0, CeedElemRestriction *elem_restr_p0); // Set up libCEED for a given degree -PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, Ceed ceed, - AppCtx app_ctx, +PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, + Ceed ceed, AppCtx app_ctx, ProblemData problem_data, CeedData ceed_data); #endif // setuplibceed_h diff --git a/examples/Hdiv-mixed/include/setup-solvers.h b/examples/Hdiv-mixed/include/setup-solvers.h index f615d287e9..c7a40a1722 100644 --- a/examples/Hdiv-mixed/include/setup-solvers.h +++ b/examples/Hdiv-mixed/include/setup-solvers.h @@ -4,9 +4,11 @@ #include #include +#include "petscvec.h" #include "structs.h" PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, + VecType vec_type, OperatorApplyContext ctx_jacobian); PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual); @@ -15,14 +17,9 @@ PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, PetscErrorCode ApplyMatOp(Mat A, Vec X, Vec Y); PetscErrorCode SNESFormResidual(SNES snes, Vec X, Vec Y, void *ctx); PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, void *ctx); -PetscErrorCode PDESolver(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, - VecType vec_type, SNES snes, KSP ksp, Vec *U); -PetscErrorCode ComputeL2Error(DM dm, Ceed ceed, CeedData ceed_data, Vec U, +PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, + SNES snes, KSP ksp, Vec *U); +PetscErrorCode ComputeL2Error(CeedData ceed_data, AppCtx app_ctx, Vec U, CeedScalar *l2_error_u, CeedScalar *l2_error_p); -PetscErrorCode PrintOutput(Ceed ceed, AppCtx app_ctx, PetscBool has_ts, - CeedMemType mem_type_backend, - TS ts, SNES snes, KSP ksp, - Vec U, CeedScalar l2_error_u, - CeedScalar l2_error_p); #endif // setup_solvers_h diff --git a/examples/Hdiv-mixed/include/setup-ts.h b/examples/Hdiv-mixed/include/setup-ts.h index 4da1ea4d18..d461ba5675 100644 --- a/examples/Hdiv-mixed/include/setup-ts.h +++ b/examples/Hdiv-mixed/include/setup-ts.h @@ -6,11 +6,8 @@ #include "structs.h" -PetscErrorCode CreateInitialConditions(CeedData ceed_data, - Vec U, VecType vec_type, - OperatorApplyContext ctx_initial_u0, - OperatorApplyContext ctx_initial_p0, - OperatorApplyContext ctx_residual_ut); +PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, + VecType vec_type, Vec U); PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual_ut); PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm, Ceed ceed, @@ -19,7 +16,7 @@ PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_initial_p0); PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, void *ctx_residual_ut); -PetscErrorCode TSSolveRichard(DM dm, CeedData ceed_data, AppCtx app_ctx, - Vec *U, TS *ts); +PetscErrorCode TSSolveRichard(CeedData ceed_data, AppCtx app_ctx, + TS ts, Vec *U); #endif // setup_ts_h diff --git a/examples/Hdiv-mixed/include/structs.h b/examples/Hdiv-mixed/include/structs.h index 9713e485eb..63d97eddce 100644 --- a/examples/Hdiv-mixed/include/structs.h +++ b/examples/Hdiv-mixed/include/structs.h @@ -4,35 +4,20 @@ #include #include -// Application context from user command line options -typedef struct AppCtx_ *AppCtx; -struct AppCtx_ { - MPI_Comm comm; - // Degree of polynomial (1 only), extra quadrature pts - PetscInt degree; - PetscInt q_extra; - // For applying traction BCs - PetscInt bc_pressure_count; - PetscInt bc_faces[16]; // face ID - PetscScalar bc_pressure_value[16]; - // Problem type arguments - PetscFunctionList problems; - char problem_name[PETSC_MAX_PATH_LEN]; - CeedScalar t_final, t; -}; - // PETSc operator contexts typedef struct OperatorApplyContext_ *OperatorApplyContext; struct OperatorApplyContext_ { MPI_Comm comm; Vec X_loc, Y_loc, X_t_loc; - CeedVector x_ceed, y_ceed, x_t_ceed; - CeedOperator op_apply; + CeedVector x_ceed, y_ceed, x_t_ceed, x_coord, rhs_ceed_H1; + CeedOperator op_apply, op_rhs_H1; DM dm; Ceed ceed; CeedScalar t, dt; CeedContextFieldLabel solution_time_label, final_time_label, - timestep_label; ; + timestep_label; + CeedElemRestriction elem_restr_u_H1; + VecType vec_type; }; // libCEED data struct @@ -41,32 +26,54 @@ struct CeedData_ { CeedBasis basis_x, basis_u, basis_p, basis_u_face; CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_U_i, elem_restr_p, elem_restr_p_i, elem_restr_u0, - elem_restr_p0; + elem_restr_p0, elem_restr_u_H1; CeedQFunction qf_residual, qf_jacobian, qf_error, qf_ics_u, qf_ics_p, - qf_rhs_u0, qf_rhs_p0; + qf_rhs_u0, qf_rhs_p0, qf_rhs_H1, qf_post_mass; CeedOperator op_residual, op_jacobian, op_error, op_ics_u, op_ics_p, - op_rhs_u0, op_rhs_p0; + op_rhs_u0, op_rhs_p0, op_rhs_H1, op_post_mass; CeedVector x_ceed, y_ceed, x_coord, x_t_ceed, rhs_u0_ceed, - u0_ceed, v0_ceed, rhs_p0_ceed, p0_ceed, q0_ceed; - OperatorApplyContext ctx_residual, ctx_jacobian, ctx_error, ctx_residual_ut, - ctx_initial_u0, ctx_initial_p0; + u0_ceed, v0_ceed, rhs_p0_ceed, p0_ceed, q0_ceed, + rhs_ceed_H1, u_ceed, up_ceed, vp_ceed; CeedInt num_elem; }; +// Application context from user command line options +typedef struct AppCtx_ *AppCtx; +struct AppCtx_ { + char ceed_resource[PETSC_MAX_PATH_LEN]; // libCEED backend + MPI_Comm comm; + // Degree of polynomial (1 only), extra quadrature pts + PetscInt degree; + PetscInt q_extra; + // For applying traction BCs + PetscInt bc_pressure_count; + PetscInt bc_faces[16]; // face ID + PetscScalar bc_pressure_value[16]; + // Problem type arguments + PetscFunctionList problems; + char problem_name[PETSC_MAX_PATH_LEN]; + CeedScalar t_final, t; + PetscBool view_solution, quartic; + char output_dir[PETSC_MAX_PATH_LEN]; + PetscInt output_freq; + OperatorApplyContext ctx_residual, ctx_jacobian, ctx_error, ctx_residual_ut, + ctx_initial_u0, ctx_initial_p0, ctx_Hdiv, ctx_H1; +}; + // Problem specific data typedef struct ProblemData_ *ProblemData; struct ProblemData_ { CeedQFunctionUser true_solution, residual, jacobian, error, ics_u, ics_p, - bc_pressure, rhs_u0, rhs_p0; + bc_pressure, rhs_u0, rhs_p0, post_rhs, post_mass; const char *true_solution_loc, *residual_loc, *jacobian_loc, *error_loc, *bc_pressure_loc, *ics_u_loc, *ics_p_loc, *rhs_u0_loc, - *rhs_p0_loc; + *rhs_p0_loc, *post_rhs_loc, *post_mass_loc; CeedQuadMode quadrature_mode; CeedInt elem_node, dim, q_data_size_face; CeedQFunctionContext true_qfunction_ctx, error_qfunction_ctx, residual_qfunction_ctx, jacobian_qfunction_ctx, rhs_u0_qfunction_ctx ; - PetscBool has_ts; + PetscBool has_ts, view_solution, quartic; }; #endif // structs_h \ No newline at end of file diff --git a/examples/Hdiv-mixed/main.c b/examples/Hdiv-mixed/main.c index cf8a851b5e..814fd086f8 100644 --- a/examples/Hdiv-mixed/main.c +++ b/examples/Hdiv-mixed/main.c @@ -24,10 +24,14 @@ // Build with: make // Run with: // ./main -pc_type svd -problem darcy2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 +// ./main -pc_type none -problem darcy2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 -ksp_type minres // ./main -pc_type svd -problem darcy3d -dm_plex_dim 3 -dm_plex_box_faces 4,4,4 // ./main -pc_type svd -problem darcy3d -dm_plex_filename /path to the mesh file +// ./main -pc_type svd -problem richard2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 +// (boundary is not working) // ./main -pc_type svd -problem darcy2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 -bc_pressure 1 // ./main -pc_type svd -problem darcy2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 -bc_pressure 1,2,3,4 +// ./main -pc_type svd -problem darcy3d -dm_plex_dim 3 -dm_plex_box_faces 4,4,4 -view_solution -dm_plex_box_lower 0,0,0 -dm_plex_box_upper 1,0.1,1 const char help[] = "Solve H(div)-mixed problem using PETSc and libCEED\n"; #include "main.h" @@ -38,56 +42,99 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- PetscCall( PetscInitialize(&argc, &argv, NULL, help) ); + // --------------------------------------------------------------------------- + // Create structs + // --------------------------------------------------------------------------- + AppCtx app_ctx; + PetscCall( PetscCalloc1(1, &app_ctx) ); + + ProblemData problem_data = NULL; + PetscCall( PetscCalloc1(1, &problem_data) ); + + CeedData ceed_data; + PetscCall( PetscCalloc1(1, &ceed_data) ); + + OperatorApplyContext ctx_jacobian, ctx_residual, ctx_residual_ut, + ctx_initial_u0, ctx_initial_p0, + ctx_error, ctx_Hdiv, ctx_H1; + PetscCall( PetscCalloc1(1, &ctx_jacobian) ); + PetscCall( PetscCalloc1(1, &ctx_residual) ); + PetscCall( PetscCalloc1(1, &ctx_residual_ut) ); + PetscCall( PetscCalloc1(1, &ctx_initial_u0) ); + PetscCall( PetscCalloc1(1, &ctx_initial_p0) ); + PetscCall( PetscCalloc1(1, &ctx_error) ); + PetscCall( PetscCalloc1(1, &ctx_Hdiv) ); + PetscCall( PetscCalloc1(1, &ctx_H1) ); + // Context for Darcy problem + app_ctx->ctx_residual = ctx_residual; + app_ctx->ctx_jacobian = ctx_jacobian; + // Context for Richards problem + app_ctx->ctx_residual_ut = ctx_residual_ut; + // Context for initial velocity + app_ctx->ctx_initial_u0 = ctx_initial_u0; + // Context for initial pressure + app_ctx->ctx_initial_p0 = ctx_initial_p0; + // Context for MMS + app_ctx->ctx_error = ctx_error; + // Context for post-processing + app_ctx->ctx_Hdiv = ctx_Hdiv; + app_ctx->ctx_H1 = ctx_H1; + // --------------------------------------------------------------------------- // Initialize libCEED // --------------------------------------------------------------------------- // -- Initialize backend Ceed ceed; CeedInit("/cpu/self/ref/serial", &ceed); + //CeedInit(app_ctx->ceed_resource, &ceed); CeedMemType mem_type_backend; CeedGetPreferredMemType(ceed, &mem_type_backend); - VecType vec_type = NULL; + VecType vec_type = NULL; + MatType mat_type = NULL; switch (mem_type_backend) { case CEED_MEM_HOST: vec_type = VECSTANDARD; break; case CEED_MEM_DEVICE: { const char *resolved; CeedGetResource(ceed, &resolved); if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; - else if (strstr(resolved, "/gpu/hip/occa")) - vec_type = VECSTANDARD; // https://github.com/CEED/libCEED/issues/678 - else if (strstr(resolved, "/gpu/hip")) vec_type = VECHIP; + else if (strstr(resolved, "/gpu/hip")) vec_type = VECKOKKOS; else vec_type = VECSTANDARD; } } + if (strstr(vec_type, VECCUDA)) mat_type = MATAIJCUSPARSE; + else if (strstr(vec_type, VECKOKKOS)) mat_type = MATAIJKOKKOS; + else mat_type = MATAIJ; + // -- Process general command line options + MPI_Comm comm = PETSC_COMM_WORLD; // --------------------------------------------------------------------------- - // Create structs + // Create DM // --------------------------------------------------------------------------- - AppCtx app_ctx; - PetscCall( PetscCalloc1(1, &app_ctx) ); - - ProblemData problem_data = NULL; - PetscCall( PetscCalloc1(1, &problem_data) ); + DM dm, dm_u0, dm_p0, dm_H1; + // DM for mixed problem + PetscCall( CreateDM(comm, mat_type, vec_type, &dm) ); + // DM for projecting initial velocity to Hdiv space + PetscCall( CreateDM(comm, mat_type, vec_type, &dm_u0) ); + // DM for projecting initial pressure in L2 + PetscCall( CreateDM(comm, mat_type, vec_type, &dm_p0) ); + // DM for projecting solution U into H1 space for PetscViewer + PetscCall( CreateDM(comm, mat_type, vec_type, &dm_H1) ); + // TODO: add mesh option + // perturb dm to have smooth random mesh + //PetscCall( PerturbVerticesSmooth(dm) ); + //PetscCall( PerturbVerticesSmooth(dm_H1) ); - CeedData ceed_data; - PetscCall( PetscCalloc1(1, &ceed_data) ); + // perturb dm to have random mesh + //PetscCall(PerturbVerticesRandom(dm) ); + //PetscCall(PerturbVerticesRandom(dm_H1) ); - OperatorApplyContext ctx_residual_ut, ctx_initial_u0, ctx_initial_p0; - PetscCall( PetscCalloc1(1, &ctx_residual_ut) ); - PetscCall( PetscCalloc1(1, &ctx_initial_u0) ); - PetscCall( PetscCalloc1(1, &ctx_initial_p0) ); - ceed_data->ctx_residual_ut = ctx_residual_ut; - ceed_data->ctx_initial_u0 = ctx_initial_u0; - ceed_data->ctx_initial_p0 = ctx_initial_p0; // --------------------------------------------------------------------------- // Process command line options // --------------------------------------------------------------------------- // -- Register problems to be available on the command line PetscCall( RegisterProblems_Hdiv(app_ctx) ); - // -- Process general command line options - MPI_Comm comm = PETSC_COMM_WORLD; app_ctx->comm = comm; PetscCall( ProcessCommandLineOptions(app_ctx) ); @@ -95,46 +142,68 @@ int main(int argc, char **argv) { // Choose the problem from the list of registered problems // --------------------------------------------------------------------------- { - PetscErrorCode (*p)(Ceed, ProblemData, void *); + PetscErrorCode (*p)(Ceed, ProblemData, DM, void *); PetscCall( PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p) ); if (!p) SETERRQ(PETSC_COMM_SELF, 1, "Problem '%s' not found", app_ctx->problem_name); - PetscCall( (*p)(ceed, problem_data, &app_ctx) ); + PetscCall( (*p)(ceed, problem_data, dm, &app_ctx) ); } // --------------------------------------------------------------------------- - // Create DM - // --------------------------------------------------------------------------- - DM dm, dm_u0, dm_p0; - PetscCall( CreateDM(comm, vec_type, &dm) ); - PetscCall( CreateDM(comm, vec_type, &dm_u0) ); - PetscCall( CreateDM(comm, vec_type, &dm_p0) ); - // TODO: add mesh option - // perturb to have smooth random mesh - //PetscCall( PerturbVerticesSmooth(dm) ); - - // --------------------------------------------------------------------------- - // Setup FE + // Setup FE for H(div) mixed-problem and H1 projection in post-processing.c // --------------------------------------------------------------------------- - SetupFE(comm, dm, dm_u0, dm_p0); + PetscCall( SetupFEHdiv(comm, dm, dm_u0, dm_p0) ); + PetscCall( SetupFEH1(problem_data, app_ctx, dm_H1) ); // --------------------------------------------------------------------------- - // Create local Force vector + // Create global unkown solution // --------------------------------------------------------------------------- - Vec U; // U=[p,u], U0=u0 + Vec U; // U=[p,u] PetscCall( DMCreateGlobalVector(dm, &U) ); - PetscCall( VecZeroEntries(U) ); // --------------------------------------------------------------------------- // Setup libCEED // --------------------------------------------------------------------------- // -- Set up libCEED objects - PetscCall( SetupLibceed(dm, dm_u0, dm_p0, ceed, app_ctx, problem_data, - ceed_data) ); - //CeedVectorView(force_ceed, "%12.8f", stdout); - //PetscCall( DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, - // bc_pressure) ); + PetscCall( SetupLibceed(dm, dm_u0, dm_p0, dm_H1, ceed, app_ctx, + problem_data, ceed_data) ); + + // --------------------------------------------------------------------------- + // Setup pressure boundary conditions + // --------------------------------------------------------------------------- + // --Create empty local vector for libCEED + Vec P_loc; + PetscInt P_loc_size; + CeedScalar *p0; + CeedVector P_ceed; + PetscMemType pressure_mem_type; + PetscCall( DMCreateLocalVector(dm, &P_loc) ); + PetscCall( VecGetSize(P_loc, &P_loc_size) ); + PetscCall( VecZeroEntries(P_loc) ); + PetscCall( VecGetArrayAndMemType(P_loc, &p0, &pressure_mem_type) ); + CeedVectorCreate(ceed, P_loc_size, &P_ceed); + CeedVectorSetArray(P_ceed, MemTypeP2C(pressure_mem_type), CEED_USE_POINTER, + p0); + // -- Apply operator to create local pressure vector on boundary + PetscCall( DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, + P_ceed) ); + //CeedVectorView(P_ceed, "%12.8f", stdout); + // -- Map local to global + Vec P; + CeedVectorTakeArray(P_ceed, MemTypeP2C(pressure_mem_type), NULL); + PetscCall( VecRestoreArrayAndMemType(P_loc, &p0) ); + PetscCall( DMCreateGlobalVector(dm, &P) ); + PetscCall( VecZeroEntries(P) ); + PetscCall( DMLocalToGlobal(dm, P_loc, ADD_VALUES, P) ); + + // --------------------------------------------------------------------------- + // Setup context for projection problem; post-processing.c + // --------------------------------------------------------------------------- + PetscCall( SetupProjectVelocityCtx_Hdiv(comm, dm, ceed, ceed_data, + app_ctx->ctx_Hdiv) ); + PetscCall( SetupProjectVelocityCtx_H1(comm, dm_H1, ceed, ceed_data, + vec_type, app_ctx->ctx_H1) ); // --------------------------------------------------------------------------- // Setup TSSolve for Richard problem @@ -142,61 +211,77 @@ int main(int argc, char **argv) { TS ts; if (problem_data->has_ts) { // --------------------------------------------------------------------------- - // Create global initial conditions + // Setup context for initial conditions // --------------------------------------------------------------------------- - SetupResidualOperatorCtx_U0(comm, dm_u0, ceed, ceed_data, - ceed_data->ctx_initial_u0); - SetupResidualOperatorCtx_P0(comm, dm_p0, ceed, ceed_data, - ceed_data->ctx_initial_p0); - SetupResidualOperatorCtx_Ut(comm, dm, ceed, ceed_data, - ceed_data->ctx_residual_ut); - CreateInitialConditions(ceed_data, U, vec_type, - ceed_data->ctx_initial_u0, - ceed_data->ctx_initial_p0, - ceed_data->ctx_residual_ut); + PetscCall( SetupResidualOperatorCtx_U0(comm, dm_u0, ceed, ceed_data, + app_ctx->ctx_initial_u0) ); + PetscCall( SetupResidualOperatorCtx_P0(comm, dm_p0, ceed, ceed_data, + app_ctx->ctx_initial_p0) ); + PetscCall( SetupResidualOperatorCtx_Ut(comm, dm, ceed, ceed_data, + app_ctx->ctx_residual_ut) ); + PetscCall( CreateInitialConditions(ceed_data, app_ctx, vec_type, U) ); //VecView(U, PETSC_VIEWER_STDOUT_WORLD); // Solve Richards problem - PetscCall( VecZeroEntries(ceed_data->ctx_residual_ut->X_loc) ); - PetscCall( VecZeroEntries(ceed_data->ctx_residual_ut->X_t_loc) ); - PetscCall( TSSolveRichard(dm, ceed_data, app_ctx, - &U, &ts) ); + PetscCall( TSCreate(comm, &ts) ); + PetscCall( VecZeroEntries(app_ctx->ctx_residual_ut->X_loc) ); + PetscCall( VecZeroEntries(app_ctx->ctx_residual_ut->X_t_loc) ); + PetscCall( TSSolveRichard(ceed_data, app_ctx, ts, &U) ); //VecView(U, PETSC_VIEWER_STDOUT_WORLD); } + // --------------------------------------------------------------------------- + // Setup SNES for Darcy problem + // --------------------------------------------------------------------------- SNES snes; KSP ksp; if (!problem_data->has_ts) { - // --------------------------------------------------------------------------- - // Setup SNES for Darcy problem - // --------------------------------------------------------------------------- + PetscCall( SetupJacobianOperatorCtx(dm, ceed, ceed_data, vec_type, + app_ctx->ctx_jacobian) ); + PetscCall( SetupResidualOperatorCtx(dm, ceed, ceed_data, + app_ctx->ctx_residual) ); // Create SNES PetscCall( SNESCreate(comm, &snes) ); PetscCall( SNESGetKSP(snes, &ksp) ); - PetscCall( PDESolver(comm, dm, ceed, ceed_data, vec_type, snes, ksp, &U) ); + PetscCall( PDESolver(ceed_data, app_ctx, snes, ksp, &U) ); //VecView(U, PETSC_VIEWER_STDOUT_WORLD); } // --------------------------------------------------------------------------- // Compute L2 error of mms problem // --------------------------------------------------------------------------- + PetscCall( SetupErrorOperatorCtx(dm, ceed, ceed_data, app_ctx->ctx_error) ); CeedScalar l2_error_u, l2_error_p; - PetscCall( ComputeL2Error(dm, ceed, ceed_data, U, &l2_error_u, - &l2_error_p) ); + PetscCall( ComputeL2Error(ceed_data, app_ctx, U, + &l2_error_u, &l2_error_p) ); // --------------------------------------------------------------------------- - // Print output results + // Print solver iterations and final norms // --------------------------------------------------------------------------- - PetscCall( PrintOutput(ceed, app_ctx, problem_data->has_ts, mem_type_backend, + PetscCall( PrintOutput(dm, ceed, app_ctx, problem_data->has_ts, + mem_type_backend, ts, snes, ksp, U, l2_error_u, l2_error_p) ); // --------------------------------------------------------------------------- // Save solution (paraview) // --------------------------------------------------------------------------- - PetscViewer viewer; - PetscCall( PetscViewerVTKOpen(comm,"solution.vtu",FILE_MODE_WRITE,&viewer) ); - PetscCall( VecView(U, viewer) ); - PetscCall( PetscViewerDestroy(&viewer) ); + if (app_ctx->view_solution) { + PetscViewer viewer_p; + PetscCall( PetscViewerVTKOpen(comm,"darcy_pressure.vtu",FILE_MODE_WRITE, + &viewer_p) ); + PetscCall( VecView(U, viewer_p) ); + PetscCall( PetscViewerDestroy(&viewer_p) ); + + Vec U_H1; // velocity in H1 space for post-processing + PetscCall( DMCreateGlobalVector(dm_H1, &U_H1) ); + PetscCall( ProjectVelocity(app_ctx, U, &U_H1) ); + PetscViewer viewer_u; + PetscCall( PetscViewerVTKOpen(comm,"darcy_velocity.vtu",FILE_MODE_WRITE, + &viewer_u) ); + PetscCall( VecView(U_H1, viewer_u) ); + PetscCall( PetscViewerDestroy(&viewer_u) ); + PetscCall( VecDestroy(&U_H1) ); + } // --------------------------------------------------------------------------- // Free objects // --------------------------------------------------------------------------- @@ -205,7 +290,9 @@ int main(int argc, char **argv) { PetscCall( DMDestroy(&dm) ); PetscCall( DMDestroy(&dm_u0) ); PetscCall( DMDestroy(&dm_p0) ); + PetscCall( DMDestroy(&dm_H1) ); PetscCall( VecDestroy(&U) ); + PetscCall( CtxVecDestroy(app_ctx) ); if (problem_data->has_ts) { PetscCall( TSDestroy(&ts) ); } else { @@ -222,6 +309,11 @@ int main(int argc, char **argv) { PetscCall( PetscFree(ctx_initial_u0) ); PetscCall( PetscFree(ctx_initial_p0) ); PetscCall( PetscFree(ctx_residual_ut) ); + PetscCall( PetscFree(ctx_residual) ); + PetscCall( PetscFree(ctx_jacobian) ); + PetscCall( PetscFree(ctx_error) ); + PetscCall( PetscFree(ctx_H1) ); + PetscCall( PetscFree(ctx_Hdiv) ); // Free libCEED objects //CeedVectorDestroy(&bc_pressure); diff --git a/examples/Hdiv-mixed/main.h b/examples/Hdiv-mixed/main.h index 1b85c9f8be..7033f49f38 100644 --- a/examples/Hdiv-mixed/main.h +++ b/examples/Hdiv-mixed/main.h @@ -11,5 +11,6 @@ #include "include/setup-matops.h" #include "include/setup-solvers.h" #include "include/setup-ts.h" +#include "include/post-processing.h" #endif // MAIN_H \ No newline at end of file diff --git a/examples/Hdiv-mixed/problems/darcy2d.c b/examples/Hdiv-mixed/problems/darcy2d.c index fca8b2bdd3..16d61dbb62 100644 --- a/examples/Hdiv-mixed/problems/darcy2d.c +++ b/examples/Hdiv-mixed/problems/darcy2d.c @@ -21,9 +21,13 @@ #include "../qfunctions/darcy-true2d.h" #include "../qfunctions/darcy-system2d.h" #include "../qfunctions/darcy-error2d.h" +#include "../qfunctions/post-processing2d.h" +#include "../qfunctions/darcy-true-quartic2d.h" +#include "../qfunctions/darcy-system-quartic2d.h" //#include "../qfunctions/pressure-boundary2d.h" -PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, void *ctx) { +PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, DM dm, + void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; DARCYContext darcy_ctx; CeedQFunctionContext darcy_context; @@ -49,7 +53,22 @@ PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, void *ctx) { problem_data->error_loc = DarcyError2D_loc; //problem_data->bc_pressure = BCPressure2D; //problem_data->bc_pressure_loc = BCPressure2D_loc; + problem_data->post_rhs = PostProcessingRhs2D; + problem_data->post_rhs_loc = PostProcessingRhs2D_loc; + problem_data->post_mass = PostProcessingMass2D; + problem_data->post_mass_loc = PostProcessingMass2D_loc; problem_data->has_ts = PETSC_FALSE; + problem_data->view_solution = app_ctx->view_solution; + problem_data->quartic = app_ctx->quartic; + + if (app_ctx->quartic) { + problem_data->true_solution = DarcyTrueQuartic2D; + problem_data->true_solution_loc = DarcyTrueQuartic2D_loc; + problem_data->residual = DarcySystemQuartic2D; + problem_data->residual_loc = DarcySystemQuartic2D_loc; + problem_data->jacobian = JacobianDarcySystemQuartic2D; + problem_data->jacobian_loc = JacobianDarcySystemQuartic2D_loc; + } // ------------------------------------------------------ // Command line Options @@ -68,11 +87,17 @@ PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, void *ctx) { b_a, &b_a, NULL)); PetscOptionsEnd(); + PetscReal domain_min[2], domain_max[2], domain_size[2]; + PetscCall( DMGetBoundingBox(dm, domain_min, domain_max) ); + for (PetscInt i=0; i<2; i++) domain_size[i] = domain_max[i] - domain_min[i]; + darcy_ctx->kappa = kappa; darcy_ctx->rho_a0 = rho_a0; darcy_ctx->g = g; darcy_ctx->alpha_a = alpha_a; darcy_ctx->b_a = b_a; + darcy_ctx->lx = domain_size[0]; + darcy_ctx->ly = domain_size[1]; CeedQFunctionContextCreate(ceed, &darcy_context); CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, diff --git a/examples/Hdiv-mixed/problems/darcy3d.c b/examples/Hdiv-mixed/problems/darcy3d.c index 96fba6b598..0003bb010e 100644 --- a/examples/Hdiv-mixed/problems/darcy3d.c +++ b/examples/Hdiv-mixed/problems/darcy3d.c @@ -21,9 +21,11 @@ #include "../qfunctions/darcy-true3d.h" #include "../qfunctions/darcy-system3d.h" #include "../qfunctions/darcy-error3d.h" +#include "../qfunctions/post-processing3d.h" //#include "../qfunctions/pressure-boundary3d.h" -PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, void *ctx) { +PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, DM dm, + void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; DARCYContext darcy_ctx; CeedQFunctionContext darcy_context; @@ -49,8 +51,12 @@ PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, void *ctx) { problem_data->error_loc = DarcyError3D_loc; //problem_data->bc_pressure = BCPressure3D; //problem_data->bc_pressure_loc = BCPressure3D_loc; + problem_data->post_rhs = PostProcessingRhs3D; + problem_data->post_rhs_loc = PostProcessingRhs3D_loc; + problem_data->post_mass = PostProcessingMass3D; + problem_data->post_mass_loc = PostProcessingMass3D_loc; problem_data->has_ts = PETSC_FALSE; - + problem_data->view_solution = app_ctx->view_solution; // ------------------------------------------------------ // Command line Options // ------------------------------------------------------ @@ -68,11 +74,18 @@ PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, void *ctx) { b_a, &b_a, NULL)); PetscOptionsEnd(); + PetscReal domain_min[3], domain_max[3], domain_size[3]; + PetscCall( DMGetBoundingBox(dm, domain_min, domain_max) ); + for (PetscInt i=0; i<3; i++) domain_size[i] = domain_max[i] - domain_min[i]; + darcy_ctx->kappa = kappa; darcy_ctx->rho_a0 = rho_a0; darcy_ctx->g = g; darcy_ctx->alpha_a = alpha_a; darcy_ctx->b_a = b_a; + darcy_ctx->lx = domain_size[0]; + darcy_ctx->ly = domain_size[1]; + darcy_ctx->lz = domain_size[2]; CeedQFunctionContextCreate(ceed, &darcy_context); CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, diff --git a/examples/Hdiv-mixed/problems/register-problem.c b/examples/Hdiv-mixed/problems/register-problem.c index 28fc386fbe..5e4ffe7b43 100644 --- a/examples/Hdiv-mixed/problems/register-problem.c +++ b/examples/Hdiv-mixed/problems/register-problem.c @@ -34,6 +34,8 @@ PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx) { // 4) richard PetscCall( PetscFunctionListAdd(&app_ctx->problems, "richard2d", Hdiv_RICHARD2D) ); + PetscCall( PetscFunctionListAdd(&app_ctx->problems, "richard3d", + Hdiv_RICHARD3D) ); PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mixed/problems/richard2d.c b/examples/Hdiv-mixed/problems/richard2d.c index 7240a5375a..aa9bbd2c6d 100644 --- a/examples/Hdiv-mixed/problems/richard2d.c +++ b/examples/Hdiv-mixed/problems/richard2d.c @@ -22,10 +22,12 @@ #include "../qfunctions/richard-true2d.h" #include "../qfunctions/richard-ics2d.h" #include "../qfunctions/darcy-error2d.h" +#include "../qfunctions/post-processing2d.h" //#include "../qfunctions/pressure-boundary2d.h" #include "petscsystypes.h" -PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx) { +PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, DM dm, + void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; RICHARDContext richard_ctx; CeedQFunctionContext richard_context; @@ -59,7 +61,12 @@ PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx) { problem_data->error_loc = DarcyError2D_loc; //problem_data->bc_pressure = BCPressure2D; //problem_data->bc_pressure_loc = BCPressure2D_loc; + problem_data->post_rhs = PostProcessingRhs2D; + problem_data->post_rhs_loc = PostProcessingRhs2D_loc; + problem_data->post_mass = PostProcessingMass2D; + problem_data->post_mass_loc = PostProcessingMass2D_loc; problem_data->has_ts = PETSC_TRUE; + problem_data->view_solution = app_ctx->view_solution; // ------------------------------------------------------ // Command line Options @@ -85,6 +92,10 @@ PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx) { app_ctx->t_final, &app_ctx->t_final, NULL)); PetscOptionsEnd(); + PetscReal domain_min[2], domain_max[2], domain_size[2]; + PetscCall( DMGetBoundingBox(dm, domain_min, domain_max) ); + for (PetscInt i=0; i<2; i++) domain_size[i] = domain_max[i] - domain_min[i]; + richard_ctx->kappa = kappa; richard_ctx->alpha_a = alpha_a; richard_ctx->b_a = b_a; @@ -95,6 +106,9 @@ PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, void *ctx) { richard_ctx->gamma = 5.; richard_ctx->t = 0.; richard_ctx->t_final = app_ctx->t_final; + richard_ctx->lx = domain_size[0]; + richard_ctx->ly = domain_size[1]; + CeedQFunctionContextCreate(ceed, &richard_context); CeedQFunctionContextSetData(richard_context, CEED_MEM_HOST, CEED_COPY_VALUES, sizeof(*richard_ctx), richard_ctx); diff --git a/examples/Hdiv-mixed/problems/richard3d.c b/examples/Hdiv-mixed/problems/richard3d.c new file mode 100644 index 0000000000..9559138526 --- /dev/null +++ b/examples/Hdiv-mixed/problems/richard3d.c @@ -0,0 +1,134 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Utility functions for setting up Richard problem in 3D + +#include "../include/register-problem.h" +#include "../qfunctions/richard-system3d.h" +#include "../qfunctions/richard-true3d.h" +#include "../qfunctions/richard-ics3d.h" +#include "../qfunctions/darcy-error3d.h" +#include "../qfunctions/post-processing3d.h" +//#include "../qfunctions/pressure-boundary2d.h" +#include "petscsystypes.h" + +PetscErrorCode Hdiv_RICHARD3D(Ceed ceed, ProblemData problem_data, DM dm, + void *ctx) { + AppCtx app_ctx = *(AppCtx *)ctx; + RICHARDContext richard_ctx; + CeedQFunctionContext richard_context; + + PetscFunctionBeginUser; + + PetscCall( PetscCalloc1(1, &richard_ctx) ); + + // ------------------------------------------------------ + // SET UP POISSON_QUAD2D + // ------------------------------------------------------ + problem_data->dim = 3; + problem_data->elem_node = 8; + problem_data->q_data_size_face = 4; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->true_solution = RichardTrue3D; + problem_data->true_solution_loc = RichardTrue3D_loc; + problem_data->rhs_u0 = RichardRhsU03D; + problem_data->rhs_u0_loc = RichardRhsU03D_loc; + problem_data->ics_u = RichardICsU3D; + problem_data->ics_u_loc = RichardICsU3D_loc; + problem_data->rhs_p0 = RichardRhsP03D; + problem_data->rhs_p0_loc = RichardRhsP03D_loc; + problem_data->ics_p = RichardICsP3D; + problem_data->ics_p_loc = RichardICsP3D_loc; + problem_data->residual = RichardSystem3D; + problem_data->residual_loc = RichardSystem3D_loc; + //problem_data->jacobian = JacobianRichardSystem2D; + //problem_data->jacobian_loc = JacobianRichardSystem2D_loc; + problem_data->error = DarcyError3D; + problem_data->error_loc = DarcyError3D_loc; + //problem_data->bc_pressure = BCPressure2D; + //problem_data->bc_pressure_loc = BCPressure2D_loc; + problem_data->post_rhs = PostProcessingRhs3D; + problem_data->post_rhs_loc = PostProcessingRhs3D_loc; + problem_data->post_mass = PostProcessingMass3D; + problem_data->post_mass_loc = PostProcessingMass3D_loc; + problem_data->has_ts = PETSC_TRUE; + problem_data->view_solution = app_ctx->view_solution; + + // ------------------------------------------------------ + // Command line Options + // ------------------------------------------------------ + CeedScalar kappa = 10., alpha_a = 1., b_a = 10., rho_a0 = 998.2, + beta = 0., g = 9.8, p0 = 101325; + + PetscOptionsBegin(app_ctx->comm, NULL, "Options for Hdiv-mixed problem", NULL); + PetscCall( PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, + kappa, &kappa, NULL)); + PetscCall( PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", + NULL, + alpha_a, &alpha_a, NULL)); + PetscCall( PetscOptionsScalar("-b_a", "Parameter for relative permeability", + NULL, + b_a, &b_a, NULL)); + PetscCall( PetscOptionsScalar("-rho_a0", "Density at p0", NULL, + rho_a0, &rho_a0, NULL)); + PetscCall( PetscOptionsScalar("-beta", "Water compressibility", NULL, + beta, &beta, NULL)); + app_ctx->t_final = 0.5; + PetscCall( PetscOptionsScalar("-t_final", "End time", NULL, + app_ctx->t_final, &app_ctx->t_final, NULL)); + PetscOptionsEnd(); + + PetscReal domain_min[3], domain_max[3], domain_size[3]; + PetscCall( DMGetBoundingBox(dm, domain_min, domain_max) ); + for (PetscInt i=0; i<3; i++) domain_size[i] = domain_max[i] - domain_min[i]; + + richard_ctx->kappa = kappa; + richard_ctx->alpha_a = alpha_a; + richard_ctx->b_a = b_a; + richard_ctx->rho_a0 = rho_a0; + richard_ctx->beta = beta; + richard_ctx->g = g; + richard_ctx->p0 = p0; + richard_ctx->gamma = 5.; + richard_ctx->t = 0.; + richard_ctx->t_final = app_ctx->t_final; + richard_ctx->lx = domain_size[0]; + richard_ctx->ly = domain_size[1]; + richard_ctx->lz = domain_size[2]; + + CeedQFunctionContextCreate(ceed, &richard_context); + CeedQFunctionContextSetData(richard_context, CEED_MEM_HOST, CEED_COPY_VALUES, + sizeof(*richard_ctx), richard_ctx); + //CeedQFunctionContextSetDataDestroy(richard_context, CEED_MEM_HOST, + // FreeContextPetsc); + CeedQFunctionContextRegisterDouble(richard_context, "time", + offsetof(struct RICHARDContext_, t), 1, "current solver time"); + CeedQFunctionContextRegisterDouble(richard_context, "final_time", + offsetof(struct RICHARDContext_, t_final), 1, "final time"); + CeedQFunctionContextRegisterDouble(richard_context, "time_step", + offsetof(struct RICHARDContext_, dt), 1, "time step"); + problem_data->true_qfunction_ctx = richard_context; + CeedQFunctionContextReferenceCopy(richard_context, + &problem_data->rhs_u0_qfunction_ctx); + CeedQFunctionContextReferenceCopy(richard_context, + &problem_data->residual_qfunction_ctx); + CeedQFunctionContextReferenceCopy(richard_context, + &problem_data->error_qfunction_ctx); + PetscCall( PetscFree(richard_ctx) ); + + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mixed/qfunctions/darcy-error2d.h b/examples/Hdiv-mixed/qfunctions/darcy-error2d.h index e5ccbc5719..62c574337b 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-error2d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-error2d.h @@ -35,6 +35,7 @@ struct DARCYContext_ { CeedScalar g; CeedScalar rho_a0; CeedScalar alpha_a, b_a; + CeedScalar lx, ly; }; #endif CEED_QFUNCTION(DarcyError2D)(void *ctx, const CeedInt Q, diff --git a/examples/Hdiv-mixed/qfunctions/darcy-error3d.h b/examples/Hdiv-mixed/qfunctions/darcy-error3d.h index 63af1b1e48..18ae3810ea 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-error3d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-error3d.h @@ -36,6 +36,7 @@ struct DARCYContext_ { CeedScalar g; CeedScalar rho_a0; CeedScalar alpha_a, b_a; + CeedScalar lx, ly, lz; }; #endif CEED_QFUNCTION(DarcyError3D)(void *ctx, const CeedInt Q, diff --git a/examples/Hdiv-mixed/qfunctions/darcy-system-quartic2d.h b/examples/Hdiv-mixed/qfunctions/darcy-system-quartic2d.h new file mode 100644 index 0000000000..9a43d406bf --- /dev/null +++ b/examples/Hdiv-mixed/qfunctions/darcy-system-quartic2d.h @@ -0,0 +1,170 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Darcy problem 2D (quad element) using PETSc + +#ifndef DARCY_SYSTEM_QUARTIC2D_H +#define DARCY_SYSTEM_QUARTIC2D_H + +#include +#include +#include "utils.h" + +// ----------------------------------------------------------------------------- +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -grad(\psi) in \Omega +// -\div(u) = -f in \Omega +// p = p_b on \Gamma_D +// u.n = u_b on \Gamma_N +// +// Weak form: Find (u, \psi) \in VxQ (V=H(div), Q=L^2) on \Omega +// (v, u) -(\div(v), \psi) = -_{\Gamma_D} +// -(q, \div(u)) + (q, f) = 0 +// +// This QFunction setup the mixed form of the above equation +// Inputs: +// w : weight of quadrature +// J : dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// u : basis_u at quadrature points +// div(u) : divergence of basis_u at quadrature points +// p : basis_p at quadrature points +// f : force vector created in true qfunction +// +// Output: +// v : (v, K^{-1}/rho*k_r u) = \int (v^T * K^{-1}/rho*k_r*u detJ*w)dX ==> \int (v^T J^T * K^{-1}/rho*k_r *J*u*w/detJ)dX +// -(v, rho*g_u) = \int (v^T * rho*g_u detJ*w)dX ==> \int (v^T J^T * rho*g_u*w) dX +// div(v) : -(\div(v), \psi) = -\int (div(v)^T * \psi *w) dX +// q : -(q, \div(u)) = -\int (q^T * div(u) * w) dX +// (q, f) = \int( q^T * f * w*detJ )dX +// ----------------------------------------------------------------------------- +#ifndef DARCY_CTX +#define DARCY_CTX +typedef struct DARCYContext_ *DARCYContext; +struct DARCYContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; + CeedScalar lx, ly; +}; +#endif +// ----------------------------------------------------------------------------- +// Residual evaluation for Darcy problem +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(DarcySystemQuartic2D)(void *ctx, CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*div_u) = (const CeedScalar(*))in[3], + (*p) = (const CeedScalar(*))in[4], + (*f) = in[5]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], + (*div_v) = (CeedScalar(*))out[1], + (*q) = (CeedScalar(*))out[2]; + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i #include +#include "ceed/ceed-f64.h" #include "utils.h" // ----------------------------------------------------------------------------- @@ -68,6 +69,7 @@ struct DARCYContext_ { CeedScalar g; CeedScalar rho_a0; CeedScalar alpha_a, b_a; + CeedScalar lx, ly; }; #endif // ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/qfunctions/darcy-system3d.h b/examples/Hdiv-mixed/qfunctions/darcy-system3d.h index 3d8af38444..403dcbcbd3 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-system3d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-system3d.h @@ -68,6 +68,7 @@ struct DARCYContext_ { CeedScalar g; CeedScalar rho_a0; CeedScalar alpha_a, b_a; + CeedScalar lx, ly, lz; }; #endif // ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/qfunctions/darcy-true-quartic2d.h b/examples/Hdiv-mixed/qfunctions/darcy-true-quartic2d.h new file mode 100644 index 0000000000..ceeabc0efc --- /dev/null +++ b/examples/Hdiv-mixed/qfunctions/darcy-true-quartic2d.h @@ -0,0 +1,98 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Compute true solution of the H(div) example using PETSc + +#ifndef DARCY_TRUE_QUARTIC2D_H +#define DARCY_TRUE_QUARTIC2D_H + +#include +#include +#include "utils.h" + +// ----------------------------------------------------------------------------- +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -\grad(psi) on \Omega +// \div(u) = f on \Omega +// p = p0 on \Gamma_D +// u.n = g on \Gamma_N +// Weak form: Find (u,p) \in VxQ (V=H(div), Q=L^2) on \Omega +// (v, u) - (\div(v), psi) = -_{\Gamma_D} +// -(q, \div(u)) = -(q, f) +// +// This QFunction setup the true solution and forcing f of the above equation +// Inputs: +// coords: physical coordinate +// +// Output: +// true_force : = div(u) +// true_solution : = [\psi, u] where \psi, u are the exact solution solution +// ----------------------------------------------------------------------------- +#ifndef DARCY_CTX +#define DARCY_CTX +typedef struct DARCYContext_ *DARCYContext; +struct DARCYContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; + CeedScalar lx, ly; +}; +#endif +CEED_QFUNCTION(DarcyTrueQuartic2D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*coords) = in[0]; + // Outputs + CeedScalar (*true_force) = out[0], (*true_solution) = out[1]; + // Context + DARCYContext context = (DARCYContext)ctx; + const CeedScalar lx = context->lx; + const CeedScalar ly = context->ly; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; ikappa; const CeedScalar alpha_a = context->alpha_a; const CeedScalar b_a = context->b_a; + const CeedScalar lx = context->lx; + const CeedScalar ly = context->ly; + // Quadrature Point Loop CeedPragmaSIMD for (CeedInt i=0; ikappa; const CeedScalar alpha_a = context->alpha_a; const CeedScalar b_a = context->b_a; + const CeedScalar lx = context->lx; + const CeedScalar ly = context->ly; + const CeedScalar lz = context->lz; // Quadrature Point Loop CeedPragmaSIMD for (CeedInt i=0; i +#include +#include "ceed/ceed-f64.h" +#include "utils.h" + +// ----------------------------------------------------------------------------- +// We solve (v, u) = (v, uh), to project Hdiv to L2 space +// This QFunction create post_rhs = (v, uh) +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(PostProcessingRhs2D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + // Outputs + CeedScalar (*post_rhs) = out[0]; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i rhs = J*u*w + CeedScalar u1[2] = {u[0][i], u[1][i]}, rhs[2]; + AlphaMatVecMult2x2(w[i], J, u1, rhs); + + post_rhs[i+0*Q] = rhs[0]; + post_rhs[i+1*Q] = rhs[1]; + } // End of Quadrature Point Loop + return 0; +} + +// ----------------------------------------------------------------------------- +// We solve (v, u) = (v, uh), to project Hdiv to L2 space +// This QFunction create mass matrix (v, u), then we solve using ksp to have +// projected uh in L2 space and use it for post-processing +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(PostProcessingMass2D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#include +#include "ceed/ceed-f64.h" +#include "utils.h" + +// ----------------------------------------------------------------------------- +// We solve (v, u) = (v, uh), to project Hdiv to L2 space +// This QFunction create post_rhs = (v, uh) +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(PostProcessingRhs3D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + // Outputs + CeedScalar (*post_rhs) = out[0]; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i rhs = J*u*w + CeedScalar u1[3] = {u[0][i], u[1][i], u[2][i]}, rhs[3]; + AlphaMatVecMult3x3(w[i], J, u1, rhs); + + post_rhs[i+0*Q] = rhs[0]; + post_rhs[i+1*Q] = rhs[1]; + post_rhs[i+2*Q] = rhs[2]; + } // End of Quadrature Point Loop + return 0; +} + +// ----------------------------------------------------------------------------- +// We solve (v, u) = (v, uh), to project Hdiv to L2 space +// This QFunction create mass matrix (v, u), then we solve using ksp to have +// projected uh in L2 space and use it for post-processing +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(PostProcessingMass3D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#include +#include "ceed/ceed-f64.h" +#include "utils.h" + +#ifndef RICHARD_CTX +#define RICHARD_CTX +typedef struct RICHARDContext_ *RICHARDContext; +struct RICHARDContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; + CeedScalar beta, p0; + CeedScalar t, t_final, dt; + CeedScalar gamma; + CeedScalar lx, ly, lz; +}; +#endif + +// ----------------------------------------------------------------------------- +// We solve (v, u) = (v, ue) at t=0, to project ue to Hdiv space +// This QFunction create rhs_u0 = (v, ue) +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(RichardRhsU03D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*coords) = in[1], + (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[2]; + // Outputs + CeedScalar (*rhs_u0) = out[0]; + // Context + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; i +#include +#include "ceed/ceed-f64.h" +#include "utils.h" + +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega x [0,T] +// -\div(u) = -f + d (rho*\theta)/dt in \Omega x [0,T] +// p = p_b on \Gamma_D x [0,T] +// u.n = u_b on \Gamma_N x [0,T] +// p = p_0 in \Omega, t = 0 +// +// Where rho = rho_a/rho_a0, rho_a = rho_a0*exp(\beta * (p - p0)), p0 = 101325 Pa is atmospheric pressure +// rho_a0 is the density at p_0, g_u = g/norm(g) where g is gravity. +// k_r = b_a + alpha_a * (\psi - x2), where \psi = p / (rho_a0 * norm(g)) and x2 is vertical axis +// +// Weak form: Find (u, \psi) \in VxQ (V=H(div), Q=L^2) on \Omega +// (v, K^{-1}/rho*k_r * u) -(v, rho*g_u) -(\div(v), \psi) = -_{\Gamma_D} +// -(q, \div(u)) + (q, f) -(q, d (rho*\theta)/dt ) = 0 +// +// We solve MMS for K = kappa*I and beta=0 ==> rho=1 and \theta = alpha_a*\psi, so +// -(q, d (rho*\theta)/dt ) = -alpha_a*(q, d(\psi)/dt ) +// +// This QFunction setup the mixed form of the above equation +// Inputs: +// w : weight of quadrature +// J : dx/dX. x physical coordinate, X reference coordinate [-1,1]^dim +// u : basis_u at quadrature points +// div(u) : divergence of basis_u at quadrature points +// p : basis_p at quadrature points +// U_t : time derivative of U = [p, u] +// +// Output: +// v : (v, K^{-1}/rho*k_r u) = \int (v^T * K^{-1}/rho*k_r*u detJ*w)dX ==> \int (v^T J^T * K^{-1}/rho*k_r *J*u*w/detJ)dX +// -(v, rho*g_u) = \int (v^T * rho*g_u detJ*w)dX ==> \int (v^T J^T * rho*g_u*w) dX +// div(v) : -(\div(v), \psi) = -\int (div(v)^T * \psi *w) dX +// q : -(q, \div(u)) = -\int (q^T * div(u) * w) dX +// (q, f) = \int( q^T * f * w*detJ )dX +// -alpha_a*(q, d\psi/dt) = -alpha_a \int (q^T * \psi_t*w*detJ)dX +// +// ----------------------------------------------------------------------------- +#ifndef RICHARD_CTX +#define RICHARD_CTX +typedef struct RICHARDContext_ *RICHARDContext; +struct RICHARDContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; + CeedScalar beta, p0; + CeedScalar t, t_final, dt; + CeedScalar gamma; + CeedScalar lx, ly, lz; +}; +#endif +// ----------------------------------------------------------------------------- +// Residual evaluation for Richard problem +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(RichardSystem3D)(void *ctx, CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*w) = in[0], + (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], + (*div_u) = (const CeedScalar(*))in[3], + (*p) = (const CeedScalar(*))in[4], + (*f) = in[5], + (*coords) = in[6], + (*p_t) = (const CeedScalar(*))in[7]; + + // Outputs + CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], + (*div_v) = (CeedScalar(*))out[1], + (*q) = (CeedScalar(*))out[2]; + // Context + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + //const CeedScalar beta = context->beta; + //const CeedScalar p0 = context->p0; // atmospheric pressure + const CeedScalar gamma = context->gamma; + CeedScalar t = context->t; + //CeedScalar dt = context->dt; + + // *INDENT-ON* + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; ikappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar beta = context->beta; + const CeedScalar g = context->g; + const CeedScalar p0 = context->p0;// atmospheric pressure + // *INDENT-ON* + +// Quadrature Point Loop +CeedPragmaSIMD +for (CeedInt i=0; i +#include +#include "utils.h" + +// See Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +// ----------------------------------------------------------------------------- +// Strong form: +// u = -rho*k_r*K *[grad(\psi) - rho*g_u] in \Omega x [0,T] +// -\div(u) = -f + d (rho*theta)/dt in \Omega x [0,T] +// p = p_b on \Gamma_D x [0,T] +// u.n = u_b on \Gamma_N x [0,T] +// p = p_0 in \Omega, t = 0 +// +// Where rho = rho_a/rho_a0, rho_a = rho_a0*exp(\beta * (p - p0)), p0 = 101325 Pa is atmospheric pressure +// rho_a0 is the density at p_0, g_u = g/norm(g) where g is gravity. +// k_r = b_a + alpha_a * (\psi - x2), where \psi = p / (rho_a0 * norm(g)) and x2 is vertical axis +// +// Weak form: Find (u, \psi) \in VxQ (V=H(div), Q=L^2) on \Omega +// (v, K^{-1}/rho*k_r * u) -(v, rho*g_u) -(\div(v), \psi) = -_{\Gamma_D} +// -(q, \div(u)) + (q, f) -(q, d (rho*\theta)/dt ) = 0 +// +// We solve MMS for K = kappa*I and beta=0 ==> rho=1 and \theta = alpha_a*\psi, so +// -(q, d (rho*\theta)/dt ) = -alpha_a*(q, d(\psi)/dt ) +// +// This QFunction setup the true solution and forcing f of the above equation +// Inputs: +// coords: physical coordinate +// +// Output: +// true_force : = div(u) + d (rho*theta)/dt +// true_solution : = [\psi, u] where \psi, u are the exact solution solution +// ----------------------------------------------------------------------------- +// We have 3 experiment parameters as described in Table 1:P1, P2, P3 +// Matthew Farthing, Christopher Kees, Cass Miller (2003) +// https://www.sciencedirect.com/science/article/pii/S0309170802001872 +#ifndef RICHARD_CTX +#define RICHARD_CTX +typedef struct RICHARDContext_ *RICHARDContext; +struct RICHARDContext_ { + CeedScalar kappa; + CeedScalar g; + CeedScalar rho_a0; + CeedScalar alpha_a, b_a; + CeedScalar beta, p0; + CeedScalar t, t_final, dt; + CeedScalar gamma; + CeedScalar lx, ly, lz; +}; +#endif + +// ----------------------------------------------------------------------------- +// True solution for Richard problem +// ----------------------------------------------------------------------------- +CEED_QFUNCTION(RichardTrue3D)(void *ctx, const CeedInt Q, + const CeedScalar *const *in, + CeedScalar *const *out) { + // *INDENT-OFF* + // Inputs + const CeedScalar (*coords) = in[0]; + // Outputs + CeedScalar (*true_force) = out[0], (*true_solution) = out[1]; + // Context + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + const CeedScalar gamma = context->gamma; + CeedScalar t_final = context->t_final; + + // Quadrature Point Loop + CeedPragmaSIMD + for (CeedInt i=0; icomm, NULL, "H(div) examples in PETSc with libCEED", NULL); + PetscCall( PetscOptionsString("-ceed", "CEED resource specifier", + NULL, app_ctx->ceed_resource, app_ctx->ceed_resource, + sizeof(app_ctx->ceed_resource), &ceed_flag) ); + + // Provide default ceed resource if not specified + if (!ceed_flag) { + const char *ceed_resource = "/cpu/self"; + strncpy(app_ctx->ceed_resource, ceed_resource, 10); + } + PetscCall( PetscOptionsFList("-problem", "Problem to solve", NULL, app_ctx->problems, app_ctx->problem_name, app_ctx->problem_name, sizeof(app_ctx->problem_name), @@ -44,12 +55,33 @@ PetscErrorCode ProcessCommandLineOptions(AppCtx app_ctx) { app_ctx->q_extra = 0; PetscCall( PetscOptionsInt("-q_extra", "Number of extra quadrature points", NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL) ); + app_ctx->view_solution = PETSC_FALSE; + PetscCall( PetscOptionsBool("-view_solution", + "View solution in Paraview", + NULL, app_ctx->view_solution, + &(app_ctx->view_solution), NULL) ); + app_ctx->quartic = PETSC_FALSE; + PetscCall( PetscOptionsBool("-quartic", + "To test PetscViewer", + NULL, app_ctx->quartic, + &(app_ctx->quartic), NULL) ); + + PetscCall( PetscStrncpy(app_ctx->output_dir, ".", 2) ); + PetscCall( PetscOptionsString("-output_dir", "Output directory", + NULL, app_ctx->output_dir, app_ctx->output_dir, + sizeof(app_ctx->output_dir), NULL) ); + + app_ctx->output_freq = 10; + PetscCall( PetscOptionsInt("-output_freq", + "Frequency of output, in number of steps", + NULL, app_ctx->output_freq, &app_ctx->output_freq, NULL) ); app_ctx->bc_pressure_count = 16; // we can set one face by: -bc_faces 1 OR multiple faces by :-bc_faces 1,2,3 PetscCall( PetscOptionsIntArray("-bc_faces", "Face IDs to apply pressure BC", NULL, app_ctx->bc_faces, &app_ctx->bc_pressure_count, NULL) ); + PetscOptionsEnd(); PetscFunctionReturn(0); diff --git a/examples/Hdiv-mixed/src/post-processing.c b/examples/Hdiv-mixed/src/post-processing.c new file mode 100644 index 0000000000..df94038ed4 --- /dev/null +++ b/examples/Hdiv-mixed/src/post-processing.c @@ -0,0 +1,272 @@ +#include "../include/post-processing.h" +#include "../include/setup-solvers.h" +#include "ceed/ceed.h" +// ----------------------------------------------------------------------------- +// This function print the output +// ----------------------------------------------------------------------------- +PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, + CeedMemType mem_type_backend, + TS ts, SNES snes, KSP ksp, + Vec U, CeedScalar l2_error_u, CeedScalar l2_error_p) { + + PetscFunctionBeginUser; + + const char *used_resource; + CeedGetResource(ceed, &used_resource); + char hostname[PETSC_MAX_PATH_LEN]; + PetscCall( PetscGetHostName(hostname, sizeof hostname) ); + PetscInt comm_size; + PetscCall( MPI_Comm_size(app_ctx->comm, &comm_size) ); + PetscCall( PetscPrintf(app_ctx->comm, + "\n-- Mixed H(div) Example - libCEED + PETSc --\n" + " MPI:\n" + " Hostname : %s\n" + " Total ranks : %d\n" + " libCEED:\n" + " libCEED Backend : %s\n" + " libCEED Backend MemType : %s\n", + hostname, comm_size, used_resource, CeedMemTypes[mem_type_backend]) ); + + MatType mat_type; + VecType vec_type; + PetscCall( DMGetMatType(dm, &mat_type) ); + PetscCall( DMGetVecType(dm, &vec_type) ); + PetscCall( PetscPrintf(app_ctx->comm, + " PETSc:\n" + " DM MatType : %s\n" + " DM VecType : %s\n", + mat_type, vec_type) ); + + PetscInt U_l_size, U_g_size; + PetscCall( VecGetSize(U, &U_g_size) ); + PetscCall( VecGetLocalSize(U, &U_l_size) ); + PetscCall( PetscPrintf(app_ctx->comm, + " Problem:\n" + " Problem Name : %s\n" + " Global nodes (u + p) : %" PetscInt_FMT "\n" + " Owned nodes (u + p) : %" PetscInt_FMT "\n", + app_ctx->problem_name, U_g_size, U_l_size + ) ); + // --TS + if (has_ts) { + PetscInt ts_steps; + TSType ts_type; + TSConvergedReason ts_reason; + PetscCall( TSGetStepNumber(ts, &ts_steps) ); + PetscCall( TSGetType(ts, &ts_type) ); + PetscCall( TSGetConvergedReason(ts, &ts_reason) ); + PetscCall( PetscPrintf(app_ctx->comm, + " TS:\n" + " TS Type : %s\n" + " TS Convergence : %s\n" + " Number of TS steps : %" PetscInt_FMT "\n" + " Final time : %g\n", + ts_type, TSConvergedReasons[ts_reason], + ts_steps, (double)app_ctx->t_final) ); + + PetscCall( TSGetSNES(ts, &snes) ); + } + // -- SNES + PetscInt its, snes_its = 0; + PetscCall( SNESGetIterationNumber(snes, &its) ); + snes_its += its; + SNESType snes_type; + SNESConvergedReason snes_reason; + PetscReal snes_rnorm; + PetscCall( SNESGetType(snes, &snes_type) ); + PetscCall( SNESGetConvergedReason(snes, &snes_reason) ); + PetscCall( SNESGetFunctionNorm(snes, &snes_rnorm) ); + PetscCall( PetscPrintf(app_ctx->comm, + " SNES:\n" + " SNES Type : %s\n" + " SNES Convergence : %s\n" + " Total SNES Iterations : %" PetscInt_FMT "\n" + " Final rnorm : %e\n", + snes_type, SNESConvergedReasons[snes_reason], + snes_its, (double)snes_rnorm) ); + if (!has_ts) { + PetscInt ksp_its = 0; + PetscCall( SNESGetLinearSolveIterations(snes, &its) ); + ksp_its += its; + KSPType ksp_type; + KSPConvergedReason ksp_reason; + PetscReal ksp_rnorm; + PC pc; + PCType pc_type; + PetscCall( KSPGetPC(ksp, &pc) ); + PetscCall( PCGetType(pc, &pc_type) ); + PetscCall( KSPGetType(ksp, &ksp_type) ); + PetscCall( KSPGetConvergedReason(ksp, &ksp_reason) ); + PetscCall( KSPGetIterationNumber(ksp, &ksp_its) ); + PetscCall( KSPGetResidualNorm(ksp, &ksp_rnorm) ); + PetscCall( PetscPrintf(app_ctx->comm, + " KSP:\n" + " KSP Type : %s\n" + " PC Type : %s\n" + " KSP Convergence : %s\n" + " Total KSP Iterations : %" PetscInt_FMT "\n" + " Final rnorm : %e\n", + ksp_type, pc_type, KSPConvergedReasons[ksp_reason], ksp_its, + (double)ksp_rnorm ) ); + } + + PetscCall( PetscPrintf(app_ctx->comm, + " L2 Error (MMS):\n" + " L2 error of u and p : %e, %e\n", + (double)l2_error_u, + (double)l2_error_p) ); + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// Setup operator context data for initial condition, u field +// ----------------------------------------------------------------------------- +PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, + CeedData ceed_data, + OperatorApplyContext ctx_Hdiv) { + PetscFunctionBeginUser; + + ctx_Hdiv->comm = comm; + ctx_Hdiv->dm = dm; + PetscCall( DMCreateLocalVector(dm, &ctx_Hdiv->X_loc) ); + ctx_Hdiv->x_ceed = ceed_data->u_ceed; + //ctx_project_velocity->y_ceed = ceed_data->v0_ceed; + ctx_Hdiv->ceed = ceed; + //ctx_project_velocity->op_apply = ceed_data->op_ics_u; + + PetscFunctionReturn(0); +} + +PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, + CeedData ceed_data, VecType vec_type, + OperatorApplyContext ctx_H1) { + PetscFunctionBeginUser; + + ctx_H1->comm = comm; + ctx_H1->dm = dm_H1; + PetscCall( DMCreateLocalVector(dm_H1, &ctx_H1->X_loc) ); + PetscCall( VecDuplicate(ctx_H1->X_loc, &ctx_H1->Y_loc) ); + ctx_H1->x_ceed = ceed_data->up_ceed; + ctx_H1->y_ceed = ceed_data->vp_ceed; + ctx_H1->x_coord = ceed_data->x_coord; + ctx_H1->ceed = ceed; + ctx_H1->op_apply = ceed_data->op_post_mass; + ctx_H1->op_rhs_H1 = ceed_data->op_rhs_H1; + ctx_H1->elem_restr_u_H1 = ceed_data->elem_restr_u_H1; + ctx_H1->vec_type = vec_type; + PetscFunctionReturn(0); +} +// ----------------------------------------------------------------------------- +// This function print the output +// ----------------------------------------------------------------------------- +PetscErrorCode ProjectVelocity(AppCtx app_ctx, + Vec U, Vec *U_H1) { + + PetscFunctionBeginUser; + const PetscScalar *x; + PetscMemType x_mem_type; + + // ---------------------------------------------- + // Create local rhs for u field + // ---------------------------------------------- + Vec rhs_loc_H1; + PetscScalar *ru; + PetscMemType ru_mem_type; + PetscCall( DMCreateLocalVector(app_ctx->ctx_H1->dm, &rhs_loc_H1) ); + PetscCall( VecZeroEntries(rhs_loc_H1) ); + PetscCall( VecGetArrayAndMemType(rhs_loc_H1, &ru, &ru_mem_type) ); + CeedElemRestrictionCreateVector(app_ctx->ctx_H1->elem_restr_u_H1, + &app_ctx->ctx_H1->rhs_ceed_H1, + NULL); + CeedVectorSetArray(app_ctx->ctx_H1->rhs_ceed_H1, MemTypeP2C(ru_mem_type), + CEED_USE_POINTER, ru); + + // Global-to-local: map final U in Hdiv space to local vector + PetscCall( DMGlobalToLocal(app_ctx->ctx_Hdiv->dm, + U, INSERT_VALUES, app_ctx->ctx_Hdiv->X_loc) ); + // Place Hdiv PETSc vectors in CEED vectors + PetscCall( VecGetArrayReadAndMemType(app_ctx->ctx_Hdiv->X_loc, + &x, &x_mem_type) ); + CeedVectorSetArray(app_ctx->ctx_Hdiv->x_ceed, MemTypeP2C(x_mem_type), + CEED_USE_POINTER, (PetscScalar *)x); + + // Apply operator to create RHS for u field + CeedOperatorApply(app_ctx->ctx_H1->op_rhs_H1, app_ctx->ctx_H1->x_coord, + app_ctx->ctx_H1->rhs_ceed_H1, CEED_REQUEST_IMMEDIATE); + + // Restore Hdiv vector + CeedVectorTakeArray(app_ctx->ctx_Hdiv->x_ceed, + MemTypeP2C(x_mem_type), NULL); + PetscCall( VecRestoreArrayReadAndMemType(app_ctx->ctx_Hdiv->X_loc, &x) ); + + // ---------------------------------------------- + // Create global rhs for u field + // ---------------------------------------------- + Vec rhs_H1; + CeedVectorTakeArray(app_ctx->ctx_H1->rhs_ceed_H1, MemTypeP2C(ru_mem_type), + NULL); + PetscCall( VecRestoreArrayAndMemType(rhs_loc_H1, &ru) ); + PetscCall( DMCreateGlobalVector(app_ctx->ctx_H1->dm, &rhs_H1) ); + PetscCall( VecZeroEntries(rhs_H1) ); + PetscCall( DMLocalToGlobal(app_ctx->ctx_H1->dm, rhs_loc_H1, ADD_VALUES, + rhs_H1) ); + + // ---------------------------------------------- + // Solve for U_H1, M*U_H1 = rhs_H1 + // ---------------------------------------------- + PetscInt UH1_g_size, UH1_l_size; + PetscCall( VecGetSize(*U_H1, &UH1_g_size) ); + // Local size for matShell + PetscCall( VecGetLocalSize(*U_H1, &UH1_l_size) ); + + // Operator + Mat mat_ksp_projection; + // -- Form Action of residual on u + PetscCall( MatCreateShell(app_ctx->comm, UH1_l_size, UH1_l_size, UH1_g_size, + UH1_g_size, app_ctx->ctx_H1, &mat_ksp_projection) ); + PetscCall( MatShellSetOperation(mat_ksp_projection, MATOP_MULT, + (void (*)(void))ApplyMatOp) ); + PetscCall( MatShellSetVecType(mat_ksp_projection, app_ctx->ctx_H1->vec_type) ); + + KSP ksp_projection; + PetscCall( KSPCreate(app_ctx->ctx_H1->comm, &ksp_projection) ); + PetscCall( KSPSetOperators(ksp_projection, mat_ksp_projection, + mat_ksp_projection) ); + PetscCall( KSPSetFromOptions(ksp_projection) ); + PetscCall( KSPSetUp(ksp_projection) ); + PetscCall( VecZeroEntries(*U_H1) ); + PetscCall( KSPSolve(ksp_projection, rhs_H1, *U_H1) ); + + // Clean up + PetscCall( VecDestroy(&rhs_loc_H1) ); + PetscCall( VecDestroy(&rhs_H1) ); + PetscCall( MatDestroy(&mat_ksp_projection) ); + PetscCall( KSPDestroy(&ksp_projection) ); + CeedVectorDestroy(&app_ctx->ctx_H1->rhs_ceed_H1); + + PetscFunctionReturn(0); +}; + + +PetscErrorCode CtxVecDestroy(AppCtx app_ctx) { + + PetscFunctionBegin; + PetscCall( VecDestroy(&app_ctx->ctx_H1->X_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_H1->Y_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_Hdiv->X_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_initial_u0->X_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_initial_u0->Y_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_initial_p0->X_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_initial_p0->Y_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_residual_ut->X_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_residual_ut->X_t_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_residual_ut->Y_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_jacobian->Y_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_jacobian->X_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_residual->Y_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_residual->X_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_error->Y_loc) ); + PetscCall( VecDestroy(&app_ctx->ctx_error->X_loc) ); + PetscFunctionReturn(0); +} +// ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/src/setup-dm.c b/examples/Hdiv-mixed/src/setup-dm.c index 91b0758219..dff3592bec 100644 --- a/examples/Hdiv-mixed/src/setup-dm.c +++ b/examples/Hdiv-mixed/src/setup-dm.c @@ -4,13 +4,15 @@ // --------------------------------------------------------------------------- // Setup DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDM(MPI_Comm comm, VecType vec_type, DM *dm) { +PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, + VecType vec_type, DM *dm) { PetscFunctionBeginUser; // Create DMPLEX PetscCall( DMCreate(comm, dm) ); PetscCall( DMSetType(*dm, DMPLEX) ); + PetscCall( DMSetMatType(*dm, mat_type) ); PetscCall( DMSetVecType(*dm, vec_type) ); // Set Tensor elements PetscCall( PetscOptionsSetValue(NULL, "-dm_plex_simplex", "0") ); @@ -39,19 +41,86 @@ PetscErrorCode PerturbVerticesSmooth(DM dm) { for(v=vStart; vdegree + 2 + app_ctx->q_extra; + PetscBool is_simplex = PETSC_TRUE; + PetscFunctionBeginUser; + + // Check if simplex or tensor-product element + PetscCall( DMPlexIsSimplex(dm_H1, &is_simplex) ); + // Create FE space + PetscCall( PetscFECreateLagrange(app_ctx->comm, problem_data->dim, + problem_data->dim, is_simplex, + app_ctx->degree, q_degree, &fe) ); + PetscCall( PetscObjectSetName((PetscObject)fe, "U") ); + PetscCall( DMAddField(dm_H1, NULL, (PetscObject)fe) ); + PetscCall( DMCreateDS(dm_H1) ); + + { + // create FE field for coordinates + // PetscFE fe_coords; + // PetscInt num_comp_coord; + // PetscCall( DMGetCoordinateDim(dm_H1, &num_comp_coord) ); + // PetscCall( PetscFECreateLagrange(app_ctx->comm, problem_data->dim, + // num_comp_coord, + // is_simplex, 1, q_degree, + // &fe_coords) ); + // PetscCall( DMProjectCoordinates(dm_H1, fe_coords) ); + // PetscCall( PetscFEDestroy(&fe_coords) ); + } + PetscCall(DMPlexSetClosurePermutationTensor(dm_H1, PETSC_DETERMINE, NULL)); + // Cleanup + PetscCall( PetscFEDestroy(&fe) ); + + // Empty name for conserved field (because there is only one field) + PetscSection section; + PetscCall( DMGetLocalSection(dm_H1, §ion) ); + PetscCall( PetscSectionSetFieldName(section, 0, "Velocity") ); + if (problem_data->dim == 2) { + PetscCall( PetscSectionSetComponentName(section, 0, 0, "Velocity_X") ); + PetscCall( PetscSectionSetComponentName(section, 0, 1, "Velocity_Y") ); + } else { + PetscCall( PetscSectionSetComponentName(section, 0, 0, "Velocity_X") ); + PetscCall( PetscSectionSetComponentName(section, 0, 1, "Velocity_Y") ); + PetscCall( PetscSectionSetComponentName(section, 0, 2, "Velocity_Z") ); + } PetscFunctionReturn(0); }; \ No newline at end of file diff --git a/examples/Hdiv-mixed/src/setup-libceed.c b/examples/Hdiv-mixed/src/setup-libceed.c index ba26d0e35c..8748a60173 100644 --- a/examples/Hdiv-mixed/src/setup-libceed.c +++ b/examples/Hdiv-mixed/src/setup-libceed.c @@ -66,6 +66,18 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data) { //Operators CeedOperatorDestroy(&ceed_data->op_jacobian); } + + // data for post-processing + if(problem_data->view_solution) { + CeedVectorDestroy(&ceed_data->up_ceed); + CeedVectorDestroy(&ceed_data->vp_ceed); + CeedVectorDestroy(&ceed_data->u_ceed); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_u_H1); + CeedQFunctionDestroy(&ceed_data->qf_rhs_H1); + CeedOperatorDestroy(&ceed_data->op_rhs_H1); + CeedQFunctionDestroy(&ceed_data->qf_post_mass); + CeedOperatorDestroy(&ceed_data->op_post_mass); + } PetscCall( PetscFree(ceed_data) ); PetscFunctionReturn(0); @@ -235,8 +247,8 @@ PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, // ----------------------------------------------------------------------------- // Set up libCEED on the fine grid for a given degree // ----------------------------------------------------------------------------- -PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, Ceed ceed, - AppCtx app_ctx, +PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, + Ceed ceed, AppCtx app_ctx, ProblemData problem_data, CeedData ceed_data) { CeedInt P = app_ctx->degree + 1; @@ -273,10 +285,10 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, Ceed ceed, L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); CeedBasisCreateH1(ceed, CEED_TOPOLOGY_QUAD, num_comp_p, 1, num_qpts, interp_p, grad, q_ref,q_weights, &ceed_data->basis_p); - //HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, - // CEED_GAUSS_LOBATTO); - //CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, - // interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); + HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, + CEED_GAUSS_LOBATTO); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, + interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); } else { HdivBasisHex(Q, q_ref, q_weights, interp_u, div, problem_data->quadrature_mode); CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, @@ -284,9 +296,9 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, Ceed ceed, L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); CeedBasisCreateH1(ceed, CEED_TOPOLOGY_HEX, num_comp_p, 1, num_qpts, interp_p, grad, q_ref,q_weights, &ceed_data->basis_p); - //HdivBasisHex(Q, q_ref, q_weights, interp_u, div, CEED_GAUSS_LOBATTO); - //CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, - // interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); + HdivBasisHex(Q, q_ref, q_weights, interp_u, div, CEED_GAUSS_LOBATTO); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, + interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); } CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, @@ -349,9 +361,9 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, Ceed ceed, if (problem_data->has_ts) { double final_time = app_ctx->t_final; CeedOperatorContextGetFieldLabel(op_true, "final_time", - &ceed_data->ctx_residual_ut->final_time_label); + &app_ctx->ctx_residual_ut->final_time_label); CeedOperatorContextSetDouble(op_true, - ceed_data->ctx_residual_ut->final_time_label, &final_time); + app_ctx->ctx_residual_ut->final_time_label, &final_time); } CeedOperatorSetField(op_true, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); @@ -523,13 +535,13 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, Ceed ceed, CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_residual); if (problem_data->has_ts) { - //double t = ceed_data->ctx_residual_ut->t; + //double t = app_ctx->ctx_residual_ut->t; CeedOperatorContextGetFieldLabel(op_residual, "time", - &ceed_data->ctx_residual_ut->solution_time_label); + &app_ctx->ctx_residual_ut->solution_time_label); //CeedOperatorContextGetFieldLabel(op_residual, "time_step", - // &ceed_data->ctx_residual_ut->timestep_label); + // &app_ctx->ctx_residual_ut->timestep_label); //CeedOperatorContextSetDouble(op_residual, - // ceed_data->ctx_residual_ut->solution_time_label, &t); + // app_ctx->ctx_residual_ut->solution_time_label, &t); } CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); @@ -646,6 +658,72 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, Ceed ceed, ceed_data->qf_error = qf_error; ceed_data->op_error = op_error; + if (app_ctx->view_solution) { + // -- Post processing + PetscCall( CreateRestrictionFromPlex(ceed, dm_H1, height, domain_label, + value, &ceed_data->elem_restr_u_H1) ); + // --------------------------------------------------------------------------- + // Setup RHS for post processing + // --------------------------------------------------------------------------- + // -- Operator action variables: we use them in post-processing.c + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->u_ceed, + NULL); + CeedQFunction qf_rhs_H1; + CeedOperator op_rhs_H1; + // Create the q-function that sets up the RHS + CeedQFunctionCreateInterior(ceed, 1, problem_data->post_rhs, + problem_data->post_rhs_loc, &qf_rhs_H1); + CeedQFunctionAddInput(qf_rhs_H1, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_rhs_H1, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_rhs_H1, "u_post", dim, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_rhs_H1, "rhs_post", dim, CEED_EVAL_INTERP); + // Create the operator that builds the RHS + CeedOperatorCreate(ceed, qf_rhs_H1, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_rhs_H1); + CeedOperatorSetField(op_rhs_H1, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_rhs_H1, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_H1, "u_post", ceed_data->elem_restr_u, + ceed_data->basis_u, ceed_data->u_ceed); + CeedOperatorSetField(op_rhs_H1, "rhs_post", ceed_data->elem_restr_u_H1, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + // -- Save libCEED data to apply operator in post-processing.c + ceed_data->qf_rhs_H1 = qf_rhs_H1; + ceed_data->op_rhs_H1 = op_rhs_H1; + // --------------------------------------------------------------------------- + // Setup qfunction for initial conditions u0 + // --------------------------------------------------------------------------- + CeedQFunction qf_post_mass; + CeedOperator op_post_mass; + CeedQFunctionCreateInterior(ceed, 1, problem_data->post_mass, + problem_data->post_mass_loc, &qf_post_mass); + CeedQFunctionAddInput(qf_post_mass, "weight", 1, CEED_EVAL_WEIGHT); + CeedQFunctionAddInput(qf_post_mass, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_post_mass, "u", dim, CEED_EVAL_INTERP); + CeedQFunctionAddOutput(qf_post_mass, "v", dim, CEED_EVAL_INTERP); + // Create the operator that builds the initial conditions + CeedOperatorCreate(ceed, qf_post_mass, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, + &op_post_mass); + CeedOperatorSetField(op_post_mass, "weight", CEED_ELEMRESTRICTION_NONE, + ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_post_mass, "dx", ceed_data->elem_restr_x, + ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_post_mass, "u", ceed_data->elem_restr_u_H1, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_post_mass, "v", ceed_data->elem_restr_u_H1, + ceed_data->basis_x, CEED_VECTOR_ACTIVE); + // -- Save libCEED data to apply operator in post-processing.c + ceed_data->qf_post_mass = qf_post_mass; + ceed_data->op_post_mass = op_post_mass; + // -- Operator action variables: we use them in post-processing.c + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u_H1, + &ceed_data->up_ceed, + NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u_H1, + &ceed_data->vp_ceed, + NULL); + } // -- Cleanup CeedVectorDestroy(&true_vec); CeedVectorDestroy(&true_force); diff --git a/examples/Hdiv-mixed/src/setup-solvers.c b/examples/Hdiv-mixed/src/setup-solvers.c index 4f47c4fcc4..b62ea16be0 100644 --- a/examples/Hdiv-mixed/src/setup-solvers.c +++ b/examples/Hdiv-mixed/src/setup-solvers.c @@ -1,11 +1,13 @@ #include "../include/setup-solvers.h" #include "../include/setup-matops.h" #include "../include/setup-libceed.h" +#include "petscvec.h" // ----------------------------------------------------------------------------- // Setup operator context data // ----------------------------------------------------------------------------- PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, + VecType vec_type, OperatorApplyContext ctx_jacobian) { PetscFunctionBeginUser; @@ -16,7 +18,7 @@ PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, ctx_jacobian->y_ceed = ceed_data->y_ceed; ctx_jacobian->ceed = ceed; ctx_jacobian->op_apply = ceed_data->op_jacobian; - + ctx_jacobian->vec_type = vec_type; PetscFunctionReturn(0); } @@ -106,44 +108,39 @@ PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, // --------------------------------------------------------------------------- // Setup Solver // --------------------------------------------------------------------------- -PetscErrorCode PDESolver(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, - VecType vec_type, SNES snes, KSP ksp, Vec *U_g) { +PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, + SNES snes, KSP ksp, Vec *U) { PetscInt U_l_size, U_g_size; PetscFunctionBeginUser; - // Create global unknown solution U_g - PetscCall( DMCreateGlobalVector(dm, U_g) ); - PetscCall( VecGetSize(*U_g, &U_g_size) ); + // Create global unknown solution U + PetscCall( VecGetSize(*U, &U_g_size) ); // Local size for matShell - PetscCall( VecGetLocalSize(*U_g, &U_l_size) ); + PetscCall( VecGetLocalSize(*U, &U_l_size) ); Vec R; - PetscCall( VecDuplicate(*U_g, &R) ); + PetscCall( VecDuplicate(*U, &R) ); // --------------------------------------------------------------------------- // Setup SNES // --------------------------------------------------------------------------- // Operator Mat mat_jacobian; - PetscCall( PetscCalloc1(1, &ceed_data->ctx_jacobian) ); - SetupJacobianOperatorCtx(dm, ceed, ceed_data, ceed_data->ctx_jacobian); - PetscCall( SNESSetDM(snes, ceed_data->ctx_jacobian->dm) ); + PetscCall( SNESSetDM(snes, app_ctx->ctx_jacobian->dm) ); // -- Form Action of Jacobian on delta_u - PetscCall( MatCreateShell(comm, U_l_size, U_l_size, U_g_size, - U_g_size, ceed_data->ctx_jacobian, &mat_jacobian) ); + PetscCall( MatCreateShell(app_ctx->comm, U_l_size, U_l_size, U_g_size, + U_g_size, app_ctx->ctx_jacobian, &mat_jacobian) ); PetscCall( MatShellSetOperation(mat_jacobian, MATOP_MULT, (void (*)(void))ApplyMatOp) ); - PetscCall( MatShellSetVecType(mat_jacobian, vec_type) ); + PetscCall( MatShellSetVecType(mat_jacobian, app_ctx->ctx_jacobian->vec_type) ); // Set SNES residual evaluation function - PetscCall( PetscCalloc1(1, &ceed_data->ctx_residual) ); - SetupResidualOperatorCtx(dm, ceed, ceed_data, ceed_data->ctx_residual); PetscCall( SNESSetFunction(snes, R, SNESFormResidual, - ceed_data->ctx_residual) ); + app_ctx->ctx_residual) ); // -- SNES Jacobian PetscCall( SNESSetJacobian(snes, mat_jacobian, mat_jacobian, - SNESFormJacobian, ceed_data->ctx_jacobian) ); + SNESFormJacobian, app_ctx->ctx_jacobian) ); // Setup KSP PetscCall( KSPSetFromOptions(ksp) ); @@ -156,18 +153,12 @@ PetscErrorCode PDESolver(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, PetscCall( SNESSetFromOptions(snes) ); // Solve - PetscCall( VecSet(*U_g, 0.0)); - PetscCall( SNESSolve(snes, NULL, *U_g)); + PetscCall( VecSet(*U, 0.0)); + PetscCall( SNESSolve(snes, NULL, *U)); // Free PETSc objects PetscCall( MatDestroy(&mat_jacobian) ); PetscCall( VecDestroy(&R) ); - PetscCall( VecDestroy(&ceed_data->ctx_jacobian->Y_loc) ); - PetscCall( VecDestroy(&ceed_data->ctx_jacobian->X_loc) ); - PetscCall( VecDestroy(&ceed_data->ctx_residual->Y_loc) ); - PetscCall( VecDestroy(&ceed_data->ctx_residual->X_loc) ); - PetscCall( PetscFree(ceed_data->ctx_jacobian) ); - PetscCall( PetscFree(ceed_data->ctx_residual) ); PetscFunctionReturn(0); }; @@ -175,42 +166,38 @@ PetscErrorCode PDESolver(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, // ----------------------------------------------------------------------------- // This function calculates the L2 error in the final solution // ----------------------------------------------------------------------------- -PetscErrorCode ComputeL2Error(DM dm, Ceed ceed, CeedData ceed_data, Vec U, - CeedScalar *l2_error_u, - CeedScalar *l2_error_p) { +PetscErrorCode ComputeL2Error(CeedData ceed_data, AppCtx app_ctx, Vec U, + CeedScalar *l2_error_u, CeedScalar *l2_error_p) { PetscScalar *x; PetscMemType mem_type; CeedVector collocated_error; PetscFunctionBeginUser; - PetscCall( PetscCalloc1(1, &ceed_data->ctx_error) ); - SetupErrorOperatorCtx(dm, ceed, ceed_data, ceed_data->ctx_error); - CeedInt c_start, c_end, dim, num_elem, num_qpts; - PetscCall( DMGetDimension(ceed_data->ctx_error->dm, &dim) ); + CeedInt dim, num_elem, num_qpts; + PetscCall( DMGetDimension(app_ctx->ctx_error->dm, &dim) ); CeedBasisGetNumQuadraturePoints(ceed_data->basis_u, &num_qpts); - PetscCall( DMPlexGetHeightStratum(ceed_data->ctx_error->dm, 0, &c_start, - &c_end) ); - num_elem = c_end -c_start; - CeedVectorCreate(ceed, num_elem*num_qpts*(dim+1), &collocated_error); + num_elem = ceed_data->num_elem; + CeedVectorCreate(app_ctx->ctx_error->ceed, num_elem*num_qpts*(dim+1), + &collocated_error); // Global-to-local - PetscCall( DMGlobalToLocal(ceed_data->ctx_error->dm, U, INSERT_VALUES, - ceed_data->ctx_error->X_loc) ); + PetscCall( DMGlobalToLocal(app_ctx->ctx_error->dm, U, INSERT_VALUES, + app_ctx->ctx_error->X_loc) ); // Setup CEED vector - PetscCall( VecGetArrayAndMemType(ceed_data->ctx_error->X_loc, &x, &mem_type) ); - CeedVectorSetArray(ceed_data->ctx_error->x_ceed, MemTypeP2C(mem_type), + PetscCall( VecGetArrayAndMemType(app_ctx->ctx_error->X_loc, &x, &mem_type) ); + CeedVectorSetArray(app_ctx->ctx_error->x_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, x); // Apply CEED operator - CeedOperatorApply(ceed_data->ctx_error->op_apply, ceed_data->ctx_error->x_ceed, + CeedOperatorApply(app_ctx->ctx_error->op_apply, app_ctx->ctx_error->x_ceed, collocated_error, CEED_REQUEST_IMMEDIATE); // Restore PETSc vector - CeedVectorTakeArray(ceed_data->ctx_error->x_ceed, MemTypeP2C(mem_type), NULL); - PetscCall( VecRestoreArrayReadAndMemType(ceed_data->ctx_error->X_loc, + CeedVectorTakeArray(app_ctx->ctx_error->x_ceed, MemTypeP2C(mem_type), NULL); + PetscCall( VecRestoreArrayReadAndMemType(app_ctx->ctx_error->X_loc, (const PetscScalar **)&x) ); // Compute L2 error for each field CeedInt cent_qpts = num_qpts / 2; @@ -220,8 +207,8 @@ PetscErrorCode ComputeL2Error(DM dm, Ceed ceed, CeedData ceed_data, Vec U, length_p = num_elem; length_u = num_elem*num_qpts*dim; CeedScalar e_u[length_u], e_p[length_p]; - CeedVectorCreate(ceed_data->ctx_error->ceed, length_p, &collocated_error_p); - CeedVectorCreate(ceed_data->ctx_error->ceed, length_u, &collocated_error_u); + CeedVectorCreate(app_ctx->ctx_error->ceed, length_p, &collocated_error_p); + CeedVectorCreate(app_ctx->ctx_error->ceed, length_u, &collocated_error_u); // E_U is ordered as [p_0,u_0/.../p_n,u_n] for 0 to n elements // For each element p_0 size is num_qpts, and u_0 is dim*num_qpts CeedVectorGetArrayRead(collocated_error, CEED_MEM_HOST, &E_U); @@ -254,124 +241,8 @@ PetscErrorCode ComputeL2Error(DM dm, Ceed ceed, CeedData ceed_data, Vec U, CeedVectorDestroy(&collocated_error); CeedVectorDestroy(&collocated_error_u); CeedVectorDestroy(&collocated_error_p); - PetscCall( VecDestroy(&ceed_data->ctx_error->Y_loc) ); - PetscCall( VecDestroy(&ceed_data->ctx_error->X_loc) ); - PetscCall( PetscFree(ceed_data->ctx_error) ); PetscFunctionReturn(0); }; // ----------------------------------------------------------------------------- -// This function print the output -// ----------------------------------------------------------------------------- -PetscErrorCode PrintOutput(Ceed ceed, AppCtx app_ctx, PetscBool has_ts, - CeedMemType mem_type_backend, - TS ts, SNES snes, KSP ksp, - Vec U, CeedScalar l2_error_u, CeedScalar l2_error_p) { - - PetscFunctionBeginUser; - - const char *used_resource; - CeedGetResource(ceed, &used_resource); - char hostname[PETSC_MAX_PATH_LEN]; - PetscCall( PetscGetHostName(hostname, sizeof hostname) ); - PetscInt comm_size; - PetscCall( MPI_Comm_size(app_ctx->comm, &comm_size) ); - PetscCall( PetscPrintf(app_ctx->comm, - "\n-- Mixed H(div) Example - libCEED + PETSc --\n" - " MPI:\n" - " Hostname : %s\n" - " Total ranks : %d\n" - " libCEED:\n" - " libCEED Backend : %s\n" - " libCEED Backend MemType : %s\n", - hostname, comm_size, used_resource, CeedMemTypes[mem_type_backend]) ); - - VecType vecType; - PetscCall( VecGetType(U, &vecType) ); - PetscCall( PetscPrintf(app_ctx->comm, - " PETSc:\n" - " PETSc Vec Type : %s\n", - vecType) ); - - PetscInt U_l_size, U_g_size; - PetscCall( VecGetSize(U, &U_g_size) ); - PetscCall( VecGetLocalSize(U, &U_l_size) ); - PetscCall( PetscPrintf(app_ctx->comm, - " Problem:\n" - " Problem Name : %s\n" - " Global nodes (u + p) : %" PetscInt_FMT "\n" - " Owned nodes (u + p) : %" PetscInt_FMT "\n", - app_ctx->problem_name, U_g_size, U_l_size - ) ); - // --TS - if (has_ts) { - PetscInt ts_steps; - TSType ts_type; - TSConvergedReason ts_reason; - PetscCall( TSGetStepNumber(ts, &ts_steps) ); - PetscCall( TSGetType(ts, &ts_type) ); - PetscCall( TSGetConvergedReason(ts, &ts_reason) ); - PetscCall( PetscPrintf(app_ctx->comm, - " TS:\n" - " TS Type : %s\n" - " TS Convergence : %s\n" - " Number of TS steps : %" PetscInt_FMT "\n" - " Final time : %g\n", - ts_type, TSConvergedReasons[ts_reason], - ts_steps, (double)app_ctx->t_final) ); - - PetscCall( TSGetSNES(ts, &snes) ); - } - // -- SNES - PetscInt its, snes_its = 0; - PetscCall( SNESGetIterationNumber(snes, &its) ); - snes_its += its; - SNESType snes_type; - SNESConvergedReason snes_reason; - PetscReal snes_rnorm; - PetscCall( SNESGetType(snes, &snes_type) ); - PetscCall( SNESGetConvergedReason(snes, &snes_reason) ); - PetscCall( SNESGetFunctionNorm(snes, &snes_rnorm) ); - PetscCall( PetscPrintf(app_ctx->comm, - " SNES:\n" - " SNES Type : %s\n" - " SNES Convergence : %s\n" - " Total SNES Iterations : %" PetscInt_FMT "\n" - " Final rnorm : %e\n", - snes_type, SNESConvergedReasons[snes_reason], - snes_its, (double)snes_rnorm) ); - if (!has_ts) { - PetscInt ksp_its = 0; - PetscCall( SNESGetLinearSolveIterations(snes, &its) ); - ksp_its += its; - KSPType ksp_type; - KSPConvergedReason ksp_reason; - PetscReal ksp_rnorm; - PC pc; - PCType pc_type; - PetscCall( KSPGetPC(ksp, &pc) ); - PetscCall( PCGetType(pc, &pc_type) ); - PetscCall( KSPGetType(ksp, &ksp_type) ); - PetscCall( KSPGetConvergedReason(ksp, &ksp_reason) ); - PetscCall( KSPGetIterationNumber(ksp, &ksp_its) ); - PetscCall( KSPGetResidualNorm(ksp, &ksp_rnorm) ); - PetscCall( PetscPrintf(app_ctx->comm, - " KSP:\n" - " KSP Type : %s\n" - " PC Type : %s\n" - " KSP Convergence : %s\n" - " Total KSP Iterations : %" PetscInt_FMT "\n" - " Final rnorm : %e\n", - ksp_type, pc_type, KSPConvergedReasons[ksp_reason], ksp_its, - (double)ksp_rnorm ) ); - } - - PetscCall( PetscPrintf(app_ctx->comm, - " L2 Error (MMS):\n" - " L2 error of u and p : %e, %e\n", - (double)l2_error_u, - (double)l2_error_p) ); - PetscFunctionReturn(0); -}; -// ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/src/setup-ts.c b/examples/Hdiv-mixed/src/setup-ts.c index dd0728951b..24aaec705d 100644 --- a/examples/Hdiv-mixed/src/setup-ts.c +++ b/examples/Hdiv-mixed/src/setup-ts.c @@ -2,6 +2,7 @@ #include "../include/setup-matops.h" #include "../include/setup-libceed.h" #include "../include/setup-solvers.h" +#include "../include/post-processing.h" #include "ceed/ceed.h" #include "petscerror.h" #include "petscsystypes.h" @@ -72,11 +73,8 @@ PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, // ----------------------------------------------------------------------------- // Create global initial conditions vector // ----------------------------------------------------------------------------- -PetscErrorCode CreateInitialConditions(CeedData ceed_data, - Vec U, VecType vec_type, - OperatorApplyContext ctx_initial_u0, - OperatorApplyContext ctx_initial_p0, - OperatorApplyContext ctx_residual_ut) { +PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, + VecType vec_type, Vec U) { PetscFunctionBeginUser; // ---------------------------------------------- @@ -85,7 +83,7 @@ PetscErrorCode CreateInitialConditions(CeedData ceed_data, Vec rhs_u_loc; PetscScalar *ru; PetscMemType ru_mem_type; - PetscCall( DMCreateLocalVector(ctx_initial_u0->dm, &rhs_u_loc) ); + PetscCall( DMCreateLocalVector(app_ctx->ctx_initial_u0->dm, &rhs_u_loc) ); PetscCall( VecZeroEntries(rhs_u_loc) ); PetscCall( VecGetArrayAndMemType(rhs_u_loc, &ru, &ru_mem_type) ); CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, @@ -104,15 +102,16 @@ PetscErrorCode CreateInitialConditions(CeedData ceed_data, Vec rhs_u0; CeedVectorTakeArray(ceed_data->rhs_u0_ceed, MemTypeP2C(ru_mem_type), NULL); PetscCall( VecRestoreArrayAndMemType(rhs_u_loc, &ru) ); - PetscCall( DMCreateGlobalVector(ctx_initial_u0->dm, &rhs_u0) ); + PetscCall( DMCreateGlobalVector(app_ctx->ctx_initial_u0->dm, &rhs_u0) ); PetscCall( VecZeroEntries(rhs_u0) ); - PetscCall( DMLocalToGlobal(ctx_initial_u0->dm, rhs_u_loc, ADD_VALUES, rhs_u0) ); + PetscCall( DMLocalToGlobal(app_ctx->ctx_initial_u0->dm, rhs_u_loc, ADD_VALUES, + rhs_u0) ); // ---------------------------------------------- // Solve for U0, M*U0 = rhs_u0 // ---------------------------------------------- Vec U0; - PetscCall( DMCreateGlobalVector(ctx_initial_u0->dm, &U0) ); + PetscCall( DMCreateGlobalVector(app_ctx->ctx_initial_u0->dm, &U0) ); PetscCall( VecZeroEntries(U0) ); PetscInt U0_g_size, U0_l_size; PetscCall( VecGetSize(U0, &U0_g_size) ); @@ -122,14 +121,14 @@ PetscErrorCode CreateInitialConditions(CeedData ceed_data, // Operator Mat mat_ksp_u0; // -- Form Action of residual on u - PetscCall( MatCreateShell(ctx_initial_u0->comm, U0_l_size, U0_l_size, U0_g_size, - U0_g_size, ceed_data->ctx_initial_u0, &mat_ksp_u0) ); + PetscCall( MatCreateShell(app_ctx->comm, U0_l_size, U0_l_size, U0_g_size, + U0_g_size, app_ctx->ctx_initial_u0, &mat_ksp_u0) ); PetscCall( MatShellSetOperation(mat_ksp_u0, MATOP_MULT, (void (*)(void))ApplyMatOp) ); PetscCall( MatShellSetVecType(mat_ksp_u0, vec_type) ); KSP ksp_u0; - PetscCall( KSPCreate(ctx_initial_u0->comm, &ksp_u0) ); + PetscCall( KSPCreate(app_ctx->comm, &ksp_u0) ); PetscCall( KSPSetOperators(ksp_u0, mat_ksp_u0, mat_ksp_u0) ); PetscCall( KSPSetFromOptions(ksp_u0) ); PetscCall( KSPSetUp(ksp_u0) ); @@ -141,7 +140,7 @@ PetscErrorCode CreateInitialConditions(CeedData ceed_data, Vec rhs_p_loc; PetscScalar *rp; PetscMemType rp_mem_type; - PetscCall( DMCreateLocalVector(ctx_initial_p0->dm, &rhs_p_loc) ); + PetscCall( DMCreateLocalVector(app_ctx->ctx_initial_p0->dm, &rhs_p_loc) ); PetscCall( VecZeroEntries(rhs_p_loc) ); PetscCall( VecGetArrayAndMemType(rhs_p_loc, &rp, &rp_mem_type) ); CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, @@ -160,15 +159,16 @@ PetscErrorCode CreateInitialConditions(CeedData ceed_data, Vec rhs_p0; CeedVectorTakeArray(ceed_data->rhs_p0_ceed, MemTypeP2C(rp_mem_type), NULL); PetscCall( VecRestoreArrayAndMemType(rhs_p_loc, &rp) ); - PetscCall( DMCreateGlobalVector(ctx_initial_p0->dm, &rhs_p0) ); + PetscCall( DMCreateGlobalVector(app_ctx->ctx_initial_p0->dm, &rhs_p0) ); PetscCall( VecZeroEntries(rhs_p0) ); - PetscCall( DMLocalToGlobal(ctx_initial_p0->dm, rhs_p_loc, ADD_VALUES, rhs_p0) ); + PetscCall( DMLocalToGlobal(app_ctx->ctx_initial_p0->dm, rhs_p_loc, ADD_VALUES, + rhs_p0) ); // ---------------------------------------------- // Solve for P0, M*P0 = rhs_p0 // ---------------------------------------------- Vec P0; - PetscCall( DMCreateGlobalVector(ctx_initial_p0->dm, &P0) ); + PetscCall( DMCreateGlobalVector(app_ctx->ctx_initial_p0->dm, &P0) ); PetscCall( VecZeroEntries(P0) ); PetscInt P0_g_size, P0_l_size; PetscCall( VecGetSize(P0, &P0_g_size) ); @@ -178,14 +178,14 @@ PetscErrorCode CreateInitialConditions(CeedData ceed_data, // Operator Mat mat_ksp_p0; // -- Form Action of residual on u - PetscCall( MatCreateShell(ctx_initial_p0->comm, P0_l_size, P0_l_size, P0_g_size, - P0_g_size, ceed_data->ctx_initial_p0, &mat_ksp_p0) ); + PetscCall( MatCreateShell(app_ctx->comm, P0_l_size, P0_l_size, P0_g_size, + P0_g_size, app_ctx->ctx_initial_p0, &mat_ksp_p0) ); PetscCall( MatShellSetOperation(mat_ksp_p0, MATOP_MULT, (void (*)(void))ApplyMatOp) ); PetscCall( MatShellSetVecType(mat_ksp_p0, vec_type) ); KSP ksp_p0; - PetscCall( KSPCreate(ctx_initial_p0->comm, &ksp_p0) ); + PetscCall( KSPCreate(app_ctx->comm, &ksp_p0) ); PetscCall( KSPSetOperators(ksp_p0, mat_ksp_p0, mat_ksp_p0) ); PetscCall( KSPSetFromOptions(ksp_p0) ); PetscCall( KSPSetUp(ksp_p0) ); @@ -195,44 +195,41 @@ PetscErrorCode CreateInitialConditions(CeedData ceed_data, // Create final initial conditions U // ---------------------------------------------- // Global-to-local for U0, P0 - PetscCall( DMGlobalToLocal(ctx_initial_u0->dm, U0, INSERT_VALUES, - ctx_initial_u0->X_loc) ); - PetscCall( DMGlobalToLocal(ctx_initial_p0->dm, P0, INSERT_VALUES, - ctx_initial_p0->X_loc) ); + PetscCall( DMGlobalToLocal(app_ctx->ctx_initial_u0->dm, U0, INSERT_VALUES, + app_ctx->ctx_initial_u0->X_loc) ); + PetscCall( DMGlobalToLocal(app_ctx->ctx_initial_p0->dm, P0, INSERT_VALUES, + app_ctx->ctx_initial_p0->X_loc) ); // Get array u0,p0 const PetscScalar *u0, *p0; - PetscCall( VecGetArrayRead(ctx_initial_u0->X_loc, &u0) ); - PetscCall( VecGetArrayRead(ctx_initial_p0->X_loc, &p0) ); + PetscCall( VecGetArrayRead(app_ctx->ctx_initial_u0->X_loc, &u0) ); + PetscCall( VecGetArrayRead(app_ctx->ctx_initial_p0->X_loc, &p0) ); // Get array of local vector U = [p,u] PetscScalar *u; PetscInt U_l_size; PetscCall( VecGetLocalSize(U, &U_l_size) ); - PetscCall( VecZeroEntries(ctx_residual_ut->X_loc) ); - PetscCall( VecGetArray(ctx_residual_ut->X_loc, &u) ); + PetscCall( VecZeroEntries(app_ctx->ctx_residual_ut->X_loc) ); + PetscCall( VecGetArray(app_ctx->ctx_residual_ut->X_loc, &u) ); for (PetscInt i = 0; inum_elem; i++) { u[i] = p0[i]; } for (PetscInt i = ceed_data->num_elem; inum_elem]; } - PetscCall( VecRestoreArray(ctx_residual_ut->X_loc, &u) ); - PetscCall( VecRestoreArrayRead(ctx_initial_p0->X_loc, &p0) ); - PetscCall( VecRestoreArrayRead(ctx_initial_u0->X_loc, &u0) ); - PetscCall( DMLocalToGlobal(ctx_residual_ut->dm, ctx_residual_ut->X_loc, + PetscCall( VecRestoreArray(app_ctx->ctx_residual_ut->X_loc, &u) ); + PetscCall( VecRestoreArrayRead(app_ctx->ctx_initial_p0->X_loc, &p0) ); + PetscCall( VecRestoreArrayRead(app_ctx->ctx_initial_u0->X_loc, &u0) ); + PetscCall( DMLocalToGlobal(app_ctx->ctx_residual_ut->dm, + app_ctx->ctx_residual_ut->X_loc, ADD_VALUES, U) ); // Clean up PetscCall( VecDestroy(&rhs_u_loc) ); PetscCall( VecDestroy(&rhs_u0) ); PetscCall( VecDestroy(&U0) ); - PetscCall( VecDestroy(&ctx_initial_u0->X_loc) ); - PetscCall( VecDestroy(&ctx_initial_u0->Y_loc) ); PetscCall( VecDestroy(&rhs_p_loc) ); PetscCall( VecDestroy(&rhs_p0) ); PetscCall( VecDestroy(&P0) ); - PetscCall( VecDestroy(&ctx_initial_p0->X_loc) ); - PetscCall( VecDestroy(&ctx_initial_p0->Y_loc) ); PetscCall( MatDestroy(&mat_ksp_p0) ); PetscCall( MatDestroy(&mat_ksp_u0) ); PetscCall( KSPDestroy(&ksp_p0) ); @@ -295,39 +292,95 @@ PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, PetscFunctionReturn(0); } +PetscErrorCode WriteOutput(Vec U, PetscInt steps, + PetscScalar time, AppCtx app_ctx) { + char output_filename[PETSC_MAX_PATH_LEN]; + PetscViewer viewer_p, viewer_u; + PetscMPIInt rank; + PetscFunctionBeginUser; + + // Create output directory + MPI_Comm_rank(app_ctx->comm, &rank); + if (!rank) {PetscCall( PetscMkdir(app_ctx->output_dir) );} + + // Build file name + PetscCall( PetscSNPrintf(output_filename, sizeof output_filename, + "%s/richard_pressure-%03" PetscInt_FMT ".vtu", + app_ctx->output_dir, + steps) ); + PetscCall(PetscViewerVTKOpen(app_ctx->comm, output_filename, + FILE_MODE_WRITE, &viewer_p)); + PetscCall(VecView(U, viewer_p)); + PetscCall(PetscViewerDestroy(&viewer_p)); + + // Project velocity to H1 + Vec U_H1; // velocity in H1 space for post-processing + PetscCall( DMCreateGlobalVector(app_ctx->ctx_H1->dm, &U_H1) ); + PetscCall( ProjectVelocity(app_ctx, U, &U_H1) ); + // Build file name + PetscCall( PetscSNPrintf(output_filename, sizeof output_filename, + "%s/richard_velocity-%03" PetscInt_FMT ".vtu", + app_ctx->output_dir, + steps) ); + PetscCall(PetscViewerVTKOpen(app_ctx->comm, output_filename, + FILE_MODE_WRITE, &viewer_u)); + PetscCall(VecView(U_H1, viewer_u)); + PetscCall(PetscViewerDestroy(&viewer_u)); + PetscCall( VecDestroy(&U_H1) ); + PetscFunctionReturn(0); +} + +// User provided TS Monitor +PetscErrorCode TSMonitorRichard(TS ts, PetscInt steps, PetscReal time, + Vec U, void *ctx) { + AppCtx app_ctx = (AppCtx)ctx; + + PetscFunctionBeginUser; + + // Print every 'output_freq' steps + if (app_ctx->output_freq <= 0 + || steps % app_ctx->output_freq != 0) + PetscFunctionReturn(0); + + PetscCall( WriteOutput(U, steps, time, app_ctx) ); + + PetscFunctionReturn(0); +} + // TS: Create, setup, and solve -PetscErrorCode TSSolveRichard(DM dm, CeedData ceed_data, AppCtx app_ctx, - Vec *U, TS *ts) { - MPI_Comm comm = app_ctx->comm; +PetscErrorCode TSSolveRichard(CeedData ceed_data, AppCtx app_ctx, + TS ts, Vec *U) { TSAdapt adapt; PetscFunctionBeginUser; - PetscCall( TSCreate(comm, ts) ); - PetscCall( TSSetDM(*ts, dm) ); - PetscCall( TSSetType(*ts, TSBDF) ); - PetscCall( TSSetIFunction(*ts, NULL, TSFormIResidual, - ceed_data->ctx_residual_ut) ); + PetscCall( TSSetDM(ts, app_ctx->ctx_residual_ut->dm) ); + PetscCall( TSSetType(ts, TSBDF) ); + PetscCall( TSSetIFunction(ts, NULL, TSFormIResidual, + app_ctx->ctx_residual_ut) ); - PetscCall( TSSetMaxTime(*ts, app_ctx->t_final) ); - PetscCall( TSSetExactFinalTime(*ts, TS_EXACTFINALTIME_STEPOVER) ); - PetscCall( TSSetTimeStep(*ts, 1.e-2) ); - PetscCall( TSGetAdapt(*ts, &adapt) ); + PetscCall( TSSetMaxTime(ts, app_ctx->t_final) ); + PetscCall( TSSetExactFinalTime(ts, TS_EXACTFINALTIME_STEPOVER) ); + PetscCall( TSSetTimeStep(ts, 1.e-2) ); + PetscCall( TSGetAdapt(ts, &adapt) ); PetscCall( TSAdaptSetStepLimits(adapt, 1.e-12, 1.e2) ); - PetscCall( TSSetFromOptions(*ts) ); - ceed_data->ctx_residual_ut->t = -1.0; + PetscCall( TSSetFromOptions(ts) ); + app_ctx->ctx_residual_ut->t = -1.0; //ceed_data->ctx_residual_ut->dt = -1.0; + if (app_ctx->view_solution) { + PetscCall( TSMonitorSet(ts, TSMonitorRichard, app_ctx, NULL) ); + } // Solve PetscScalar start_time; - PetscCall( TSGetTime(*ts, &start_time) ); + PetscCall( TSGetTime(ts, &start_time) ); - PetscCall(TSSetTime(*ts, start_time)); - PetscCall(TSSetStepNumber(*ts, 0)); + PetscCall(TSSetTime(ts, start_time)); + PetscCall(TSSetStepNumber(ts, 0)); - PetscCall( PetscBarrier((PetscObject) *ts) ); - PetscCall( TSSolve(*ts, *U) ); + PetscCall( PetscBarrier((PetscObject) ts) ); + PetscCall( TSSolve(ts, *U) ); PetscScalar final_time; - PetscCall( TSGetSolveTime(*ts, &final_time) ); + PetscCall( TSGetSolveTime(ts, &final_time) ); app_ctx->t_final = final_time; PetscFunctionReturn(0); diff --git a/interface/ceed-operator.c b/interface/ceed-operator.c index 4d7ed19e6f..8f2bf988eb 100644 --- a/interface/ceed-operator.c +++ b/interface/ceed-operator.c @@ -76,10 +76,6 @@ static int CeedOperatorCheckField(Ceed ceed, CeedQFunctionField qf_field, CeedEl CeedEvalModes[eval_mode]); // LCOV_EXCL_STOP } - bool is_oriented; - ierr = CeedElemRestrictionIsOriented(r, &is_oriented); CeedChk(ierr); - CeedInt scale_r; - scale_r = is_oriented ? qf_field->size : 1; // Field size switch (eval_mode) { case CEED_EVAL_NONE: From 501fabecdacbf4e4e72abe7b22d16a857a867688 Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Mon, 14 Nov 2022 17:45:52 -0700 Subject: [PATCH 09/15] Tried to assemble Jacobian --- examples/Hdiv-mixed/src/setup-solvers.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/examples/Hdiv-mixed/src/setup-solvers.c b/examples/Hdiv-mixed/src/setup-solvers.c index b62ea16be0..3ea1f631f1 100644 --- a/examples/Hdiv-mixed/src/setup-solvers.c +++ b/examples/Hdiv-mixed/src/setup-solvers.c @@ -92,9 +92,31 @@ PetscErrorCode SNESFormResidual(SNES snes, Vec X, Vec Y, void *ctx_residual) { // ----------------------------------------------------------------------------- PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, void *ctx_jacobian) { - + OperatorApplyContext ctx = (OperatorApplyContext)ctx_jacobian; PetscFunctionBeginUser; + Mat A; + PetscCall(DMCreateMatrix(ctx->dm, &A)); + // Assemble matrix analytically + PetscCount num_entries; + CeedInt *rows, *cols; + CeedVector coo_values; + CeedOperatorLinearAssembleSymbolic(ctx->op_apply, &num_entries, &rows, + &cols); + PetscCall(MatSetPreallocationCOO(A, num_entries, rows, cols)); + free(rows); + free(cols); + CeedVectorCreate(ctx->ceed, num_entries, &coo_values); + CeedOperatorLinearAssemble(ctx->op_apply, coo_values); + const CeedScalar *values; + CeedVectorGetArrayRead(coo_values, CEED_MEM_HOST, &values); + PetscCall(MatSetValuesCOO(A, values, ADD_VALUES)); + CeedVectorRestoreArrayRead(coo_values, &values); + MatView(A, PETSC_VIEWER_STDOUT_WORLD); + //CeedVectorView(coo_values, "%12.8f", stdout); + CeedVectorDestroy(&coo_values); + PetscCall( MatDestroy(&A) ); + // J_pre might be AIJ (e.g., when using coloring), so we need to assemble it PetscCall( MatAssemblyBegin(J_pre, MAT_FINAL_ASSEMBLY) ); PetscCall( MatAssemblyEnd(J_pre, MAT_FINAL_ASSEMBLY) ); From 33fc16e9dd38a832cc07b7049b7d0c833eab3bd3 Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Sat, 19 Nov 2022 09:44:21 -0700 Subject: [PATCH 10/15] make format and rebased --- examples/Hdiv-mass/basis/Hdiv-hex.h | 246 ++++--- examples/Hdiv-mass/basis/Hdiv-quad.h | 75 +-- examples/Hdiv-mass/include/cl-options.h | 2 +- examples/Hdiv-mass/include/matops.h | 5 +- examples/Hdiv-mass/include/petsc-macros.h | 16 +- examples/Hdiv-mass/include/problems.h | 2 +- examples/Hdiv-mass/include/setup-dm.h | 8 +- examples/Hdiv-mass/include/setup-libceed.h | 13 +- examples/Hdiv-mass/include/structs.h | 17 +- examples/Hdiv-mass/main.c | 235 ++++--- examples/Hdiv-mass/main.h | 8 +- examples/Hdiv-mass/problems/mass2d.c | 33 +- examples/Hdiv-mass/problems/mass3d.c | 33 +- .../Hdiv-mass/qfunctions/poisson-error2d.h | 35 +- .../Hdiv-mass/qfunctions/poisson-error3d.h | 39 +- .../Hdiv-mass/qfunctions/poisson-mass2d.h | 28 +- .../Hdiv-mass/qfunctions/poisson-mass3d.h | 30 +- examples/Hdiv-mass/qfunctions/poisson-rhs2d.h | 42 +- examples/Hdiv-mass/qfunctions/poisson-rhs3d.h | 52 +- examples/Hdiv-mass/qfunctions/utils.h | 79 +-- examples/Hdiv-mass/src/cl-options.c | 32 +- examples/Hdiv-mass/src/matops.c | 55 +- examples/Hdiv-mass/src/setup-dm.c | 71 +- examples/Hdiv-mass/src/setup-libceed.c | 270 ++++---- examples/Hdiv-mixed/basis/Hdiv-hex.h | 246 ++++--- examples/Hdiv-mixed/basis/Hdiv-quad.h | 75 +-- examples/Hdiv-mixed/basis/L2-P0.h | 52 +- examples/Hdiv-mixed/include/cl-options.h | 2 +- examples/Hdiv-mixed/include/petsc-macros.h | 16 +- examples/Hdiv-mixed/include/post-processing.h | 20 +- .../Hdiv-mixed/include/register-problem.h | 14 +- examples/Hdiv-mixed/include/setup-boundary.h | 13 +- examples/Hdiv-mixed/include/setup-dm.h | 8 +- examples/Hdiv-mixed/include/setup-fe.h | 8 +- examples/Hdiv-mixed/include/setup-libceed.h | 18 +- examples/Hdiv-mixed/include/setup-matops.h | 8 +- examples/Hdiv-mixed/include/setup-solvers.h | 18 +- examples/Hdiv-mixed/include/setup-ts.h | 20 +- examples/Hdiv-mixed/include/structs.h | 88 ++- examples/Hdiv-mixed/main.c | 245 ++++--- examples/Hdiv-mixed/main.h | 12 +- examples/Hdiv-mixed/problems/darcy2d.c | 115 ++-- examples/Hdiv-mixed/problems/darcy3d.c | 101 ++- .../Hdiv-mixed/problems/register-problem.c | 16 +- examples/Hdiv-mixed/problems/richard2d.c | 141 ++-- examples/Hdiv-mixed/problems/richard3d.c | 143 ++-- .../Hdiv-mixed/qfunctions/darcy-error2d.h | 50 +- .../Hdiv-mixed/qfunctions/darcy-error3d.h | 55 +- .../qfunctions/darcy-system-quartic2d.h | 71 +- .../Hdiv-mixed/qfunctions/darcy-system2d.h | 133 ++-- .../Hdiv-mixed/qfunctions/darcy-system3d.h | 142 ++-- .../qfunctions/darcy-true-quartic2d.h | 45 +- examples/Hdiv-mixed/qfunctions/darcy-true2d.h | 63 +- examples/Hdiv-mixed/qfunctions/darcy-true3d.h | 75 +-- .../Hdiv-mixed/qfunctions/post-processing2d.h | 64 +- .../Hdiv-mixed/qfunctions/post-processing3d.h | 70 +- .../qfunctions/pressure-boundary2d.h | 20 +- .../qfunctions/pressure-boundary3d.h | 19 +- .../Hdiv-mixed/qfunctions/richard-ics2d.h | 138 ++-- .../Hdiv-mixed/qfunctions/richard-ics3d.h | 153 ++--- .../Hdiv-mixed/qfunctions/richard-system2d.h | 91 ++- .../Hdiv-mixed/qfunctions/richard-system3d.h | 95 ++- .../Hdiv-mixed/qfunctions/richard-true2d.h | 67 +- .../Hdiv-mixed/qfunctions/richard-true3d.h | 78 +-- examples/Hdiv-mixed/qfunctions/utils.h | 79 +-- examples/Hdiv-mixed/src/cl-options.c | 48 +- examples/Hdiv-mixed/src/post-processing.c | 322 ++++----- examples/Hdiv-mixed/src/setup-boundary.c | 72 +- examples/Hdiv-mixed/src/setup-dm.c | 153 ++--- examples/Hdiv-mixed/src/setup-fe.c | 136 ++-- examples/Hdiv-mixed/src/setup-libceed.c | 631 +++++++----------- examples/Hdiv-mixed/src/setup-matops.c | 38 +- examples/Hdiv-mixed/src/setup-solvers.c | 171 +++-- examples/Hdiv-mixed/src/setup-ts.c | 369 +++++----- tests/t330-basis.h | 2 +- 75 files changed, 2839 insertions(+), 3416 deletions(-) diff --git a/examples/Hdiv-mass/basis/Hdiv-hex.h b/examples/Hdiv-mass/basis/Hdiv-hex.h index a01b1a3ef3..9259bede85 100644 --- a/examples/Hdiv-mass/basis/Hdiv-hex.h +++ b/examples/Hdiv-mass/basis/Hdiv-hex.h @@ -16,119 +16,115 @@ // To see how the nodal basis is constructed visit: // https://github.com/rezgarshakeri/H-div-Tests -int NodalHdivBasisHex(CeedScalar *x, CeedScalar *Bx, CeedScalar *By, - CeedScalar *Bz) { - - Bx[ 0] = 0.0625*x[0]*x[0] - 0.0625 ; - By[ 0] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; - Bz[ 0] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bx[ 1] = 0.0625 - 0.0625*x[0]*x[0] ; - By[ 1] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; - Bz[ 1] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bx[ 2] = 0.0625*x[0]*x[0] - 0.0625 ; - By[ 2] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; - Bz[ 2] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bx[ 3] = 0.0625 - 0.0625*x[0]*x[0] ; - By[ 3] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; - Bz[ 3] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] + - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bx[ 4] = 0.0625*x[0]*x[0] - 0.0625 ; - By[ 4] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; - Bz[ 4] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] - - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bx[ 5] = 0.0625 - 0.0625*x[0]*x[0] ; - By[ 5] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; - Bz[ 5] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bx[ 6] = 0.0625*x[0]*x[0] - 0.0625 ; - By[ 6] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; - Bz[ 6] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bx[ 7] = 0.0625 - 0.0625*x[0]*x[0] ; - By[ 7] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; - Bz[ 7] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bx[ 8] = 0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] - 0.0625*x[2] + 0.0625 ; - By[ 8] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bz[ 8] = 0.0625*x[2]*x[2] - 0.0625 ; - Bx[ 9] = -0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] + 0.0625*x[2] - 0.0625 ; - By[ 9] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bz[ 9] = 0.0625*x[2]*x[2] - 0.0625 ; - Bx[10] = -0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] + 0.0625*x[2] + 0.0625 ; - By[10] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; - Bz[10] = 0.0625 - 0.0625*x[2]*x[2] ; - Bx[11] = 0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] - 0.0625*x[2] - 0.0625 ; - By[11] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; - Bz[11] = 0.0625 - 0.0625*x[2]*x[2] ; - Bx[12] = 0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] - 0.0625*x[2] + 0.0625 ; - By[12] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; - Bz[12] = 0.0625*x[2]*x[2] - 0.0625 ; - Bx[13] = -0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] + 0.0625*x[2] - 0.0625 ; - By[13] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; - Bz[13] = 0.0625*x[2]*x[2] - 0.0625 ; - Bx[14] = -0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] + 0.0625*x[2] + 0.0625 ; - By[14] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bz[14] = 0.0625 - 0.0625*x[2]*x[2] ; - Bx[15] = 0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] - 0.0625*x[2] - 0.0625 ; - By[15] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bz[15] = 0.0625 - 0.0625*x[2]*x[2] ; - Bx[16] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] - 0.125*x[1] - 0.125*x[2] + 0.125 ; - By[16] = 0.0625*x[1]*x[1] - 0.0625 ; - Bz[16] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; - Bx[17] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; - By[17] = 0.0625 - 0.0625*x[1]*x[1] ; - Bz[17] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; - Bx[18] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; - By[18] = 0.0625*x[1]*x[1] - 0.0625 ; - Bz[18] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; - Bx[19] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - By[19] = 0.0625 - 0.0625*x[1]*x[1] ; - Bz[19] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; - Bx[20] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - By[20] = 0.0625*x[1]*x[1] - 0.0625 ; - Bz[20] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; - Bx[21] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; - By[21] = 0.0625 - 0.0625*x[1]*x[1] ; - Bz[21] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; - Bx[22] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; - By[22] = 0.0625*x[1]*x[1] - 0.0625 ; - Bz[22] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; - Bx[23] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] - 0.125*x[1] - 0.125*x[2] - 0.125 ; - By[23] = 0.0625 - 0.0625*x[1]*x[1] ; - Bz[23] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; +int NodalHdivBasisHex(CeedScalar *x, CeedScalar *Bx, CeedScalar *By, CeedScalar *Bz) { + Bx[0] = 0.0625 * x[0] * x[0] - 0.0625; + By[0] = -0.0625 * x[0] * x[1] * x[1] + 0.0625 * x[0] + 0.0625 * x[1] * x[1] - 0.0625; + Bz[0] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bx[1] = 0.0625 - 0.0625 * x[0] * x[0]; + By[1] = 0.0625 * x[0] * x[1] * x[1] - 0.0625 * x[0] + 0.0625 * x[1] * x[1] - 0.0625; + Bz[1] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] - 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bx[2] = 0.0625 * x[0] * x[0] - 0.0625; + By[2] = 0.0625 * x[0] * x[1] * x[1] - 0.0625 * x[0] - 0.0625 * x[1] * x[1] + 0.0625; + Bz[2] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] - 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bx[3] = 0.0625 - 0.0625 * x[0] * x[0]; + By[3] = -0.0625 * x[0] * x[1] * x[1] + 0.0625 * x[0] - 0.0625 * x[1] * x[1] + 0.0625; + Bz[3] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] - 0.125 * x[0] + 0.125 * x[1] * x[2] - 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bx[4] = 0.0625 * x[0] * x[0] - 0.0625; + By[4] = -0.0625 * x[0] * x[1] * x[1] + 0.0625 * x[0] + 0.0625 * x[1] * x[1] - 0.0625; + Bz[4] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] - 0.125 * x[0] - 0.125 * x[1] * x[2] - 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bx[5] = 0.0625 - 0.0625 * x[0] * x[0]; + By[5] = 0.0625 * x[0] * x[1] * x[1] - 0.0625 * x[0] + 0.0625 * x[1] * x[1] - 0.0625; + Bz[5] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] - 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bx[6] = 0.0625 * x[0] * x[0] - 0.0625; + By[6] = 0.0625 * x[0] * x[1] * x[1] - 0.0625 * x[0] - 0.0625 * x[1] * x[1] + 0.0625; + Bz[6] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] - 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bx[7] = 0.0625 - 0.0625 * x[0] * x[0]; + By[7] = -0.0625 * x[0] * x[1] * x[1] + 0.0625 * x[0] - 0.0625 * x[1] * x[1] + 0.0625; + Bz[7] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bx[8] = 0.0625 * x[0] * x[0] * x[2] - 0.0625 * x[0] * x[0] - 0.0625 * x[2] + 0.0625; + By[8] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] - 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bz[8] = 0.0625 * x[2] * x[2] - 0.0625; + Bx[9] = -0.0625 * x[0] * x[0] * x[2] + 0.0625 * x[0] * x[0] + 0.0625 * x[2] - 0.0625; + By[9] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bz[9] = 0.0625 * x[2] * x[2] - 0.0625; + Bx[10] = -0.0625 * x[0] * x[0] * x[2] - 0.0625 * x[0] * x[0] + 0.0625 * x[2] + 0.0625; + By[10] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] - 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] - 0.125 * x[2] - + 0.125; + Bz[10] = 0.0625 - 0.0625 * x[2] * x[2]; + Bx[11] = 0.0625 * x[0] * x[0] * x[2] + 0.0625 * x[0] * x[0] - 0.0625 * x[2] - 0.0625; + By[11] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] - + 0.125 * x[2] - 0.125; + Bz[11] = 0.0625 - 0.0625 * x[2] * x[2]; + Bx[12] = 0.0625 * x[0] * x[0] * x[2] - 0.0625 * x[0] * x[0] - 0.0625 * x[2] + 0.0625; + By[12] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] - + 0.125 * x[2] + 0.125; + Bz[12] = 0.0625 * x[2] * x[2] - 0.0625; + Bx[13] = -0.0625 * x[0] * x[0] * x[2] + 0.0625 * x[0] * x[0] + 0.0625 * x[2] - 0.0625; + By[13] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] - 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] - 0.125 * x[2] + + 0.125; + Bz[13] = 0.0625 * x[2] * x[2] - 0.0625; + Bx[14] = -0.0625 * x[0] * x[0] * x[2] - 0.0625 * x[0] * x[0] + 0.0625 * x[2] + 0.0625; + By[14] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bz[14] = 0.0625 - 0.0625 * x[2] * x[2]; + Bx[15] = 0.0625 * x[0] * x[0] * x[2] + 0.0625 * x[0] * x[0] - 0.0625 * x[2] - 0.0625; + By[15] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] - 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + + 0.125 * x[2] + 0.125; + Bz[15] = 0.0625 - 0.0625 * x[2] * x[2]; + Bx[16] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] - 0.125 * x[1] - 0.125 * x[2] + + 0.125; + By[16] = 0.0625 * x[1] * x[1] - 0.0625; + Bz[16] = -0.0625 * x[1] * x[2] * x[2] + 0.0625 * x[1] + 0.0625 * x[2] * x[2] - 0.0625; + Bx[17] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] - + 0.125 * x[2] + 0.125; + By[17] = 0.0625 - 0.0625 * x[1] * x[1]; + Bz[17] = 0.0625 * x[1] * x[2] * x[2] - 0.0625 * x[1] + 0.0625 * x[2] * x[2] - 0.0625; + Bx[18] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] - 0.125 * x[1] + + 0.125 * x[2] + 0.125; + By[18] = 0.0625 * x[1] * x[1] - 0.0625; + Bz[18] = 0.0625 * x[1] * x[2] * x[2] - 0.0625 * x[1] - 0.0625 * x[2] * x[2] + 0.0625; + Bx[19] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] + + 0.125; + By[19] = 0.0625 - 0.0625 * x[1] * x[1]; + Bz[19] = -0.0625 * x[1] * x[2] * x[2] + 0.0625 * x[1] - 0.0625 * x[2] * x[2] + 0.0625; + Bx[20] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + By[20] = 0.0625 * x[1] * x[1] - 0.0625; + Bz[20] = -0.0625 * x[1] * x[2] * x[2] + 0.0625 * x[1] + 0.0625 * x[2] * x[2] - 0.0625; + Bx[21] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] - 0.125 * x[1] + + 0.125 * x[2] - 0.125; + By[21] = 0.0625 - 0.0625 * x[1] * x[1]; + Bz[21] = 0.0625 * x[1] * x[2] * x[2] - 0.0625 * x[1] + 0.0625 * x[2] * x[2] - 0.0625; + Bx[22] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] - + 0.125 * x[2] - 0.125; + By[22] = 0.0625 * x[1] * x[1] - 0.0625; + Bz[22] = 0.0625 * x[1] * x[2] * x[2] - 0.0625 * x[1] - 0.0625 * x[2] * x[2] + 0.0625; + Bx[23] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] - 0.125 * x[1] - 0.125 * x[2] - + 0.125; + By[23] = 0.0625 - 0.0625 * x[1] * x[1]; + Bz[23] = -0.0625 * x[1] * x[2] * x[2] + 0.0625 * x[1] - 0.0625 * x[2] * x[2] + 0.0625; return 0; } -static void HdivBasisHex(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, - CeedScalar *interp, CeedScalar *div, CeedQuadMode quad_mode) { - +static void HdivBasisHex(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, CeedScalar *interp, CeedScalar *div, CeedQuadMode quad_mode) { // Get 1D quadrature on [-1,1] CeedScalar q_ref_1d[Q], q_weight_1d[Q]; switch (quad_mode) { - case CEED_GAUSS: - CeedGaussQuadrature(Q, q_ref_1d, q_weight_1d); - break; - case CEED_GAUSS_LOBATTO: - CeedLobattoQuadrature(Q, q_ref_1d, q_weight_1d); - break; + case CEED_GAUSS: + CeedGaussQuadrature(Q, q_ref_1d, q_weight_1d); + break; + case CEED_GAUSS_LOBATTO: + CeedLobattoQuadrature(Q, q_ref_1d, q_weight_1d); + break; } // Divergence operator; Divergence of nodal basis for ref element @@ -136,27 +132,25 @@ static void HdivBasisHex(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, // Loop over quadrature points CeedScalar Bx[24], By[24], Bz[24]; CeedScalar x[3]; - for (CeedInt k=0; k #include #include #include -#include + #include "../include/structs.h" // --------------------------------------------------------------------------- // Set-up DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDistributedDM(MPI_Comm comm, ProblemData *problem_data, - DM *dm); +PetscErrorCode CreateDistributedDM(MPI_Comm comm, ProblemData *problem_data, DM *dm); -#endif // setupdm_h +#endif // setupdm_h diff --git a/examples/Hdiv-mass/include/setup-libceed.h b/examples/Hdiv-mass/include/setup-libceed.h index bced0f69cc..7be8d75464 100644 --- a/examples/Hdiv-mass/include/setup-libceed.h +++ b/examples/Hdiv-mass/include/setup-libceed.h @@ -10,15 +10,10 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data); // Utility function - essential BC dofs are encoded in closure indices as -(i+1) PetscInt Involute(PetscInt i); // Utility function to create local CEED restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, - CeedInt height, DMLabel domain_label, CeedInt value, - CeedElemRestriction *elem_restr); +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr); // Utility function to create local CEED Oriented restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, - CeedInt P, CeedElemRestriction *elem_restr_oriented); +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, CeedElemRestriction *elem_restr_oriented); // Set up libCEED for a given degree -PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, - ProblemData *problem_data, PetscInt U_g_size, - PetscInt U_loc_size, CeedData ceed_data, +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *problem_data, PetscInt U_g_size, PetscInt U_loc_size, CeedData ceed_data, CeedVector rhs_ceed, CeedVector *target); -#endif // setuplibceed_h +#endif // setuplibceed_h diff --git a/examples/Hdiv-mass/include/structs.h b/examples/Hdiv-mass/include/structs.h index b4a8754e65..69f65d916b 100644 --- a/examples/Hdiv-mass/include/structs.h +++ b/examples/Hdiv-mass/include/structs.h @@ -8,8 +8,8 @@ typedef struct AppCtx_ *AppCtx; struct AppCtx_ { // libCEED arguments - PetscInt degree; - PetscInt q_extra; + PetscInt degree; + PetscInt q_extra; // Problem type arguments PetscFunctionList problems; char problem_name[PETSC_MAX_PATH_LEN]; @@ -19,8 +19,7 @@ struct AppCtx_ { typedef struct CeedData_ *CeedData; struct CeedData_ { CeedBasis basis_x, basis_u; - CeedElemRestriction elem_restr_x, elem_restr_u, - elem_restr_u_i; + CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_u_i; CeedQFunction qf_residual, qf_error; CeedOperator op_residual, op_error; CeedVector x_ceed, y_ceed; @@ -52,8 +51,8 @@ struct PH3DContext_ { // Struct that contains all enums and structs used for the physics of all problems typedef struct Physics_ *Physics; struct Physics_ { - PQ2DContext pq2d_ctx; - PH3DContext ph3d_ctx; + PQ2DContext pq2d_ctx; + PH3DContext ph3d_ctx; }; // PETSc user data @@ -72,11 +71,11 @@ struct User_ { // Problem specific data typedef struct { CeedQFunctionUser setup_rhs, residual, setup_error; - const char *setup_rhs_loc, *residual_loc, *setup_error_loc; + const char *setup_rhs_loc, *residual_loc, *setup_error_loc; CeedQuadMode quadrature_mode; CeedInt elem_node, dim; - PetscErrorCode (*setup_ctx)(Ceed, CeedData, Physics); + PetscErrorCode (*setup_ctx)(Ceed, CeedData, Physics); } ProblemData; -#endif // structs_h \ No newline at end of file +#endif // structs_h \ No newline at end of file diff --git a/examples/Hdiv-mass/main.c b/examples/Hdiv-mass/main.c index ddd66fbb70..338e745cfb 100644 --- a/examples/Hdiv-mass/main.c +++ b/examples/Hdiv-mass/main.c @@ -43,19 +43,24 @@ int main(int argc, char **argv) { // Create structs // --------------------------------------------------------------------------- AppCtx app_ctx; - ierr = PetscCalloc1(1, &app_ctx); CHKERRQ(ierr); + ierr = PetscCalloc1(1, &app_ctx); + CHKERRQ(ierr); ProblemData *problem_data = NULL; - ierr = PetscCalloc1(1, &problem_data); CHKERRQ(ierr); + ierr = PetscCalloc1(1, &problem_data); + CHKERRQ(ierr); User user; - ierr = PetscCalloc1(1, &user); CHKERRQ(ierr); + ierr = PetscCalloc1(1, &user); + CHKERRQ(ierr); CeedData ceed_data; - ierr = PetscCalloc1(1, &ceed_data); CHKERRQ(ierr); + ierr = PetscCalloc1(1, &ceed_data); + CHKERRQ(ierr); Physics phys_ctx; - ierr = PetscCalloc1(1, &phys_ctx); CHKERRQ(ierr); + ierr = PetscCalloc1(1, &phys_ctx); + CHKERRQ(ierr); user->app_ctx = app_ctx; user->phys = phys_ctx; @@ -64,11 +69,13 @@ int main(int argc, char **argv) { // Process command line options // --------------------------------------------------------------------------- // -- Register problems to be available on the command line - ierr = RegisterProblems_Hdiv(app_ctx); CHKERRQ(ierr); + ierr = RegisterProblems_Hdiv(app_ctx); + CHKERRQ(ierr); // -- Process general command line options MPI_Comm comm = PETSC_COMM_WORLD; - ierr = ProcessCommandLineOptions(comm, app_ctx); CHKERRQ(ierr); + ierr = ProcessCommandLineOptions(comm, app_ctx); + CHKERRQ(ierr); // --------------------------------------------------------------------------- // Choose the problem from the list of registered problems @@ -77,9 +84,9 @@ int main(int argc, char **argv) { PetscErrorCode (*p)(ProblemData *, void *); ierr = PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p); CHKERRQ(ierr); - if (!p) SETERRQ(PETSC_COMM_SELF, 1, "Problem '%s' not found", - app_ctx->problem_name); - ierr = (*p)(problem_data, &user); CHKERRQ(ierr); + if (!p) SETERRQ(PETSC_COMM_SELF, 1, "Problem '%s' not found", app_ctx->problem_name); + ierr = (*p)(problem_data, &user); + CHKERRQ(ierr); } // --------------------------------------------------------------------------- @@ -94,48 +101,60 @@ int main(int argc, char **argv) { // Set-up DM // --------------------------------------------------------------------------- // PETSc objects - DM dm; - VecType vec_type; - ierr = CreateDistributedDM(comm, problem_data, &dm); CHKERRQ(ierr); - ierr = DMGetVecType(dm, &vec_type); CHKERRQ(ierr); - if (!vec_type) { // Not yet set by user -dm_vec_type + DM dm; + VecType vec_type; + ierr = CreateDistributedDM(comm, problem_data, &dm); + CHKERRQ(ierr); + ierr = DMGetVecType(dm, &vec_type); + CHKERRQ(ierr); + if (!vec_type) { // Not yet set by user -dm_vec_type switch (mem_type_backend) { - case CEED_MEM_HOST: vec_type = VECSTANDARD; break; - case CEED_MEM_DEVICE: { - const char *resolved; - CeedGetResource(ceed, &resolved); - if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; - else if (strstr(resolved, "/gpu/hip/occa")) - vec_type = VECSTANDARD; // https://github.com/CEED/libCEED/issues/678 - else if (strstr(resolved, "/gpu/hip")) vec_type = VECHIP; - else vec_type = VECSTANDARD; - } + case CEED_MEM_HOST: + vec_type = VECSTANDARD; + break; + case CEED_MEM_DEVICE: { + const char *resolved; + CeedGetResource(ceed, &resolved); + if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; + else if (strstr(resolved, "/gpu/hip/occa")) vec_type = VECSTANDARD; // https://github.com/CEED/libCEED/issues/678 + else if (strstr(resolved, "/gpu/hip")) vec_type = VECHIP; + else vec_type = VECSTANDARD; + } } - ierr = DMSetVecType(dm, vec_type); CHKERRQ(ierr); + ierr = DMSetVecType(dm, vec_type); + CHKERRQ(ierr); } // --------------------------------------------------------------------------- // Create global, local solution, local rhs vector // --------------------------------------------------------------------------- - Vec U_g, U_loc; - PetscInt U_l_size, U_g_size, U_loc_size; + Vec U_g, U_loc; + PetscInt U_l_size, U_g_size, U_loc_size; // Create global and local solution vectors - ierr = DMCreateGlobalVector(dm, &U_g); CHKERRQ(ierr); - ierr = VecGetSize(U_g, &U_g_size); CHKERRQ(ierr); + ierr = DMCreateGlobalVector(dm, &U_g); + CHKERRQ(ierr); + ierr = VecGetSize(U_g, &U_g_size); + CHKERRQ(ierr); // Local size for matShell - ierr = VecGetLocalSize(U_g, &U_l_size); CHKERRQ(ierr); + ierr = VecGetLocalSize(U_g, &U_l_size); + CHKERRQ(ierr); // Create local unknown vector U_loc - ierr = DMCreateLocalVector(dm, &U_loc); CHKERRQ(ierr); + ierr = DMCreateLocalVector(dm, &U_loc); + CHKERRQ(ierr); // Local size for libCEED - ierr = VecGetSize(U_loc, &U_loc_size); CHKERRQ(ierr); + ierr = VecGetSize(U_loc, &U_loc_size); + CHKERRQ(ierr); // Get RHS vector - Vec rhs_loc; + Vec rhs_loc; PetscScalar *r; - CeedVector rhs_ceed, target; + CeedVector rhs_ceed, target; PetscMemType mem_type; - ierr = VecDuplicate(U_loc, &rhs_loc); CHKERRQ(ierr); - ierr = VecZeroEntries(rhs_loc); CHKERRQ(ierr); - ierr = VecGetArrayAndMemType(rhs_loc, &r, &mem_type); CHKERRQ(ierr); + ierr = VecDuplicate(U_loc, &rhs_loc); + CHKERRQ(ierr); + ierr = VecZeroEntries(rhs_loc); + CHKERRQ(ierr); + ierr = VecGetArrayAndMemType(rhs_loc, &r, &mem_type); + CHKERRQ(ierr); CeedVectorCreate(ceed, U_l_size, &rhs_ceed); CeedVectorSetArray(rhs_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, r); @@ -143,103 +162,135 @@ int main(int argc, char **argv) { // Setup libCEED // --------------------------------------------------------------------------- // -- Set up libCEED objects - ierr = SetupLibceed(dm, ceed, app_ctx, problem_data, U_g_size, - U_loc_size, ceed_data, rhs_ceed, &target); CHKERRQ(ierr); + ierr = SetupLibceed(dm, ceed, app_ctx, problem_data, U_g_size, U_loc_size, ceed_data, rhs_ceed, &target); + CHKERRQ(ierr); // --------------------------------------------------------------------------- // Gather RHS // --------------------------------------------------------------------------- Vec rhs; CeedVectorTakeArray(rhs_ceed, MemTypeP2C(mem_type), NULL); - ierr = VecRestoreArrayAndMemType(rhs_loc, &r); CHKERRQ(ierr); - ierr = VecDuplicate(U_g, &rhs); CHKERRQ(ierr); - ierr = VecZeroEntries(rhs); CHKERRQ(ierr); - ierr = DMLocalToGlobal(dm, rhs_loc, ADD_VALUES, rhs); CHKERRQ(ierr); - //VecView(rhs, PETSC_VIEWER_STDOUT_WORLD); + ierr = VecRestoreArrayAndMemType(rhs_loc, &r); + CHKERRQ(ierr); + ierr = VecDuplicate(U_g, &rhs); + CHKERRQ(ierr); + ierr = VecZeroEntries(rhs); + CHKERRQ(ierr); + ierr = DMLocalToGlobal(dm, rhs_loc, ADD_VALUES, rhs); + CHKERRQ(ierr); + // VecView(rhs, PETSC_VIEWER_STDOUT_WORLD); // --------------------------------------------------------------------------- // Setup Mat, KSP // --------------------------------------------------------------------------- - user->comm = comm; - user->dm = dm; + user->comm = comm; + user->dm = dm; user->X_loc = U_loc; - ierr = VecDuplicate(U_loc, &user->Y_loc); CHKERRQ(ierr); - user->x_ceed = ceed_data->x_ceed; - user->y_ceed = ceed_data->y_ceed; + ierr = VecDuplicate(U_loc, &user->Y_loc); + CHKERRQ(ierr); + user->x_ceed = ceed_data->x_ceed; + user->y_ceed = ceed_data->y_ceed; user->op_apply = ceed_data->op_residual; user->op_error = ceed_data->op_error; - user->ceed = ceed; + user->ceed = ceed; // Operator Mat mat; - ierr = MatCreateShell(comm, U_l_size, U_l_size, U_g_size, U_g_size, - user, &mat); CHKERRQ(ierr); - ierr = MatShellSetOperation(mat, MATOP_MULT, - (void(*)(void))MatMult_Ceed); CHKERRQ(ierr); - ierr = MatShellSetVecType(mat, vec_type); CHKERRQ(ierr); + ierr = MatCreateShell(comm, U_l_size, U_l_size, U_g_size, U_g_size, user, &mat); + CHKERRQ(ierr); + ierr = MatShellSetOperation(mat, MATOP_MULT, (void (*)(void))MatMult_Ceed); + CHKERRQ(ierr); + ierr = MatShellSetVecType(mat, vec_type); + CHKERRQ(ierr); KSP ksp; - ierr = KSPCreate(comm, &ksp); CHKERRQ(ierr); - ierr = KSPSetOperators(ksp, mat, mat); CHKERRQ(ierr); - ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr); - ierr = KSPSetUp(ksp); CHKERRQ(ierr); - ierr = KSPSolve(ksp, rhs, U_g); CHKERRQ(ierr); - //printf("U_g\n"); - //VecView(U_g, PETSC_VIEWER_STDOUT_WORLD); - // --------------------------------------------------------------------------- - // Compute pointwise L2 maximum error - // --------------------------------------------------------------------------- + ierr = KSPCreate(comm, &ksp); + CHKERRQ(ierr); + ierr = KSPSetOperators(ksp, mat, mat); + CHKERRQ(ierr); + ierr = KSPSetFromOptions(ksp); + CHKERRQ(ierr); + ierr = KSPSetUp(ksp); + CHKERRQ(ierr); + ierr = KSPSolve(ksp, rhs, U_g); + CHKERRQ(ierr); + // printf("U_g\n"); + // VecView(U_g, PETSC_VIEWER_STDOUT_WORLD); + // --------------------------------------------------------------------------- + // Compute pointwise L2 maximum error + // --------------------------------------------------------------------------- CeedScalar l2_error; - ierr = ComputeError(user, U_g, target, &l2_error); CHKERRQ(ierr); + ierr = ComputeError(user, U_g, target, &l2_error); + CHKERRQ(ierr); // --------------------------------------------------------------------------- // Output results // --------------------------------------------------------------------------- - KSPType ksp_type; + KSPType ksp_type; KSPConvergedReason reason; - PetscReal rnorm; - PetscInt its; - ierr = KSPGetType(ksp, &ksp_type); CHKERRQ(ierr); - ierr = KSPGetConvergedReason(ksp, &reason); CHKERRQ(ierr); - ierr = KSPGetIterationNumber(ksp, &its); CHKERRQ(ierr); - ierr = KSPGetResidualNorm(ksp, &rnorm); CHKERRQ(ierr); + PetscReal rnorm; + PetscInt its; + ierr = KSPGetType(ksp, &ksp_type); + CHKERRQ(ierr); + ierr = KSPGetConvergedReason(ksp, &reason); + CHKERRQ(ierr); + ierr = KSPGetIterationNumber(ksp, &its); + CHKERRQ(ierr); + ierr = KSPGetResidualNorm(ksp, &rnorm); + CHKERRQ(ierr); ierr = PetscPrintf(comm, " KSP:\n" " KSP Type : %s\n" " KSP Convergence : %s\n" - " Total KSP Iterations : %" PetscInt_FMT "\n" + " Total KSP Iterations : %" PetscInt_FMT + "\n" " Final rnorm : %e\n" " L2 Error : %e\n", - ksp_type, KSPConvergedReasons[reason], its, - (double)rnorm, (double)l2_error); CHKERRQ(ierr); + ksp_type, KSPConvergedReasons[reason], its, (double)rnorm, (double)l2_error); + CHKERRQ(ierr); // --------------------------------------------------------------------------- // Free objects // --------------------------------------------------------------------------- // Free PETSc objects - ierr = DMDestroy(&dm); CHKERRQ(ierr); - ierr = VecDestroy(&U_g); CHKERRQ(ierr); - ierr = VecDestroy(&U_loc); CHKERRQ(ierr); - ierr = VecDestroy(&rhs); CHKERRQ(ierr); - ierr = VecDestroy(&rhs_loc); CHKERRQ(ierr); - ierr = VecDestroy(&user->Y_loc); CHKERRQ(ierr); - ierr = MatDestroy(&mat); CHKERRQ(ierr); - ierr = KSPDestroy(&ksp); CHKERRQ(ierr); + ierr = DMDestroy(&dm); + CHKERRQ(ierr); + ierr = VecDestroy(&U_g); + CHKERRQ(ierr); + ierr = VecDestroy(&U_loc); + CHKERRQ(ierr); + ierr = VecDestroy(&rhs); + CHKERRQ(ierr); + ierr = VecDestroy(&rhs_loc); + CHKERRQ(ierr); + ierr = VecDestroy(&user->Y_loc); + CHKERRQ(ierr); + ierr = MatDestroy(&mat); + CHKERRQ(ierr); + ierr = KSPDestroy(&ksp); + CHKERRQ(ierr); // -- Function list - ierr = PetscFunctionListDestroy(&app_ctx->problems); CHKERRQ(ierr); + ierr = PetscFunctionListDestroy(&app_ctx->problems); + CHKERRQ(ierr); // -- Structs - ierr = PetscFree(app_ctx); CHKERRQ(ierr); - ierr = PetscFree(problem_data); CHKERRQ(ierr); - ierr = PetscFree(user); CHKERRQ(ierr); - ierr = PetscFree(phys_ctx->pq2d_ctx); CHKERRQ(ierr); - ierr = PetscFree(phys_ctx); CHKERRQ(ierr); + ierr = PetscFree(app_ctx); + CHKERRQ(ierr); + ierr = PetscFree(problem_data); + CHKERRQ(ierr); + ierr = PetscFree(user); + CHKERRQ(ierr); + ierr = PetscFree(phys_ctx->pq2d_ctx); + CHKERRQ(ierr); + ierr = PetscFree(phys_ctx); + CHKERRQ(ierr); // Free libCEED objects CeedVectorDestroy(&rhs_ceed); CeedVectorDestroy(&target); - ierr = CeedDataDestroy(ceed_data); CHKERRQ(ierr); + ierr = CeedDataDestroy(ceed_data); + CHKERRQ(ierr); CeedDestroy(&ceed); return PetscFinalize(); diff --git a/examples/Hdiv-mass/main.h b/examples/Hdiv-mass/main.h index da283f2d04..c5ecde00d9 100644 --- a/examples/Hdiv-mass/main.h +++ b/examples/Hdiv-mass/main.h @@ -2,10 +2,10 @@ #ifndef MAIN_H #define MAIN_H -#include "include/setup-libceed.h" -#include "include/setup-dm.h" #include "include/cl-options.h" -#include "include/problems.h" #include "include/matops.h" +#include "include/problems.h" +#include "include/setup-dm.h" +#include "include/setup-libceed.h" -#endif // MAIN_H \ No newline at end of file +#endif // MAIN_H \ No newline at end of file diff --git a/examples/Hdiv-mass/problems/mass2d.c b/examples/Hdiv-mass/problems/mass2d.c index 4049b42d5d..2e58f87045 100644 --- a/examples/Hdiv-mass/problems/mass2d.c +++ b/examples/Hdiv-mass/problems/mass2d.c @@ -17,33 +17,34 @@ /// @file /// Utility functions for setting up POISSON_QUAD2D -#include "../include/setup-libceed.h" #include "../include/problems.h" -#include "../qfunctions/poisson-rhs2d.h" -#include "../qfunctions/poisson-mass2d.h" +#include "../include/setup-libceed.h" #include "../qfunctions/poisson-error2d.h" +#include "../qfunctions/poisson-mass2d.h" +#include "../qfunctions/poisson-rhs2d.h" // Hdiv_POISSON_MASS2D is registered in cl-option.c PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx) { - User user = *(User *)ctx; - MPI_Comm comm = PETSC_COMM_WORLD; - PetscInt ierr; + User user = *(User *)ctx; + MPI_Comm comm = PETSC_COMM_WORLD; + PetscInt ierr; PetscFunctionBeginUser; - ierr = PetscCalloc1(1, &user->phys->pq2d_ctx); CHKERRQ(ierr); + ierr = PetscCalloc1(1, &user->phys->pq2d_ctx); + CHKERRQ(ierr); // ------------------------------------------------------ // SET UP POISSON_QUAD2D // ------------------------------------------------------ - problem_data->dim = 2; - problem_data->elem_node = 4; - problem_data->quadrature_mode = CEED_GAUSS; - problem_data->setup_rhs = SetupRhs2D; - problem_data->setup_rhs_loc = SetupRhs2D_loc; - problem_data->residual = SetupMass2D; - problem_data->residual_loc = SetupMass2D_loc; - problem_data->setup_error = SetupError2D; - problem_data->setup_error_loc = SetupError2D_loc; + problem_data->dim = 2; + problem_data->elem_node = 4; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->setup_rhs = SetupRhs2D; + problem_data->setup_rhs_loc = SetupRhs2D_loc; + problem_data->residual = SetupMass2D; + problem_data->residual_loc = SetupMass2D_loc; + problem_data->setup_error = SetupError2D; + problem_data->setup_error_loc = SetupError2D_loc; // ------------------------------------------------------ // Command line Options diff --git a/examples/Hdiv-mass/problems/mass3d.c b/examples/Hdiv-mass/problems/mass3d.c index a86d3c5600..2997ebb572 100644 --- a/examples/Hdiv-mass/problems/mass3d.c +++ b/examples/Hdiv-mass/problems/mass3d.c @@ -17,33 +17,34 @@ /// @file /// Utility functions for setting up POISSON_QUAD2D -#include "../include/setup-libceed.h" #include "../include/problems.h" -#include "../qfunctions/poisson-rhs3d.h" -#include "../qfunctions/poisson-mass3d.h" +#include "../include/setup-libceed.h" #include "../qfunctions/poisson-error3d.h" +#include "../qfunctions/poisson-mass3d.h" +#include "../qfunctions/poisson-rhs3d.h" // Hdiv_POISSON_MASS2D is registered in cl-option.c PetscErrorCode Hdiv_POISSON_MASS3D(ProblemData *problem_data, void *ctx) { - User user = *(User *)ctx; - MPI_Comm comm = PETSC_COMM_WORLD; - PetscInt ierr; + User user = *(User *)ctx; + MPI_Comm comm = PETSC_COMM_WORLD; + PetscInt ierr; PetscFunctionBeginUser; - ierr = PetscCalloc1(1, &user->phys->ph3d_ctx); CHKERRQ(ierr); + ierr = PetscCalloc1(1, &user->phys->ph3d_ctx); + CHKERRQ(ierr); // ------------------------------------------------------ // SET UP POISSON_QUAD3D // ------------------------------------------------------ - problem_data->dim = 3; - problem_data->elem_node = 8; - problem_data->quadrature_mode = CEED_GAUSS; - problem_data->setup_rhs = SetupRhs3D; - problem_data->setup_rhs_loc = SetupRhs3D_loc; - problem_data->residual = SetupMass3D; - problem_data->residual_loc = SetupMass3D_loc; - problem_data->setup_error = SetupError3D; - problem_data->setup_error_loc = SetupError3D_loc; + problem_data->dim = 3; + problem_data->elem_node = 8; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->setup_rhs = SetupRhs3D; + problem_data->setup_rhs_loc = SetupRhs3D_loc; + problem_data->residual = SetupMass3D; + problem_data->residual_loc = SetupMass3D_loc; + problem_data->setup_error = SetupError3D; + problem_data->setup_error_loc = SetupError3D_loc; // ------------------------------------------------------ // Command line Options diff --git a/examples/Hdiv-mass/qfunctions/poisson-error2d.h b/examples/Hdiv-mass/qfunctions/poisson-error2d.h index ad80fe4be4..f48574f93c 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-error2d.h +++ b/examples/Hdiv-mass/qfunctions/poisson-error2d.h @@ -21,38 +21,37 @@ #define ERROR2D_H #include + #include "utils.h" // ----------------------------------------------------------------------------- // Compuet error // ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupError2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { +CEED_QFUNCTION(SetupError2D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[0], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[1], - (*target) = in[2], (*w) = in[3]; + const CeedScalar(*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[0], (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[1], + (*target) = in[2], (*w) = in[3]; // Outputs - CeedScalar (*error) = out[0]; + CeedScalar(*error) = out[0]; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i + #include "utils.h" // ----------------------------------------------------------------------------- // Compuet error // ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupError3D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { +CEED_QFUNCTION(SetupError3D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[0], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[1], - (*target) = in[2], (*w) = in[3]; + const CeedScalar(*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[0], (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[1], + (*target) = in[2], (*w) = in[3]; // Outputs - CeedScalar (*error) = out[0]; + CeedScalar(*error) = out[0]; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i + #include "utils.h" // ----------------------------------------------------------------------------- // This QFunction applies the mass operator for a vector field of 2 components. @@ -35,28 +36,27 @@ // Note we need to apply Piola map on the basis, which is J*u/detJ // So (v,u) = \int (v^T * u detJ*w) ==> \int (v^T J^T*J*u*w/detJ) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupMass2D)(void *ctx, CeedInt Q, const CeedScalar *const *in, - CeedScalar *const *out) { +CEED_QFUNCTION(SetupMass2D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + const CeedScalar(*w) = in[0], (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; // *INDENT-ON* // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i + #include "utils.h" // ----------------------------------------------------------------------------- // This QFunction applies the mass operator for a vector field of 2 components. @@ -35,43 +36,42 @@ // Note we need to apply Piola map on the basis, which is J*u/detJ // So (v,u) = \int (v^T * u detJ*w) ==> \int (v^T J^T*J*u*w/detJ) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupMass3D)(void *ctx, CeedInt Q, const CeedScalar *const *in, - CeedScalar *const *out) { +CEED_QFUNCTION(SetupMass3D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + const CeedScalar(*w) = in[0], (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; // *INDENT-ON* // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i + #include "utils.h" #ifndef M_PI -#define M_PI 3.14159265358979323846 +#define M_PI 3.14159265358979323846 #endif // ----------------------------------------------------------------------------- // This QFunction sets up the rhs and true solution for the problem @@ -40,41 +41,38 @@ // Note we need to apply Piola map on the basis, which is J*u/detJ // So (v,ue) = \int (v^T * ue detJ*w) ==> \int (v^T J^T* ue * w) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupRhs2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { +CEED_QFUNCTION(SetupRhs2D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*coords) = in[0], - (*w) = in[1], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[2]; + const CeedScalar(*coords) = in[0], (*w) = in[1], (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[2]; // Outputs - //CeedScalar (*rhs)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; - CeedScalar (*true_soln) = out[0], (*rhs) = out[1]; + // CeedScalar (*rhs)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + CeedScalar(*true_soln) = out[0], (*rhs) = out[1]; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i + #include "utils.h" #ifndef M_PI -#define M_PI 3.14159265358979323846 +#define M_PI 3.14159265358979323846 #endif // ----------------------------------------------------------------------------- // This QFunction sets up the rhs and true solution for the problem @@ -39,46 +40,41 @@ // Note we need to apply Piola map on the basis, which is J*u/detJ // So (v,ue) = \int (v^T * ue detJ*w) ==> \int (v^T J^T* ue * w) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(SetupRhs3D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { +CEED_QFUNCTION(SetupRhs3D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // *INDENT-OFF* // Inputs - const CeedScalar (*coords) = in[0], - (*w) = in[1], - (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[2]; + const CeedScalar(*coords) = in[0], (*w) = in[1], (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[2]; // Outputs - //CeedScalar (*rhs)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; - CeedScalar (*true_soln) = out[0], (*rhs) = out[1]; + // CeedScalar (*rhs)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + CeedScalar(*true_soln) = out[0], (*rhs) = out[1]; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i +#include "ceed/ceed-f64.h" + #define PI_DOUBLE 3.14159265358979323846 // ----------------------------------------------------------------------------- // Compute alpha * A * B = C // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatMatMult3x3(const CeedScalar alpha, - const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { +CEED_QFUNCTION_HELPER int AlphaMatMatMult3x3(const CeedScalar alpha, const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { for (CeedInt j = 0; j < 3; j++) { for (CeedInt k = 0; k < 3; k++) { C[j][k] = 0; @@ -29,8 +29,8 @@ CEED_QFUNCTION_HELPER int AlphaMatMatMult3x3(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute alpha * A^T * B = C // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult3x3(const CeedScalar alpha, - const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { +CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult3x3(const CeedScalar alpha, const CeedScalar A[3][3], const CeedScalar B[3][3], + CeedScalar C[3][3]) { for (CeedInt j = 0; j < 3; j++) { for (CeedInt k = 0; k < 3; k++) { C[j][k] = 0; @@ -48,26 +48,24 @@ CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult3x3(const CeedScalar alpha, // ----------------------------------------------------------------------------- CEED_QFUNCTION_HELPER CeedScalar MatDet3x3(const CeedScalar A[3][3]) { // Compute det(A) - const CeedScalar B11 = A[1][1]*A[2][2] - A[1][2]*A[2][1]; - const CeedScalar B12 = A[0][2]*A[2][1] - A[0][1]*A[2][2]; - const CeedScalar B13 = A[0][1]*A[1][2] - A[0][2]*A[1][1]; - return A[0][0]*B11 + A[1][0]*B12 + A[2][0]*B13; - + const CeedScalar B11 = A[1][1] * A[2][2] - A[1][2] * A[2][1]; + const CeedScalar B12 = A[0][2] * A[2][1] - A[0][1] * A[2][2]; + const CeedScalar B13 = A[0][1] * A[1][2] - A[0][2] * A[1][1]; + return A[0][0] * B11 + A[1][0] * B12 + A[2][0] * B13; }; // ----------------------------------------------------------------------------- // Compute inverse of 3x3 symmetric matrix // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int MatInverse3x3(const CeedScalar A[3][3], - const CeedScalar det_A, CeedScalar A_inv[3][3]) { +CEED_QFUNCTION_HELPER int MatInverse3x3(const CeedScalar A[3][3], const CeedScalar det_A, CeedScalar A_inv[3][3]) { // Compute A^(-1) : A-Inverse CeedScalar B[6] = { - A[1][1] * A[2][2] - A[1][2] * A[2][1], /* *NOPAD* */ - A[0][0] * A[2][2] - A[0][2] * A[2][0], /* *NOPAD* */ - A[0][0] * A[1][1] - A[0][1] * A[1][0], /* *NOPAD* */ - A[0][2] * A[1][0] - A[0][0] * A[1][2], /* *NOPAD* */ - A[0][1] * A[1][2] - A[0][2] * A[1][1], /* *NOPAD* */ - A[0][2] * A[2][1] - A[0][1] * A[2][2] /* *NOPAD* */ + A[1][1] * A[2][2] - A[1][2] * A[2][1], /* *NOPAD* */ + A[0][0] * A[2][2] - A[0][2] * A[2][0], /* *NOPAD* */ + A[0][0] * A[1][1] - A[0][1] * A[1][0], /* *NOPAD* */ + A[0][2] * A[1][0] - A[0][0] * A[1][2], /* *NOPAD* */ + A[0][1] * A[1][2] - A[0][2] * A[1][1], /* *NOPAD* */ + A[0][2] * A[2][1] - A[0][1] * A[2][2] /* *NOPAD* */ }; CeedScalar A_inv1[6]; for (CeedInt m = 0; m < 6; m++) { @@ -88,13 +86,11 @@ CEED_QFUNCTION_HELPER int MatInverse3x3(const CeedScalar A[3][3], // ----------------------------------------------------------------------------- // Compute matrix-vector product: alpha*A*u // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatVecMult3x3(const CeedScalar alpha, - const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { +CEED_QFUNCTION_HELPER int AlphaMatVecMult3x3(const CeedScalar alpha, const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { // Compute v = alpha*A*u for (CeedInt k = 0; k < 3; k++) { v[k] = 0; - for (CeedInt m = 0; m < 3; m++) - v[k] += A[k][m] * u[m] * alpha; + for (CeedInt m = 0; m < 3; m++) v[k] += A[k][m] * u[m] * alpha; } return 0; @@ -103,13 +99,11 @@ CEED_QFUNCTION_HELPER int AlphaMatVecMult3x3(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute matrix-vector product: alpha*A^T*u // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult3x3(const CeedScalar alpha, - const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { +CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult3x3(const CeedScalar alpha, const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { // Compute v = alpha*A^T*u for (CeedInt k = 0; k < 3; k++) { v[k] = 0; - for (CeedInt m = 0; m < 3; m++) - v[k] += A[m][k] * u[m] * alpha; + for (CeedInt m = 0; m < 3; m++) v[k] += A[m][k] * u[m] * alpha; } return 0; @@ -118,8 +112,7 @@ CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult3x3(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute alpha * A * B = C // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatMatMult2x2(const CeedScalar alpha, - const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { +CEED_QFUNCTION_HELPER int AlphaMatMatMult2x2(const CeedScalar alpha, const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { for (CeedInt j = 0; j < 2; j++) { for (CeedInt k = 0; k < 2; k++) { C[j][k] = 0; @@ -135,8 +128,8 @@ CEED_QFUNCTION_HELPER int AlphaMatMatMult2x2(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute alpha * A^T * B = C // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult2x2(const CeedScalar alpha, - const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { +CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult2x2(const CeedScalar alpha, const CeedScalar A[2][2], const CeedScalar B[2][2], + CeedScalar C[2][2]) { for (CeedInt j = 0; j < 2; j++) { for (CeedInt k = 0; k < 2; k++) { C[j][k] = 0; @@ -154,20 +147,18 @@ CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult2x2(const CeedScalar alpha, // ----------------------------------------------------------------------------- CEED_QFUNCTION_HELPER CeedScalar MatDet2x2(const CeedScalar A[2][2]) { // Compute det(A) - return A[0][0]*A[1][1] - A[1][0]*A[0][1]; - + return A[0][0] * A[1][1] - A[1][0] * A[0][1]; }; // ----------------------------------------------------------------------------- // Compute inverse of 2x2 symmetric matrix // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int MatInverse2x2(const CeedScalar A[2][2], - const CeedScalar det_A, CeedScalar A_inv[2][2]) { +CEED_QFUNCTION_HELPER int MatInverse2x2(const CeedScalar A[2][2], const CeedScalar det_A, CeedScalar A_inv[2][2]) { // Compute A^(-1) : A-Inverse - A_inv[0][0] = A[1][1]/ det_A; - A_inv[0][1] = -A[0][1]/ det_A; - A_inv[1][0] = -A[1][0]/ det_A; - A_inv[1][1] = A[0][0]/ det_A; + A_inv[0][0] = A[1][1] / det_A; + A_inv[0][1] = -A[0][1] / det_A; + A_inv[1][0] = -A[1][0] / det_A; + A_inv[1][1] = A[0][0] / det_A; return 0; }; @@ -175,13 +166,11 @@ CEED_QFUNCTION_HELPER int MatInverse2x2(const CeedScalar A[2][2], // ----------------------------------------------------------------------------- // Compute matrix-vector product: alpha*A*u // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatVecMult2x2(const CeedScalar alpha, - const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { +CEED_QFUNCTION_HELPER int AlphaMatVecMult2x2(const CeedScalar alpha, const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { // Compute v = alpha*A*u for (CeedInt k = 0; k < 2; k++) { v[k] = 0; - for (CeedInt m = 0; m < 2; m++) - v[k] += A[k][m] * u[m] * alpha; + for (CeedInt m = 0; m < 2; m++) v[k] += A[k][m] * u[m] * alpha; } return 0; @@ -190,13 +179,11 @@ CEED_QFUNCTION_HELPER int AlphaMatVecMult2x2(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute matrix-vector product: alpha*A^T*u // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult2x2(const CeedScalar alpha, - const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { +CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult2x2(const CeedScalar alpha, const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { // Compute v = alpha*A^T*u for (CeedInt k = 0; k < 2; k++) { v[k] = 0; - for (CeedInt m = 0; m < 2; m++) - v[k] += A[m][k] * u[m] * alpha; + for (CeedInt m = 0; m < 2; m++) v[k] += A[m][k] * u[m] * alpha; } return 0; diff --git a/examples/Hdiv-mass/src/cl-options.c b/examples/Hdiv-mass/src/cl-options.c index 250d96d79a..105de000df 100644 --- a/examples/Hdiv-mass/src/cl-options.c +++ b/examples/Hdiv-mass/src/cl-options.c @@ -18,19 +18,20 @@ /// Command line option processing for H(div) example using PETSc #include "../include/cl-options.h" + #include "../include/problems.h" // Register problems to be available on the command line PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx) { app_ctx->problems = NULL; - PetscErrorCode ierr; + PetscErrorCode ierr; PetscFunctionBeginUser; // 1) poisson-quad2d (Hdiv_POISSON_MASS2D is created in poisson-mass2d.c) - ierr = PetscFunctionListAdd(&app_ctx->problems, "mass2d", - Hdiv_POISSON_MASS2D); CHKERRQ(ierr); + ierr = PetscFunctionListAdd(&app_ctx->problems, "mass2d", Hdiv_POISSON_MASS2D); + CHKERRQ(ierr); // 2) poisson-hex3d - ierr = PetscFunctionListAdd(&app_ctx->problems, "mass3d", - Hdiv_POISSON_MASS3D); CHKERRQ(ierr); + ierr = PetscFunctionListAdd(&app_ctx->problems, "mass3d", Hdiv_POISSON_MASS3D); + CHKERRQ(ierr); // 3) poisson-prism3d @@ -41,26 +42,23 @@ PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx) { // Process general command line options PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx) { - - PetscBool problem_flag = PETSC_FALSE; + PetscBool problem_flag = PETSC_FALSE; PetscErrorCode ierr; PetscFunctionBeginUser; - PetscOptionsBegin(comm, NULL, - "H(div) examples in PETSc with libCEED", NULL); + PetscOptionsBegin(comm, NULL, "H(div) examples in PETSc with libCEED", NULL); - ierr = PetscOptionsFList("-problem", "Problem to solve", NULL, - app_ctx->problems, - app_ctx->problem_name, app_ctx->problem_name, sizeof(app_ctx->problem_name), - &problem_flag); CHKERRQ(ierr); + ierr = PetscOptionsFList("-problem", "Problem to solve", NULL, app_ctx->problems, app_ctx->problem_name, app_ctx->problem_name, + sizeof(app_ctx->problem_name), &problem_flag); + CHKERRQ(ierr); app_ctx->degree = 1; - ierr = PetscOptionsInt("-degree", "Polynomial degree of finite elements", - NULL, app_ctx->degree, &app_ctx->degree, NULL); CHKERRQ(ierr); + ierr = PetscOptionsInt("-degree", "Polynomial degree of finite elements", NULL, app_ctx->degree, &app_ctx->degree, NULL); + CHKERRQ(ierr); app_ctx->q_extra = 0; - ierr = PetscOptionsInt("-q_extra", "Number of extra quadrature points", - NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL); CHKERRQ(ierr); + ierr = PetscOptionsInt("-q_extra", "Number of extra quadrature points", NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL); + CHKERRQ(ierr); // Provide default problem if not specified if (!problem_flag) { diff --git a/examples/Hdiv-mass/src/matops.c b/examples/Hdiv-mass/src/matops.c index 5a31719a35..45c39f0160 100644 --- a/examples/Hdiv-mass/src/matops.c +++ b/examples/Hdiv-mass/src/matops.c @@ -1,4 +1,5 @@ #include "../include/matops.h" + #include "../include/setup-libceed.h" // ----------------------------------------------------------------------------- @@ -7,35 +8,39 @@ // ----------------------------------------------------------------------------- PetscErrorCode ApplyLocal_Ceed(User user, Vec X, Vec Y) { PetscErrorCode ierr; - PetscScalar *x, *y; - PetscMemType x_mem_type, y_mem_type; + PetscScalar *x, *y; + PetscMemType x_mem_type, y_mem_type; PetscFunctionBeginUser; // Global-to-local - ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); CHKERRQ(ierr); + ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); + CHKERRQ(ierr); // Setup libCEED vectors - ierr = VecGetArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x, - &x_mem_type); CHKERRQ(ierr); - ierr = VecGetArrayAndMemType(user->Y_loc, &y, &y_mem_type); CHKERRQ(ierr); + ierr = VecGetArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x, &x_mem_type); + CHKERRQ(ierr); + ierr = VecGetArrayAndMemType(user->Y_loc, &y, &y_mem_type); + CHKERRQ(ierr); CeedVectorSetArray(user->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, x); CeedVectorSetArray(user->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); // Apply libCEED operator - CeedOperatorApply(user->op_apply, user->x_ceed, user->y_ceed, - CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(user->op_apply, user->x_ceed, user->y_ceed, CEED_REQUEST_IMMEDIATE); // Restore PETSc vectors CeedVectorTakeArray(user->x_ceed, MemTypeP2C(x_mem_type), NULL); CeedVectorTakeArray(user->y_ceed, MemTypeP2C(y_mem_type), NULL); ierr = VecRestoreArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x); CHKERRQ(ierr); - ierr = VecRestoreArrayAndMemType(user->Y_loc, &y); CHKERRQ(ierr); + ierr = VecRestoreArrayAndMemType(user->Y_loc, &y); + CHKERRQ(ierr); // Local-to-global - ierr = VecZeroEntries(Y); CHKERRQ(ierr); - ierr = DMLocalToGlobal(user->dm, user->Y_loc, ADD_VALUES, Y); CHKERRQ(ierr); + ierr = VecZeroEntries(Y); + CHKERRQ(ierr); + ierr = DMLocalToGlobal(user->dm, user->Y_loc, ADD_VALUES, Y); + CHKERRQ(ierr); PetscFunctionReturn(0); }; @@ -45,14 +50,16 @@ PetscErrorCode ApplyLocal_Ceed(User user, Vec X, Vec Y) { // ----------------------------------------------------------------------------- PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y) { PetscErrorCode ierr; - User user; + User user; PetscFunctionBeginUser; - ierr = MatShellGetContext(A, &user); CHKERRQ(ierr); + ierr = MatShellGetContext(A, &user); + CHKERRQ(ierr); // libCEED for local action of residual evaluator - ierr = ApplyLocal_Ceed(user, X, Y); CHKERRQ(ierr); + ierr = ApplyLocal_Ceed(user, X, Y); + CHKERRQ(ierr); PetscFunctionReturn(0); }; @@ -60,28 +67,28 @@ PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y) { // ----------------------------------------------------------------------------- // This function calculates the error in the final solution // ----------------------------------------------------------------------------- -PetscErrorCode ComputeError(User user, Vec X, CeedVector target, - CeedScalar *l2_error) { +PetscErrorCode ComputeError(User user, Vec X, CeedVector target, CeedScalar *l2_error) { PetscErrorCode ierr; - PetscScalar *x; - PetscMemType mem_type; - CeedVector collocated_error; - CeedSize length; + PetscScalar *x; + PetscMemType mem_type; + CeedVector collocated_error; + CeedSize length; PetscFunctionBeginUser; CeedVectorGetLength(target, &length); CeedVectorCreate(user->ceed, length, &collocated_error); // Global-to-local - ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); CHKERRQ(ierr); + ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); + CHKERRQ(ierr); // Setup CEED vector - ierr = VecGetArrayAndMemType(user->X_loc, &x, &mem_type); CHKERRQ(ierr); + ierr = VecGetArrayAndMemType(user->X_loc, &x, &mem_type); + CHKERRQ(ierr); CeedVectorSetArray(user->x_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, x); // Apply CEED operator - CeedOperatorApply(user->op_error, user->x_ceed, collocated_error, - CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(user->op_error, user->x_ceed, collocated_error, CEED_REQUEST_IMMEDIATE); // Restore PETSc vector CeedVectorTakeArray(user->x_ceed, MemTypeP2C(mem_type), NULL); ierr = VecRestoreArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x); diff --git a/examples/Hdiv-mass/src/setup-dm.c b/examples/Hdiv-mass/src/setup-dm.c index 7023fdbe6d..a48ee93606 100644 --- a/examples/Hdiv-mass/src/setup-dm.c +++ b/examples/Hdiv-mass/src/setup-dm.c @@ -3,47 +3,66 @@ // --------------------------------------------------------------------------- // Set-up DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDistributedDM(MPI_Comm comm, ProblemData *problem_data, - DM *dm) { - PetscErrorCode ierr; +PetscErrorCode CreateDistributedDM(MPI_Comm comm, ProblemData *problem_data, DM *dm) { + PetscErrorCode ierr; PetscSection sec; PetscInt dofs_per_face; PetscInt p_start, p_end; - PetscInt c_start, c_end; // cells - PetscInt f_start, f_end; // faces - PetscInt v_start, v_end; // vertices + PetscInt c_start, c_end; // cells + PetscInt f_start, f_end; // faces + PetscInt v_start, v_end; // vertices PetscFunctionBeginUser; // Create DMPLEX - ierr = DMCreate(comm, dm); CHKERRQ(ierr); - ierr = DMSetType(*dm, DMPLEX); CHKERRQ(ierr); + ierr = DMCreate(comm, dm); + CHKERRQ(ierr); + ierr = DMSetType(*dm, DMPLEX); + CHKERRQ(ierr); // Set Tensor elements - ierr = PetscOptionsSetValue(NULL, "-dm_plex_simplex", "0"); CHKERRQ(ierr); + ierr = PetscOptionsSetValue(NULL, "-dm_plex_simplex", "0"); + CHKERRQ(ierr); // Set CL options - ierr = DMSetFromOptions(*dm); CHKERRQ(ierr); - ierr = DMViewFromOptions(*dm, NULL, "-dm_view"); CHKERRQ(ierr); + ierr = DMSetFromOptions(*dm); + CHKERRQ(ierr); + ierr = DMViewFromOptions(*dm, NULL, "-dm_view"); + CHKERRQ(ierr); // Get plex limits - ierr = DMPlexGetChart(*dm, &p_start, &p_end); CHKERRQ(ierr); - ierr = DMPlexGetHeightStratum(*dm, 0, &c_start, &c_end); CHKERRQ(ierr); - ierr = DMPlexGetHeightStratum(*dm, 1, &f_start, &f_end); CHKERRQ(ierr); - ierr = DMPlexGetDepthStratum(*dm, 0, &v_start, &v_end); CHKERRQ(ierr); + ierr = DMPlexGetChart(*dm, &p_start, &p_end); + CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(*dm, 0, &c_start, &c_end); + CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(*dm, 1, &f_start, &f_end); + CHKERRQ(ierr); + ierr = DMPlexGetDepthStratum(*dm, 0, &v_start, &v_end); + CHKERRQ(ierr); // Create section - ierr = PetscSectionCreate(comm, &sec); CHKERRQ(ierr); - ierr = PetscSectionSetNumFields(sec,1); CHKERRQ(ierr); - ierr = PetscSectionSetFieldName(sec,0,"Velocity"); CHKERRQ(ierr); - ierr = PetscSectionSetFieldComponents(sec,0,1); CHKERRQ(ierr); - ierr = PetscSectionSetChart(sec,p_start,p_end); CHKERRQ(ierr); + ierr = PetscSectionCreate(comm, &sec); + CHKERRQ(ierr); + ierr = PetscSectionSetNumFields(sec, 1); + CHKERRQ(ierr); + ierr = PetscSectionSetFieldName(sec, 0, "Velocity"); + CHKERRQ(ierr); + ierr = PetscSectionSetFieldComponents(sec, 0, 1); + CHKERRQ(ierr); + ierr = PetscSectionSetChart(sec, p_start, p_end); + CHKERRQ(ierr); // Setup dofs per face for (PetscInt f = f_start; f < f_end; f++) { - ierr = DMPlexGetConeSize(*dm, f, &dofs_per_face); CHKERRQ(ierr); - ierr = PetscSectionSetFieldDof(sec, f, 0, dofs_per_face); CHKERRQ(ierr); - ierr = PetscSectionSetDof (sec, f, dofs_per_face); CHKERRQ(ierr); + ierr = DMPlexGetConeSize(*dm, f, &dofs_per_face); + CHKERRQ(ierr); + ierr = PetscSectionSetFieldDof(sec, f, 0, dofs_per_face); + CHKERRQ(ierr); + ierr = PetscSectionSetDof(sec, f, dofs_per_face); + CHKERRQ(ierr); } - ierr = PetscSectionSetUp(sec); CHKERRQ(ierr); - ierr = DMSetSection(*dm,sec); CHKERRQ(ierr); - ierr = PetscSectionDestroy(&sec); CHKERRQ(ierr); + ierr = PetscSectionSetUp(sec); + CHKERRQ(ierr); + ierr = DMSetSection(*dm, sec); + CHKERRQ(ierr); + ierr = PetscSectionDestroy(&sec); + CHKERRQ(ierr); PetscFunctionReturn(0); }; \ No newline at end of file diff --git a/examples/Hdiv-mass/src/setup-libceed.c b/examples/Hdiv-mass/src/setup-libceed.c index 767ceca67a..fd5c29b7b6 100644 --- a/examples/Hdiv-mass/src/setup-libceed.c +++ b/examples/Hdiv-mass/src/setup-libceed.c @@ -1,14 +1,13 @@ #include "../include/setup-libceed.h" -#include "../include/petsc-macros.h" -#include "../basis/Hdiv-quad.h" + #include "../basis/Hdiv-hex.h" +#include "../basis/Hdiv-quad.h" +#include "../include/petsc-macros.h" // ----------------------------------------------------------------------------- // Convert PETSc MemType to libCEED MemType // ----------------------------------------------------------------------------- -CeedMemType MemTypeP2C(PetscMemType mem_type) { - return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; -} +CeedMemType MemTypeP2C(PetscMemType mem_type) { return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; } // ----------------------------------------------------------------------------- // Destroy libCEED objects // ----------------------------------------------------------------------------- @@ -33,7 +32,8 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data) { // Operators CeedOperatorDestroy(&ceed_data->op_residual); CeedOperatorDestroy(&ceed_data->op_error); - ierr = PetscFree(ceed_data); CHKERRQ(ierr); + ierr = PetscFree(ceed_data); + CHKERRQ(ierr); PetscFunctionReturn(0); }; @@ -41,28 +41,23 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data) { // ----------------------------------------------------------------------------- // Utility function - essential BC dofs are encoded in closure indices as -(i+1) // ----------------------------------------------------------------------------- -PetscInt Involute(PetscInt i) { - return i >= 0 ? i : -(i + 1); -}; +PetscInt Involute(PetscInt i) { return i >= 0 ? i : -(i + 1); }; // ----------------------------------------------------------------------------- // Get CEED restriction data from DMPlex // ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, - DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { - PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { + PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; PetscErrorCode ierr; PetscFunctionBeginUser; - ierr = DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, - &elem_size, &num_comp, &num_dof, &elem_restr_offsets); + ierr = DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, &elem_size, &num_comp, &num_dof, &elem_restr_offsets); CHKERRQ(ierr); - CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, - 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, - elem_restr_offsets, elem_restr); - ierr = PetscFree(elem_restr_offsets); CHKERRQ(ierr); + CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets, elem_restr); + ierr = PetscFree(elem_restr_offsets); + CHKERRQ(ierr); PetscFunctionReturn(0); }; @@ -70,182 +65,176 @@ PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, // ----------------------------------------------------------------------------- // Get Oriented CEED restriction data from DMPlex // ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, - CeedInt P, CeedElemRestriction *elem_restr_oriented) { - PetscSection section; - PetscInt p, num_elem, num_dof, *restr_indices, elem_offset, num_fields, - dim, c_start, c_end; - Vec U_loc; - PetscErrorCode ierr; - const PetscInt *ornt; // this is for orientation of dof +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, CeedElemRestriction *elem_restr_oriented) { + PetscSection section; + PetscInt p, num_elem, num_dof, *restr_indices, elem_offset, num_fields, dim, c_start, c_end; + Vec U_loc; + PetscErrorCode ierr; + const PetscInt *ornt; // this is for orientation of dof PetscFunctionBeginUser; - ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr); - ierr = DMGetLocalSection(dm, §ion); CHKERRQ(ierr); - ierr = PetscSectionGetNumFields(section, &num_fields); CHKERRQ(ierr); - PetscInt num_comp[num_fields], field_offsets[num_fields+1]; + ierr = DMGetDimension(dm, &dim); + CHKERRQ(ierr); + ierr = DMGetLocalSection(dm, §ion); + CHKERRQ(ierr); + ierr = PetscSectionGetNumFields(section, &num_fields); + CHKERRQ(ierr); + PetscInt num_comp[num_fields], field_offsets[num_fields + 1]; field_offsets[0] = 0; for (PetscInt f = 0; f < num_fields; f++) { - ierr = PetscSectionGetFieldComponents(section, f, &num_comp[f]); CHKERRQ(ierr); - field_offsets[f+1] = field_offsets[f] + num_comp[f]; + ierr = PetscSectionGetFieldComponents(section, f, &num_comp[f]); + CHKERRQ(ierr); + field_offsets[f + 1] = field_offsets[f] + num_comp[f]; } - ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); + CHKERRQ(ierr); num_elem = c_end - c_start; - ierr = PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), - &restr_indices); CHKERRQ(ierr); - bool *orient_indices; // to flip the dof - ierr = PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), &orient_indices); + ierr = PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &restr_indices); + CHKERRQ(ierr); + bool *orient_indices; // to flip the dof + ierr = PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &orient_indices); CHKERRQ(ierr); for (p = 0, elem_offset = 0; p < num_elem; p++) { PetscInt num_indices, *indices, faces_per_elem, dofs_per_face; - ierr = DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, - &num_indices, &indices, NULL, NULL); + ierr = DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL); CHKERRQ(ierr); - ierr = DMPlexGetConeOrientation(dm, p, &ornt); CHKERRQ(ierr); + ierr = DMPlexGetConeOrientation(dm, p, &ornt); + CHKERRQ(ierr); // Get number of faces per element - ierr = DMPlexGetConeSize(dm, p, &faces_per_elem); CHKERRQ(ierr); + ierr = DMPlexGetConeSize(dm, p, &faces_per_elem); + CHKERRQ(ierr); dofs_per_face = faces_per_elem - 2; for (PetscInt f = 0; f < faces_per_elem; f++) { for (PetscInt i = 0; i < dofs_per_face; i++) { - PetscInt ii = dofs_per_face*f + i; + PetscInt ii = dofs_per_face * f + i; // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. - PetscInt loc = Involute(indices[ii*num_comp[0]]); + PetscInt loc = Involute(indices[ii * num_comp[0]]); restr_indices[elem_offset] = loc; // Set orientation orient_indices[elem_offset] = ornt[f] < 0; elem_offset++; } } - ierr = DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, - &num_indices, &indices, NULL, NULL); + ierr = DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL); CHKERRQ(ierr); } - //if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) - // SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, - // "ElemRestriction of size (%" PetscInt_FMT ",%" PetscInt_FMT ") - // initialized %" PetscInt_FMT "nodes", num_elem, - // dim*PetscPowInt(P, dim),elem_offset); - ierr = DMGetLocalVector(dm, &U_loc); CHKERRQ(ierr); - ierr = VecGetLocalSize(U_loc, &num_dof); CHKERRQ(ierr); - ierr = DMRestoreLocalVector(dm, &U_loc); CHKERRQ(ierr); + // if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) + // SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, + // "ElemRestriction of size (%" PetscInt_FMT ",%" PetscInt_FMT ") + // initialized %" PetscInt_FMT "nodes", num_elem, + // dim*PetscPowInt(P, dim),elem_offset); + ierr = DMGetLocalVector(dm, &U_loc); + CHKERRQ(ierr); + ierr = VecGetLocalSize(U_loc, &num_dof); + CHKERRQ(ierr); + ierr = DMRestoreLocalVector(dm, &U_loc); + CHKERRQ(ierr); // dof per element in Hdiv is dim*P^dim, for linear element P=2 - CeedElemRestrictionCreateOriented(ceed, num_elem, dim*PetscPowInt(P, dim), - field_offsets[num_fields], - 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, - restr_indices, orient_indices, - elem_restr_oriented); - ierr = PetscFree(restr_indices); CHKERRQ(ierr); - ierr = PetscFree(orient_indices); CHKERRQ(ierr); + CeedElemRestrictionCreateOriented(ceed, num_elem, dim * PetscPowInt(P, dim), field_offsets[num_fields], 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, + restr_indices, orient_indices, elem_restr_oriented); + ierr = PetscFree(restr_indices); + CHKERRQ(ierr); + ierr = PetscFree(orient_indices); + CHKERRQ(ierr); PetscFunctionReturn(0); }; // ----------------------------------------------------------------------------- // Set up libCEED on the fine grid for a given degree // ----------------------------------------------------------------------------- -PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, - ProblemData *problem_data, PetscInt U_g_size, - PetscInt U_loc_size, CeedData ceed_data, +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *problem_data, PetscInt U_g_size, PetscInt U_loc_size, CeedData ceed_data, CeedVector rhs_ceed, CeedVector *target) { - int ierr; - CeedInt P = app_ctx->degree + 1; + int ierr; + CeedInt P = app_ctx->degree + 1; // Number of quadratures in 1D, q_extra is set in cl-options.c - CeedInt Q = P + 1 + app_ctx->q_extra; - CeedInt dim, num_comp_x, num_comp_u; - //CeedInt elem_node = problem_data->elem_node; - DM dm_coord; - Vec coords; - PetscInt c_start, c_end, num_elem; + CeedInt Q = P + 1 + app_ctx->q_extra; + CeedInt dim, num_comp_x, num_comp_u; + // CeedInt elem_node = problem_data->elem_node; + DM dm_coord; + Vec coords; + PetscInt c_start, c_end, num_elem; const PetscScalar *coordArray; - CeedVector x_coord; - CeedQFunction qf_setup_rhs, qf_residual, qf_error; - CeedOperator op_setup_rhs, op_residual, op_error; + CeedVector x_coord; + CeedQFunction qf_setup_rhs, qf_residual, qf_error; + CeedOperator op_setup_rhs, op_residual, op_error; PetscFunctionBeginUser; // --------------------------------------------------------------------------- // libCEED bases:Hdiv basis_u and Lagrange basis_x // --------------------------------------------------------------------------- - dim = problem_data->dim; + dim = problem_data->dim; num_comp_x = dim; - num_comp_u = 1; // one vector dof + num_comp_u = 1; // one vector dof // Number of quadratures per element - CeedInt num_qpts = PetscPowInt(Q, dim); - CeedInt P_u = dim*PetscPowInt(P, dim); // dof per element - CeedScalar q_ref[dim*num_qpts], q_weights[num_qpts]; - CeedScalar div[P_u*num_qpts], interp[dim*P_u*num_qpts]; + CeedInt num_qpts = PetscPowInt(Q, dim); + CeedInt P_u = dim * PetscPowInt(P, dim); // dof per element + CeedScalar q_ref[dim * num_qpts], q_weights[num_qpts]; + CeedScalar div[P_u * num_qpts], interp[dim * P_u * num_qpts]; if (dim == 2) { HdivBasisQuad(Q, q_ref, q_weights, interp, div, problem_data->quadrature_mode); - CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, - interp, div, q_ref, q_weights, &ceed_data->basis_u); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, interp, div, q_ref, q_weights, &ceed_data->basis_u); } else { HdivBasisHex(Q, q_ref, q_weights, interp, div, problem_data->quadrature_mode); - CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, - interp, div, q_ref, q_weights, &ceed_data->basis_u); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, interp, div, q_ref, q_weights, &ceed_data->basis_u); } - CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, - problem_data->quadrature_mode, &ceed_data->basis_x); + CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, problem_data->quadrature_mode, &ceed_data->basis_x); // --------------------------------------------------------------------------- // libCEED restrictions // --------------------------------------------------------------------------- - ierr = DMGetCoordinateDM(dm, &dm_coord); CHKERRQ(ierr); + ierr = DMGetCoordinateDM(dm, &dm_coord); + CHKERRQ(ierr); ierr = DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL); CHKERRQ(ierr); - CeedInt height = 0; // 0 means no boundary conditions - DMLabel domain_label = 0; - PetscInt value = 0; + CeedInt height = 0; // 0 means no boundary conditions + DMLabel domain_label = 0; + PetscInt value = 0; // -- Coordinate restriction - ierr = CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, - value, &ceed_data->elem_restr_x); CHKERRQ(ierr); + ierr = CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, value, &ceed_data->elem_restr_x); + CHKERRQ(ierr); // -- Solution and projected true solution restriction - ierr = CreateRestrictionFromPlexOriented(ceed, dm, - P, &ceed_data->elem_restr_u); + ierr = CreateRestrictionFromPlexOriented(ceed, dm, P, &ceed_data->elem_restr_u); CHKERRQ(ierr); // -- Geometric ceed_data restriction - ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); CHKERRQ(ierr); + ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); + CHKERRQ(ierr); num_elem = c_end - c_start; - CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, dim, - dim*num_elem*num_qpts, - CEED_STRIDES_BACKEND, &ceed_data->elem_restr_u_i); + CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, dim, dim * num_elem * num_qpts, CEED_STRIDES_BACKEND, &ceed_data->elem_restr_u_i); // --------------------------------------------------------------------------- // Element coordinates // --------------------------------------------------------------------------- - ierr = DMGetCoordinatesLocal(dm, &coords); CHKERRQ(ierr); - ierr = VecGetArrayRead(coords, &coordArray); CHKERRQ(ierr); + ierr = DMGetCoordinatesLocal(dm, &coords); + CHKERRQ(ierr); + ierr = VecGetArrayRead(coords, &coordArray); + CHKERRQ(ierr); CeedElemRestrictionCreateVector(ceed_data->elem_restr_x, &x_coord, NULL); - CeedVectorSetArray(x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, - (PetscScalar *)coordArray); - ierr = VecRestoreArrayRead(coords, &coordArray); CHKERRQ(ierr); + CeedVectorSetArray(x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, (PetscScalar *)coordArray); + ierr = VecRestoreArrayRead(coords, &coordArray); + CHKERRQ(ierr); // --------------------------------------------------------------------------- // Setup RHS and true solution // --------------------------------------------------------------------------- - CeedVectorCreate(ceed, num_elem*num_qpts*dim, target); + CeedVectorCreate(ceed, num_elem * num_qpts * dim, target); // Create the q-function that sets up the RHS and true solution - CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_rhs, - problem_data->setup_rhs_loc, &qf_setup_rhs); + CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_rhs, problem_data->setup_rhs_loc, &qf_setup_rhs); CeedQFunctionAddInput(qf_setup_rhs, "x", num_comp_x, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_setup_rhs, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_setup_rhs, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_setup_rhs, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddOutput(qf_setup_rhs, "true_soln", dim, CEED_EVAL_NONE); CeedQFunctionAddOutput(qf_setup_rhs, "rhs", dim, CEED_EVAL_INTERP); // Create the operator that builds the RHS and true solution - CeedOperatorCreate(ceed, qf_setup_rhs, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_setup_rhs); - CeedOperatorSetField(op_setup_rhs, "x", ceed_data->elem_restr_x, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_setup_rhs, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_setup_rhs, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_setup_rhs, "true_soln", ceed_data->elem_restr_u_i, - CEED_BASIS_COLLOCATED, *target); - CeedOperatorSetField(op_setup_rhs, "rhs", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_setup_rhs, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_setup_rhs); + CeedOperatorSetField(op_setup_rhs, "x", ceed_data->elem_restr_x, ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup_rhs, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_setup_rhs, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_setup_rhs, "true_soln", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, *target); + CeedOperatorSetField(op_setup_rhs, "rhs", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); // Setup RHS and true solution CeedOperatorApply(op_setup_rhs, x_coord, rhs_ceed, CEED_REQUEST_IMMEDIATE); @@ -262,24 +251,18 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // Create the QFunction and Operator that computes the residual of the PDE. // --------------------------------------------------------------------------- // -- QFunction - CeedQFunctionCreateInterior(ceed, 1, problem_data->residual, - problem_data->residual_loc, &qf_residual); + CeedQFunctionCreateInterior(ceed, 1, problem_data->residual, problem_data->residual_loc, &qf_residual); CeedQFunctionAddInput(qf_residual, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_residual, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_residual, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_residual, "u", dim, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_residual, "v", dim, CEED_EVAL_INTERP); // -- Operator - CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_residual); - CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_residual, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, x_coord); - CeedOperatorSetField(op_residual, "u", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_residual, "v", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_residual); + CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_residual, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, x_coord); + CeedOperatorSetField(op_residual, "u", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "v", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in matops.c ceed_data->qf_residual = qf_residual; @@ -289,26 +272,19 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, // Setup Error Qfunction // --------------------------------------------------------------------------- // Create the q-function that sets up the error - CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_error, - problem_data->setup_error_loc, &qf_error); - CeedQFunctionAddInput(qf_error, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_error, problem_data->setup_error_loc, &qf_error); + CeedQFunctionAddInput(qf_error, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_error, "u", dim, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_error, "true_soln", dim, CEED_EVAL_NONE); CeedQFunctionAddInput(qf_error, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddOutput(qf_error, "error", dim, CEED_EVAL_NONE); // Create the operator that builds the error - CeedOperatorCreate(ceed, qf_error, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_error); - CeedOperatorSetField(op_error, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, x_coord); - CeedOperatorSetField(op_error, "u", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_error, "true_soln", ceed_data->elem_restr_u_i, - CEED_BASIS_COLLOCATED, *target); - CeedOperatorSetField(op_error, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_u_i, - CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_error, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_error); + CeedOperatorSetField(op_error, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, x_coord); + CeedOperatorSetField(op_error, "u", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_error, "true_soln", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, *target); + CeedOperatorSetField(op_error, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in matops.c ceed_data->qf_error = qf_error; ceed_data->op_error = op_error; diff --git a/examples/Hdiv-mixed/basis/Hdiv-hex.h b/examples/Hdiv-mixed/basis/Hdiv-hex.h index e14b873dfb..1777981e11 100644 --- a/examples/Hdiv-mixed/basis/Hdiv-hex.h +++ b/examples/Hdiv-mixed/basis/Hdiv-hex.h @@ -18,119 +18,115 @@ // To see how the nodal basis is constructed visit: // https://github.com/rezgarshakeri/H-div-Tests -int NodalHdivBasisHex(CeedScalar *x, CeedScalar *Bx, CeedScalar *By, - CeedScalar *Bz) { - - Bx[ 0] = 0.0625*x[0]*x[0] - 0.0625 ; - By[ 0] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; - Bz[ 0] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bx[ 1] = 0.0625 - 0.0625*x[0]*x[0] ; - By[ 1] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; - Bz[ 1] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bx[ 2] = 0.0625*x[0]*x[0] - 0.0625 ; - By[ 2] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; - Bz[ 2] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bx[ 3] = 0.0625 - 0.0625*x[0]*x[0] ; - By[ 3] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; - Bz[ 3] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] + - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bx[ 4] = 0.0625*x[0]*x[0] - 0.0625 ; - By[ 4] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; - Bz[ 4] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] - - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bx[ 5] = 0.0625 - 0.0625*x[0]*x[0] ; - By[ 5] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] + 0.0625*x[1]*x[1] - 0.0625 ; - Bz[ 5] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bx[ 6] = 0.0625*x[0]*x[0] - 0.0625 ; - By[ 6] = 0.0625*x[0]*x[1]*x[1] - 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; - Bz[ 6] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bx[ 7] = 0.0625 - 0.0625*x[0]*x[0] ; - By[ 7] = -0.0625*x[0]*x[1]*x[1] + 0.0625*x[0] - 0.0625*x[1]*x[1] + 0.0625 ; - Bz[ 7] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bx[ 8] = 0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] - 0.0625*x[2] + 0.0625 ; - By[ 8] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bz[ 8] = 0.0625*x[2]*x[2] - 0.0625 ; - Bx[ 9] = -0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] + 0.0625*x[2] - 0.0625 ; - By[ 9] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - Bz[ 9] = 0.0625*x[2]*x[2] - 0.0625 ; - Bx[10] = -0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] + 0.0625*x[2] + 0.0625 ; - By[10] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; - Bz[10] = 0.0625 - 0.0625*x[2]*x[2] ; - Bx[11] = 0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] - 0.0625*x[2] - 0.0625 ; - By[11] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; - Bz[11] = 0.0625 - 0.0625*x[2]*x[2] ; - Bx[12] = 0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] - 0.0625*x[2] + 0.0625 ; - By[12] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; - Bz[12] = 0.0625*x[2]*x[2] - 0.0625 ; - Bx[13] = -0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] + 0.0625*x[2] - 0.0625 ; - By[13] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] - 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; - Bz[13] = 0.0625*x[2]*x[2] - 0.0625 ; - Bx[14] = -0.0625*x[0]*x[0]*x[2] - 0.0625*x[0]*x[0] + 0.0625*x[2] + 0.0625 ; - By[14] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bz[14] = 0.0625 - 0.0625*x[2]*x[2] ; - Bx[15] = 0.0625*x[0]*x[0]*x[2] + 0.0625*x[0]*x[0] - 0.0625*x[2] - 0.0625 ; - By[15] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] - 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - Bz[15] = 0.0625 - 0.0625*x[2]*x[2] ; - Bx[16] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] - 0.125*x[1] - 0.125*x[2] + 0.125 ; - By[16] = 0.0625*x[1]*x[1] - 0.0625 ; - Bz[16] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; - Bx[17] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] + 0.125 ; - By[17] = 0.0625 - 0.0625*x[1]*x[1] ; - Bz[17] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; - Bx[18] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] + 0.125 ; - By[18] = 0.0625*x[1]*x[1] - 0.0625 ; - Bz[18] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; - Bx[19] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] + - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] + 0.125 ; - By[19] = 0.0625 - 0.0625*x[1]*x[1] ; - Bz[19] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; - Bx[20] = 0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] + 0.125*x[1] + 0.125*x[2] - 0.125 ; - By[20] = 0.0625*x[1]*x[1] - 0.0625 ; - Bz[20] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; - Bx[21] = -0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] - 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] - 0.125*x[1] + 0.125*x[2] - 0.125 ; - By[21] = 0.0625 - 0.0625*x[1]*x[1] ; - Bz[21] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] + 0.0625*x[2]*x[2] - 0.0625 ; - Bx[22] = -0.125*x[0]*x[1]*x[2] - 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - + 0.125*x[1]*x[2] + 0.125*x[1] - 0.125*x[2] - 0.125 ; - By[22] = 0.0625*x[1]*x[1] - 0.0625 ; - Bz[22] = 0.0625*x[1]*x[2]*x[2] - 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; - Bx[23] = 0.125*x[0]*x[1]*x[2] + 0.125*x[0]*x[1] + 0.125*x[0]*x[2] + 0.125*x[0] - - 0.125*x[1]*x[2] - 0.125*x[1] - 0.125*x[2] - 0.125 ; - By[23] = 0.0625 - 0.0625*x[1]*x[1] ; - Bz[23] = -0.0625*x[1]*x[2]*x[2] + 0.0625*x[1] - 0.0625*x[2]*x[2] + 0.0625 ; +int NodalHdivBasisHex(CeedScalar *x, CeedScalar *Bx, CeedScalar *By, CeedScalar *Bz) { + Bx[0] = 0.0625 * x[0] * x[0] - 0.0625; + By[0] = -0.0625 * x[0] * x[1] * x[1] + 0.0625 * x[0] + 0.0625 * x[1] * x[1] - 0.0625; + Bz[0] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bx[1] = 0.0625 - 0.0625 * x[0] * x[0]; + By[1] = 0.0625 * x[0] * x[1] * x[1] - 0.0625 * x[0] + 0.0625 * x[1] * x[1] - 0.0625; + Bz[1] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] - 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bx[2] = 0.0625 * x[0] * x[0] - 0.0625; + By[2] = 0.0625 * x[0] * x[1] * x[1] - 0.0625 * x[0] - 0.0625 * x[1] * x[1] + 0.0625; + Bz[2] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] - 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bx[3] = 0.0625 - 0.0625 * x[0] * x[0]; + By[3] = -0.0625 * x[0] * x[1] * x[1] + 0.0625 * x[0] - 0.0625 * x[1] * x[1] + 0.0625; + Bz[3] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] - 0.125 * x[0] + 0.125 * x[1] * x[2] - 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bx[4] = 0.0625 * x[0] * x[0] - 0.0625; + By[4] = -0.0625 * x[0] * x[1] * x[1] + 0.0625 * x[0] + 0.0625 * x[1] * x[1] - 0.0625; + Bz[4] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] - 0.125 * x[0] - 0.125 * x[1] * x[2] - 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bx[5] = 0.0625 - 0.0625 * x[0] * x[0]; + By[5] = 0.0625 * x[0] * x[1] * x[1] - 0.0625 * x[0] + 0.0625 * x[1] * x[1] - 0.0625; + Bz[5] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] - 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bx[6] = 0.0625 * x[0] * x[0] - 0.0625; + By[6] = 0.0625 * x[0] * x[1] * x[1] - 0.0625 * x[0] - 0.0625 * x[1] * x[1] + 0.0625; + Bz[6] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] - 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bx[7] = 0.0625 - 0.0625 * x[0] * x[0]; + By[7] = -0.0625 * x[0] * x[1] * x[1] + 0.0625 * x[0] - 0.0625 * x[1] * x[1] + 0.0625; + Bz[7] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bx[8] = 0.0625 * x[0] * x[0] * x[2] - 0.0625 * x[0] * x[0] - 0.0625 * x[2] + 0.0625; + By[8] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] - 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bz[8] = 0.0625 * x[2] * x[2] - 0.0625; + Bx[9] = -0.0625 * x[0] * x[0] * x[2] + 0.0625 * x[0] * x[0] + 0.0625 * x[2] - 0.0625; + By[9] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + Bz[9] = 0.0625 * x[2] * x[2] - 0.0625; + Bx[10] = -0.0625 * x[0] * x[0] * x[2] - 0.0625 * x[0] * x[0] + 0.0625 * x[2] + 0.0625; + By[10] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] - 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] - 0.125 * x[2] - + 0.125; + Bz[10] = 0.0625 - 0.0625 * x[2] * x[2]; + Bx[11] = 0.0625 * x[0] * x[0] * x[2] + 0.0625 * x[0] * x[0] - 0.0625 * x[2] - 0.0625; + By[11] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] - + 0.125 * x[2] - 0.125; + Bz[11] = 0.0625 - 0.0625 * x[2] * x[2]; + Bx[12] = 0.0625 * x[0] * x[0] * x[2] - 0.0625 * x[0] * x[0] - 0.0625 * x[2] + 0.0625; + By[12] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] - + 0.125 * x[2] + 0.125; + Bz[12] = 0.0625 * x[2] * x[2] - 0.0625; + Bx[13] = -0.0625 * x[0] * x[0] * x[2] + 0.0625 * x[0] * x[0] + 0.0625 * x[2] - 0.0625; + By[13] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] - 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] - 0.125 * x[2] + + 0.125; + Bz[13] = 0.0625 * x[2] * x[2] - 0.0625; + Bx[14] = -0.0625 * x[0] * x[0] * x[2] - 0.0625 * x[0] * x[0] + 0.0625 * x[2] + 0.0625; + By[14] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] + + 0.125; + Bz[14] = 0.0625 - 0.0625 * x[2] * x[2]; + Bx[15] = 0.0625 * x[0] * x[0] * x[2] + 0.0625 * x[0] * x[0] - 0.0625 * x[2] - 0.0625; + By[15] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] - 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + + 0.125 * x[2] + 0.125; + Bz[15] = 0.0625 - 0.0625 * x[2] * x[2]; + Bx[16] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] - 0.125 * x[1] - 0.125 * x[2] + + 0.125; + By[16] = 0.0625 * x[1] * x[1] - 0.0625; + Bz[16] = -0.0625 * x[1] * x[2] * x[2] + 0.0625 * x[1] + 0.0625 * x[2] * x[2] - 0.0625; + Bx[17] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] - + 0.125 * x[2] + 0.125; + By[17] = 0.0625 - 0.0625 * x[1] * x[1]; + Bz[17] = 0.0625 * x[1] * x[2] * x[2] - 0.0625 * x[1] + 0.0625 * x[2] * x[2] - 0.0625; + Bx[18] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] - 0.125 * x[1] + + 0.125 * x[2] + 0.125; + By[18] = 0.0625 * x[1] * x[1] - 0.0625; + Bz[18] = 0.0625 * x[1] * x[2] * x[2] - 0.0625 * x[1] - 0.0625 * x[2] * x[2] + 0.0625; + Bx[19] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] + + 0.125; + By[19] = 0.0625 - 0.0625 * x[1] * x[1]; + Bz[19] = -0.0625 * x[1] * x[2] * x[2] + 0.0625 * x[1] - 0.0625 * x[2] * x[2] + 0.0625; + Bx[20] = 0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] + 0.125 * x[1] + 0.125 * x[2] - + 0.125; + By[20] = 0.0625 * x[1] * x[1] - 0.0625; + Bz[20] = -0.0625 * x[1] * x[2] * x[2] + 0.0625 * x[1] + 0.0625 * x[2] * x[2] - 0.0625; + Bx[21] = -0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] - 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] - 0.125 * x[1] + + 0.125 * x[2] - 0.125; + By[21] = 0.0625 - 0.0625 * x[1] * x[1]; + Bz[21] = 0.0625 * x[1] * x[2] * x[2] - 0.0625 * x[1] + 0.0625 * x[2] * x[2] - 0.0625; + Bx[22] = -0.125 * x[0] * x[1] * x[2] - 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] + 0.125 * x[1] * x[2] + 0.125 * x[1] - + 0.125 * x[2] - 0.125; + By[22] = 0.0625 * x[1] * x[1] - 0.0625; + Bz[22] = 0.0625 * x[1] * x[2] * x[2] - 0.0625 * x[1] - 0.0625 * x[2] * x[2] + 0.0625; + Bx[23] = 0.125 * x[0] * x[1] * x[2] + 0.125 * x[0] * x[1] + 0.125 * x[0] * x[2] + 0.125 * x[0] - 0.125 * x[1] * x[2] - 0.125 * x[1] - 0.125 * x[2] - + 0.125; + By[23] = 0.0625 - 0.0625 * x[1] * x[1]; + Bz[23] = -0.0625 * x[1] * x[2] * x[2] + 0.0625 * x[1] - 0.0625 * x[2] * x[2] + 0.0625; return 0; } -static void HdivBasisHex(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, - CeedScalar *interp, CeedScalar *div, CeedQuadMode quad_mode) { - +static void HdivBasisHex(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, CeedScalar *interp, CeedScalar *div, CeedQuadMode quad_mode) { // Get 1D quadrature on [-1,1] CeedScalar q_ref_1d[Q], q_weight_1d[Q]; switch (quad_mode) { - case CEED_GAUSS: - CeedGaussQuadrature(Q, q_ref_1d, q_weight_1d); - break; - case CEED_GAUSS_LOBATTO: - CeedLobattoQuadrature(Q, q_ref_1d, q_weight_1d); - break; + case CEED_GAUSS: + CeedGaussQuadrature(Q, q_ref_1d, q_weight_1d); + break; + case CEED_GAUSS_LOBATTO: + CeedLobattoQuadrature(Q, q_ref_1d, q_weight_1d); + break; } // Divergence operator; Divergence of nodal basis for ref element @@ -138,27 +134,27 @@ static void HdivBasisHex(CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, // Loop over quadrature points CeedScalar Bx[24], By[24], Bz[24]; CeedScalar x[3]; - for (CeedInt k=0; k #include -#include "structs.h" #include "../include/setup-libceed.h" -PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, - CeedMemType mem_type_backend, - TS ts, SNES snes, KSP ksp, - Vec U, CeedScalar l2_error_u, - CeedScalar l2_error_p); -PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, - CeedData ceed_data, OperatorApplyContext ctx_Hdiv); -PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, - CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_H1); -PetscErrorCode ProjectVelocity(AppCtx app_ctx, - Vec U, Vec *U_H1); +#include "structs.h" +PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, CeedMemType mem_type_backend, TS ts, SNES snes, KSP ksp, Vec U, + CeedScalar l2_error_u, CeedScalar l2_error_p); +PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_Hdiv); +PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_H1); +PetscErrorCode ProjectVelocity(AppCtx app_ctx, Vec U, Vec *U_H1); PetscErrorCode CtxVecDestroy(AppCtx app_ctx); -#endif // post_processing_h +#endif // post_processing_h diff --git a/examples/Hdiv-mixed/include/register-problem.h b/examples/Hdiv-mixed/include/register-problem.h index 9d54e26794..88e4b2a34a 100644 --- a/examples/Hdiv-mixed/include/register-problem.h +++ b/examples/Hdiv-mixed/include/register-problem.h @@ -9,22 +9,18 @@ PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx); // Set up problems function prototype // ----------------------------------------------------------------------------- // 1) darcy2d -PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, DM dm, - void *ctx); +PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx); // 2) darcy3d -PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, DM dm, - void *ctx); +PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx); // 3) darcy3dprism // 4) richard -PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, DM dm, - void *ctx); +PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx); -PetscErrorCode Hdiv_RICHARD3D(Ceed ceed, ProblemData problem_data, DM dm, - void *ctx); +PetscErrorCode Hdiv_RICHARD3D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx); extern int FreeContextPetsc(void *); -#endif // register_problems_h +#endif // register_problems_h diff --git a/examples/Hdiv-mixed/include/setup-boundary.h b/examples/Hdiv-mixed/include/setup-boundary.h index 21e62d536f..c62fb92982 100644 --- a/examples/Hdiv-mixed/include/setup-boundary.h +++ b/examples/Hdiv-mixed/include/setup-boundary.h @@ -1,10 +1,11 @@ #ifndef setup_boundary_h #define setup_boundary_h +#include #include #include #include -#include + #include "structs.h" // --------------------------------------------------------------------------- @@ -16,10 +17,6 @@ PetscErrorCode CreateBCLabel(DM dm, const char name[]); // Add Dirichlet boundaries to DM // --------------------------------------------------------------------------- PetscErrorCode DMAddBoundariesDirichlet(DM dm); -PetscErrorCode BoundaryDirichletMMS(PetscInt dim, PetscReal t, - const PetscReal coords[], - PetscInt num_comp_u, PetscScalar *u, void *ctx); -PetscErrorCode DMAddBoundariesPressure(Ceed ceed, CeedData ceed_data, - AppCtx app_ctx, ProblemData problem_data, DM dm, - CeedVector bc_pressure); -#endif // setup_boundary_h +PetscErrorCode BoundaryDirichletMMS(PetscInt dim, PetscReal t, const PetscReal coords[], PetscInt num_comp_u, PetscScalar *u, void *ctx); +PetscErrorCode DMAddBoundariesPressure(Ceed ceed, CeedData ceed_data, AppCtx app_ctx, ProblemData problem_data, DM dm, CeedVector bc_pressure); +#endif // setup_boundary_h diff --git a/examples/Hdiv-mixed/include/setup-dm.h b/examples/Hdiv-mixed/include/setup-dm.h index b61ac7e17d..71469cbd37 100644 --- a/examples/Hdiv-mixed/include/setup-dm.h +++ b/examples/Hdiv-mixed/include/setup-dm.h @@ -1,18 +1,18 @@ #ifndef setupdm_h #define setupdm_h +#include #include #include #include -#include + #include "structs.h" // --------------------------------------------------------------------------- // Setup DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, - VecType vec_type, DM *dm); +PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, VecType vec_type, DM *dm); PetscErrorCode PerturbVerticesSmooth(DM dm); PetscErrorCode PerturbVerticesRandom(DM dm); -#endif // setupdm_h +#endif // setupdm_h diff --git a/examples/Hdiv-mixed/include/setup-fe.h b/examples/Hdiv-mixed/include/setup-fe.h index 24dcda84ab..382b047740 100644 --- a/examples/Hdiv-mixed/include/setup-fe.h +++ b/examples/Hdiv-mixed/include/setup-fe.h @@ -1,16 +1,16 @@ #ifndef setupfe_h #define setupfe_h +#include #include #include #include -#include + #include "structs.h" // --------------------------------------------------------------------------- // Setup FE // --------------------------------------------------------------------------- PetscErrorCode SetupFEHdiv(MPI_Comm comm, DM dm, DM dm_u0, DM dm_p0); -PetscErrorCode SetupFEH1(ProblemData problem_data, - AppCtx app_ctx, DM dm_H1); -#endif // setupfe_h +PetscErrorCode SetupFEH1(ProblemData problem_data, AppCtx app_ctx, DM dm_H1); +#endif // setupfe_h diff --git a/examples/Hdiv-mixed/include/setup-libceed.h b/examples/Hdiv-mixed/include/setup-libceed.h index aac44d76fd..3b509b4a65 100644 --- a/examples/Hdiv-mixed/include/setup-libceed.h +++ b/examples/Hdiv-mixed/include/setup-libceed.h @@ -10,17 +10,11 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data); // Utility function - essential BC dofs are encoded in closure indices as -(i+1) PetscInt Involute(PetscInt i); // Utility function to create local CEED restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, - CeedInt height, DMLabel domain_label, CeedInt value, - CeedElemRestriction *elem_restr); +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr); // Utility function to create local CEED Oriented restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, - DM dm_p0, CeedInt P, - CeedElemRestriction *elem_restr_u, CeedElemRestriction *elem_restr_p, - CeedElemRestriction *elem_restr_u0, CeedElemRestriction *elem_restr_p0); +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u, + CeedElemRestriction *elem_restr_p, CeedElemRestriction *elem_restr_u0, + CeedElemRestriction *elem_restr_p0); // Set up libCEED for a given degree -PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, - Ceed ceed, AppCtx app_ctx, - ProblemData problem_data, - CeedData ceed_data); -#endif // setuplibceed_h +PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, Ceed ceed, AppCtx app_ctx, ProblemData problem_data, CeedData ceed_data); +#endif // setuplibceed_h diff --git a/examples/Hdiv-mixed/include/setup-matops.h b/examples/Hdiv-mixed/include/setup-matops.h index 7c21ed22d4..dad7737099 100644 --- a/examples/Hdiv-mixed/include/setup-matops.h +++ b/examples/Hdiv-mixed/include/setup-matops.h @@ -6,9 +6,7 @@ #include "structs.h" -PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, - OperatorApplyContext op_apply_ctx); -PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, - OperatorApplyContext op_apply_ctx); +PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx); +PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx); -#endif // setup_matops_h +#endif // setup_matops_h diff --git a/examples/Hdiv-mixed/include/setup-solvers.h b/examples/Hdiv-mixed/include/setup-solvers.h index c7a40a1722..5502172408 100644 --- a/examples/Hdiv-mixed/include/setup-solvers.h +++ b/examples/Hdiv-mixed/include/setup-solvers.h @@ -7,19 +7,13 @@ #include "petscvec.h" #include "structs.h" -PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, - VecType vec_type, - OperatorApplyContext ctx_jacobian); -PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, - OperatorApplyContext ctx_residual); -PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, - OperatorApplyContext ctx_error); +PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_jacobian); +PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual); +PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_error); PetscErrorCode ApplyMatOp(Mat A, Vec X, Vec Y); PetscErrorCode SNESFormResidual(SNES snes, Vec X, Vec Y, void *ctx); PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, void *ctx); -PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, - SNES snes, KSP ksp, Vec *U); -PetscErrorCode ComputeL2Error(CeedData ceed_data, AppCtx app_ctx, Vec U, - CeedScalar *l2_error_u, CeedScalar *l2_error_p); +PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, SNES snes, KSP ksp, Vec *U); +PetscErrorCode ComputeL2Error(CeedData ceed_data, AppCtx app_ctx, Vec U, CeedScalar *l2_error_u, CeedScalar *l2_error_p); -#endif // setup_solvers_h +#endif // setup_solvers_h diff --git a/examples/Hdiv-mixed/include/setup-ts.h b/examples/Hdiv-mixed/include/setup-ts.h index d461ba5675..d490ba9bab 100644 --- a/examples/Hdiv-mixed/include/setup-ts.h +++ b/examples/Hdiv-mixed/include/setup-ts.h @@ -6,17 +6,11 @@ #include "structs.h" -PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, - VecType vec_type, Vec U); -PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, - CeedData ceed_data, OperatorApplyContext ctx_residual_ut); -PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm, Ceed ceed, - CeedData ceed_data, OperatorApplyContext ctx_initial_u0); -PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm, Ceed ceed, - CeedData ceed_data, OperatorApplyContext ctx_initial_p0); -PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, - void *ctx_residual_ut); -PetscErrorCode TSSolveRichard(CeedData ceed_data, AppCtx app_ctx, - TS ts, Vec *U); +PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, VecType vec_type, Vec U); +PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual_ut); +PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_initial_u0); +PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_initial_p0); +PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, void *ctx_residual_ut); +PetscErrorCode TSSolveRichard(CeedData ceed_data, AppCtx app_ctx, TS ts, Vec *U); -#endif // setup_ts_h +#endif // setup_ts_h diff --git a/examples/Hdiv-mixed/include/structs.h b/examples/Hdiv-mixed/include/structs.h index 63d97eddce..1c70274d64 100644 --- a/examples/Hdiv-mixed/include/structs.h +++ b/examples/Hdiv-mixed/include/structs.h @@ -7,73 +7,61 @@ // PETSc operator contexts typedef struct OperatorApplyContext_ *OperatorApplyContext; struct OperatorApplyContext_ { - MPI_Comm comm; - Vec X_loc, Y_loc, X_t_loc; - CeedVector x_ceed, y_ceed, x_t_ceed, x_coord, rhs_ceed_H1; - CeedOperator op_apply, op_rhs_H1; - DM dm; - Ceed ceed; - CeedScalar t, dt; - CeedContextFieldLabel solution_time_label, final_time_label, - timestep_label; - CeedElemRestriction elem_restr_u_H1; - VecType vec_type; + MPI_Comm comm; + Vec X_loc, Y_loc, X_t_loc; + CeedVector x_ceed, y_ceed, x_t_ceed, x_coord, rhs_ceed_H1; + CeedOperator op_apply, op_rhs_H1; + DM dm; + Ceed ceed; + CeedScalar t, dt; + CeedContextFieldLabel solution_time_label, final_time_label, timestep_label; + CeedElemRestriction elem_restr_u_H1; + VecType vec_type; }; // libCEED data struct typedef struct CeedData_ *CeedData; struct CeedData_ { - CeedBasis basis_x, basis_u, basis_p, basis_u_face; - CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_U_i, - elem_restr_p, elem_restr_p_i, elem_restr_u0, - elem_restr_p0, elem_restr_u_H1; - CeedQFunction qf_residual, qf_jacobian, qf_error, qf_ics_u, qf_ics_p, - qf_rhs_u0, qf_rhs_p0, qf_rhs_H1, qf_post_mass; - CeedOperator op_residual, op_jacobian, op_error, op_ics_u, op_ics_p, - op_rhs_u0, op_rhs_p0, op_rhs_H1, op_post_mass; - CeedVector x_ceed, y_ceed, x_coord, x_t_ceed, rhs_u0_ceed, - u0_ceed, v0_ceed, rhs_p0_ceed, p0_ceed, q0_ceed, - rhs_ceed_H1, u_ceed, up_ceed, vp_ceed; - CeedInt num_elem; + CeedBasis basis_x, basis_u, basis_p, basis_u_face; + CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_U_i, elem_restr_p, elem_restr_p_i, elem_restr_u0, elem_restr_p0, elem_restr_u_H1; + CeedQFunction qf_residual, qf_jacobian, qf_error, qf_ics_u, qf_ics_p, qf_rhs_u0, qf_rhs_p0, qf_rhs_H1, qf_post_mass; + CeedOperator op_residual, op_jacobian, op_error, op_ics_u, op_ics_p, op_rhs_u0, op_rhs_p0, op_rhs_H1, op_post_mass; + CeedVector x_ceed, y_ceed, x_coord, x_t_ceed, rhs_u0_ceed, u0_ceed, v0_ceed, rhs_p0_ceed, p0_ceed, q0_ceed, rhs_ceed_H1, u_ceed, up_ceed, vp_ceed; + CeedInt num_elem; }; // Application context from user command line options typedef struct AppCtx_ *AppCtx; struct AppCtx_ { - char ceed_resource[PETSC_MAX_PATH_LEN]; // libCEED backend - MPI_Comm comm; + char ceed_resource[PETSC_MAX_PATH_LEN]; // libCEED backend + MPI_Comm comm; // Degree of polynomial (1 only), extra quadrature pts - PetscInt degree; - PetscInt q_extra; + PetscInt degree; + PetscInt q_extra; // For applying traction BCs - PetscInt bc_pressure_count; - PetscInt bc_faces[16]; // face ID - PetscScalar bc_pressure_value[16]; + PetscInt bc_pressure_count; + PetscInt bc_faces[16]; // face ID + PetscScalar bc_pressure_value[16]; // Problem type arguments - PetscFunctionList problems; - char problem_name[PETSC_MAX_PATH_LEN]; - CeedScalar t_final, t; - PetscBool view_solution, quartic; - char output_dir[PETSC_MAX_PATH_LEN]; - PetscInt output_freq; - OperatorApplyContext ctx_residual, ctx_jacobian, ctx_error, ctx_residual_ut, - ctx_initial_u0, ctx_initial_p0, ctx_Hdiv, ctx_H1; + PetscFunctionList problems; + char problem_name[PETSC_MAX_PATH_LEN]; + CeedScalar t_final, t; + PetscBool view_solution, quartic; + char output_dir[PETSC_MAX_PATH_LEN]; + PetscInt output_freq; + OperatorApplyContext ctx_residual, ctx_jacobian, ctx_error, ctx_residual_ut, ctx_initial_u0, ctx_initial_p0, ctx_Hdiv, ctx_H1; }; // Problem specific data typedef struct ProblemData_ *ProblemData; struct ProblemData_ { - CeedQFunctionUser true_solution, residual, jacobian, error, ics_u, ics_p, - bc_pressure, rhs_u0, rhs_p0, post_rhs, post_mass; - const char *true_solution_loc, *residual_loc, *jacobian_loc, - *error_loc, *bc_pressure_loc, *ics_u_loc, *ics_p_loc, *rhs_u0_loc, - *rhs_p0_loc, *post_rhs_loc, *post_mass_loc; - CeedQuadMode quadrature_mode; - CeedInt elem_node, dim, q_data_size_face; - CeedQFunctionContext true_qfunction_ctx, error_qfunction_ctx, - residual_qfunction_ctx, jacobian_qfunction_ctx, - rhs_u0_qfunction_ctx ; - PetscBool has_ts, view_solution, quartic; + CeedQFunctionUser true_solution, residual, jacobian, error, ics_u, ics_p, bc_pressure, rhs_u0, rhs_p0, post_rhs, post_mass; + const char *true_solution_loc, *residual_loc, *jacobian_loc, *error_loc, *bc_pressure_loc, *ics_u_loc, *ics_p_loc, *rhs_u0_loc, *rhs_p0_loc, + *post_rhs_loc, *post_mass_loc; + CeedQuadMode quadrature_mode; + CeedInt elem_node, dim, q_data_size_face; + CeedQFunctionContext true_qfunction_ctx, error_qfunction_ctx, residual_qfunction_ctx, jacobian_qfunction_ctx, rhs_u0_qfunction_ctx; + PetscBool has_ts, view_solution, quartic; }; -#endif // structs_h \ No newline at end of file +#endif // structs_h \ No newline at end of file diff --git a/examples/Hdiv-mixed/main.c b/examples/Hdiv-mixed/main.c index 814fd086f8..a0df25b77e 100644 --- a/examples/Hdiv-mixed/main.c +++ b/examples/Hdiv-mixed/main.c @@ -40,31 +40,29 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // Initialize PETSc // --------------------------------------------------------------------------- - PetscCall( PetscInitialize(&argc, &argv, NULL, help) ); + PetscCall(PetscInitialize(&argc, &argv, NULL, help)); // --------------------------------------------------------------------------- // Create structs // --------------------------------------------------------------------------- AppCtx app_ctx; - PetscCall( PetscCalloc1(1, &app_ctx) ); + PetscCall(PetscCalloc1(1, &app_ctx)); ProblemData problem_data = NULL; - PetscCall( PetscCalloc1(1, &problem_data) ); + PetscCall(PetscCalloc1(1, &problem_data)); CeedData ceed_data; - PetscCall( PetscCalloc1(1, &ceed_data) ); + PetscCall(PetscCalloc1(1, &ceed_data)); - OperatorApplyContext ctx_jacobian, ctx_residual, ctx_residual_ut, - ctx_initial_u0, ctx_initial_p0, - ctx_error, ctx_Hdiv, ctx_H1; - PetscCall( PetscCalloc1(1, &ctx_jacobian) ); - PetscCall( PetscCalloc1(1, &ctx_residual) ); - PetscCall( PetscCalloc1(1, &ctx_residual_ut) ); - PetscCall( PetscCalloc1(1, &ctx_initial_u0) ); - PetscCall( PetscCalloc1(1, &ctx_initial_p0) ); - PetscCall( PetscCalloc1(1, &ctx_error) ); - PetscCall( PetscCalloc1(1, &ctx_Hdiv) ); - PetscCall( PetscCalloc1(1, &ctx_H1) ); + OperatorApplyContext ctx_jacobian, ctx_residual, ctx_residual_ut, ctx_initial_u0, ctx_initial_p0, ctx_error, ctx_Hdiv, ctx_H1; + PetscCall(PetscCalloc1(1, &ctx_jacobian)); + PetscCall(PetscCalloc1(1, &ctx_residual)); + PetscCall(PetscCalloc1(1, &ctx_residual_ut)); + PetscCall(PetscCalloc1(1, &ctx_initial_u0)); + PetscCall(PetscCalloc1(1, &ctx_initial_p0)); + PetscCall(PetscCalloc1(1, &ctx_error)); + PetscCall(PetscCalloc1(1, &ctx_Hdiv)); + PetscCall(PetscCalloc1(1, &ctx_H1)); // Context for Darcy problem app_ctx->ctx_residual = ctx_residual; app_ctx->ctx_jacobian = ctx_jacobian; @@ -78,7 +76,7 @@ int main(int argc, char **argv) { app_ctx->ctx_error = ctx_error; // Context for post-processing app_ctx->ctx_Hdiv = ctx_Hdiv; - app_ctx->ctx_H1 = ctx_H1; + app_ctx->ctx_H1 = ctx_H1; // --------------------------------------------------------------------------- // Initialize libCEED @@ -86,21 +84,23 @@ int main(int argc, char **argv) { // -- Initialize backend Ceed ceed; CeedInit("/cpu/self/ref/serial", &ceed); - //CeedInit(app_ctx->ceed_resource, &ceed); + // CeedInit(app_ctx->ceed_resource, &ceed); CeedMemType mem_type_backend; CeedGetPreferredMemType(ceed, &mem_type_backend); VecType vec_type = NULL; MatType mat_type = NULL; switch (mem_type_backend) { - case CEED_MEM_HOST: vec_type = VECSTANDARD; break; - case CEED_MEM_DEVICE: { - const char *resolved; - CeedGetResource(ceed, &resolved); - if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; - else if (strstr(resolved, "/gpu/hip")) vec_type = VECKOKKOS; - else vec_type = VECSTANDARD; - } + case CEED_MEM_HOST: + vec_type = VECSTANDARD; + break; + case CEED_MEM_DEVICE: { + const char *resolved; + CeedGetResource(ceed, &resolved); + if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; + else if (strstr(resolved, "/gpu/hip")) vec_type = VECKOKKOS; + else vec_type = VECSTANDARD; + } } if (strstr(vec_type, VECCUDA)) mat_type = MATAIJCUSPARSE; else if (strstr(vec_type, VECKOKKOS)) mat_type = MATAIJKOKKOS; @@ -111,99 +111,92 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // Create DM // --------------------------------------------------------------------------- - DM dm, dm_u0, dm_p0, dm_H1; + DM dm, dm_u0, dm_p0, dm_H1; // DM for mixed problem - PetscCall( CreateDM(comm, mat_type, vec_type, &dm) ); + PetscCall(CreateDM(comm, mat_type, vec_type, &dm)); // DM for projecting initial velocity to Hdiv space - PetscCall( CreateDM(comm, mat_type, vec_type, &dm_u0) ); + PetscCall(CreateDM(comm, mat_type, vec_type, &dm_u0)); // DM for projecting initial pressure in L2 - PetscCall( CreateDM(comm, mat_type, vec_type, &dm_p0) ); + PetscCall(CreateDM(comm, mat_type, vec_type, &dm_p0)); // DM for projecting solution U into H1 space for PetscViewer - PetscCall( CreateDM(comm, mat_type, vec_type, &dm_H1) ); + PetscCall(CreateDM(comm, mat_type, vec_type, &dm_H1)); // TODO: add mesh option // perturb dm to have smooth random mesh - //PetscCall( PerturbVerticesSmooth(dm) ); - //PetscCall( PerturbVerticesSmooth(dm_H1) ); + // PetscCall( PerturbVerticesSmooth(dm) ); + // PetscCall( PerturbVerticesSmooth(dm_H1) ); // perturb dm to have random mesh - //PetscCall(PerturbVerticesRandom(dm) ); - //PetscCall(PerturbVerticesRandom(dm_H1) ); + // PetscCall(PerturbVerticesRandom(dm) ); + // PetscCall(PerturbVerticesRandom(dm_H1) ); // --------------------------------------------------------------------------- // Process command line options // --------------------------------------------------------------------------- // -- Register problems to be available on the command line - PetscCall( RegisterProblems_Hdiv(app_ctx) ); + PetscCall(RegisterProblems_Hdiv(app_ctx)); app_ctx->comm = comm; - PetscCall( ProcessCommandLineOptions(app_ctx) ); + PetscCall(ProcessCommandLineOptions(app_ctx)); // --------------------------------------------------------------------------- // Choose the problem from the list of registered problems // --------------------------------------------------------------------------- { PetscErrorCode (*p)(Ceed, ProblemData, DM, void *); - PetscCall( PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, - &p) ); - if (!p) SETERRQ(PETSC_COMM_SELF, 1, "Problem '%s' not found", - app_ctx->problem_name); - PetscCall( (*p)(ceed, problem_data, dm, &app_ctx) ); + PetscCall(PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p)); + if (!p) SETERRQ(PETSC_COMM_SELF, 1, "Problem '%s' not found", app_ctx->problem_name); + PetscCall((*p)(ceed, problem_data, dm, &app_ctx)); } // --------------------------------------------------------------------------- // Setup FE for H(div) mixed-problem and H1 projection in post-processing.c // --------------------------------------------------------------------------- - PetscCall( SetupFEHdiv(comm, dm, dm_u0, dm_p0) ); - PetscCall( SetupFEH1(problem_data, app_ctx, dm_H1) ); + PetscCall(SetupFEHdiv(comm, dm, dm_u0, dm_p0)); + PetscCall(SetupFEH1(problem_data, app_ctx, dm_H1)); // --------------------------------------------------------------------------- // Create global unkown solution // --------------------------------------------------------------------------- - Vec U; // U=[p,u] - PetscCall( DMCreateGlobalVector(dm, &U) ); + Vec U; // U=[p,u] + PetscCall(DMCreateGlobalVector(dm, &U)); // --------------------------------------------------------------------------- // Setup libCEED // --------------------------------------------------------------------------- // -- Set up libCEED objects - PetscCall( SetupLibceed(dm, dm_u0, dm_p0, dm_H1, ceed, app_ctx, - problem_data, ceed_data) ); + PetscCall(SetupLibceed(dm, dm_u0, dm_p0, dm_H1, ceed, app_ctx, problem_data, ceed_data)); // --------------------------------------------------------------------------- // Setup pressure boundary conditions // --------------------------------------------------------------------------- // --Create empty local vector for libCEED - Vec P_loc; - PetscInt P_loc_size; - CeedScalar *p0; - CeedVector P_ceed; + Vec P_loc; + PetscInt P_loc_size; + CeedScalar *p0; + CeedVector P_ceed; PetscMemType pressure_mem_type; - PetscCall( DMCreateLocalVector(dm, &P_loc) ); - PetscCall( VecGetSize(P_loc, &P_loc_size) ); - PetscCall( VecZeroEntries(P_loc) ); - PetscCall( VecGetArrayAndMemType(P_loc, &p0, &pressure_mem_type) ); + PetscCall(DMCreateLocalVector(dm, &P_loc)); + PetscCall(VecGetSize(P_loc, &P_loc_size)); + PetscCall(VecZeroEntries(P_loc)); + PetscCall(VecGetArrayAndMemType(P_loc, &p0, &pressure_mem_type)); CeedVectorCreate(ceed, P_loc_size, &P_ceed); - CeedVectorSetArray(P_ceed, MemTypeP2C(pressure_mem_type), CEED_USE_POINTER, - p0); + CeedVectorSetArray(P_ceed, MemTypeP2C(pressure_mem_type), CEED_USE_POINTER, p0); // -- Apply operator to create local pressure vector on boundary - PetscCall( DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, - P_ceed) ); - //CeedVectorView(P_ceed, "%12.8f", stdout); - // -- Map local to global + PetscCall(DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, P_ceed)); + // CeedVectorView(P_ceed, "%12.8f", stdout); + // -- Map local to global Vec P; CeedVectorTakeArray(P_ceed, MemTypeP2C(pressure_mem_type), NULL); - PetscCall( VecRestoreArrayAndMemType(P_loc, &p0) ); - PetscCall( DMCreateGlobalVector(dm, &P) ); - PetscCall( VecZeroEntries(P) ); - PetscCall( DMLocalToGlobal(dm, P_loc, ADD_VALUES, P) ); + PetscCall(VecRestoreArrayAndMemType(P_loc, &p0)); + PetscCall(DMCreateGlobalVector(dm, &P)); + PetscCall(VecZeroEntries(P)); + PetscCall(DMLocalToGlobal(dm, P_loc, ADD_VALUES, P)); // --------------------------------------------------------------------------- // Setup context for projection problem; post-processing.c // --------------------------------------------------------------------------- - PetscCall( SetupProjectVelocityCtx_Hdiv(comm, dm, ceed, ceed_data, - app_ctx->ctx_Hdiv) ); - PetscCall( SetupProjectVelocityCtx_H1(comm, dm_H1, ceed, ceed_data, - vec_type, app_ctx->ctx_H1) ); + PetscCall(SetupProjectVelocityCtx_Hdiv(comm, dm, ceed, ceed_data, app_ctx->ctx_Hdiv)); + PetscCall(SetupProjectVelocityCtx_H1(comm, dm_H1, ceed, ceed_data, vec_type, app_ctx->ctx_H1)); // --------------------------------------------------------------------------- // Setup TSSolve for Richard problem @@ -213,110 +206,100 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // Setup context for initial conditions // --------------------------------------------------------------------------- - PetscCall( SetupResidualOperatorCtx_U0(comm, dm_u0, ceed, ceed_data, - app_ctx->ctx_initial_u0) ); - PetscCall( SetupResidualOperatorCtx_P0(comm, dm_p0, ceed, ceed_data, - app_ctx->ctx_initial_p0) ); - PetscCall( SetupResidualOperatorCtx_Ut(comm, dm, ceed, ceed_data, - app_ctx->ctx_residual_ut) ); - PetscCall( CreateInitialConditions(ceed_data, app_ctx, vec_type, U) ); - //VecView(U, PETSC_VIEWER_STDOUT_WORLD); - // Solve Richards problem - PetscCall( TSCreate(comm, &ts) ); - PetscCall( VecZeroEntries(app_ctx->ctx_residual_ut->X_loc) ); - PetscCall( VecZeroEntries(app_ctx->ctx_residual_ut->X_t_loc) ); - PetscCall( TSSolveRichard(ceed_data, app_ctx, ts, &U) ); - //VecView(U, PETSC_VIEWER_STDOUT_WORLD); + PetscCall(SetupResidualOperatorCtx_U0(comm, dm_u0, ceed, ceed_data, app_ctx->ctx_initial_u0)); + PetscCall(SetupResidualOperatorCtx_P0(comm, dm_p0, ceed, ceed_data, app_ctx->ctx_initial_p0)); + PetscCall(SetupResidualOperatorCtx_Ut(comm, dm, ceed, ceed_data, app_ctx->ctx_residual_ut)); + PetscCall(CreateInitialConditions(ceed_data, app_ctx, vec_type, U)); + // VecView(U, PETSC_VIEWER_STDOUT_WORLD); + // Solve Richards problem + PetscCall(TSCreate(comm, &ts)); + PetscCall(VecZeroEntries(app_ctx->ctx_residual_ut->X_loc)); + PetscCall(VecZeroEntries(app_ctx->ctx_residual_ut->X_t_loc)); + PetscCall(TSSolveRichard(ceed_data, app_ctx, ts, &U)); + // VecView(U, PETSC_VIEWER_STDOUT_WORLD); } // --------------------------------------------------------------------------- // Setup SNES for Darcy problem // --------------------------------------------------------------------------- SNES snes; - KSP ksp; + KSP ksp; if (!problem_data->has_ts) { - PetscCall( SetupJacobianOperatorCtx(dm, ceed, ceed_data, vec_type, - app_ctx->ctx_jacobian) ); - PetscCall( SetupResidualOperatorCtx(dm, ceed, ceed_data, - app_ctx->ctx_residual) ); + PetscCall(SetupJacobianOperatorCtx(dm, ceed, ceed_data, vec_type, app_ctx->ctx_jacobian)); + PetscCall(SetupResidualOperatorCtx(dm, ceed, ceed_data, app_ctx->ctx_residual)); // Create SNES - PetscCall( SNESCreate(comm, &snes) ); - PetscCall( SNESGetKSP(snes, &ksp) ); - PetscCall( PDESolver(ceed_data, app_ctx, snes, ksp, &U) ); - //VecView(U, PETSC_VIEWER_STDOUT_WORLD); + PetscCall(SNESCreate(comm, &snes)); + PetscCall(SNESGetKSP(snes, &ksp)); + PetscCall(PDESolver(ceed_data, app_ctx, snes, ksp, &U)); + // VecView(U, PETSC_VIEWER_STDOUT_WORLD); } // --------------------------------------------------------------------------- // Compute L2 error of mms problem // --------------------------------------------------------------------------- - PetscCall( SetupErrorOperatorCtx(dm, ceed, ceed_data, app_ctx->ctx_error) ); + PetscCall(SetupErrorOperatorCtx(dm, ceed, ceed_data, app_ctx->ctx_error)); CeedScalar l2_error_u, l2_error_p; - PetscCall( ComputeL2Error(ceed_data, app_ctx, U, - &l2_error_u, &l2_error_p) ); + PetscCall(ComputeL2Error(ceed_data, app_ctx, U, &l2_error_u, &l2_error_p)); // --------------------------------------------------------------------------- // Print solver iterations and final norms // --------------------------------------------------------------------------- - PetscCall( PrintOutput(dm, ceed, app_ctx, problem_data->has_ts, - mem_type_backend, - ts, snes, ksp, U, l2_error_u, l2_error_p) ); + PetscCall(PrintOutput(dm, ceed, app_ctx, problem_data->has_ts, mem_type_backend, ts, snes, ksp, U, l2_error_u, l2_error_p)); // --------------------------------------------------------------------------- // Save solution (paraview) // --------------------------------------------------------------------------- if (app_ctx->view_solution) { PetscViewer viewer_p; - PetscCall( PetscViewerVTKOpen(comm,"darcy_pressure.vtu",FILE_MODE_WRITE, - &viewer_p) ); - PetscCall( VecView(U, viewer_p) ); - PetscCall( PetscViewerDestroy(&viewer_p) ); + PetscCall(PetscViewerVTKOpen(comm, "darcy_pressure.vtu", FILE_MODE_WRITE, &viewer_p)); + PetscCall(VecView(U, viewer_p)); + PetscCall(PetscViewerDestroy(&viewer_p)); - Vec U_H1; // velocity in H1 space for post-processing - PetscCall( DMCreateGlobalVector(dm_H1, &U_H1) ); - PetscCall( ProjectVelocity(app_ctx, U, &U_H1) ); + Vec U_H1; // velocity in H1 space for post-processing + PetscCall(DMCreateGlobalVector(dm_H1, &U_H1)); + PetscCall(ProjectVelocity(app_ctx, U, &U_H1)); PetscViewer viewer_u; - PetscCall( PetscViewerVTKOpen(comm,"darcy_velocity.vtu",FILE_MODE_WRITE, - &viewer_u) ); - PetscCall( VecView(U_H1, viewer_u) ); - PetscCall( PetscViewerDestroy(&viewer_u) ); - PetscCall( VecDestroy(&U_H1) ); + PetscCall(PetscViewerVTKOpen(comm, "darcy_velocity.vtu", FILE_MODE_WRITE, &viewer_u)); + PetscCall(VecView(U_H1, viewer_u)); + PetscCall(PetscViewerDestroy(&viewer_u)); + PetscCall(VecDestroy(&U_H1)); } // --------------------------------------------------------------------------- // Free objects // --------------------------------------------------------------------------- // Free PETSc objects - PetscCall( DMDestroy(&dm) ); - PetscCall( DMDestroy(&dm_u0) ); - PetscCall( DMDestroy(&dm_p0) ); - PetscCall( DMDestroy(&dm_H1) ); - PetscCall( VecDestroy(&U) ); - PetscCall( CtxVecDestroy(app_ctx) ); + PetscCall(DMDestroy(&dm)); + PetscCall(DMDestroy(&dm_u0)); + PetscCall(DMDestroy(&dm_p0)); + PetscCall(DMDestroy(&dm_H1)); + PetscCall(VecDestroy(&U)); + PetscCall(CtxVecDestroy(app_ctx)); if (problem_data->has_ts) { - PetscCall( TSDestroy(&ts) ); + PetscCall(TSDestroy(&ts)); } else { - PetscCall( SNESDestroy(&snes) ); + PetscCall(SNESDestroy(&snes)); } - PetscCall( CeedDataDestroy(ceed_data, problem_data) ); + PetscCall(CeedDataDestroy(ceed_data, problem_data)); // -- Function list - PetscCall( PetscFunctionListDestroy(&app_ctx->problems) ); + PetscCall(PetscFunctionListDestroy(&app_ctx->problems)); // -- Structs - PetscCall( PetscFree(app_ctx) ); - PetscCall( PetscFree(problem_data) ); - PetscCall( PetscFree(ctx_initial_u0) ); - PetscCall( PetscFree(ctx_initial_p0) ); - PetscCall( PetscFree(ctx_residual_ut) ); - PetscCall( PetscFree(ctx_residual) ); - PetscCall( PetscFree(ctx_jacobian) ); - PetscCall( PetscFree(ctx_error) ); - PetscCall( PetscFree(ctx_H1) ); - PetscCall( PetscFree(ctx_Hdiv) ); + PetscCall(PetscFree(app_ctx)); + PetscCall(PetscFree(problem_data)); + PetscCall(PetscFree(ctx_initial_u0)); + PetscCall(PetscFree(ctx_initial_p0)); + PetscCall(PetscFree(ctx_residual_ut)); + PetscCall(PetscFree(ctx_residual)); + PetscCall(PetscFree(ctx_jacobian)); + PetscCall(PetscFree(ctx_error)); + PetscCall(PetscFree(ctx_H1)); + PetscCall(PetscFree(ctx_Hdiv)); // Free libCEED objects - //CeedVectorDestroy(&bc_pressure); + // CeedVectorDestroy(&bc_pressure); CeedDestroy(&ceed); return PetscFinalize(); diff --git a/examples/Hdiv-mixed/main.h b/examples/Hdiv-mixed/main.h index 7033f49f38..8f3e6abe05 100644 --- a/examples/Hdiv-mixed/main.h +++ b/examples/Hdiv-mixed/main.h @@ -2,15 +2,15 @@ #ifndef MAIN_H #define MAIN_H -#include "include/setup-libceed.h" -#include "include/setup-dm.h" -#include "include/setup-fe.h" -#include "include/setup-boundary.h" #include "include/cl-options.h" +#include "include/post-processing.h" #include "include/register-problem.h" +#include "include/setup-boundary.h" +#include "include/setup-dm.h" +#include "include/setup-fe.h" +#include "include/setup-libceed.h" #include "include/setup-matops.h" #include "include/setup-solvers.h" #include "include/setup-ts.h" -#include "include/post-processing.h" -#endif // MAIN_H \ No newline at end of file +#endif // MAIN_H \ No newline at end of file diff --git a/examples/Hdiv-mixed/problems/darcy2d.c b/examples/Hdiv-mixed/problems/darcy2d.c index 16d61dbb62..4864993661 100644 --- a/examples/Hdiv-mixed/problems/darcy2d.c +++ b/examples/Hdiv-mixed/problems/darcy2d.c @@ -18,56 +18,55 @@ /// Utility functions for setting up Darcy problem in 2D #include "../include/register-problem.h" -#include "../qfunctions/darcy-true2d.h" -#include "../qfunctions/darcy-system2d.h" #include "../qfunctions/darcy-error2d.h" -#include "../qfunctions/post-processing2d.h" -#include "../qfunctions/darcy-true-quartic2d.h" #include "../qfunctions/darcy-system-quartic2d.h" +#include "../qfunctions/darcy-system2d.h" +#include "../qfunctions/darcy-true-quartic2d.h" +#include "../qfunctions/darcy-true2d.h" +#include "../qfunctions/post-processing2d.h" //#include "../qfunctions/pressure-boundary2d.h" -PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, DM dm, - void *ctx) { +PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; DARCYContext darcy_ctx; CeedQFunctionContext darcy_context; PetscFunctionBeginUser; - PetscCall( PetscCalloc1(1, &darcy_ctx) ); + PetscCall(PetscCalloc1(1, &darcy_ctx)); // ------------------------------------------------------ // SET UP POISSON_QUAD2D // ------------------------------------------------------ - problem_data->dim = 2; - problem_data->elem_node = 4; - problem_data->q_data_size_face = 3; - problem_data->quadrature_mode = CEED_GAUSS; - problem_data->true_solution = DarcyTrue2D; - problem_data->true_solution_loc = DarcyTrue2D_loc; - problem_data->residual = DarcySystem2D; - problem_data->residual_loc = DarcySystem2D_loc; - problem_data->jacobian = JacobianDarcySystem2D; - problem_data->jacobian_loc = JacobianDarcySystem2D_loc; - problem_data->error = DarcyError2D; - problem_data->error_loc = DarcyError2D_loc; - //problem_data->bc_pressure = BCPressure2D; - //problem_data->bc_pressure_loc = BCPressure2D_loc; - problem_data->post_rhs = PostProcessingRhs2D; - problem_data->post_rhs_loc = PostProcessingRhs2D_loc; - problem_data->post_mass = PostProcessingMass2D; - problem_data->post_mass_loc = PostProcessingMass2D_loc; - problem_data->has_ts = PETSC_FALSE; - problem_data->view_solution = app_ctx->view_solution; - problem_data->quartic = app_ctx->quartic; + problem_data->dim = 2; + problem_data->elem_node = 4; + problem_data->q_data_size_face = 3; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->true_solution = DarcyTrue2D; + problem_data->true_solution_loc = DarcyTrue2D_loc; + problem_data->residual = DarcySystem2D; + problem_data->residual_loc = DarcySystem2D_loc; + problem_data->jacobian = JacobianDarcySystem2D; + problem_data->jacobian_loc = JacobianDarcySystem2D_loc; + problem_data->error = DarcyError2D; + problem_data->error_loc = DarcyError2D_loc; + // problem_data->bc_pressure = BCPressure2D; + // problem_data->bc_pressure_loc = BCPressure2D_loc; + problem_data->post_rhs = PostProcessingRhs2D; + problem_data->post_rhs_loc = PostProcessingRhs2D_loc; + problem_data->post_mass = PostProcessingMass2D; + problem_data->post_mass_loc = PostProcessingMass2D_loc; + problem_data->has_ts = PETSC_FALSE; + problem_data->view_solution = app_ctx->view_solution; + problem_data->quartic = app_ctx->quartic; if (app_ctx->quartic) { - problem_data->true_solution = DarcyTrueQuartic2D; - problem_data->true_solution_loc = DarcyTrueQuartic2D_loc; - problem_data->residual = DarcySystemQuartic2D; - problem_data->residual_loc = DarcySystemQuartic2D_loc; - problem_data->jacobian = JacobianDarcySystemQuartic2D; - problem_data->jacobian_loc = JacobianDarcySystemQuartic2D_loc; + problem_data->true_solution = DarcyTrueQuartic2D; + problem_data->true_solution_loc = DarcyTrueQuartic2D_loc; + problem_data->residual = DarcySystemQuartic2D; + problem_data->residual_loc = DarcySystemQuartic2D_loc; + problem_data->jacobian = JacobianDarcySystemQuartic2D; + problem_data->jacobian_loc = JacobianDarcySystemQuartic2D_loc; } // ------------------------------------------------------ @@ -75,44 +74,34 @@ PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, DM dm, // ------------------------------------------------------ CeedScalar kappa = 10., rho_a0 = 998.2, g = 9.8, alpha_a = 1., b_a = 10.; PetscOptionsBegin(app_ctx->comm, NULL, "Options for Hdiv-mixed problem", NULL); - PetscCall( PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, - kappa, &kappa, NULL)); - PetscCall( PetscOptionsScalar("-rho_a0", "Density at p0", NULL, - rho_a0, &rho_a0, NULL)); - PetscCall( PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", - NULL, - alpha_a, &alpha_a, NULL)); - PetscCall( PetscOptionsScalar("-b_a", "Parameter for relative permeability", - NULL, - b_a, &b_a, NULL)); + PetscCall(PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, kappa, &kappa, NULL)); + PetscCall(PetscOptionsScalar("-rho_a0", "Density at p0", NULL, rho_a0, &rho_a0, NULL)); + PetscCall(PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", NULL, alpha_a, &alpha_a, NULL)); + PetscCall(PetscOptionsScalar("-b_a", "Parameter for relative permeability", NULL, b_a, &b_a, NULL)); PetscOptionsEnd(); PetscReal domain_min[2], domain_max[2], domain_size[2]; - PetscCall( DMGetBoundingBox(dm, domain_min, domain_max) ); - for (PetscInt i=0; i<2; i++) domain_size[i] = domain_max[i] - domain_min[i]; + PetscCall(DMGetBoundingBox(dm, domain_min, domain_max)); + for (PetscInt i = 0; i < 2; i++) domain_size[i] = domain_max[i] - domain_min[i]; - darcy_ctx->kappa = kappa; - darcy_ctx->rho_a0 = rho_a0; - darcy_ctx->g = g; + darcy_ctx->kappa = kappa; + darcy_ctx->rho_a0 = rho_a0; + darcy_ctx->g = g; darcy_ctx->alpha_a = alpha_a; - darcy_ctx->b_a = b_a; - darcy_ctx->lx = domain_size[0]; - darcy_ctx->ly = domain_size[1]; + darcy_ctx->b_a = b_a; + darcy_ctx->lx = domain_size[0]; + darcy_ctx->ly = domain_size[1]; CeedQFunctionContextCreate(ceed, &darcy_context); - CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, - sizeof(*darcy_ctx), darcy_ctx); - //CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, - // FreeContextPetsc); + CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, sizeof(*darcy_ctx), darcy_ctx); + // CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, + // FreeContextPetsc); problem_data->true_qfunction_ctx = darcy_context; - CeedQFunctionContextReferenceCopy(darcy_context, - &problem_data->residual_qfunction_ctx); - CeedQFunctionContextReferenceCopy(darcy_context, - &problem_data->jacobian_qfunction_ctx); - CeedQFunctionContextReferenceCopy(darcy_context, - &problem_data->error_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, &problem_data->residual_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, &problem_data->jacobian_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, &problem_data->error_qfunction_ctx); - PetscCall( PetscFree(darcy_ctx) ); + PetscCall(PetscFree(darcy_ctx)); PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mixed/problems/darcy3d.c b/examples/Hdiv-mixed/problems/darcy3d.c index 0003bb010e..4cdbc75a71 100644 --- a/examples/Hdiv-mixed/problems/darcy3d.c +++ b/examples/Hdiv-mixed/problems/darcy3d.c @@ -18,89 +18,78 @@ /// Utility functions for setting up Darcy problem in 3D #include "../include/register-problem.h" -#include "../qfunctions/darcy-true3d.h" -#include "../qfunctions/darcy-system3d.h" #include "../qfunctions/darcy-error3d.h" +#include "../qfunctions/darcy-system3d.h" +#include "../qfunctions/darcy-true3d.h" #include "../qfunctions/post-processing3d.h" //#include "../qfunctions/pressure-boundary3d.h" -PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, DM dm, - void *ctx) { - AppCtx app_ctx = *(AppCtx *)ctx; +PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx) { + AppCtx app_ctx = *(AppCtx *)ctx; DARCYContext darcy_ctx; CeedQFunctionContext darcy_context; PetscFunctionBeginUser; - PetscCall( PetscCalloc1(1, &darcy_ctx) ); + PetscCall(PetscCalloc1(1, &darcy_ctx)); // ------------------------------------------------------ // SET UP POISSON_QUAD2D // ------------------------------------------------------ - problem_data->dim = 3; - problem_data->elem_node = 8; - problem_data->q_data_size_face = 4; - problem_data->quadrature_mode = CEED_GAUSS; - problem_data->true_solution = DarcyTrue3D; - problem_data->true_solution_loc = DarcyTrue3D_loc; - problem_data->residual = DarcySystem3D; - problem_data->residual_loc = DarcySystem3D_loc; - problem_data->jacobian = JacobianDarcySystem3D; - problem_data->jacobian_loc = JacobianDarcySystem3D_loc; - problem_data->error = DarcyError3D; - problem_data->error_loc = DarcyError3D_loc; - //problem_data->bc_pressure = BCPressure3D; - //problem_data->bc_pressure_loc = BCPressure3D_loc; - problem_data->post_rhs = PostProcessingRhs3D; - problem_data->post_rhs_loc = PostProcessingRhs3D_loc; - problem_data->post_mass = PostProcessingMass3D; - problem_data->post_mass_loc = PostProcessingMass3D_loc; - problem_data->has_ts = PETSC_FALSE; - problem_data->view_solution = app_ctx->view_solution; + problem_data->dim = 3; + problem_data->elem_node = 8; + problem_data->q_data_size_face = 4; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->true_solution = DarcyTrue3D; + problem_data->true_solution_loc = DarcyTrue3D_loc; + problem_data->residual = DarcySystem3D; + problem_data->residual_loc = DarcySystem3D_loc; + problem_data->jacobian = JacobianDarcySystem3D; + problem_data->jacobian_loc = JacobianDarcySystem3D_loc; + problem_data->error = DarcyError3D; + problem_data->error_loc = DarcyError3D_loc; + // problem_data->bc_pressure = BCPressure3D; + // problem_data->bc_pressure_loc = BCPressure3D_loc; + problem_data->post_rhs = PostProcessingRhs3D; + problem_data->post_rhs_loc = PostProcessingRhs3D_loc; + problem_data->post_mass = PostProcessingMass3D; + problem_data->post_mass_loc = PostProcessingMass3D_loc; + problem_data->has_ts = PETSC_FALSE; + problem_data->view_solution = app_ctx->view_solution; // ------------------------------------------------------ // Command line Options // ------------------------------------------------------ CeedScalar kappa = 1., rho_a0 = 998.2, g = 9.8, alpha_a = 1., b_a = 10.; PetscOptionsBegin(app_ctx->comm, NULL, "Options for Hdiv-mixed problem", NULL); - PetscCall( PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, - kappa, &kappa, NULL)); - PetscCall( PetscOptionsScalar("-rho_a0", "Density at p0", NULL, - rho_a0, &rho_a0, NULL)); - PetscCall( PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", - NULL, - alpha_a, &alpha_a, NULL)); - PetscCall( PetscOptionsScalar("-b_a", "Parameter for relative permeability", - NULL, - b_a, &b_a, NULL)); + PetscCall(PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, kappa, &kappa, NULL)); + PetscCall(PetscOptionsScalar("-rho_a0", "Density at p0", NULL, rho_a0, &rho_a0, NULL)); + PetscCall(PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", NULL, alpha_a, &alpha_a, NULL)); + PetscCall(PetscOptionsScalar("-b_a", "Parameter for relative permeability", NULL, b_a, &b_a, NULL)); PetscOptionsEnd(); PetscReal domain_min[3], domain_max[3], domain_size[3]; - PetscCall( DMGetBoundingBox(dm, domain_min, domain_max) ); - for (PetscInt i=0; i<3; i++) domain_size[i] = domain_max[i] - domain_min[i]; + PetscCall(DMGetBoundingBox(dm, domain_min, domain_max)); + for (PetscInt i = 0; i < 3; i++) domain_size[i] = domain_max[i] - domain_min[i]; - darcy_ctx->kappa = kappa; - darcy_ctx->rho_a0 = rho_a0; - darcy_ctx->g = g; + darcy_ctx->kappa = kappa; + darcy_ctx->rho_a0 = rho_a0; + darcy_ctx->g = g; darcy_ctx->alpha_a = alpha_a; - darcy_ctx->b_a = b_a; - darcy_ctx->lx = domain_size[0]; - darcy_ctx->ly = domain_size[1]; - darcy_ctx->lz = domain_size[2]; + darcy_ctx->b_a = b_a; + darcy_ctx->lx = domain_size[0]; + darcy_ctx->ly = domain_size[1]; + darcy_ctx->lz = domain_size[2]; CeedQFunctionContextCreate(ceed, &darcy_context); - CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, - sizeof(*darcy_ctx), darcy_ctx); - //CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, - // FreeContextPetsc); + CeedQFunctionContextSetData(darcy_context, CEED_MEM_HOST, CEED_COPY_VALUES, sizeof(*darcy_ctx), darcy_ctx); + // CeedQFunctionContextSetDataDestroy(darcy_context, CEED_MEM_HOST, + // FreeContextPetsc); problem_data->true_qfunction_ctx = darcy_context; - CeedQFunctionContextReferenceCopy(darcy_context, - &problem_data->residual_qfunction_ctx); - CeedQFunctionContextReferenceCopy(darcy_context, - &problem_data->jacobian_qfunction_ctx); - CeedQFunctionContextReferenceCopy(darcy_context, - &problem_data->error_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, &problem_data->residual_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, &problem_data->jacobian_qfunction_ctx); + CeedQFunctionContextReferenceCopy(darcy_context, &problem_data->error_qfunction_ctx); - PetscCall( PetscFree(darcy_ctx) ); + PetscCall(PetscFree(darcy_ctx)); PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mixed/problems/register-problem.c b/examples/Hdiv-mixed/problems/register-problem.c index 5e4ffe7b43..994e4f4605 100644 --- a/examples/Hdiv-mixed/problems/register-problem.c +++ b/examples/Hdiv-mixed/problems/register-problem.c @@ -24,25 +24,19 @@ PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx) { app_ctx->problems = NULL; PetscFunctionBeginUser; // 1) darcy2d (Hdiv_DARCY2D is created in darcy2d.c) - PetscCall( PetscFunctionListAdd(&app_ctx->problems, "darcy2d", - Hdiv_DARCY2D) ); + PetscCall(PetscFunctionListAdd(&app_ctx->problems, "darcy2d", Hdiv_DARCY2D)); // 2) darcy3d (Hdiv_DARCY3D is created in dacry3d.c) - PetscCall( PetscFunctionListAdd(&app_ctx->problems, "darcy3d", - Hdiv_DARCY3D) ); + PetscCall(PetscFunctionListAdd(&app_ctx->problems, "darcy3d", Hdiv_DARCY3D)); // 3) darcy3d-prism // 4) richard - PetscCall( PetscFunctionListAdd(&app_ctx->problems, "richard2d", - Hdiv_RICHARD2D) ); - PetscCall( PetscFunctionListAdd(&app_ctx->problems, "richard3d", - Hdiv_RICHARD3D) ); + PetscCall(PetscFunctionListAdd(&app_ctx->problems, "richard2d", Hdiv_RICHARD2D)); + PetscCall(PetscFunctionListAdd(&app_ctx->problems, "richard3d", Hdiv_RICHARD3D)); PetscFunctionReturn(0); } - // Free a plain data context that was allocated using PETSc; returning libCEED error codes int FreeContextPetsc(void *data) { - if (PetscFree(data)) return CeedError(NULL, CEED_ERROR_ACCESS, - "PetscFree failed"); + if (PetscFree(data)) return CeedError(NULL, CEED_ERROR_ACCESS, "PetscFree failed"); return CEED_ERROR_SUCCESS; } \ No newline at end of file diff --git a/examples/Hdiv-mixed/problems/richard2d.c b/examples/Hdiv-mixed/problems/richard2d.c index aa9bbd2c6d..a2b4cad337 100644 --- a/examples/Hdiv-mixed/problems/richard2d.c +++ b/examples/Hdiv-mixed/problems/richard2d.c @@ -18,116 +18,99 @@ /// Utility functions for setting up Richard problem in 2D #include "../include/register-problem.h" -#include "../qfunctions/richard-system2d.h" -#include "../qfunctions/richard-true2d.h" -#include "../qfunctions/richard-ics2d.h" #include "../qfunctions/darcy-error2d.h" #include "../qfunctions/post-processing2d.h" +#include "../qfunctions/richard-ics2d.h" +#include "../qfunctions/richard-system2d.h" +#include "../qfunctions/richard-true2d.h" //#include "../qfunctions/pressure-boundary2d.h" #include "petscsystypes.h" -PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, DM dm, - void *ctx) { +PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; RICHARDContext richard_ctx; CeedQFunctionContext richard_context; PetscFunctionBeginUser; - PetscCall( PetscCalloc1(1, &richard_ctx) ); + PetscCall(PetscCalloc1(1, &richard_ctx)); // ------------------------------------------------------ // SET UP POISSON_QUAD2D // ------------------------------------------------------ - problem_data->dim = 2; - problem_data->elem_node = 4; - problem_data->q_data_size_face = 3; - problem_data->quadrature_mode = CEED_GAUSS; - problem_data->true_solution = RichardTrue2D; - problem_data->true_solution_loc = RichardTrue2D_loc; - problem_data->rhs_u0 = RichardRhsU02D; - problem_data->rhs_u0_loc = RichardRhsU02D_loc; - problem_data->ics_u = RichardICsU2D; - problem_data->ics_u_loc = RichardICsU2D_loc; - problem_data->rhs_p0 = RichardRhsP02D; - problem_data->rhs_p0_loc = RichardRhsP02D_loc; - problem_data->ics_p = RichardICsP2D; - problem_data->ics_p_loc = RichardICsP2D_loc; - problem_data->residual = RichardSystem2D; - problem_data->residual_loc = RichardSystem2D_loc; - //problem_data->jacobian = JacobianRichardSystem2D; - //problem_data->jacobian_loc = JacobianRichardSystem2D_loc; - problem_data->error = DarcyError2D; - problem_data->error_loc = DarcyError2D_loc; - //problem_data->bc_pressure = BCPressure2D; - //problem_data->bc_pressure_loc = BCPressure2D_loc; - problem_data->post_rhs = PostProcessingRhs2D; - problem_data->post_rhs_loc = PostProcessingRhs2D_loc; - problem_data->post_mass = PostProcessingMass2D; - problem_data->post_mass_loc = PostProcessingMass2D_loc; - problem_data->has_ts = PETSC_TRUE; - problem_data->view_solution = app_ctx->view_solution; + problem_data->dim = 2; + problem_data->elem_node = 4; + problem_data->q_data_size_face = 3; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->true_solution = RichardTrue2D; + problem_data->true_solution_loc = RichardTrue2D_loc; + problem_data->rhs_u0 = RichardRhsU02D; + problem_data->rhs_u0_loc = RichardRhsU02D_loc; + problem_data->ics_u = RichardICsU2D; + problem_data->ics_u_loc = RichardICsU2D_loc; + problem_data->rhs_p0 = RichardRhsP02D; + problem_data->rhs_p0_loc = RichardRhsP02D_loc; + problem_data->ics_p = RichardICsP2D; + problem_data->ics_p_loc = RichardICsP2D_loc; + problem_data->residual = RichardSystem2D; + problem_data->residual_loc = RichardSystem2D_loc; + // problem_data->jacobian = JacobianRichardSystem2D; + // problem_data->jacobian_loc = JacobianRichardSystem2D_loc; + problem_data->error = DarcyError2D; + problem_data->error_loc = DarcyError2D_loc; + // problem_data->bc_pressure = BCPressure2D; + // problem_data->bc_pressure_loc = BCPressure2D_loc; + problem_data->post_rhs = PostProcessingRhs2D; + problem_data->post_rhs_loc = PostProcessingRhs2D_loc; + problem_data->post_mass = PostProcessingMass2D; + problem_data->post_mass_loc = PostProcessingMass2D_loc; + problem_data->has_ts = PETSC_TRUE; + problem_data->view_solution = app_ctx->view_solution; // ------------------------------------------------------ // Command line Options // ------------------------------------------------------ - CeedScalar kappa = 10., alpha_a = 1., b_a = 10., rho_a0 = 998.2, - beta = 0., g = 9.8, p0 = 101325; + CeedScalar kappa = 10., alpha_a = 1., b_a = 10., rho_a0 = 998.2, beta = 0., g = 9.8, p0 = 101325; PetscOptionsBegin(app_ctx->comm, NULL, "Options for Hdiv-mixed problem", NULL); - PetscCall( PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, - kappa, &kappa, NULL)); - PetscCall( PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", - NULL, - alpha_a, &alpha_a, NULL)); - PetscCall( PetscOptionsScalar("-b_a", "Parameter for relative permeability", - NULL, - b_a, &b_a, NULL)); - PetscCall( PetscOptionsScalar("-rho_a0", "Density at p0", NULL, - rho_a0, &rho_a0, NULL)); - PetscCall( PetscOptionsScalar("-beta", "Water compressibility", NULL, - beta, &beta, NULL)); + PetscCall(PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, kappa, &kappa, NULL)); + PetscCall(PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", NULL, alpha_a, &alpha_a, NULL)); + PetscCall(PetscOptionsScalar("-b_a", "Parameter for relative permeability", NULL, b_a, &b_a, NULL)); + PetscCall(PetscOptionsScalar("-rho_a0", "Density at p0", NULL, rho_a0, &rho_a0, NULL)); + PetscCall(PetscOptionsScalar("-beta", "Water compressibility", NULL, beta, &beta, NULL)); app_ctx->t_final = 0.5; - PetscCall( PetscOptionsScalar("-t_final", "End time", NULL, - app_ctx->t_final, &app_ctx->t_final, NULL)); + PetscCall(PetscOptionsScalar("-t_final", "End time", NULL, app_ctx->t_final, &app_ctx->t_final, NULL)); PetscOptionsEnd(); PetscReal domain_min[2], domain_max[2], domain_size[2]; - PetscCall( DMGetBoundingBox(dm, domain_min, domain_max) ); - for (PetscInt i=0; i<2; i++) domain_size[i] = domain_max[i] - domain_min[i]; + PetscCall(DMGetBoundingBox(dm, domain_min, domain_max)); + for (PetscInt i = 0; i < 2; i++) domain_size[i] = domain_max[i] - domain_min[i]; - richard_ctx->kappa = kappa; + richard_ctx->kappa = kappa; richard_ctx->alpha_a = alpha_a; - richard_ctx->b_a = b_a; - richard_ctx->rho_a0 = rho_a0; - richard_ctx->beta = beta; - richard_ctx->g = g; - richard_ctx->p0 = p0; - richard_ctx->gamma = 5.; - richard_ctx->t = 0.; + richard_ctx->b_a = b_a; + richard_ctx->rho_a0 = rho_a0; + richard_ctx->beta = beta; + richard_ctx->g = g; + richard_ctx->p0 = p0; + richard_ctx->gamma = 5.; + richard_ctx->t = 0.; richard_ctx->t_final = app_ctx->t_final; - richard_ctx->lx = domain_size[0]; - richard_ctx->ly = domain_size[1]; + richard_ctx->lx = domain_size[0]; + richard_ctx->ly = domain_size[1]; CeedQFunctionContextCreate(ceed, &richard_context); - CeedQFunctionContextSetData(richard_context, CEED_MEM_HOST, CEED_COPY_VALUES, - sizeof(*richard_ctx), richard_ctx); - //CeedQFunctionContextSetDataDestroy(richard_context, CEED_MEM_HOST, - // FreeContextPetsc); - CeedQFunctionContextRegisterDouble(richard_context, "time", - offsetof(struct RICHARDContext_, t), 1, "current solver time"); - CeedQFunctionContextRegisterDouble(richard_context, "final_time", - offsetof(struct RICHARDContext_, t_final), 1, "final time"); - CeedQFunctionContextRegisterDouble(richard_context, "time_step", - offsetof(struct RICHARDContext_, dt), 1, "time step"); + CeedQFunctionContextSetData(richard_context, CEED_MEM_HOST, CEED_COPY_VALUES, sizeof(*richard_ctx), richard_ctx); + // CeedQFunctionContextSetDataDestroy(richard_context, CEED_MEM_HOST, + // FreeContextPetsc); + CeedQFunctionContextRegisterDouble(richard_context, "time", offsetof(struct RICHARDContext_, t), 1, "current solver time"); + CeedQFunctionContextRegisterDouble(richard_context, "final_time", offsetof(struct RICHARDContext_, t_final), 1, "final time"); + CeedQFunctionContextRegisterDouble(richard_context, "time_step", offsetof(struct RICHARDContext_, dt), 1, "time step"); problem_data->true_qfunction_ctx = richard_context; - CeedQFunctionContextReferenceCopy(richard_context, - &problem_data->rhs_u0_qfunction_ctx); - CeedQFunctionContextReferenceCopy(richard_context, - &problem_data->residual_qfunction_ctx); - CeedQFunctionContextReferenceCopy(richard_context, - &problem_data->error_qfunction_ctx); - PetscCall( PetscFree(richard_ctx) ); + CeedQFunctionContextReferenceCopy(richard_context, &problem_data->rhs_u0_qfunction_ctx); + CeedQFunctionContextReferenceCopy(richard_context, &problem_data->residual_qfunction_ctx); + CeedQFunctionContextReferenceCopy(richard_context, &problem_data->error_qfunction_ctx); + PetscCall(PetscFree(richard_ctx)); PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mixed/problems/richard3d.c b/examples/Hdiv-mixed/problems/richard3d.c index 9559138526..92663aa2c3 100644 --- a/examples/Hdiv-mixed/problems/richard3d.c +++ b/examples/Hdiv-mixed/problems/richard3d.c @@ -18,117 +18,100 @@ /// Utility functions for setting up Richard problem in 3D #include "../include/register-problem.h" -#include "../qfunctions/richard-system3d.h" -#include "../qfunctions/richard-true3d.h" -#include "../qfunctions/richard-ics3d.h" #include "../qfunctions/darcy-error3d.h" #include "../qfunctions/post-processing3d.h" +#include "../qfunctions/richard-ics3d.h" +#include "../qfunctions/richard-system3d.h" +#include "../qfunctions/richard-true3d.h" //#include "../qfunctions/pressure-boundary2d.h" #include "petscsystypes.h" -PetscErrorCode Hdiv_RICHARD3D(Ceed ceed, ProblemData problem_data, DM dm, - void *ctx) { +PetscErrorCode Hdiv_RICHARD3D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; RICHARDContext richard_ctx; CeedQFunctionContext richard_context; PetscFunctionBeginUser; - PetscCall( PetscCalloc1(1, &richard_ctx) ); + PetscCall(PetscCalloc1(1, &richard_ctx)); // ------------------------------------------------------ // SET UP POISSON_QUAD2D // ------------------------------------------------------ - problem_data->dim = 3; - problem_data->elem_node = 8; - problem_data->q_data_size_face = 4; - problem_data->quadrature_mode = CEED_GAUSS; - problem_data->true_solution = RichardTrue3D; - problem_data->true_solution_loc = RichardTrue3D_loc; - problem_data->rhs_u0 = RichardRhsU03D; - problem_data->rhs_u0_loc = RichardRhsU03D_loc; - problem_data->ics_u = RichardICsU3D; - problem_data->ics_u_loc = RichardICsU3D_loc; - problem_data->rhs_p0 = RichardRhsP03D; - problem_data->rhs_p0_loc = RichardRhsP03D_loc; - problem_data->ics_p = RichardICsP3D; - problem_data->ics_p_loc = RichardICsP3D_loc; - problem_data->residual = RichardSystem3D; - problem_data->residual_loc = RichardSystem3D_loc; - //problem_data->jacobian = JacobianRichardSystem2D; - //problem_data->jacobian_loc = JacobianRichardSystem2D_loc; - problem_data->error = DarcyError3D; - problem_data->error_loc = DarcyError3D_loc; - //problem_data->bc_pressure = BCPressure2D; - //problem_data->bc_pressure_loc = BCPressure2D_loc; - problem_data->post_rhs = PostProcessingRhs3D; - problem_data->post_rhs_loc = PostProcessingRhs3D_loc; - problem_data->post_mass = PostProcessingMass3D; - problem_data->post_mass_loc = PostProcessingMass3D_loc; - problem_data->has_ts = PETSC_TRUE; - problem_data->view_solution = app_ctx->view_solution; + problem_data->dim = 3; + problem_data->elem_node = 8; + problem_data->q_data_size_face = 4; + problem_data->quadrature_mode = CEED_GAUSS; + problem_data->true_solution = RichardTrue3D; + problem_data->true_solution_loc = RichardTrue3D_loc; + problem_data->rhs_u0 = RichardRhsU03D; + problem_data->rhs_u0_loc = RichardRhsU03D_loc; + problem_data->ics_u = RichardICsU3D; + problem_data->ics_u_loc = RichardICsU3D_loc; + problem_data->rhs_p0 = RichardRhsP03D; + problem_data->rhs_p0_loc = RichardRhsP03D_loc; + problem_data->ics_p = RichardICsP3D; + problem_data->ics_p_loc = RichardICsP3D_loc; + problem_data->residual = RichardSystem3D; + problem_data->residual_loc = RichardSystem3D_loc; + // problem_data->jacobian = JacobianRichardSystem2D; + // problem_data->jacobian_loc = JacobianRichardSystem2D_loc; + problem_data->error = DarcyError3D; + problem_data->error_loc = DarcyError3D_loc; + // problem_data->bc_pressure = BCPressure2D; + // problem_data->bc_pressure_loc = BCPressure2D_loc; + problem_data->post_rhs = PostProcessingRhs3D; + problem_data->post_rhs_loc = PostProcessingRhs3D_loc; + problem_data->post_mass = PostProcessingMass3D; + problem_data->post_mass_loc = PostProcessingMass3D_loc; + problem_data->has_ts = PETSC_TRUE; + problem_data->view_solution = app_ctx->view_solution; // ------------------------------------------------------ // Command line Options // ------------------------------------------------------ - CeedScalar kappa = 10., alpha_a = 1., b_a = 10., rho_a0 = 998.2, - beta = 0., g = 9.8, p0 = 101325; + CeedScalar kappa = 10., alpha_a = 1., b_a = 10., rho_a0 = 998.2, beta = 0., g = 9.8, p0 = 101325; PetscOptionsBegin(app_ctx->comm, NULL, "Options for Hdiv-mixed problem", NULL); - PetscCall( PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, - kappa, &kappa, NULL)); - PetscCall( PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", - NULL, - alpha_a, &alpha_a, NULL)); - PetscCall( PetscOptionsScalar("-b_a", "Parameter for relative permeability", - NULL, - b_a, &b_a, NULL)); - PetscCall( PetscOptionsScalar("-rho_a0", "Density at p0", NULL, - rho_a0, &rho_a0, NULL)); - PetscCall( PetscOptionsScalar("-beta", "Water compressibility", NULL, - beta, &beta, NULL)); + PetscCall(PetscOptionsScalar("-kappa", "Hydraulic Conductivity", NULL, kappa, &kappa, NULL)); + PetscCall(PetscOptionsScalar("-alpha_a", "Parameter for relative permeability", NULL, alpha_a, &alpha_a, NULL)); + PetscCall(PetscOptionsScalar("-b_a", "Parameter for relative permeability", NULL, b_a, &b_a, NULL)); + PetscCall(PetscOptionsScalar("-rho_a0", "Density at p0", NULL, rho_a0, &rho_a0, NULL)); + PetscCall(PetscOptionsScalar("-beta", "Water compressibility", NULL, beta, &beta, NULL)); app_ctx->t_final = 0.5; - PetscCall( PetscOptionsScalar("-t_final", "End time", NULL, - app_ctx->t_final, &app_ctx->t_final, NULL)); + PetscCall(PetscOptionsScalar("-t_final", "End time", NULL, app_ctx->t_final, &app_ctx->t_final, NULL)); PetscOptionsEnd(); PetscReal domain_min[3], domain_max[3], domain_size[3]; - PetscCall( DMGetBoundingBox(dm, domain_min, domain_max) ); - for (PetscInt i=0; i<3; i++) domain_size[i] = domain_max[i] - domain_min[i]; + PetscCall(DMGetBoundingBox(dm, domain_min, domain_max)); + for (PetscInt i = 0; i < 3; i++) domain_size[i] = domain_max[i] - domain_min[i]; - richard_ctx->kappa = kappa; + richard_ctx->kappa = kappa; richard_ctx->alpha_a = alpha_a; - richard_ctx->b_a = b_a; - richard_ctx->rho_a0 = rho_a0; - richard_ctx->beta = beta; - richard_ctx->g = g; - richard_ctx->p0 = p0; - richard_ctx->gamma = 5.; - richard_ctx->t = 0.; + richard_ctx->b_a = b_a; + richard_ctx->rho_a0 = rho_a0; + richard_ctx->beta = beta; + richard_ctx->g = g; + richard_ctx->p0 = p0; + richard_ctx->gamma = 5.; + richard_ctx->t = 0.; richard_ctx->t_final = app_ctx->t_final; - richard_ctx->lx = domain_size[0]; - richard_ctx->ly = domain_size[1]; - richard_ctx->lz = domain_size[2]; + richard_ctx->lx = domain_size[0]; + richard_ctx->ly = domain_size[1]; + richard_ctx->lz = domain_size[2]; CeedQFunctionContextCreate(ceed, &richard_context); - CeedQFunctionContextSetData(richard_context, CEED_MEM_HOST, CEED_COPY_VALUES, - sizeof(*richard_ctx), richard_ctx); - //CeedQFunctionContextSetDataDestroy(richard_context, CEED_MEM_HOST, - // FreeContextPetsc); - CeedQFunctionContextRegisterDouble(richard_context, "time", - offsetof(struct RICHARDContext_, t), 1, "current solver time"); - CeedQFunctionContextRegisterDouble(richard_context, "final_time", - offsetof(struct RICHARDContext_, t_final), 1, "final time"); - CeedQFunctionContextRegisterDouble(richard_context, "time_step", - offsetof(struct RICHARDContext_, dt), 1, "time step"); + CeedQFunctionContextSetData(richard_context, CEED_MEM_HOST, CEED_COPY_VALUES, sizeof(*richard_ctx), richard_ctx); + // CeedQFunctionContextSetDataDestroy(richard_context, CEED_MEM_HOST, + // FreeContextPetsc); + CeedQFunctionContextRegisterDouble(richard_context, "time", offsetof(struct RICHARDContext_, t), 1, "current solver time"); + CeedQFunctionContextRegisterDouble(richard_context, "final_time", offsetof(struct RICHARDContext_, t_final), 1, "final time"); + CeedQFunctionContextRegisterDouble(richard_context, "time_step", offsetof(struct RICHARDContext_, dt), 1, "time step"); problem_data->true_qfunction_ctx = richard_context; - CeedQFunctionContextReferenceCopy(richard_context, - &problem_data->rhs_u0_qfunction_ctx); - CeedQFunctionContextReferenceCopy(richard_context, - &problem_data->residual_qfunction_ctx); - CeedQFunctionContextReferenceCopy(richard_context, - &problem_data->error_qfunction_ctx); - PetscCall( PetscFree(richard_ctx) ); + CeedQFunctionContextReferenceCopy(richard_context, &problem_data->rhs_u0_qfunction_ctx); + CeedQFunctionContextReferenceCopy(richard_context, &problem_data->residual_qfunction_ctx); + CeedQFunctionContextReferenceCopy(richard_context, &problem_data->error_qfunction_ctx); + PetscCall(PetscFree(richard_ctx)); PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mixed/qfunctions/darcy-error2d.h b/examples/Hdiv-mixed/qfunctions/darcy-error2d.h index 62c574337b..8e5c6de83f 100644 --- a/examples/Hdiv-mixed/qfunctions/darcy-error2d.h +++ b/examples/Hdiv-mixed/qfunctions/darcy-error2d.h @@ -20,8 +20,9 @@ #ifndef DARCY_ERROR2D_H #define DARCY_ERROR2D_H -#include #include +#include + #include "utils.h" // ----------------------------------------------------------------------------- @@ -38,43 +39,38 @@ struct DARCYContext_ { CeedScalar lx, ly; }; #endif -CEED_QFUNCTION(DarcyError2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(DarcyError2D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], - (*p) = (const CeedScalar(*))in[3], - (*true_soln) = in[4]; + const CeedScalar(*w) = in[0], (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], (*p) = (const CeedScalar(*))in[3], (*true_soln) = in[4]; // Outputs - CeedScalar (*error) = out[0]; + CeedScalar(*error) = out[0]; // Context - DARCYContext context = (DARCYContext)ctx; - //const CeedScalar kappa = context->kappa; - const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar g = context->g; + DARCYContext context = (DARCYContext)ctx; + // const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include -#include "utils.h" +#include +#include "utils.h" // ----------------------------------------------------------------------------- // Compuet error @@ -39,45 +39,40 @@ struct DARCYContext_ { CeedScalar lx, ly, lz; }; #endif -CEED_QFUNCTION(DarcyError3D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(DarcyError3D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], - (*p) = (const CeedScalar(*))in[3], - (*true_soln) = in[4]; + const CeedScalar(*w) = in[0], (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], (*p) = (const CeedScalar(*))in[3], (*true_soln) = in[4]; // Outputs - CeedScalar (*error) = out[0]; + CeedScalar(*error) = out[0]; // Context - DARCYContext context = (DARCYContext)ctx; - //const CeedScalar kappa = context->kappa; - const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar g = context->g; + DARCYContext context = (DARCYContext)ctx; + // const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "utils.h" // ----------------------------------------------------------------------------- @@ -68,32 +69,23 @@ struct DARCYContext_ { // ----------------------------------------------------------------------------- // Residual evaluation for Darcy problem // ----------------------------------------------------------------------------- -CEED_QFUNCTION(DarcySystemQuartic2D)(void *ctx, CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(DarcySystemQuartic2D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], - (*div_u) = (const CeedScalar(*))in[3], - (*p) = (const CeedScalar(*))in[4], - (*f) = in[5]; + const CeedScalar(*w) = in[0], (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], (*div_u) = (const CeedScalar(*))in[3], (*p) = (const CeedScalar(*))in[4], + (*f) = in[5]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], - (*div_v) = (CeedScalar(*))out[1], - (*q) = (CeedScalar(*))out[2]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], (*div_v) = (CeedScalar(*))out[1], (*q) = (CeedScalar(*))out[2]; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "ceed/ceed-f64.h" #include "utils.h" @@ -75,56 +76,49 @@ struct DARCYContext_ { // ----------------------------------------------------------------------------- // Residual evaluation for Darcy problem // ----------------------------------------------------------------------------- -CEED_QFUNCTION(DarcySystem2D)(void *ctx, CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(DarcySystem2D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], - (*div_u) = (const CeedScalar(*))in[3], - (*p) = (const CeedScalar(*))in[4], - (*f) = in[5], - (*coords) = in[6]; + const CeedScalar(*w) = in[0], (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], (*div_u) = (const CeedScalar(*))in[3], (*p) = (const CeedScalar(*))in[4], + (*f) = in[5], (*coords) = in[6]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], - (*div_v) = (CeedScalar(*))out[1], - (*q) = (CeedScalar(*))out[2]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], (*div_v) = (CeedScalar(*))out[1], (*q) = (CeedScalar(*))out[2]; // Context - DARCYContext context = (DARCYContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar g = context->g; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; ikappa; - const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar g = context->g; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; + CeedScalar(*dv)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], (*div_dv) = (CeedScalar(*))out[1], (*dq) = (CeedScalar(*))out[2]; + + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "utils.h" // ----------------------------------------------------------------------------- @@ -74,59 +75,51 @@ struct DARCYContext_ { // ----------------------------------------------------------------------------- // Residual evaluation for Darcy problem // ----------------------------------------------------------------------------- -CEED_QFUNCTION(DarcySystem3D)(void *ctx, CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(DarcySystem3D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], - (*div_u) = (const CeedScalar(*))in[3], - (*p) = (const CeedScalar(*))in[4], - (*f) = in[5], - (*coords) = in[6]; + const CeedScalar(*w) = in[0], (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], (*div_u) = (const CeedScalar(*))in[3], (*p) = (const CeedScalar(*))in[4], + (*f) = in[5], (*coords) = in[6]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], - (*div_v) = (CeedScalar(*))out[1], - (*q) = (CeedScalar(*))out[2]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], (*div_v) = (CeedScalar(*))out[1], (*q) = (CeedScalar(*))out[2]; // Context - DARCYContext context = (DARCYContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar g = context->g; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; ikappa; - const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar g = context->g; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; - // *INDENT-ON* + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "utils.h" // ----------------------------------------------------------------------------- @@ -56,43 +57,39 @@ struct DARCYContext_ { CeedScalar lx, ly; }; #endif -CEED_QFUNCTION(DarcyTrueQuartic2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(DarcyTrueQuartic2D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*coords) = in[0]; + const CeedScalar(*coords) = in[0]; // Outputs - CeedScalar (*true_force) = out[0], (*true_solution) = out[1]; + CeedScalar(*true_force) = out[0], (*true_solution) = out[1]; // Context - DARCYContext context = (DARCYContext)ctx; - const CeedScalar lx = context->lx; - const CeedScalar ly = context->ly; + DARCYContext context = (DARCYContext)ctx; + const CeedScalar lx = context->lx; + const CeedScalar ly = context->ly; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "utils.h" // ----------------------------------------------------------------------------- @@ -59,52 +60,46 @@ struct DARCYContext_ { CeedScalar lx, ly; }; #endif -CEED_QFUNCTION(DarcyTrue2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(DarcyTrue2D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*coords) = in[0]; + const CeedScalar(*coords) = in[0]; // Outputs - CeedScalar (*true_force) = out[0], (*true_solution) = out[1]; + CeedScalar(*true_force) = out[0], (*true_solution) = out[1]; // Context - DARCYContext context = (DARCYContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; - const CeedScalar lx = context->lx; - const CeedScalar ly = context->ly; + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + const CeedScalar lx = context->lx; + const CeedScalar ly = context->ly; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "utils.h" // ----------------------------------------------------------------------------- @@ -59,59 +60,51 @@ struct DARCYContext_ { CeedScalar lx, ly, lz; }; #endif -CEED_QFUNCTION(DarcyTrue3D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(DarcyTrue3D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*coords) = in[0]; + const CeedScalar(*coords) = in[0]; // Outputs - CeedScalar (*true_force) = out[0], (*true_soln) = out[1]; + CeedScalar(*true_force) = out[0], (*true_soln) = out[1]; // Context - DARCYContext context = (DARCYContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; - const CeedScalar lx = context->lx; - const CeedScalar ly = context->ly; - const CeedScalar lz = context->lz; + DARCYContext context = (DARCYContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + const CeedScalar lx = context->lx; + const CeedScalar ly = context->ly; + const CeedScalar lz = context->lz; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "ceed/ceed-f64.h" #include "utils.h" @@ -29,69 +30,60 @@ // We solve (v, u) = (v, uh), to project Hdiv to L2 space // This QFunction create post_rhs = (v, uh) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(PostProcessingRhs2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(PostProcessingRhs2D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + const CeedScalar(*w) = in[0], (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; // Outputs - CeedScalar (*post_rhs) = out[0]; + CeedScalar(*post_rhs) = out[0]; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i rhs = J*u*w + // 2) rhs = (v, uh) = uh*w*det_J ==> rhs = J*u*w CeedScalar u1[2] = {u[0][i], u[1][i]}, rhs[2]; AlphaMatVecMult2x2(w[i], J, u1, rhs); - post_rhs[i+0*Q] = rhs[0]; - post_rhs[i+1*Q] = rhs[1]; - } // End of Quadrature Point Loop + post_rhs[i + 0 * Q] = rhs[0]; + post_rhs[i + 1 * Q] = rhs[1]; + } // End of Quadrature Point Loop return 0; } // ----------------------------------------------------------------------------- // We solve (v, u) = (v, uh), to project Hdiv to L2 space -// This QFunction create mass matrix (v, u), then we solve using ksp to have +// This QFunction create mass matrix (v, u), then we solve using ksp to have // projected uh in L2 space and use it for post-processing // ----------------------------------------------------------------------------- -CEED_QFUNCTION(PostProcessingMass2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(PostProcessingMass2D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + const CeedScalar(*w) = in[0], (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "ceed/ceed-f64.h" #include "utils.h" @@ -29,72 +30,63 @@ // We solve (v, u) = (v, uh), to project Hdiv to L2 space // This QFunction create post_rhs = (v, uh) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(PostProcessingRhs3D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(PostProcessingRhs3D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + const CeedScalar(*w) = in[0], (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; // Outputs - CeedScalar (*post_rhs) = out[0]; + CeedScalar(*post_rhs) = out[0]; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i rhs = J*u*w + // 2) rhs = (v, uh) = uh*w*det_J ==> rhs = J*u*w CeedScalar u1[3] = {u[0][i], u[1][i], u[2][i]}, rhs[3]; AlphaMatVecMult3x3(w[i], J, u1, rhs); - post_rhs[i+0*Q] = rhs[0]; - post_rhs[i+1*Q] = rhs[1]; - post_rhs[i+2*Q] = rhs[2]; - } // End of Quadrature Point Loop + post_rhs[i + 0 * Q] = rhs[0]; + post_rhs[i + 1 * Q] = rhs[1]; + post_rhs[i + 2 * Q] = rhs[2]; + } // End of Quadrature Point Loop return 0; } // ----------------------------------------------------------------------------- // We solve (v, u) = (v, uh), to project Hdiv to L2 space -// This QFunction create mass matrix (v, u), then we solve using ksp to have +// This QFunction create mass matrix (v, u), then we solve using ksp to have // projected uh in L2 space and use it for post-processing // ----------------------------------------------------------------------------- -CEED_QFUNCTION(PostProcessingMass3D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(PostProcessingMass3D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; + const CeedScalar(*w) = in[0], (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "utils.h" // ----------------------------------------------------------------------------- // Strong form: @@ -33,24 +34,21 @@ // Note that the Piola map of the H(div) basis and physical normal "n" got canceled // and we need to multiply by the reference normal "N" on each face // ----------------------------------------------------------------------------- -CEED_QFUNCTION(BCPressure2D)(void *ctx, CeedInt Q, - const CeedScalar *const *in, CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(BCPressure2D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0]; + const CeedScalar(*w) = in[0]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; - // *INDENT-ON* + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; // User context CeedPragmaSIMD - // Quadrature Point Loop - for (CeedInt i=0; i #include +#include + #include "utils.h" // ----------------------------------------------------------------------------- // Strong form: @@ -33,23 +34,21 @@ // Note that the Piola map of the H(div) basis and physical normal "n" got canceled // and we need to multiply by the reference normal "N" on each face // ----------------------------------------------------------------------------- -CEED_QFUNCTION(BCPressure3D)(void *ctx, CeedInt Q, - const CeedScalar *const *in, CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(BCPressure3D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0]; + const CeedScalar(*w) = in[0]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0]; // *INDENT-OFF* // User context CeedPragmaSIMD - // Quadrature Point Loop - for (CeedInt i=0; i #include +#include + #include "ceed/ceed-f64.h" #include "utils.h" @@ -76,76 +77,66 @@ struct RICHARDContext_ { // We solve (v, u) = (v, ue) at t=0, to project ue to Hdiv space // This QFunction create rhs_u0 = (v, ue) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(RichardRhsU02D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(RichardRhsU02D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*coords) = in[1], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[2]; + const CeedScalar(*w) = in[0], (*coords) = in[1], (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[2]; // Outputs - CeedScalar (*rhs_u0) = out[0]; + CeedScalar(*rhs_u0) = out[0]; // Context - RICHARDContext context = (RICHARDContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "ceed/ceed-f64.h" #include "utils.h" @@ -44,81 +45,70 @@ struct RICHARDContext_ { // We solve (v, u) = (v, ue) at t=0, to project ue to Hdiv space // This QFunction create rhs_u0 = (v, ue) // ----------------------------------------------------------------------------- -CEED_QFUNCTION(RichardRhsU03D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(RichardRhsU03D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*coords) = in[1], - (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[2]; + const CeedScalar(*w) = in[0], (*coords) = in[1], (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[2]; // Outputs - CeedScalar (*rhs_u0) = out[0]; + CeedScalar(*rhs_u0) = out[0]; // Context - RICHARDContext context = (RICHARDContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "ceed/ceed-f64.h" #include "utils.h" @@ -81,68 +82,59 @@ struct RICHARDContext_ { // ----------------------------------------------------------------------------- // Residual evaluation for Richard problem // ----------------------------------------------------------------------------- -CEED_QFUNCTION(RichardSystem2D)(void *ctx, CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(RichardSystem2D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], - (*div_u) = (const CeedScalar(*))in[3], - (*p) = (const CeedScalar(*))in[4], - (*f) = in[5], - (*coords) = in[6], - (*p_t) = (const CeedScalar(*))in[7]; + const CeedScalar(*w) = in[0], (*dxdX)[2][CEED_Q_VLA] = (const CeedScalar(*)[2][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], (*div_u) = (const CeedScalar(*))in[3], (*p) = (const CeedScalar(*))in[4], + (*f) = in[5], (*coords) = in[6], (*p_t) = (const CeedScalar(*))in[7]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], - (*div_v) = (CeedScalar(*))out[1], - (*q) = (CeedScalar(*))out[2]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], (*div_v) = (CeedScalar(*))out[1], (*q) = (CeedScalar(*))out[2]; // Context - RICHARDContext context = (RICHARDContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar g = context->g; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; - //const CeedScalar beta = context->beta; - //const CeedScalar p0 = context->p0; // atmospheric pressure - const CeedScalar gamma = context->gamma; - CeedScalar t = context->t; - //CeedScalar dt = context->dt; + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + // const CeedScalar beta = context->beta; + // const CeedScalar p0 = context->p0; // atmospheric pressure + const CeedScalar gamma = context->gamma; + CeedScalar t = context->t; + // CeedScalar dt = context->dt; - // *INDENT-ON* // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; ibeta; const CeedScalar g = context->g; const CeedScalar p0 = context->p0;// atmospheric pressure - // *INDENT-ON* // Quadrature Point Loop CeedPragmaSIMD for (CeedInt i=0; i #include +#include + #include "ceed/ceed-f64.h" #include "utils.h" @@ -81,70 +82,61 @@ struct RICHARDContext_ { // ----------------------------------------------------------------------------- // Residual evaluation for Richard problem // ----------------------------------------------------------------------------- -CEED_QFUNCTION(RichardSystem3D)(void *ctx, CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(RichardSystem3D)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*w) = in[0], - (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], - (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], - (*div_u) = (const CeedScalar(*))in[3], - (*p) = (const CeedScalar(*))in[4], - (*f) = in[5], - (*coords) = in[6], - (*p_t) = (const CeedScalar(*))in[7]; + const CeedScalar(*w) = in[0], (*dxdX)[3][CEED_Q_VLA] = (const CeedScalar(*)[3][CEED_Q_VLA])in[1], + (*u)[CEED_Q_VLA] = (const CeedScalar(*)[CEED_Q_VLA])in[2], (*div_u) = (const CeedScalar(*))in[3], (*p) = (const CeedScalar(*))in[4], + (*f) = in[5], (*coords) = in[6], (*p_t) = (const CeedScalar(*))in[7]; // Outputs - CeedScalar (*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], - (*div_v) = (CeedScalar(*))out[1], - (*q) = (CeedScalar(*))out[2]; + CeedScalar(*v)[CEED_Q_VLA] = (CeedScalar(*)[CEED_Q_VLA])out[0], (*div_v) = (CeedScalar(*))out[1], (*q) = (CeedScalar(*))out[2]; // Context - RICHARDContext context = (RICHARDContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar rho_a0 = context->rho_a0; - const CeedScalar g = context->g; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; - //const CeedScalar beta = context->beta; - //const CeedScalar p0 = context->p0; // atmospheric pressure - const CeedScalar gamma = context->gamma; - CeedScalar t = context->t; - //CeedScalar dt = context->dt; + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar rho_a0 = context->rho_a0; + const CeedScalar g = context->g; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + // const CeedScalar beta = context->beta; + // const CeedScalar p0 = context->p0; // atmospheric pressure + const CeedScalar gamma = context->gamma; + CeedScalar t = context->t; + // CeedScalar dt = context->dt; - // *INDENT-ON* // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; ibeta; const CeedScalar g = context->g; const CeedScalar p0 = context->p0;// atmospheric pressure - // *INDENT-ON* // Quadrature Point Loop CeedPragmaSIMD for (CeedInt i=0; i #include +#include + #include "utils.h" // See Matthew Farthing, Christopher Kees, Cass Miller (2003) @@ -74,58 +75,52 @@ struct RICHARDContext_ { // ----------------------------------------------------------------------------- // True solution for Richard problem // ----------------------------------------------------------------------------- -CEED_QFUNCTION(RichardTrue2D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(RichardTrue2D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*coords) = in[0]; + const CeedScalar(*coords) = in[0]; // Outputs - CeedScalar (*true_force) = out[0], (*true_solution) = out[1]; + CeedScalar(*true_force) = out[0], (*true_solution) = out[1]; // Context - RICHARDContext context = (RICHARDContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; - const CeedScalar gamma = context->gamma; - CeedScalar t_final = context->t_final; + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + const CeedScalar gamma = context->gamma; + CeedScalar t_final = context->t_final; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i #include +#include + #include "utils.h" // See Matthew Farthing, Christopher Kees, Cass Miller (2003) @@ -74,64 +75,57 @@ struct RICHARDContext_ { // ----------------------------------------------------------------------------- // True solution for Richard problem // ----------------------------------------------------------------------------- -CEED_QFUNCTION(RichardTrue3D)(void *ctx, const CeedInt Q, - const CeedScalar *const *in, - CeedScalar *const *out) { - // *INDENT-OFF* +CEED_QFUNCTION(RichardTrue3D)(void *ctx, const CeedInt Q, const CeedScalar *const *in, CeedScalar *const *out) { // Inputs - const CeedScalar (*coords) = in[0]; + const CeedScalar(*coords) = in[0]; // Outputs - CeedScalar (*true_force) = out[0], (*true_solution) = out[1]; + CeedScalar(*true_force) = out[0], (*true_solution) = out[1]; // Context - RICHARDContext context = (RICHARDContext)ctx; - const CeedScalar kappa = context->kappa; - const CeedScalar alpha_a = context->alpha_a; - const CeedScalar b_a = context->b_a; - const CeedScalar gamma = context->gamma; - CeedScalar t_final = context->t_final; + RICHARDContext context = (RICHARDContext)ctx; + const CeedScalar kappa = context->kappa; + const CeedScalar alpha_a = context->alpha_a; + const CeedScalar b_a = context->b_a; + const CeedScalar gamma = context->gamma; + CeedScalar t_final = context->t_final; // Quadrature Point Loop - CeedPragmaSIMD - for (CeedInt i=0; i +#include "ceed/ceed-f64.h" + #define PI_DOUBLE 3.14159265358979323846 // ----------------------------------------------------------------------------- // Compute alpha * A * B = C // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatMatMult3x3(const CeedScalar alpha, - const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { +CEED_QFUNCTION_HELPER int AlphaMatMatMult3x3(const CeedScalar alpha, const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { for (CeedInt j = 0; j < 3; j++) { for (CeedInt k = 0; k < 3; k++) { C[j][k] = 0; @@ -29,8 +29,8 @@ CEED_QFUNCTION_HELPER int AlphaMatMatMult3x3(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute alpha * A^T * B = C // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult3x3(const CeedScalar alpha, - const CeedScalar A[3][3], const CeedScalar B[3][3], CeedScalar C[3][3]) { +CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult3x3(const CeedScalar alpha, const CeedScalar A[3][3], const CeedScalar B[3][3], + CeedScalar C[3][3]) { for (CeedInt j = 0; j < 3; j++) { for (CeedInt k = 0; k < 3; k++) { C[j][k] = 0; @@ -48,26 +48,24 @@ CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult3x3(const CeedScalar alpha, // ----------------------------------------------------------------------------- CEED_QFUNCTION_HELPER CeedScalar MatDet3x3(const CeedScalar A[3][3]) { // Compute det(A) - const CeedScalar B11 = A[1][1]*A[2][2] - A[1][2]*A[2][1]; - const CeedScalar B12 = A[0][2]*A[2][1] - A[0][1]*A[2][2]; - const CeedScalar B13 = A[0][1]*A[1][2] - A[0][2]*A[1][1]; - return A[0][0]*B11 + A[1][0]*B12 + A[2][0]*B13; - + const CeedScalar B11 = A[1][1] * A[2][2] - A[1][2] * A[2][1]; + const CeedScalar B12 = A[0][2] * A[2][1] - A[0][1] * A[2][2]; + const CeedScalar B13 = A[0][1] * A[1][2] - A[0][2] * A[1][1]; + return A[0][0] * B11 + A[1][0] * B12 + A[2][0] * B13; }; // ----------------------------------------------------------------------------- // Compute inverse of 3x3 symmetric matrix // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int MatInverse3x3(const CeedScalar A[3][3], - const CeedScalar det_A, CeedScalar A_inv[3][3]) { +CEED_QFUNCTION_HELPER int MatInverse3x3(const CeedScalar A[3][3], const CeedScalar det_A, CeedScalar A_inv[3][3]) { // Compute A^(-1) : A-Inverse CeedScalar B[6] = { - A[1][1] * A[2][2] - A[1][2] * A[2][1], /* *NOPAD* */ - A[0][0] * A[2][2] - A[0][2] * A[2][0], /* *NOPAD* */ - A[0][0] * A[1][1] - A[0][1] * A[1][0], /* *NOPAD* */ - A[0][2] * A[1][0] - A[0][0] * A[1][2], /* *NOPAD* */ - A[0][1] * A[1][2] - A[0][2] * A[1][1], /* *NOPAD* */ - A[0][2] * A[2][1] - A[0][1] * A[2][2] /* *NOPAD* */ + A[1][1] * A[2][2] - A[1][2] * A[2][1], /* *NOPAD* */ + A[0][0] * A[2][2] - A[0][2] * A[2][0], /* *NOPAD* */ + A[0][0] * A[1][1] - A[0][1] * A[1][0], /* *NOPAD* */ + A[0][2] * A[1][0] - A[0][0] * A[1][2], /* *NOPAD* */ + A[0][1] * A[1][2] - A[0][2] * A[1][1], /* *NOPAD* */ + A[0][2] * A[2][1] - A[0][1] * A[2][2] /* *NOPAD* */ }; CeedScalar A_inv1[6]; for (CeedInt m = 0; m < 6; m++) { @@ -88,13 +86,11 @@ CEED_QFUNCTION_HELPER int MatInverse3x3(const CeedScalar A[3][3], // ----------------------------------------------------------------------------- // Compute matrix-vector product: alpha*A*u // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatVecMult3x3(const CeedScalar alpha, - const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { +CEED_QFUNCTION_HELPER int AlphaMatVecMult3x3(const CeedScalar alpha, const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { // Compute v = alpha*A*u for (CeedInt k = 0; k < 3; k++) { v[k] = 0; - for (CeedInt m = 0; m < 3; m++) - v[k] += A[k][m] * u[m] * alpha; + for (CeedInt m = 0; m < 3; m++) v[k] += A[k][m] * u[m] * alpha; } return 0; @@ -103,13 +99,11 @@ CEED_QFUNCTION_HELPER int AlphaMatVecMult3x3(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute matrix-vector product: alpha*A^T*u // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult3x3(const CeedScalar alpha, - const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { +CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult3x3(const CeedScalar alpha, const CeedScalar A[3][3], const CeedScalar u[3], CeedScalar v[3]) { // Compute v = alpha*A^T*u for (CeedInt k = 0; k < 3; k++) { v[k] = 0; - for (CeedInt m = 0; m < 3; m++) - v[k] += A[m][k] * u[m] * alpha; + for (CeedInt m = 0; m < 3; m++) v[k] += A[m][k] * u[m] * alpha; } return 0; @@ -118,8 +112,7 @@ CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult3x3(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute alpha * A * B = C // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatMatMult2x2(const CeedScalar alpha, - const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { +CEED_QFUNCTION_HELPER int AlphaMatMatMult2x2(const CeedScalar alpha, const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { for (CeedInt j = 0; j < 2; j++) { for (CeedInt k = 0; k < 2; k++) { C[j][k] = 0; @@ -135,8 +128,8 @@ CEED_QFUNCTION_HELPER int AlphaMatMatMult2x2(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute alpha * A^T * B = C // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult2x2(const CeedScalar alpha, - const CeedScalar A[2][2], const CeedScalar B[2][2], CeedScalar C[2][2]) { +CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult2x2(const CeedScalar alpha, const CeedScalar A[2][2], const CeedScalar B[2][2], + CeedScalar C[2][2]) { for (CeedInt j = 0; j < 2; j++) { for (CeedInt k = 0; k < 2; k++) { C[j][k] = 0; @@ -154,20 +147,18 @@ CEED_QFUNCTION_HELPER int AlphaMatTransposeMatMult2x2(const CeedScalar alpha, // ----------------------------------------------------------------------------- CEED_QFUNCTION_HELPER CeedScalar MatDet2x2(const CeedScalar A[2][2]) { // Compute det(A) - return A[0][0]*A[1][1] - A[1][0]*A[0][1]; - + return A[0][0] * A[1][1] - A[1][0] * A[0][1]; }; // ----------------------------------------------------------------------------- // Compute inverse of 2x2 symmetric matrix // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int MatInverse2x2(const CeedScalar A[2][2], - const CeedScalar det_A, CeedScalar A_inv[2][2]) { +CEED_QFUNCTION_HELPER int MatInverse2x2(const CeedScalar A[2][2], const CeedScalar det_A, CeedScalar A_inv[2][2]) { // Compute A^(-1) : A-Inverse - A_inv[0][0] = A[1][1]/ det_A; - A_inv[0][1] = -A[0][1]/ det_A; - A_inv[1][0] = -A[1][0]/ det_A; - A_inv[1][1] = A[0][0]/ det_A; + A_inv[0][0] = A[1][1] / det_A; + A_inv[0][1] = -A[0][1] / det_A; + A_inv[1][0] = -A[1][0] / det_A; + A_inv[1][1] = A[0][0] / det_A; return 0; }; @@ -175,13 +166,11 @@ CEED_QFUNCTION_HELPER int MatInverse2x2(const CeedScalar A[2][2], // ----------------------------------------------------------------------------- // Compute matrix-vector product: alpha*A*u // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatVecMult2x2(const CeedScalar alpha, - const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { +CEED_QFUNCTION_HELPER int AlphaMatVecMult2x2(const CeedScalar alpha, const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { // Compute v = alpha*A*u for (CeedInt k = 0; k < 2; k++) { v[k] = 0; - for (CeedInt m = 0; m < 2; m++) - v[k] += A[k][m] * u[m] * alpha; + for (CeedInt m = 0; m < 2; m++) v[k] += A[k][m] * u[m] * alpha; } return 0; @@ -190,13 +179,11 @@ CEED_QFUNCTION_HELPER int AlphaMatVecMult2x2(const CeedScalar alpha, // ----------------------------------------------------------------------------- // Compute matrix-vector product: alpha*A^T*u // ----------------------------------------------------------------------------- -CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult2x2(const CeedScalar alpha, - const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { +CEED_QFUNCTION_HELPER int AlphaMatTransposeVecMult2x2(const CeedScalar alpha, const CeedScalar A[2][2], const CeedScalar u[2], CeedScalar v[2]) { // Compute v = alpha*A^T*u for (CeedInt k = 0; k < 2; k++) { v[k] = 0; - for (CeedInt m = 0; m < 2; m++) - v[k] += A[m][k] * u[m] * alpha; + for (CeedInt m = 0; m < 2; m++) v[k] += A[m][k] * u[m] * alpha; } return 0; diff --git a/examples/Hdiv-mixed/src/cl-options.c b/examples/Hdiv-mixed/src/cl-options.c index 85c18442e2..0d67fedb59 100644 --- a/examples/Hdiv-mixed/src/cl-options.c +++ b/examples/Hdiv-mixed/src/cl-options.c @@ -21,17 +21,14 @@ // Process general command line options PetscErrorCode ProcessCommandLineOptions(AppCtx app_ctx) { - PetscBool problem_flag = PETSC_FALSE; - PetscBool ceed_flag = PETSC_FALSE; + PetscBool ceed_flag = PETSC_FALSE; PetscFunctionBeginUser; - PetscOptionsBegin(app_ctx->comm, NULL, "H(div) examples in PETSc with libCEED", - NULL); + PetscOptionsBegin(app_ctx->comm, NULL, "H(div) examples in PETSc with libCEED", NULL); - PetscCall( PetscOptionsString("-ceed", "CEED resource specifier", - NULL, app_ctx->ceed_resource, app_ctx->ceed_resource, - sizeof(app_ctx->ceed_resource), &ceed_flag) ); + PetscCall(PetscOptionsString("-ceed", "CEED resource specifier", NULL, app_ctx->ceed_resource, app_ctx->ceed_resource, + sizeof(app_ctx->ceed_resource), &ceed_flag)); // Provide default ceed resource if not specified if (!ceed_flag) { @@ -39,48 +36,31 @@ PetscErrorCode ProcessCommandLineOptions(AppCtx app_ctx) { strncpy(app_ctx->ceed_resource, ceed_resource, 10); } - PetscCall( PetscOptionsFList("-problem", "Problem to solve", NULL, - app_ctx->problems, - app_ctx->problem_name, app_ctx->problem_name, sizeof(app_ctx->problem_name), - &problem_flag) ); + PetscCall(PetscOptionsFList("-problem", "Problem to solve", NULL, app_ctx->problems, app_ctx->problem_name, app_ctx->problem_name, + sizeof(app_ctx->problem_name), &problem_flag)); // Provide default problem if not specified if (!problem_flag) { const char *problem_name = "darcy2d"; strncpy(app_ctx->problem_name, problem_name, 16); } app_ctx->degree = 1; - PetscCall( PetscOptionsInt("-degree", "Polynomial degree of finite elements", - NULL, app_ctx->degree, &app_ctx->degree, NULL) ); + PetscCall(PetscOptionsInt("-degree", "Polynomial degree of finite elements", NULL, app_ctx->degree, &app_ctx->degree, NULL)); app_ctx->q_extra = 0; - PetscCall( PetscOptionsInt("-q_extra", "Number of extra quadrature points", - NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL) ); + PetscCall(PetscOptionsInt("-q_extra", "Number of extra quadrature points", NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL)); app_ctx->view_solution = PETSC_FALSE; - PetscCall( PetscOptionsBool("-view_solution", - "View solution in Paraview", - NULL, app_ctx->view_solution, - &(app_ctx->view_solution), NULL) ); + PetscCall(PetscOptionsBool("-view_solution", "View solution in Paraview", NULL, app_ctx->view_solution, &(app_ctx->view_solution), NULL)); app_ctx->quartic = PETSC_FALSE; - PetscCall( PetscOptionsBool("-quartic", - "To test PetscViewer", - NULL, app_ctx->quartic, - &(app_ctx->quartic), NULL) ); + PetscCall(PetscOptionsBool("-quartic", "To test PetscViewer", NULL, app_ctx->quartic, &(app_ctx->quartic), NULL)); - PetscCall( PetscStrncpy(app_ctx->output_dir, ".", 2) ); - PetscCall( PetscOptionsString("-output_dir", "Output directory", - NULL, app_ctx->output_dir, app_ctx->output_dir, - sizeof(app_ctx->output_dir), NULL) ); + PetscCall(PetscStrncpy(app_ctx->output_dir, ".", 2)); + PetscCall(PetscOptionsString("-output_dir", "Output directory", NULL, app_ctx->output_dir, app_ctx->output_dir, sizeof(app_ctx->output_dir), NULL)); app_ctx->output_freq = 10; - PetscCall( PetscOptionsInt("-output_freq", - "Frequency of output, in number of steps", - NULL, app_ctx->output_freq, &app_ctx->output_freq, NULL) ); + PetscCall(PetscOptionsInt("-output_freq", "Frequency of output, in number of steps", NULL, app_ctx->output_freq, &app_ctx->output_freq, NULL)); app_ctx->bc_pressure_count = 16; // we can set one face by: -bc_faces 1 OR multiple faces by :-bc_faces 1,2,3 - PetscCall( PetscOptionsIntArray("-bc_faces", - "Face IDs to apply pressure BC", - NULL, app_ctx->bc_faces, &app_ctx->bc_pressure_count, NULL) ); - + PetscCall(PetscOptionsIntArray("-bc_faces", "Face IDs to apply pressure BC", NULL, app_ctx->bc_faces, &app_ctx->bc_pressure_count, NULL)); PetscOptionsEnd(); diff --git a/examples/Hdiv-mixed/src/post-processing.c b/examples/Hdiv-mixed/src/post-processing.c index df94038ed4..2a99a61cf3 100644 --- a/examples/Hdiv-mixed/src/post-processing.c +++ b/examples/Hdiv-mixed/src/post-processing.c @@ -1,272 +1,244 @@ #include "../include/post-processing.h" + #include "../include/setup-solvers.h" #include "ceed/ceed.h" // ----------------------------------------------------------------------------- // This function print the output // ----------------------------------------------------------------------------- -PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, - CeedMemType mem_type_backend, - TS ts, SNES snes, KSP ksp, - Vec U, CeedScalar l2_error_u, CeedScalar l2_error_p) { - +PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, CeedMemType mem_type_backend, TS ts, SNES snes, KSP ksp, Vec U, + CeedScalar l2_error_u, CeedScalar l2_error_p) { PetscFunctionBeginUser; const char *used_resource; CeedGetResource(ceed, &used_resource); char hostname[PETSC_MAX_PATH_LEN]; - PetscCall( PetscGetHostName(hostname, sizeof hostname) ); + PetscCall(PetscGetHostName(hostname, sizeof hostname)); PetscInt comm_size; - PetscCall( MPI_Comm_size(app_ctx->comm, &comm_size) ); - PetscCall( PetscPrintf(app_ctx->comm, - "\n-- Mixed H(div) Example - libCEED + PETSc --\n" - " MPI:\n" - " Hostname : %s\n" - " Total ranks : %d\n" - " libCEED:\n" - " libCEED Backend : %s\n" - " libCEED Backend MemType : %s\n", - hostname, comm_size, used_resource, CeedMemTypes[mem_type_backend]) ); + PetscCall(MPI_Comm_size(app_ctx->comm, &comm_size)); + PetscCall(PetscPrintf(app_ctx->comm, + "\n-- Mixed H(div) Example - libCEED + PETSc --\n" + " MPI:\n" + " Hostname : %s\n" + " Total ranks : %d\n" + " libCEED:\n" + " libCEED Backend : %s\n" + " libCEED Backend MemType : %s\n", + hostname, comm_size, used_resource, CeedMemTypes[mem_type_backend])); MatType mat_type; VecType vec_type; - PetscCall( DMGetMatType(dm, &mat_type) ); - PetscCall( DMGetVecType(dm, &vec_type) ); - PetscCall( PetscPrintf(app_ctx->comm, - " PETSc:\n" - " DM MatType : %s\n" - " DM VecType : %s\n", - mat_type, vec_type) ); - - PetscInt U_l_size, U_g_size; - PetscCall( VecGetSize(U, &U_g_size) ); - PetscCall( VecGetLocalSize(U, &U_l_size) ); - PetscCall( PetscPrintf(app_ctx->comm, - " Problem:\n" - " Problem Name : %s\n" - " Global nodes (u + p) : %" PetscInt_FMT "\n" - " Owned nodes (u + p) : %" PetscInt_FMT "\n", - app_ctx->problem_name, U_g_size, U_l_size - ) ); + PetscCall(DMGetMatType(dm, &mat_type)); + PetscCall(DMGetVecType(dm, &vec_type)); + PetscCall(PetscPrintf(app_ctx->comm, + " PETSc:\n" + " DM MatType : %s\n" + " DM VecType : %s\n", + mat_type, vec_type)); + + PetscInt U_l_size, U_g_size; + PetscCall(VecGetSize(U, &U_g_size)); + PetscCall(VecGetLocalSize(U, &U_l_size)); + PetscCall(PetscPrintf(app_ctx->comm, + " Problem:\n" + " Problem Name : %s\n" + " Global nodes (u + p) : %" PetscInt_FMT "\n" + " Owned nodes (u + p) : %" PetscInt_FMT "\n", + app_ctx->problem_name, U_g_size, U_l_size)); // --TS if (has_ts) { - PetscInt ts_steps; - TSType ts_type; + PetscInt ts_steps; + TSType ts_type; TSConvergedReason ts_reason; - PetscCall( TSGetStepNumber(ts, &ts_steps) ); - PetscCall( TSGetType(ts, &ts_type) ); - PetscCall( TSGetConvergedReason(ts, &ts_reason) ); - PetscCall( PetscPrintf(app_ctx->comm, - " TS:\n" - " TS Type : %s\n" - " TS Convergence : %s\n" - " Number of TS steps : %" PetscInt_FMT "\n" - " Final time : %g\n", - ts_type, TSConvergedReasons[ts_reason], - ts_steps, (double)app_ctx->t_final) ); - - PetscCall( TSGetSNES(ts, &snes) ); + PetscCall(TSGetStepNumber(ts, &ts_steps)); + PetscCall(TSGetType(ts, &ts_type)); + PetscCall(TSGetConvergedReason(ts, &ts_reason)); + PetscCall(PetscPrintf(app_ctx->comm, + " TS:\n" + " TS Type : %s\n" + " TS Convergence : %s\n" + " Number of TS steps : %" PetscInt_FMT "\n" + " Final time : %g\n", + ts_type, TSConvergedReasons[ts_reason], ts_steps, (double)app_ctx->t_final)); + + PetscCall(TSGetSNES(ts, &snes)); } // -- SNES PetscInt its, snes_its = 0; - PetscCall( SNESGetIterationNumber(snes, &its) ); + PetscCall(SNESGetIterationNumber(snes, &its)); snes_its += its; - SNESType snes_type; + SNESType snes_type; SNESConvergedReason snes_reason; - PetscReal snes_rnorm; - PetscCall( SNESGetType(snes, &snes_type) ); - PetscCall( SNESGetConvergedReason(snes, &snes_reason) ); - PetscCall( SNESGetFunctionNorm(snes, &snes_rnorm) ); - PetscCall( PetscPrintf(app_ctx->comm, - " SNES:\n" - " SNES Type : %s\n" - " SNES Convergence : %s\n" - " Total SNES Iterations : %" PetscInt_FMT "\n" - " Final rnorm : %e\n", - snes_type, SNESConvergedReasons[snes_reason], - snes_its, (double)snes_rnorm) ); + PetscReal snes_rnorm; + PetscCall(SNESGetType(snes, &snes_type)); + PetscCall(SNESGetConvergedReason(snes, &snes_reason)); + PetscCall(SNESGetFunctionNorm(snes, &snes_rnorm)); + PetscCall(PetscPrintf(app_ctx->comm, + " SNES:\n" + " SNES Type : %s\n" + " SNES Convergence : %s\n" + " Total SNES Iterations : %" PetscInt_FMT "\n" + " Final rnorm : %e\n", + snes_type, SNESConvergedReasons[snes_reason], snes_its, (double)snes_rnorm)); if (!has_ts) { PetscInt ksp_its = 0; - PetscCall( SNESGetLinearSolveIterations(snes, &its) ); + PetscCall(SNESGetLinearSolveIterations(snes, &its)); ksp_its += its; - KSPType ksp_type; + KSPType ksp_type; KSPConvergedReason ksp_reason; - PetscReal ksp_rnorm; - PC pc; - PCType pc_type; - PetscCall( KSPGetPC(ksp, &pc) ); - PetscCall( PCGetType(pc, &pc_type) ); - PetscCall( KSPGetType(ksp, &ksp_type) ); - PetscCall( KSPGetConvergedReason(ksp, &ksp_reason) ); - PetscCall( KSPGetIterationNumber(ksp, &ksp_its) ); - PetscCall( KSPGetResidualNorm(ksp, &ksp_rnorm) ); - PetscCall( PetscPrintf(app_ctx->comm, - " KSP:\n" - " KSP Type : %s\n" - " PC Type : %s\n" - " KSP Convergence : %s\n" - " Total KSP Iterations : %" PetscInt_FMT "\n" - " Final rnorm : %e\n", - ksp_type, pc_type, KSPConvergedReasons[ksp_reason], ksp_its, - (double)ksp_rnorm ) ); + PetscReal ksp_rnorm; + PC pc; + PCType pc_type; + PetscCall(KSPGetPC(ksp, &pc)); + PetscCall(PCGetType(pc, &pc_type)); + PetscCall(KSPGetType(ksp, &ksp_type)); + PetscCall(KSPGetConvergedReason(ksp, &ksp_reason)); + PetscCall(KSPGetIterationNumber(ksp, &ksp_its)); + PetscCall(KSPGetResidualNorm(ksp, &ksp_rnorm)); + PetscCall(PetscPrintf(app_ctx->comm, + " KSP:\n" + " KSP Type : %s\n" + " PC Type : %s\n" + " KSP Convergence : %s\n" + " Total KSP Iterations : %" PetscInt_FMT "\n" + " Final rnorm : %e\n", + ksp_type, pc_type, KSPConvergedReasons[ksp_reason], ksp_its, (double)ksp_rnorm)); } - PetscCall( PetscPrintf(app_ctx->comm, - " L2 Error (MMS):\n" - " L2 error of u and p : %e, %e\n", - (double)l2_error_u, - (double)l2_error_p) ); + PetscCall(PetscPrintf(app_ctx->comm, + " L2 Error (MMS):\n" + " L2 error of u and p : %e, %e\n", + (double)l2_error_u, (double)l2_error_p)); PetscFunctionReturn(0); }; // ----------------------------------------------------------------------------- // Setup operator context data for initial condition, u field // ----------------------------------------------------------------------------- -PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, - CeedData ceed_data, - OperatorApplyContext ctx_Hdiv) { +PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_Hdiv) { PetscFunctionBeginUser; ctx_Hdiv->comm = comm; - ctx_Hdiv->dm = dm; - PetscCall( DMCreateLocalVector(dm, &ctx_Hdiv->X_loc) ); + ctx_Hdiv->dm = dm; + PetscCall(DMCreateLocalVector(dm, &ctx_Hdiv->X_loc)); ctx_Hdiv->x_ceed = ceed_data->u_ceed; - //ctx_project_velocity->y_ceed = ceed_data->v0_ceed; + // ctx_project_velocity->y_ceed = ceed_data->v0_ceed; ctx_Hdiv->ceed = ceed; - //ctx_project_velocity->op_apply = ceed_data->op_ics_u; + // ctx_project_velocity->op_apply = ceed_data->op_ics_u; PetscFunctionReturn(0); } -PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, - CeedData ceed_data, VecType vec_type, - OperatorApplyContext ctx_H1) { +PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_H1) { PetscFunctionBeginUser; ctx_H1->comm = comm; - ctx_H1->dm = dm_H1; - PetscCall( DMCreateLocalVector(dm_H1, &ctx_H1->X_loc) ); - PetscCall( VecDuplicate(ctx_H1->X_loc, &ctx_H1->Y_loc) ); - ctx_H1->x_ceed = ceed_data->up_ceed; - ctx_H1->y_ceed = ceed_data->vp_ceed; - ctx_H1->x_coord = ceed_data->x_coord; - ctx_H1->ceed = ceed; - ctx_H1->op_apply = ceed_data->op_post_mass; - ctx_H1->op_rhs_H1 = ceed_data->op_rhs_H1; + ctx_H1->dm = dm_H1; + PetscCall(DMCreateLocalVector(dm_H1, &ctx_H1->X_loc)); + PetscCall(VecDuplicate(ctx_H1->X_loc, &ctx_H1->Y_loc)); + ctx_H1->x_ceed = ceed_data->up_ceed; + ctx_H1->y_ceed = ceed_data->vp_ceed; + ctx_H1->x_coord = ceed_data->x_coord; + ctx_H1->ceed = ceed; + ctx_H1->op_apply = ceed_data->op_post_mass; + ctx_H1->op_rhs_H1 = ceed_data->op_rhs_H1; ctx_H1->elem_restr_u_H1 = ceed_data->elem_restr_u_H1; - ctx_H1->vec_type = vec_type; + ctx_H1->vec_type = vec_type; PetscFunctionReturn(0); } // ----------------------------------------------------------------------------- // This function print the output // ----------------------------------------------------------------------------- -PetscErrorCode ProjectVelocity(AppCtx app_ctx, - Vec U, Vec *U_H1) { - +PetscErrorCode ProjectVelocity(AppCtx app_ctx, Vec U, Vec *U_H1) { PetscFunctionBeginUser; const PetscScalar *x; - PetscMemType x_mem_type; + PetscMemType x_mem_type; // ---------------------------------------------- // Create local rhs for u field // ---------------------------------------------- - Vec rhs_loc_H1; + Vec rhs_loc_H1; PetscScalar *ru; PetscMemType ru_mem_type; - PetscCall( DMCreateLocalVector(app_ctx->ctx_H1->dm, &rhs_loc_H1) ); - PetscCall( VecZeroEntries(rhs_loc_H1) ); - PetscCall( VecGetArrayAndMemType(rhs_loc_H1, &ru, &ru_mem_type) ); - CeedElemRestrictionCreateVector(app_ctx->ctx_H1->elem_restr_u_H1, - &app_ctx->ctx_H1->rhs_ceed_H1, - NULL); - CeedVectorSetArray(app_ctx->ctx_H1->rhs_ceed_H1, MemTypeP2C(ru_mem_type), - CEED_USE_POINTER, ru); + PetscCall(DMCreateLocalVector(app_ctx->ctx_H1->dm, &rhs_loc_H1)); + PetscCall(VecZeroEntries(rhs_loc_H1)); + PetscCall(VecGetArrayAndMemType(rhs_loc_H1, &ru, &ru_mem_type)); + CeedElemRestrictionCreateVector(app_ctx->ctx_H1->elem_restr_u_H1, &app_ctx->ctx_H1->rhs_ceed_H1, NULL); + CeedVectorSetArray(app_ctx->ctx_H1->rhs_ceed_H1, MemTypeP2C(ru_mem_type), CEED_USE_POINTER, ru); // Global-to-local: map final U in Hdiv space to local vector - PetscCall( DMGlobalToLocal(app_ctx->ctx_Hdiv->dm, - U, INSERT_VALUES, app_ctx->ctx_Hdiv->X_loc) ); + PetscCall(DMGlobalToLocal(app_ctx->ctx_Hdiv->dm, U, INSERT_VALUES, app_ctx->ctx_Hdiv->X_loc)); // Place Hdiv PETSc vectors in CEED vectors - PetscCall( VecGetArrayReadAndMemType(app_ctx->ctx_Hdiv->X_loc, - &x, &x_mem_type) ); - CeedVectorSetArray(app_ctx->ctx_Hdiv->x_ceed, MemTypeP2C(x_mem_type), - CEED_USE_POINTER, (PetscScalar *)x); + PetscCall(VecGetArrayReadAndMemType(app_ctx->ctx_Hdiv->X_loc, &x, &x_mem_type)); + CeedVectorSetArray(app_ctx->ctx_Hdiv->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, (PetscScalar *)x); // Apply operator to create RHS for u field - CeedOperatorApply(app_ctx->ctx_H1->op_rhs_H1, app_ctx->ctx_H1->x_coord, - app_ctx->ctx_H1->rhs_ceed_H1, CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(app_ctx->ctx_H1->op_rhs_H1, app_ctx->ctx_H1->x_coord, app_ctx->ctx_H1->rhs_ceed_H1, CEED_REQUEST_IMMEDIATE); // Restore Hdiv vector - CeedVectorTakeArray(app_ctx->ctx_Hdiv->x_ceed, - MemTypeP2C(x_mem_type), NULL); - PetscCall( VecRestoreArrayReadAndMemType(app_ctx->ctx_Hdiv->X_loc, &x) ); + CeedVectorTakeArray(app_ctx->ctx_Hdiv->x_ceed, MemTypeP2C(x_mem_type), NULL); + PetscCall(VecRestoreArrayReadAndMemType(app_ctx->ctx_Hdiv->X_loc, &x)); // ---------------------------------------------- // Create global rhs for u field // ---------------------------------------------- Vec rhs_H1; - CeedVectorTakeArray(app_ctx->ctx_H1->rhs_ceed_H1, MemTypeP2C(ru_mem_type), - NULL); - PetscCall( VecRestoreArrayAndMemType(rhs_loc_H1, &ru) ); - PetscCall( DMCreateGlobalVector(app_ctx->ctx_H1->dm, &rhs_H1) ); - PetscCall( VecZeroEntries(rhs_H1) ); - PetscCall( DMLocalToGlobal(app_ctx->ctx_H1->dm, rhs_loc_H1, ADD_VALUES, - rhs_H1) ); + CeedVectorTakeArray(app_ctx->ctx_H1->rhs_ceed_H1, MemTypeP2C(ru_mem_type), NULL); + PetscCall(VecRestoreArrayAndMemType(rhs_loc_H1, &ru)); + PetscCall(DMCreateGlobalVector(app_ctx->ctx_H1->dm, &rhs_H1)); + PetscCall(VecZeroEntries(rhs_H1)); + PetscCall(DMLocalToGlobal(app_ctx->ctx_H1->dm, rhs_loc_H1, ADD_VALUES, rhs_H1)); // ---------------------------------------------- // Solve for U_H1, M*U_H1 = rhs_H1 // ---------------------------------------------- PetscInt UH1_g_size, UH1_l_size; - PetscCall( VecGetSize(*U_H1, &UH1_g_size) ); + PetscCall(VecGetSize(*U_H1, &UH1_g_size)); // Local size for matShell - PetscCall( VecGetLocalSize(*U_H1, &UH1_l_size) ); + PetscCall(VecGetLocalSize(*U_H1, &UH1_l_size)); // Operator Mat mat_ksp_projection; // -- Form Action of residual on u - PetscCall( MatCreateShell(app_ctx->comm, UH1_l_size, UH1_l_size, UH1_g_size, - UH1_g_size, app_ctx->ctx_H1, &mat_ksp_projection) ); - PetscCall( MatShellSetOperation(mat_ksp_projection, MATOP_MULT, - (void (*)(void))ApplyMatOp) ); - PetscCall( MatShellSetVecType(mat_ksp_projection, app_ctx->ctx_H1->vec_type) ); + PetscCall(MatCreateShell(app_ctx->comm, UH1_l_size, UH1_l_size, UH1_g_size, UH1_g_size, app_ctx->ctx_H1, &mat_ksp_projection)); + PetscCall(MatShellSetOperation(mat_ksp_projection, MATOP_MULT, (void (*)(void))ApplyMatOp)); + PetscCall(MatShellSetVecType(mat_ksp_projection, app_ctx->ctx_H1->vec_type)); KSP ksp_projection; - PetscCall( KSPCreate(app_ctx->ctx_H1->comm, &ksp_projection) ); - PetscCall( KSPSetOperators(ksp_projection, mat_ksp_projection, - mat_ksp_projection) ); - PetscCall( KSPSetFromOptions(ksp_projection) ); - PetscCall( KSPSetUp(ksp_projection) ); - PetscCall( VecZeroEntries(*U_H1) ); - PetscCall( KSPSolve(ksp_projection, rhs_H1, *U_H1) ); + PetscCall(KSPCreate(app_ctx->ctx_H1->comm, &ksp_projection)); + PetscCall(KSPSetOperators(ksp_projection, mat_ksp_projection, mat_ksp_projection)); + PetscCall(KSPSetFromOptions(ksp_projection)); + PetscCall(KSPSetUp(ksp_projection)); + PetscCall(VecZeroEntries(*U_H1)); + PetscCall(KSPSolve(ksp_projection, rhs_H1, *U_H1)); // Clean up - PetscCall( VecDestroy(&rhs_loc_H1) ); - PetscCall( VecDestroy(&rhs_H1) ); - PetscCall( MatDestroy(&mat_ksp_projection) ); - PetscCall( KSPDestroy(&ksp_projection) ); + PetscCall(VecDestroy(&rhs_loc_H1)); + PetscCall(VecDestroy(&rhs_H1)); + PetscCall(MatDestroy(&mat_ksp_projection)); + PetscCall(KSPDestroy(&ksp_projection)); CeedVectorDestroy(&app_ctx->ctx_H1->rhs_ceed_H1); PetscFunctionReturn(0); }; - PetscErrorCode CtxVecDestroy(AppCtx app_ctx) { - PetscFunctionBegin; - PetscCall( VecDestroy(&app_ctx->ctx_H1->X_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_H1->Y_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_Hdiv->X_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_initial_u0->X_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_initial_u0->Y_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_initial_p0->X_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_initial_p0->Y_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_residual_ut->X_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_residual_ut->X_t_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_residual_ut->Y_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_jacobian->Y_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_jacobian->X_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_residual->Y_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_residual->X_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_error->Y_loc) ); - PetscCall( VecDestroy(&app_ctx->ctx_error->X_loc) ); + PetscCall(VecDestroy(&app_ctx->ctx_H1->X_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_H1->Y_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_Hdiv->X_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_initial_u0->X_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_initial_u0->Y_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_initial_p0->X_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_initial_p0->Y_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_residual_ut->X_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_residual_ut->X_t_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_residual_ut->Y_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_jacobian->Y_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_jacobian->X_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_residual->Y_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_residual->X_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_error->Y_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_error->X_loc)); PetscFunctionReturn(0); } // ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/src/setup-boundary.c b/examples/Hdiv-mixed/src/setup-boundary.c index e3a3a859ae..81cd9bd42f 100644 --- a/examples/Hdiv-mixed/src/setup-boundary.c +++ b/examples/Hdiv-mixed/src/setup-boundary.c @@ -4,14 +4,14 @@ // Create boundary label // --------------------------------------------------------------------------- PetscErrorCode CreateBCLabel(DM dm, const char name[]) { - DMLabel label; + DMLabel label; PetscFunctionBeginUser; - PetscCall( DMCreateLabel(dm, name) ); - PetscCall( DMGetLabel(dm, name, &label) ); - PetscCall( DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, label) ); - PetscCall( DMPlexLabelComplete(dm, label) ); + PetscCall(DMCreateLabel(dm, name)); + PetscCall(DMGetLabel(dm, name, &label)); + PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, label)); + PetscCall(DMPlexLabelComplete(dm, label)); PetscFunctionReturn(0); }; @@ -20,23 +20,19 @@ PetscErrorCode CreateBCLabel(DM dm, const char name[]) { // Add Dirichlet boundaries to DM // --------------------------------------------------------------------------- PetscErrorCode DMAddBoundariesDirichlet(DM dm) { - PetscFunctionBeginUser; // BCs given by manufactured solution - PetscBool has_label; - const char *name = "MMS Face Sets"; - PetscInt face_ids[1] = {1}; - PetscCall( DMHasLabel(dm, name, &has_label) ); + PetscBool has_label; + const char *name = "MMS Face Sets"; + PetscInt face_ids[1] = {1}; + PetscCall(DMHasLabel(dm, name, &has_label)); if (!has_label) { - PetscCall( CreateBCLabel(dm, name) ); + PetscCall(CreateBCLabel(dm, name)); } DMLabel label; - PetscCall( DMGetLabel(dm, name, &label) ); - PetscCall( DMAddBoundary(dm, DM_BC_ESSENTIAL, "mms", label, 1, face_ids, 0, 0, - NULL, - (void(*)(void))BoundaryDirichletMMS, NULL, NULL, NULL) ); - + PetscCall(DMGetLabel(dm, name, &label)); + PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "mms", label, 1, face_ids, 0, 0, NULL, (void (*)(void))BoundaryDirichletMMS, NULL, NULL, NULL)); PetscFunctionReturn(0); } @@ -44,38 +40,30 @@ PetscErrorCode DMAddBoundariesDirichlet(DM dm) { // --------------------------------------------------------------------------- // Add Neumann boundaries to DM // --------------------------------------------------------------------------- -PetscErrorCode DMAddBoundariesPressure(Ceed ceed, CeedData ceed_data, - AppCtx app_ctx, ProblemData problem_data, DM dm, - CeedVector bc_pressure) { - PetscInt dim; - CeedQFunction qf_pressure; - CeedOperator op_pressure; +PetscErrorCode DMAddBoundariesPressure(Ceed ceed, CeedData ceed_data, AppCtx app_ctx, ProblemData problem_data, DM dm, CeedVector bc_pressure) { + PetscInt dim; + CeedQFunction qf_pressure; + CeedOperator op_pressure; PetscFunctionBeginUser; - PetscCall( DMGetDimension(dm, &dim) ); + PetscCall(DMGetDimension(dm, &dim)); if (app_ctx->bc_pressure_count > 0) { DMLabel domain_label; PetscCall(DMGetLabel(dm, "Face Sets", &domain_label)); // Compute contribution on each boundary face for (CeedInt i = 0; i < app_ctx->bc_pressure_count; i++) { - - CeedQFunctionCreateInterior(ceed, 1, problem_data->bc_pressure, - problem_data->bc_pressure_loc, &qf_pressure); + CeedQFunctionCreateInterior(ceed, 1, problem_data->bc_pressure, problem_data->bc_pressure_loc, &qf_pressure); CeedQFunctionAddInput(qf_pressure, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddOutput(qf_pressure, "v", dim, CEED_EVAL_INTERP); // -- Apply operator - CeedOperatorCreate(ceed, qf_pressure, NULL, NULL, - &op_pressure); - CeedOperatorSetField(op_pressure, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_pressure, "v", ceed_data->elem_restr_u, - ceed_data->basis_u_face, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_pressure, NULL, NULL, &op_pressure); + CeedOperatorSetField(op_pressure, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_pressure, "v", ceed_data->elem_restr_u, ceed_data->basis_u_face, CEED_VECTOR_ACTIVE); // ---- Compute pressure on face - CeedOperatorApplyAdd(op_pressure, ceed_data->x_coord, bc_pressure, - CEED_REQUEST_IMMEDIATE); + CeedOperatorApplyAdd(op_pressure, ceed_data->x_coord, bc_pressure, CEED_REQUEST_IMMEDIATE); // -- Cleanup CeedQFunctionDestroy(&qf_pressure); @@ -87,14 +75,12 @@ PetscErrorCode DMAddBoundariesPressure(Ceed ceed, CeedData ceed_data, } #ifndef M_PI -#define M_PI 3.14159265358979323846 +#define M_PI 3.14159265358979323846 #endif // --------------------------------------------------------------------------- // Boundary function for manufactured solution // --------------------------------------------------------------------------- -PetscErrorCode BoundaryDirichletMMS(PetscInt dim, PetscReal t, - const PetscReal coords[], - PetscInt num_comp_u, PetscScalar *u, void *ctx) { +PetscErrorCode BoundaryDirichletMMS(PetscInt dim, PetscReal t, const PetscReal coords[], PetscInt num_comp_u, PetscScalar *u, void *ctx) { PetscScalar x = coords[0]; PetscScalar y = coords[1]; PetscScalar z = coords[1]; @@ -102,12 +88,12 @@ PetscErrorCode BoundaryDirichletMMS(PetscInt dim, PetscReal t, PetscFunctionBeginUser; if (dim == 2) { - u[0] = -M_PI*cos(M_PI*x) *sin(M_PI*y) - M_PI*y; - u[1] = -M_PI*sin(M_PI*x) *cos(M_PI*y) - M_PI*x; + u[0] = -M_PI * cos(M_PI * x) * sin(M_PI * y) - M_PI * y; + u[1] = -M_PI * sin(M_PI * x) * cos(M_PI * y) - M_PI * x; } else { - u[0] = -M_PI*cos(M_PI*x) *sin(M_PI*y) *sin(M_PI*z) - M_PI*y*z; - u[1] = -M_PI*sin(M_PI*x) *cos(M_PI*y) *sin(M_PI*z) - M_PI*x*z; - u[2] = -M_PI*sin(M_PI*x) *sin(M_PI*y) *cos(M_PI*z) - M_PI*x*y; + u[0] = -M_PI * cos(M_PI * x) * sin(M_PI * y) * sin(M_PI * z) - M_PI * y * z; + u[1] = -M_PI * sin(M_PI * x) * cos(M_PI * y) * sin(M_PI * z) - M_PI * x * z; + u[2] = -M_PI * sin(M_PI * x) * sin(M_PI * y) * cos(M_PI * z) - M_PI * x * y; } PetscFunctionReturn(0); diff --git a/examples/Hdiv-mixed/src/setup-dm.c b/examples/Hdiv-mixed/src/setup-dm.c index dff3592bec..3e39ebd7e6 100644 --- a/examples/Hdiv-mixed/src/setup-dm.c +++ b/examples/Hdiv-mixed/src/setup-dm.c @@ -1,128 +1,123 @@ #include "../include/setup-dm.h" + #include "petscerror.h" // --------------------------------------------------------------------------- // Setup DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, - VecType vec_type, DM *dm) { - +PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, VecType vec_type, DM *dm) { PetscFunctionBeginUser; // Create DMPLEX - PetscCall( DMCreate(comm, dm) ); - PetscCall( DMSetType(*dm, DMPLEX) ); - PetscCall( DMSetMatType(*dm, mat_type) ); - PetscCall( DMSetVecType(*dm, vec_type) ); + PetscCall(DMCreate(comm, dm)); + PetscCall(DMSetType(*dm, DMPLEX)); + PetscCall(DMSetMatType(*dm, mat_type)); + PetscCall(DMSetVecType(*dm, vec_type)); // Set Tensor elements - PetscCall( PetscOptionsSetValue(NULL, "-dm_plex_simplex", "0") ); + PetscCall(PetscOptionsSetValue(NULL, "-dm_plex_simplex", "0")); // Set CL options - PetscCall( DMSetFromOptions(*dm) ); - PetscCall( DMViewFromOptions(*dm, NULL, "-dm_view") ); + PetscCall(DMSetFromOptions(*dm)); + PetscCall(DMViewFromOptions(*dm, NULL, "-dm_view")); PetscFunctionReturn(0); }; PetscErrorCode PerturbVerticesSmooth(DM dm) { - Vec coordinates; PetscSection coordSection; PetscScalar *coords; - PetscInt v,vStart,vEnd,offset,dim; - PetscReal x,y,z; + PetscInt v, vStart, vEnd, offset, dim; + PetscReal x, y, z; PetscFunctionBeginUser; - PetscCall( DMGetDimension(dm, &dim) ); - PetscCall( DMGetCoordinateSection(dm, &coordSection) ); - PetscCall( DMGetCoordinatesLocal(dm, &coordinates) ); - PetscCall( DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd) ); - PetscCall( VecGetArray(coordinates,&coords) ); - for(v=vStart; vdegree + 2 + app_ctx->q_extra; + PetscInt q_degree = app_ctx->degree + 2 + app_ctx->q_extra; PetscBool is_simplex = PETSC_TRUE; PetscFunctionBeginUser; // Check if simplex or tensor-product element - PetscCall( DMPlexIsSimplex(dm_H1, &is_simplex) ); + PetscCall(DMPlexIsSimplex(dm_H1, &is_simplex)); // Create FE space - PetscCall( PetscFECreateLagrange(app_ctx->comm, problem_data->dim, - problem_data->dim, is_simplex, - app_ctx->degree, q_degree, &fe) ); - PetscCall( PetscObjectSetName((PetscObject)fe, "U") ); - PetscCall( DMAddField(dm_H1, NULL, (PetscObject)fe) ); - PetscCall( DMCreateDS(dm_H1) ); + PetscCall(PetscFECreateLagrange(app_ctx->comm, problem_data->dim, problem_data->dim, is_simplex, app_ctx->degree, q_degree, &fe)); + PetscCall(PetscObjectSetName((PetscObject)fe, "U")); + PetscCall(DMAddField(dm_H1, NULL, (PetscObject)fe)); + PetscCall(DMCreateDS(dm_H1)); { // create FE field for coordinates @@ -109,19 +107,19 @@ PetscErrorCode SetupFEH1(ProblemData problem_data, } PetscCall(DMPlexSetClosurePermutationTensor(dm_H1, PETSC_DETERMINE, NULL)); // Cleanup - PetscCall( PetscFEDestroy(&fe) ); + PetscCall(PetscFEDestroy(&fe)); // Empty name for conserved field (because there is only one field) PetscSection section; - PetscCall( DMGetLocalSection(dm_H1, §ion) ); - PetscCall( PetscSectionSetFieldName(section, 0, "Velocity") ); + PetscCall(DMGetLocalSection(dm_H1, §ion)); + PetscCall(PetscSectionSetFieldName(section, 0, "Velocity")); if (problem_data->dim == 2) { - PetscCall( PetscSectionSetComponentName(section, 0, 0, "Velocity_X") ); - PetscCall( PetscSectionSetComponentName(section, 0, 1, "Velocity_Y") ); + PetscCall(PetscSectionSetComponentName(section, 0, 0, "Velocity_X")); + PetscCall(PetscSectionSetComponentName(section, 0, 1, "Velocity_Y")); } else { - PetscCall( PetscSectionSetComponentName(section, 0, 0, "Velocity_X") ); - PetscCall( PetscSectionSetComponentName(section, 0, 1, "Velocity_Y") ); - PetscCall( PetscSectionSetComponentName(section, 0, 2, "Velocity_Z") ); + PetscCall(PetscSectionSetComponentName(section, 0, 0, "Velocity_X")); + PetscCall(PetscSectionSetComponentName(section, 0, 1, "Velocity_Y")); + PetscCall(PetscSectionSetComponentName(section, 0, 2, "Velocity_Z")); } PetscFunctionReturn(0); }; \ No newline at end of file diff --git a/examples/Hdiv-mixed/src/setup-libceed.c b/examples/Hdiv-mixed/src/setup-libceed.c index 8748a60173..dc091f35e0 100644 --- a/examples/Hdiv-mixed/src/setup-libceed.c +++ b/examples/Hdiv-mixed/src/setup-libceed.c @@ -1,23 +1,22 @@ #include "../include/setup-libceed.h" -#include "../include/setup-boundary.h" -#include "../include/petsc-macros.h" -#include "../basis/Hdiv-quad.h" + +#include + #include "../basis/Hdiv-hex.h" +#include "../basis/Hdiv-quad.h" #include "../basis/L2-P0.h" +#include "../include/petsc-macros.h" +#include "../include/setup-boundary.h" #include "ceed/ceed.h" -#include // ----------------------------------------------------------------------------- // Convert PETSc MemType to libCEED MemType // ----------------------------------------------------------------------------- -CeedMemType MemTypeP2C(PetscMemType mem_type) { - return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; -} +CeedMemType MemTypeP2C(PetscMemType mem_type) { return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; } // ----------------------------------------------------------------------------- // Destroy libCEED objects // ----------------------------------------------------------------------------- PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data) { - PetscFunctionBegin; // Vectors @@ -28,7 +27,7 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data) { // Restrictions CeedElemRestrictionDestroy(&ceed_data->elem_restr_x); CeedElemRestrictionDestroy(&ceed_data->elem_restr_u); - CeedElemRestrictionDestroy(&ceed_data->elem_restr_U_i); // U = [p,u] + CeedElemRestrictionDestroy(&ceed_data->elem_restr_U_i); // U = [p,u] CeedElemRestrictionDestroy(&ceed_data->elem_restr_p); CeedElemRestrictionDestroy(&ceed_data->elem_restr_p_i); CeedElemRestrictionDestroy(&ceed_data->elem_restr_u0); @@ -54,21 +53,21 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data) { CeedQFunctionDestroy(&ceed_data->qf_ics_p); CeedOperatorDestroy(&ceed_data->op_ics_p); } - //QFunctions + // QFunctions CeedQFunctionDestroy(&ceed_data->qf_residual); CeedQFunctionDestroy(&ceed_data->qf_error); - //Operators + // Operators CeedOperatorDestroy(&ceed_data->op_residual); CeedOperatorDestroy(&ceed_data->op_error); if (!problem_data->has_ts) { - //QFunctions + // QFunctions CeedQFunctionDestroy(&ceed_data->qf_jacobian); - //Operators + // Operators CeedOperatorDestroy(&ceed_data->op_jacobian); } // data for post-processing - if(problem_data->view_solution) { + if (problem_data->view_solution) { CeedVectorDestroy(&ceed_data->up_ceed); CeedVectorDestroy(&ceed_data->vp_ceed); CeedVectorDestroy(&ceed_data->u_ceed); @@ -78,7 +77,7 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data) { CeedQFunctionDestroy(&ceed_data->qf_post_mass); CeedOperatorDestroy(&ceed_data->op_post_mass); } - PetscCall( PetscFree(ceed_data) ); + PetscCall(PetscFree(ceed_data)); PetscFunctionReturn(0); }; @@ -86,26 +85,20 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data) { // ----------------------------------------------------------------------------- // Utility function - essential BC dofs are encoded in closure indices as -(i+1) // ----------------------------------------------------------------------------- -PetscInt Involute(PetscInt i) { - return i >= 0 ? i : -(i + 1); -}; +PetscInt Involute(PetscInt i) { return i >= 0 ? i : -(i + 1); }; // ----------------------------------------------------------------------------- // Get CEED restriction data from DMPlex // ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, - DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; PetscFunctionBeginUser; - PetscCall( DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, - &elem_size, &num_comp, &num_dof, &elem_restr_offsets) ); + PetscCall(DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, &elem_size, &num_comp, &num_dof, &elem_restr_offsets)); - CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, - 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, - elem_restr_offsets, elem_restr); - PetscCall( PetscFree(elem_restr_offsets) ); + CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets, elem_restr); + PetscCall(PetscFree(elem_restr_offsets)); PetscFunctionReturn(0); }; @@ -113,267 +106,217 @@ PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, // ----------------------------------------------------------------------------- // Get Oriented CEED restriction data from DMPlex // ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, - DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u, - CeedElemRestriction *elem_restr_p, - CeedElemRestriction *elem_restr_u0, CeedElemRestriction *elem_restr_p0) { +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u, + CeedElemRestriction *elem_restr_p, CeedElemRestriction *elem_restr_u0, + CeedElemRestriction *elem_restr_p0) { PetscSection section, section_u0, section_p0; - PetscInt p, num_elem, num_dof, num_dof_u0, num_dof_p0, *restr_indices_u, - *restr_indices_p, *restr_indices_u0, *restr_indices_p0, - elem_offset, num_fields, num_fields_u0, num_fields_p0, - dim, c_start, c_end; - Vec U_loc; - const PetscInt *ornt; // this is for orientation of dof + PetscInt p, num_elem, num_dof, num_dof_u0, num_dof_p0, *restr_indices_u, *restr_indices_p, *restr_indices_u0, *restr_indices_p0, elem_offset, + num_fields, num_fields_u0, num_fields_p0, dim, c_start, c_end; + Vec U_loc; + const PetscInt *ornt; // this is for orientation of dof PetscFunctionBeginUser; // Section for mixed problem - PetscCall( DMGetDimension(dm, &dim) ); - PetscCall( DMGetLocalSection(dm, §ion) ); - PetscCall( PetscSectionGetNumFields(section, &num_fields) ); - PetscInt num_comp[num_fields], field_offsets[num_fields+1]; + PetscCall(DMGetDimension(dm, &dim)); + PetscCall(DMGetLocalSection(dm, §ion)); + PetscCall(PetscSectionGetNumFields(section, &num_fields)); + PetscInt num_comp[num_fields], field_offsets[num_fields + 1]; field_offsets[0] = 0; for (PetscInt f = 0; f < num_fields; f++) { - PetscCall( PetscSectionGetFieldComponents(section, f, &num_comp[f]) ); - field_offsets[f+1] = field_offsets[f] + num_comp[f]; + PetscCall(PetscSectionGetFieldComponents(section, f, &num_comp[f])); + field_offsets[f + 1] = field_offsets[f] + num_comp[f]; } // Section for initial conditions u0 - PetscCall( DMGetLocalSection(dm_u0, §ion_u0) ); - PetscCall( PetscSectionGetNumFields(section_u0, &num_fields_u0) ); - PetscInt num_comp_u0[num_fields_u0], field_offsets_u0[num_fields_u0+1]; + PetscCall(DMGetLocalSection(dm_u0, §ion_u0)); + PetscCall(PetscSectionGetNumFields(section_u0, &num_fields_u0)); + PetscInt num_comp_u0[num_fields_u0], field_offsets_u0[num_fields_u0 + 1]; field_offsets_u0[0] = 0; for (PetscInt f = 0; f < num_fields_u0; f++) { - PetscCall( PetscSectionGetFieldComponents(section_u0, f, &num_comp_u0[f]) ); - field_offsets_u0[f+1] = field_offsets_u0[f] + num_comp_u0[f]; + PetscCall(PetscSectionGetFieldComponents(section_u0, f, &num_comp_u0[f])); + field_offsets_u0[f + 1] = field_offsets_u0[f] + num_comp_u0[f]; } // Section for initial conditions p0 - PetscCall( DMGetLocalSection(dm_p0, §ion_p0) ); - PetscCall( PetscSectionGetNumFields(section_p0, &num_fields_p0) ); - PetscInt num_comp_p0[num_fields_p0], field_offsets_p0[num_fields_p0+1]; + PetscCall(DMGetLocalSection(dm_p0, §ion_p0)); + PetscCall(PetscSectionGetNumFields(section_p0, &num_fields_p0)); + PetscInt num_comp_p0[num_fields_p0], field_offsets_p0[num_fields_p0 + 1]; field_offsets_p0[0] = 0; for (PetscInt f = 0; f < num_fields_p0; f++) { - PetscCall( PetscSectionGetFieldComponents(section_p0, f, &num_comp_p0[f]) ); - field_offsets_p0[f+1] = field_offsets_p0[f] + num_comp_p0[f]; + PetscCall(PetscSectionGetFieldComponents(section_p0, f, &num_comp_p0[f])); + field_offsets_p0[f + 1] = field_offsets_p0[f] + num_comp_p0[f]; } - PetscCall( DMPlexGetHeightStratum(dm, 0, &c_start, &c_end) ); + PetscCall(DMPlexGetHeightStratum(dm, 0, &c_start, &c_end)); num_elem = c_end - c_start; - PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), - &restr_indices_u) ); - PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), - &restr_indices_u0) ); - PetscCall( PetscMalloc1(num_elem,&restr_indices_p) ); - PetscCall( PetscMalloc1(num_elem,&restr_indices_p0) ); - bool *orient_indices_u, *orient_indices_u0; // to flip the dof - PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), &orient_indices_u) ); - PetscCall( PetscMalloc1(num_elem*dim*PetscPowInt(P, dim), &orient_indices_u0) ); + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &restr_indices_u)); + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &restr_indices_u0)); + PetscCall(PetscMalloc1(num_elem, &restr_indices_p)); + PetscCall(PetscMalloc1(num_elem, &restr_indices_p0)); + bool *orient_indices_u, *orient_indices_u0; // to flip the dof + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &orient_indices_u)); + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &orient_indices_u0)); for (p = 0, elem_offset = 0; p < num_elem; p++) { - PetscInt num_indices, *indices, faces_per_elem, dofs_per_face, - num_indices_u0, *indices_u0, num_indices_p0, *indices_p0; - PetscCall( DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, - &num_indices, &indices, NULL, NULL) ); - PetscCall( DMPlexGetClosureIndices(dm_u0, section_u0, section_u0, p, PETSC_TRUE, - &num_indices_u0, &indices_u0, NULL, NULL) ); - PetscCall( DMPlexGetClosureIndices(dm_p0, section_p0, section_p0, p, PETSC_TRUE, - &num_indices_p0, &indices_p0, NULL, NULL) ); - restr_indices_p[p] = indices[num_indices - 1]; + PetscInt num_indices, *indices, faces_per_elem, dofs_per_face, num_indices_u0, *indices_u0, num_indices_p0, *indices_p0; + PetscCall(DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL)); + PetscCall(DMPlexGetClosureIndices(dm_u0, section_u0, section_u0, p, PETSC_TRUE, &num_indices_u0, &indices_u0, NULL, NULL)); + PetscCall(DMPlexGetClosureIndices(dm_p0, section_p0, section_p0, p, PETSC_TRUE, &num_indices_p0, &indices_p0, NULL, NULL)); + restr_indices_p[p] = indices[num_indices - 1]; restr_indices_p0[p] = indices_p0[0]; - PetscCall( DMPlexGetConeOrientation(dm, p, &ornt) ); + PetscCall(DMPlexGetConeOrientation(dm, p, &ornt)); // Get number of faces per element - PetscCall( DMPlexGetConeSize(dm, p, &faces_per_elem) ); + PetscCall(DMPlexGetConeSize(dm, p, &faces_per_elem)); dofs_per_face = faces_per_elem - 2; for (PetscInt f = 0; f < faces_per_elem; f++) { for (PetscInt i = 0; i < dofs_per_face; i++) { - PetscInt ii = dofs_per_face*f + i; + PetscInt ii = dofs_per_face * f + i; // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. - PetscInt loc = Involute(indices[ii*num_comp[0]]); + PetscInt loc = Involute(indices[ii * num_comp[0]]); restr_indices_u[elem_offset] = loc; // Set orientation orient_indices_u[elem_offset] = ornt[f] < 0; - PetscInt loc_u0 = Involute(indices_u0[ii*num_comp_u0[0]]); + PetscInt loc_u0 = Involute(indices_u0[ii * num_comp_u0[0]]); restr_indices_u0[elem_offset] = loc_u0; // Set orientation orient_indices_u0[elem_offset] = ornt[f] < 0; elem_offset++; } } - PetscCall( DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, - &num_indices, &indices, NULL, NULL) ); - PetscCall( DMPlexRestoreClosureIndices(dm_u0, section_u0, section_u0, p, - PETSC_TRUE, - &num_indices_u0, &indices_u0, NULL, NULL) ); - PetscCall( DMPlexRestoreClosureIndices(dm_p0, section_p0, section_p0, p, - PETSC_TRUE, - &num_indices_p0, &indices_p0, NULL, NULL) ); + PetscCall(DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL)); + PetscCall(DMPlexRestoreClosureIndices(dm_u0, section_u0, section_u0, p, PETSC_TRUE, &num_indices_u0, &indices_u0, NULL, NULL)); + PetscCall(DMPlexRestoreClosureIndices(dm_p0, section_p0, section_p0, p, PETSC_TRUE, &num_indices_p0, &indices_p0, NULL, NULL)); } - //if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) - // SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, - // "ElemRestriction of size (%" PetscInt_FMT ", %" PetscInt_FMT" ) - // initialized %" PetscInt_FMT " nodes", num_elem, - // dim*PetscPowInt(P, dim),elem_offset); + // if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) + // SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, + // "ElemRestriction of size (%" PetscInt_FMT ", %" PetscInt_FMT" ) + // initialized %" PetscInt_FMT " nodes", num_elem, + // dim*PetscPowInt(P, dim),elem_offset); - PetscCall( DMGetLocalVector(dm, &U_loc) ); - PetscCall( VecGetLocalSize(U_loc, &num_dof) ); - PetscCall( DMRestoreLocalVector(dm, &U_loc) ); + PetscCall(DMGetLocalVector(dm, &U_loc)); + PetscCall(VecGetLocalSize(U_loc, &num_dof)); + PetscCall(DMRestoreLocalVector(dm, &U_loc)); // dof per element in Hdiv is dim*P^dim, for linear element P=2 - CeedElemRestrictionCreateOriented(ceed, num_elem, dim*PetscPowInt(P, dim), - 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, - restr_indices_u, orient_indices_u, - elem_restr_u); - CeedElemRestrictionCreate(ceed, num_elem, 1, - 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, - restr_indices_p, elem_restr_p); - PetscCall( DMGetLocalVector(dm_u0, &U_loc) ); - PetscCall( VecGetLocalSize(U_loc, &num_dof_u0) ); - PetscCall( DMRestoreLocalVector(dm_u0, &U_loc) ); + CeedElemRestrictionCreateOriented(ceed, num_elem, dim * PetscPowInt(P, dim), 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_u, + orient_indices_u, elem_restr_u); + CeedElemRestrictionCreate(ceed, num_elem, 1, 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_p, elem_restr_p); + PetscCall(DMGetLocalVector(dm_u0, &U_loc)); + PetscCall(VecGetLocalSize(U_loc, &num_dof_u0)); + PetscCall(DMRestoreLocalVector(dm_u0, &U_loc)); // dof per element in Hdiv is dim*P^dim, for linear element P=2 - CeedElemRestrictionCreateOriented(ceed, num_elem, dim*PetscPowInt(P, dim), - 1, 1, num_dof_u0, CEED_MEM_HOST, CEED_COPY_VALUES, - restr_indices_u0, orient_indices_u0, - elem_restr_u0); - PetscCall( DMGetLocalVector(dm_p0, &U_loc) ); - PetscCall( VecGetLocalSize(U_loc, &num_dof_p0) ); - PetscCall( DMRestoreLocalVector(dm_p0, &U_loc) ); - CeedElemRestrictionCreate(ceed, num_elem, 1, - 1, 1, num_dof_p0, CEED_MEM_HOST, CEED_COPY_VALUES, - restr_indices_p0, elem_restr_p0); - PetscCall( PetscFree(restr_indices_p) ); - PetscCall( PetscFree(restr_indices_u) ); - PetscCall( PetscFree(orient_indices_u) ); - PetscCall( PetscFree(restr_indices_u0) ); - PetscCall( PetscFree(orient_indices_u0) ); - PetscCall( PetscFree(restr_indices_p0) ); + CeedElemRestrictionCreateOriented(ceed, num_elem, dim * PetscPowInt(P, dim), 1, 1, num_dof_u0, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_u0, + orient_indices_u0, elem_restr_u0); + PetscCall(DMGetLocalVector(dm_p0, &U_loc)); + PetscCall(VecGetLocalSize(U_loc, &num_dof_p0)); + PetscCall(DMRestoreLocalVector(dm_p0, &U_loc)); + CeedElemRestrictionCreate(ceed, num_elem, 1, 1, 1, num_dof_p0, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_p0, elem_restr_p0); + PetscCall(PetscFree(restr_indices_p)); + PetscCall(PetscFree(restr_indices_u)); + PetscCall(PetscFree(orient_indices_u)); + PetscCall(PetscFree(restr_indices_u0)); + PetscCall(PetscFree(orient_indices_u0)); + PetscCall(PetscFree(restr_indices_p0)); PetscFunctionReturn(0); }; // ----------------------------------------------------------------------------- // Set up libCEED on the fine grid for a given degree // ----------------------------------------------------------------------------- -PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, - Ceed ceed, AppCtx app_ctx, - ProblemData problem_data, - CeedData ceed_data) { - CeedInt P = app_ctx->degree + 1; +PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, Ceed ceed, AppCtx app_ctx, ProblemData problem_data, CeedData ceed_data) { + CeedInt P = app_ctx->degree + 1; // Number of quadratures in 1D, q_extra is set in cl-options.c - CeedInt Q = P + 1 + app_ctx->q_extra; - CeedInt dim, num_comp_x, num_comp_u, num_comp_p; - DM dm_coord; - Vec coords; - PetscInt c_start, c_end, num_elem; + CeedInt Q = P + 1 + app_ctx->q_extra; + CeedInt dim, num_comp_x, num_comp_u, num_comp_p; + DM dm_coord; + Vec coords; + PetscInt c_start, c_end, num_elem; const PetscScalar *coordArray; - CeedQFunction qf_true, qf_residual, qf_jacobian, qf_error; - CeedOperator op_true, op_residual, op_jacobian, op_error; + CeedQFunction qf_true, qf_residual, qf_jacobian, qf_error; + CeedOperator op_true, op_residual, op_jacobian, op_error; PetscFunctionBeginUser; // --------------------------------------------------------------------------- // libCEED bases:Hdiv basis_u and Lagrange basis_x // --------------------------------------------------------------------------- - dim = problem_data->dim; + dim = problem_data->dim; num_comp_x = dim; - num_comp_u = 1; // one vector dof - num_comp_p = 1; // one scalar dof + num_comp_u = 1; // one vector dof + num_comp_p = 1; // one scalar dof // Number of quadratures per element - CeedInt num_qpts = PetscPowInt(Q, dim); + CeedInt num_qpts = PetscPowInt(Q, dim); // Pressure and velocity dof per element - CeedInt P_p = 1, P_u = dim*PetscPowInt(P, dim); - CeedScalar q_ref[dim*num_qpts], q_weights[num_qpts]; - CeedScalar div[P_u*num_qpts], interp_u[dim*P_u*num_qpts], - interp_p[P_p*num_qpts], *grad=NULL; + CeedInt P_p = 1, P_u = dim * PetscPowInt(P, dim); + CeedScalar q_ref[dim * num_qpts], q_weights[num_qpts]; + CeedScalar div[P_u * num_qpts], interp_u[dim * P_u * num_qpts], interp_p[P_p * num_qpts], *grad = NULL; if (dim == 2) { - HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, - problem_data->quadrature_mode); - CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, - interp_u, div, q_ref, q_weights, &ceed_data->basis_u); + HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, problem_data->quadrature_mode); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, interp_u, div, q_ref, q_weights, &ceed_data->basis_u); L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); - CeedBasisCreateH1(ceed, CEED_TOPOLOGY_QUAD, num_comp_p, 1, num_qpts, interp_p, - grad, q_ref,q_weights, &ceed_data->basis_p); - HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, - CEED_GAUSS_LOBATTO); - CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, - interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); + CeedBasisCreateH1(ceed, CEED_TOPOLOGY_QUAD, num_comp_p, 1, num_qpts, interp_p, grad, q_ref, q_weights, &ceed_data->basis_p); + HdivBasisQuad(Q, q_ref, q_weights, interp_u, div, CEED_GAUSS_LOBATTO); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); } else { HdivBasisHex(Q, q_ref, q_weights, interp_u, div, problem_data->quadrature_mode); - CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, - interp_u, div, q_ref, q_weights, &ceed_data->basis_u); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, interp_u, div, q_ref, q_weights, &ceed_data->basis_u); L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); - CeedBasisCreateH1(ceed, CEED_TOPOLOGY_HEX, num_comp_p, 1, num_qpts, interp_p, - grad, q_ref,q_weights, &ceed_data->basis_p); + CeedBasisCreateH1(ceed, CEED_TOPOLOGY_HEX, num_comp_p, 1, num_qpts, interp_p, grad, q_ref, q_weights, &ceed_data->basis_p); HdivBasisHex(Q, q_ref, q_weights, interp_u, div, CEED_GAUSS_LOBATTO); - CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, - interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); + CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, interp_u, div, q_ref, q_weights, &ceed_data->basis_u_face); } - CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, - problem_data->quadrature_mode, &ceed_data->basis_x); + CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, problem_data->quadrature_mode, &ceed_data->basis_x); // --------------------------------------------------------------------------- // libCEED restrictions // --------------------------------------------------------------------------- - PetscCall( DMGetCoordinateDM(dm, &dm_coord) ); - PetscCall( DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL) ); - CeedInt height = 0; // 0 means no boundary conditions - DMLabel domain_label = 0; - PetscInt value = 0; + PetscCall(DMGetCoordinateDM(dm, &dm_coord)); + PetscCall(DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL)); + CeedInt height = 0; // 0 means no boundary conditions + DMLabel domain_label = 0; + PetscInt value = 0; // -- Coordinate restriction - PetscCall( CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, - value, &ceed_data->elem_restr_x) ); + PetscCall(CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, value, &ceed_data->elem_restr_x)); // -- Solution restriction - PetscCall( CreateRestrictionFromPlexOriented(ceed, dm, dm_u0, dm_p0, P, - &ceed_data->elem_restr_u, &ceed_data->elem_restr_p, - &ceed_data->elem_restr_u0, &ceed_data->elem_restr_p0) ); + PetscCall(CreateRestrictionFromPlexOriented(ceed, dm, dm_u0, dm_p0, P, &ceed_data->elem_restr_u, &ceed_data->elem_restr_p, + &ceed_data->elem_restr_u0, &ceed_data->elem_restr_p0)); // -- Geometric ceed_data restriction - PetscCall( DMPlexGetHeightStratum(dm, 0, &c_start, &c_end) ); - num_elem = c_end - c_start; + PetscCall(DMPlexGetHeightStratum(dm, 0, &c_start, &c_end)); + num_elem = c_end - c_start; ceed_data->num_elem = num_elem; - CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, (dim+1), - (dim+1)*num_elem*num_qpts, - CEED_STRIDES_BACKEND, &ceed_data->elem_restr_U_i); - CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, 1, - 1*num_elem*num_qpts, - CEED_STRIDES_BACKEND, &ceed_data->elem_restr_p_i); + CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, (dim + 1), (dim + 1) * num_elem * num_qpts, CEED_STRIDES_BACKEND, + &ceed_data->elem_restr_U_i); + CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, 1, 1 * num_elem * num_qpts, CEED_STRIDES_BACKEND, &ceed_data->elem_restr_p_i); // --------------------------------------------------------------------------- // Element coordinates // --------------------------------------------------------------------------- - PetscCall( DMGetCoordinatesLocal(dm, &coords) ); - PetscCall( VecGetArrayRead(coords, &coordArray) ); - CeedElemRestrictionCreateVector(ceed_data->elem_restr_x, &ceed_data->x_coord, - NULL); - CeedVectorSetArray(ceed_data->x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, - (PetscScalar *)coordArray); - PetscCall( VecRestoreArrayRead(coords, &coordArray) ); + PetscCall(DMGetCoordinatesLocal(dm, &coords)); + PetscCall(VecGetArrayRead(coords, &coordArray)); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_x, &ceed_data->x_coord, NULL); + CeedVectorSetArray(ceed_data->x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, (PetscScalar *)coordArray); + PetscCall(VecRestoreArrayRead(coords, &coordArray)); // --------------------------------------------------------------------------- // Setup true solution for [p,u] // --------------------------------------------------------------------------- CeedVector true_vec, true_force; - CeedVectorCreate(ceed, num_elem*num_qpts*(dim+1), &true_vec); - CeedVectorCreate(ceed, num_elem*num_qpts*1, &true_force); + CeedVectorCreate(ceed, num_elem * num_qpts * (dim + 1), &true_vec); + CeedVectorCreate(ceed, num_elem * num_qpts * 1, &true_force); // Create the q-function that sets up the RHS and true solution - CeedQFunctionCreateInterior(ceed, 1, problem_data->true_solution, - problem_data->true_solution_loc, &qf_true); + CeedQFunctionCreateInterior(ceed, 1, problem_data->true_solution, problem_data->true_solution_loc, &qf_true); CeedQFunctionSetContext(qf_true, problem_data->true_qfunction_ctx); CeedQFunctionContextDestroy(&problem_data->true_qfunction_ctx); CeedQFunctionAddInput(qf_true, "x", num_comp_x, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_true, "true force", 1, CEED_EVAL_NONE); - CeedQFunctionAddOutput(qf_true, "true solution", dim+1, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_true, "true solution", dim + 1, CEED_EVAL_NONE); // Create the operator that builds the RHS and true solution - CeedOperatorCreate(ceed, qf_true, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_true); + CeedOperatorCreate(ceed, qf_true, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_true); if (problem_data->has_ts) { double final_time = app_ctx->t_final; - CeedOperatorContextGetFieldLabel(op_true, "final_time", - &app_ctx->ctx_residual_ut->final_time_label); - CeedOperatorContextSetDouble(op_true, - app_ctx->ctx_residual_ut->final_time_label, &final_time); + CeedOperatorContextGetFieldLabel(op_true, "final_time", &app_ctx->ctx_residual_ut->final_time_label); + CeedOperatorContextSetDouble(op_true, app_ctx->ctx_residual_ut->final_time_label, &final_time); } - CeedOperatorSetField(op_true, "x", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_true, "true force", ceed_data->elem_restr_p_i, - CEED_BASIS_COLLOCATED, true_force); - CeedOperatorSetField(op_true, "true solution", ceed_data->elem_restr_U_i, - CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_true, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_true, "true force", ceed_data->elem_restr_p_i, CEED_BASIS_COLLOCATED, true_force); + CeedOperatorSetField(op_true, "true solution", ceed_data->elem_restr_U_i, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); // Setup true solution - CeedOperatorApply(op_true, ceed_data->x_coord, true_vec, - CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(op_true, ceed_data->x_coord, true_vec, CEED_REQUEST_IMMEDIATE); // --------------------------------------------------------------------------- // Setup initial conditions @@ -385,25 +328,19 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, CeedQFunction qf_rhs_u0; CeedOperator op_rhs_u0; // Create the q-function that sets up the RHS - CeedQFunctionCreateInterior(ceed, 1, problem_data->rhs_u0, - problem_data->rhs_u0_loc, &qf_rhs_u0); + CeedQFunctionCreateInterior(ceed, 1, problem_data->rhs_u0, problem_data->rhs_u0_loc, &qf_rhs_u0); CeedQFunctionSetContext(qf_rhs_u0, problem_data->rhs_u0_qfunction_ctx); CeedQFunctionContextDestroy(&problem_data->rhs_u0_qfunction_ctx); CeedQFunctionAddInput(qf_rhs_u0, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddInput(qf_rhs_u0, "x", num_comp_x, CEED_EVAL_INTERP); - CeedQFunctionAddInput(qf_rhs_u0, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_rhs_u0, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddOutput(qf_rhs_u0, "rhs_u0", dim, CEED_EVAL_INTERP); // Create the operator that builds the RHS - CeedOperatorCreate(ceed, qf_rhs_u0, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_rhs_u0); - CeedOperatorSetField(op_rhs_u0, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_rhs_u0, "x", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_rhs_u0, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_rhs_u0, "rhs_u0", ceed_data->elem_restr_u0, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_rhs_u0, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_rhs_u0); + CeedOperatorSetField(op_rhs_u0, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_rhs_u0, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_u0, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_u0, "rhs_u0", ceed_data->elem_restr_u0, ceed_data->basis_u, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in setup-ts.c ceed_data->qf_rhs_u0 = qf_rhs_u0; @@ -413,54 +350,40 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, // --------------------------------------------------------------------------- CeedQFunction qf_ics_u; CeedOperator op_ics_u; - CeedQFunctionCreateInterior(ceed, 1, problem_data->ics_u, - problem_data->ics_u_loc, &qf_ics_u); + CeedQFunctionCreateInterior(ceed, 1, problem_data->ics_u, problem_data->ics_u_loc, &qf_ics_u); CeedQFunctionAddInput(qf_ics_u, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_ics_u, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_ics_u, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_ics_u, "u", dim, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_ics_u, "v", dim, CEED_EVAL_INTERP); // Create the operator that builds the initial conditions - CeedOperatorCreate(ceed, qf_ics_u, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_ics_u); - CeedOperatorSetField(op_ics_u, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_ics_u, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_ics_u, "u", ceed_data->elem_restr_u0, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_ics_u, "v", ceed_data->elem_restr_u0, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_ics_u, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_ics_u); + CeedOperatorSetField(op_ics_u, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_ics_u, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_ics_u, "u", ceed_data->elem_restr_u0, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_ics_u, "v", ceed_data->elem_restr_u0, ceed_data->basis_u, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in setup-ts.c ceed_data->qf_ics_u = qf_ics_u; ceed_data->op_ics_u = op_ics_u; // -- Operator action variables: we use them in setup-ts.c - CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, &ceed_data->u0_ceed, - NULL); - CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, &ceed_data->v0_ceed, - NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, &ceed_data->u0_ceed, NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, &ceed_data->v0_ceed, NULL); // --------------------------------------------------------------------------- // Setup RHS for p field // --------------------------------------------------------------------------- CeedQFunction qf_rhs_p0; CeedOperator op_rhs_p0; // Create the q-function that sets up the RHS - CeedQFunctionCreateInterior(ceed, 1, problem_data->rhs_p0, - problem_data->rhs_p0_loc, &qf_rhs_p0); + CeedQFunctionCreateInterior(ceed, 1, problem_data->rhs_p0, problem_data->rhs_p0_loc, &qf_rhs_p0); CeedQFunctionAddInput(qf_rhs_p0, "weight", 1, CEED_EVAL_WEIGHT); CeedQFunctionAddInput(qf_rhs_p0, "x", num_comp_x, CEED_EVAL_INTERP); - CeedQFunctionAddInput(qf_rhs_p0, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_rhs_p0, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddOutput(qf_rhs_p0, "rhs_p0", 1, CEED_EVAL_INTERP); // Create the operator that builds the RHS - CeedOperatorCreate(ceed, qf_rhs_p0, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_rhs_p0); - CeedOperatorSetField(op_rhs_p0, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_rhs_p0, "x", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_rhs_p0, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_rhs_p0, "rhs_p0", ceed_data->elem_restr_p0, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_rhs_p0, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_rhs_p0); + CeedOperatorSetField(op_rhs_p0, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_rhs_p0, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_p0, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_p0, "rhs_p0", ceed_data->elem_restr_p0, ceed_data->basis_p, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in setup-ts.c ceed_data->qf_rhs_p0 = qf_rhs_p0; @@ -470,55 +393,43 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, // --------------------------------------------------------------------------- CeedQFunction qf_ics_p; CeedOperator op_ics_p; - CeedQFunctionCreateInterior(ceed, 1, problem_data->ics_p, - problem_data->ics_p_loc, &qf_ics_p); + CeedQFunctionCreateInterior(ceed, 1, problem_data->ics_p, problem_data->ics_p_loc, &qf_ics_p); CeedQFunctionAddInput(qf_ics_p, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_ics_p, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_ics_p, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_ics_p, "p", 1, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_ics_p, "q", 1, CEED_EVAL_INTERP); // Create the operator that builds the initial conditions - CeedOperatorCreate(ceed, qf_ics_p, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_ics_p); - CeedOperatorSetField(op_ics_p, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_ics_p, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_ics_p, "p", ceed_data->elem_restr_p0, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_ics_p, "q", ceed_data->elem_restr_p0, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_ics_p, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_ics_p); + CeedOperatorSetField(op_ics_p, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_ics_p, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_ics_p, "p", ceed_data->elem_restr_p0, ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_ics_p, "q", ceed_data->elem_restr_p0, ceed_data->basis_p, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in setup-ts.c ceed_data->qf_ics_p = qf_ics_p; ceed_data->op_ics_p = op_ics_p; // -- Operator action variables: we use them in setup-ts.c - CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, &ceed_data->p0_ceed, - NULL); - CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, &ceed_data->q0_ceed, - NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, &ceed_data->p0_ceed, NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, &ceed_data->q0_ceed, NULL); } // --------------------------------------------------------------------------- // Persistent libCEED vectors // --------------------------------------------------------------------------- // -- Operator action variables: we use them in setup-solvers.c/setup-ts.c - CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->x_ceed, - NULL); - CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->y_ceed, - NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->x_ceed, NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->y_ceed, NULL); // -- Operator action variables: we use them in setup-ts.c - CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->x_t_ceed, - NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->x_t_ceed, NULL); // Local residual evaluator // --------------------------------------------------------------------------- // Create the QFunction and Operator that computes the residual of the PDE. // --------------------------------------------------------------------------- // -- QFunction - CeedQFunctionCreateInterior(ceed, 1, problem_data->residual, - problem_data->residual_loc, &qf_residual); + CeedQFunctionCreateInterior(ceed, 1, problem_data->residual, problem_data->residual_loc, &qf_residual); CeedQFunctionSetContext(qf_residual, problem_data->residual_qfunction_ctx); CeedQFunctionContextDestroy(&problem_data->residual_qfunction_ctx); CeedQFunctionAddInput(qf_residual, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_residual, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_residual, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_residual, "u", dim, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_residual, "div_u", 1, CEED_EVAL_DIV); CeedQFunctionAddInput(qf_residual, "p", 1, CEED_EVAL_INTERP); @@ -532,41 +443,28 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, CeedQFunctionAddOutput(qf_residual, "q", 1, CEED_EVAL_INTERP); // -- Operator - CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_residual); + CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_residual); if (problem_data->has_ts) { - //double t = app_ctx->ctx_residual_ut->t; - CeedOperatorContextGetFieldLabel(op_residual, "time", - &app_ctx->ctx_residual_ut->solution_time_label); - //CeedOperatorContextGetFieldLabel(op_residual, "time_step", - // &app_ctx->ctx_residual_ut->timestep_label); - //CeedOperatorContextSetDouble(op_residual, - // app_ctx->ctx_residual_ut->solution_time_label, &t); + // double t = app_ctx->ctx_residual_ut->t; + CeedOperatorContextGetFieldLabel(op_residual, "time", &app_ctx->ctx_residual_ut->solution_time_label); + // CeedOperatorContextGetFieldLabel(op_residual, "time_step", + // &app_ctx->ctx_residual_ut->timestep_label); + // CeedOperatorContextSetDouble(op_residual, + // app_ctx->ctx_residual_ut->solution_time_label, &t); } - CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_residual, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_residual, "u", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_residual, "div_u", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_residual, "p", ceed_data->elem_restr_p, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_residual, "true force", ceed_data->elem_restr_p_i, - CEED_BASIS_COLLOCATED, true_force); - CeedOperatorSetField(op_residual, "x", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_residual, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_residual, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_residual, "u", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "div_u", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "p", ceed_data->elem_restr_p, ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "true force", ceed_data->elem_restr_p_i, CEED_BASIS_COLLOCATED, true_force); + CeedOperatorSetField(op_residual, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); if (problem_data->has_ts) { - CeedOperatorSetField(op_residual, "p_t", ceed_data->elem_restr_p, - ceed_data->basis_p, ceed_data->x_t_ceed); + CeedOperatorSetField(op_residual, "p_t", ceed_data->elem_restr_p, ceed_data->basis_p, ceed_data->x_t_ceed); } - CeedOperatorSetField(op_residual, "v", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_residual, "div_v", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_residual, "q", ceed_data->elem_restr_p, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "v", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "div_v", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_residual, "q", ceed_data->elem_restr_p, ceed_data->basis_p, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in matops.c ceed_data->qf_residual = qf_residual; ceed_data->op_residual = op_residual; @@ -574,53 +472,42 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, // --------------------------------------------------------------------------- // Add Pressure boundary condition. See setup-boundary.c // --------------------------------------------------------------------------- - //DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm); + // DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm); // Local jacobian evaluator // --------------------------------------------------------------------------- // Create the QFunction and Operator that computes the jacobian of the PDE. // --------------------------------------------------------------------------- // -- QFunction - CeedQFunctionCreateInterior(ceed, 1, problem_data->jacobian, - problem_data->jacobian_loc, &qf_jacobian); + CeedQFunctionCreateInterior(ceed, 1, problem_data->jacobian, problem_data->jacobian_loc, &qf_jacobian); CeedQFunctionSetContext(qf_jacobian, problem_data->jacobian_qfunction_ctx); CeedQFunctionContextDestroy(&problem_data->jacobian_qfunction_ctx); CeedQFunctionAddInput(qf_jacobian, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_jacobian, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_jacobian, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_jacobian, "du", dim, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_jacobian, "div_du", 1, CEED_EVAL_DIV); CeedQFunctionAddInput(qf_jacobian, "dp", 1, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_jacobian, "x", num_comp_x, CEED_EVAL_INTERP); - //CeedQFunctionAddInput(qf_jacobian, "u", dim, CEED_EVAL_INTERP); - //CeedQFunctionAddInput(qf_jacobian, "p", 1, CEED_EVAL_INTERP); + // CeedQFunctionAddInput(qf_jacobian, "u", dim, CEED_EVAL_INTERP); + // CeedQFunctionAddInput(qf_jacobian, "p", 1, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_jacobian, "dv", dim, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_jacobian, "div_dv", 1, CEED_EVAL_DIV); CeedQFunctionAddOutput(qf_jacobian, "dq", 1, CEED_EVAL_INTERP); // -- Operator - CeedOperatorCreate(ceed, qf_jacobian, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_jacobian); - CeedOperatorSetField(op_jacobian, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_jacobian, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_jacobian, "du", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "div_du", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "dp", ceed_data->elem_restr_p, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "x", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - //CeedOperatorSetField(op_jacobian, "u", ceed_data->elem_restr_u, - // ceed_data->basis_u, CEED_VECTOR_ACTIVE); - //CeedOperatorSetField(op_jacobian, "p", ceed_data->elem_restr_p, - // ceed_data->basis_p, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "dv", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "div_dv", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_jacobian, "dq", ceed_data->elem_restr_p, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_jacobian, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_jacobian); + CeedOperatorSetField(op_jacobian, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_jacobian, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_jacobian, "du", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "div_du", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dp", ceed_data->elem_restr_p, ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + // CeedOperatorSetField(op_jacobian, "u", ceed_data->elem_restr_u, + // ceed_data->basis_u, CEED_VECTOR_ACTIVE); + // CeedOperatorSetField(op_jacobian, "p", ceed_data->elem_restr_p, + // ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dv", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "div_dv", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_jacobian, "dq", ceed_data->elem_restr_p, ceed_data->basis_p, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in matops.c ceed_data->qf_jacobian = qf_jacobian; ceed_data->op_jacobian = op_jacobian; @@ -629,65 +516,49 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, // Setup Error Qfunction // --------------------------------------------------------------------------- // Create the q-function that sets up the error - CeedQFunctionCreateInterior(ceed, 1, problem_data->error, - problem_data->error_loc, &qf_error); + CeedQFunctionCreateInterior(ceed, 1, problem_data->error, problem_data->error_loc, &qf_error); CeedQFunctionSetContext(qf_error, problem_data->error_qfunction_ctx); CeedQFunctionContextDestroy(&problem_data->error_qfunction_ctx); CeedQFunctionAddInput(qf_error, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_error, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_error, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_error, "u", dim, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_error, "p", 1, CEED_EVAL_INTERP); - CeedQFunctionAddInput(qf_error, "true solution", dim+1, CEED_EVAL_NONE); - CeedQFunctionAddOutput(qf_error, "error", dim+1, CEED_EVAL_NONE); + CeedQFunctionAddInput(qf_error, "true solution", dim + 1, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_error, "error", dim + 1, CEED_EVAL_NONE); // Create the operator that builds the error - CeedOperatorCreate(ceed, qf_error, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_error); - CeedOperatorSetField(op_error, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_error, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_error, "u", ceed_data->elem_restr_u, - ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_error, "p", ceed_data->elem_restr_p, - ceed_data->basis_p, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_error, "true solution", ceed_data->elem_restr_U_i, - CEED_BASIS_COLLOCATED, true_vec); - CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_U_i, - CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_error, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_error); + CeedOperatorSetField(op_error, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_error, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_error, "u", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_error, "p", ceed_data->elem_restr_p, ceed_data->basis_p, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_error, "true solution", ceed_data->elem_restr_U_i, CEED_BASIS_COLLOCATED, true_vec); + CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_U_i, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in matops.c ceed_data->qf_error = qf_error; ceed_data->op_error = op_error; if (app_ctx->view_solution) { // -- Post processing - PetscCall( CreateRestrictionFromPlex(ceed, dm_H1, height, domain_label, - value, &ceed_data->elem_restr_u_H1) ); + PetscCall(CreateRestrictionFromPlex(ceed, dm_H1, height, domain_label, value, &ceed_data->elem_restr_u_H1)); // --------------------------------------------------------------------------- // Setup RHS for post processing // --------------------------------------------------------------------------- // -- Operator action variables: we use them in post-processing.c - CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->u_ceed, - NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->u_ceed, NULL); CeedQFunction qf_rhs_H1; CeedOperator op_rhs_H1; // Create the q-function that sets up the RHS - CeedQFunctionCreateInterior(ceed, 1, problem_data->post_rhs, - problem_data->post_rhs_loc, &qf_rhs_H1); + CeedQFunctionCreateInterior(ceed, 1, problem_data->post_rhs, problem_data->post_rhs_loc, &qf_rhs_H1); CeedQFunctionAddInput(qf_rhs_H1, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_rhs_H1, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_rhs_H1, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_rhs_H1, "u_post", dim, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_rhs_H1, "rhs_post", dim, CEED_EVAL_INTERP); // Create the operator that builds the RHS - CeedOperatorCreate(ceed, qf_rhs_H1, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_rhs_H1); - CeedOperatorSetField(op_rhs_H1, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_rhs_H1, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_rhs_H1, "u_post", ceed_data->elem_restr_u, - ceed_data->basis_u, ceed_data->u_ceed); - CeedOperatorSetField(op_rhs_H1, "rhs_post", ceed_data->elem_restr_u_H1, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_rhs_H1, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_rhs_H1); + CeedOperatorSetField(op_rhs_H1, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_rhs_H1, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_rhs_H1, "u_post", ceed_data->elem_restr_u, ceed_data->basis_u, ceed_data->u_ceed); + CeedOperatorSetField(op_rhs_H1, "rhs_post", ceed_data->elem_restr_u_H1, ceed_data->basis_x, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in post-processing.c ceed_data->qf_rhs_H1 = qf_rhs_H1; ceed_data->op_rhs_H1 = op_rhs_H1; @@ -696,33 +567,23 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, // --------------------------------------------------------------------------- CeedQFunction qf_post_mass; CeedOperator op_post_mass; - CeedQFunctionCreateInterior(ceed, 1, problem_data->post_mass, - problem_data->post_mass_loc, &qf_post_mass); + CeedQFunctionCreateInterior(ceed, 1, problem_data->post_mass, problem_data->post_mass_loc, &qf_post_mass); CeedQFunctionAddInput(qf_post_mass, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddInput(qf_post_mass, "dx", dim*dim, CEED_EVAL_GRAD); + CeedQFunctionAddInput(qf_post_mass, "dx", dim * dim, CEED_EVAL_GRAD); CeedQFunctionAddInput(qf_post_mass, "u", dim, CEED_EVAL_INTERP); CeedQFunctionAddOutput(qf_post_mass, "v", dim, CEED_EVAL_INTERP); // Create the operator that builds the initial conditions - CeedOperatorCreate(ceed, qf_post_mass, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, - &op_post_mass); - CeedOperatorSetField(op_post_mass, "weight", CEED_ELEMRESTRICTION_NONE, - ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_post_mass, "dx", ceed_data->elem_restr_x, - ceed_data->basis_x, ceed_data->x_coord); - CeedOperatorSetField(op_post_mass, "u", ceed_data->elem_restr_u_H1, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_post_mass, "v", ceed_data->elem_restr_u_H1, - ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorCreate(ceed, qf_post_mass, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_post_mass); + CeedOperatorSetField(op_post_mass, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); + CeedOperatorSetField(op_post_mass, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); + CeedOperatorSetField(op_post_mass, "u", ceed_data->elem_restr_u_H1, ceed_data->basis_x, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_post_mass, "v", ceed_data->elem_restr_u_H1, ceed_data->basis_x, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in post-processing.c ceed_data->qf_post_mass = qf_post_mass; ceed_data->op_post_mass = op_post_mass; // -- Operator action variables: we use them in post-processing.c - CeedElemRestrictionCreateVector(ceed_data->elem_restr_u_H1, - &ceed_data->up_ceed, - NULL); - CeedElemRestrictionCreateVector(ceed_data->elem_restr_u_H1, - &ceed_data->vp_ceed, - NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u_H1, &ceed_data->up_ceed, NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u_H1, &ceed_data->vp_ceed, NULL); } // -- Cleanup CeedVectorDestroy(&true_vec); diff --git a/examples/Hdiv-mixed/src/setup-matops.c b/examples/Hdiv-mixed/src/setup-matops.c index 619b189f20..d0ea7662aa 100644 --- a/examples/Hdiv-mixed/src/setup-matops.c +++ b/examples/Hdiv-mixed/src/setup-matops.c @@ -1,59 +1,49 @@ #include "../include/setup-matops.h" + #include "../include/setup-libceed.h" // ----------------------------------------------------------------------------- // Apply the local action of a libCEED operator and store result in PETSc vector // i.e. compute A X = Y // ----------------------------------------------------------------------------- -PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, - OperatorApplyContext op_apply_ctx) { +PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx) { PetscFunctionBeginUser; // Zero target vector - PetscCall( VecZeroEntries(Y) ); + PetscCall(VecZeroEntries(Y)); // Sum into target vector - PetscCall( ApplyAddLocalCeedOp(X, Y, op_apply_ctx) ); + PetscCall(ApplyAddLocalCeedOp(X, Y, op_apply_ctx)); PetscFunctionReturn(0); } -PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, - OperatorApplyContext op_apply_ctx) { - +PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx) { PetscScalar *x, *y; PetscMemType x_mem_type, y_mem_type; PetscFunctionBeginUser; // Global-to-local - PetscCall( DMGlobalToLocal(op_apply_ctx->dm, X, INSERT_VALUES, - op_apply_ctx->X_loc) ); + PetscCall(DMGlobalToLocal(op_apply_ctx->dm, X, INSERT_VALUES, op_apply_ctx->X_loc)); // Setup libCEED vectors - PetscCall( VecGetArrayReadAndMemType(op_apply_ctx->X_loc, - (const PetscScalar **)&x, - &x_mem_type) ); - PetscCall( VecGetArrayAndMemType(op_apply_ctx->Y_loc, &y, &y_mem_type) ); - CeedVectorSetArray(op_apply_ctx->x_ceed, MemTypeP2C(x_mem_type), - CEED_USE_POINTER, x); - CeedVectorSetArray(op_apply_ctx->y_ceed, MemTypeP2C(y_mem_type), - CEED_USE_POINTER, y); + PetscCall(VecGetArrayReadAndMemType(op_apply_ctx->X_loc, (const PetscScalar **)&x, &x_mem_type)); + PetscCall(VecGetArrayAndMemType(op_apply_ctx->Y_loc, &y, &y_mem_type)); + CeedVectorSetArray(op_apply_ctx->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, x); + CeedVectorSetArray(op_apply_ctx->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); // Apply libCEED operator - CeedOperatorApply(op_apply_ctx->op_apply, op_apply_ctx->x_ceed, - op_apply_ctx->y_ceed, CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(op_apply_ctx->op_apply, op_apply_ctx->x_ceed, op_apply_ctx->y_ceed, CEED_REQUEST_IMMEDIATE); // Restore PETSc vectors CeedVectorTakeArray(op_apply_ctx->x_ceed, MemTypeP2C(x_mem_type), NULL); CeedVectorTakeArray(op_apply_ctx->y_ceed, MemTypeP2C(y_mem_type), NULL); - PetscCall( VecRestoreArrayReadAndMemType(op_apply_ctx->X_loc, - (const PetscScalar **)&x) ); - PetscCall( VecRestoreArrayAndMemType(op_apply_ctx->Y_loc, &y) ); + PetscCall(VecRestoreArrayReadAndMemType(op_apply_ctx->X_loc, (const PetscScalar **)&x)); + PetscCall(VecRestoreArrayAndMemType(op_apply_ctx->Y_loc, &y)); // Local-to-global - PetscCall( DMLocalToGlobal(op_apply_ctx->dm, op_apply_ctx->Y_loc, ADD_VALUES, - Y) ); + PetscCall(DMLocalToGlobal(op_apply_ctx->dm, op_apply_ctx->Y_loc, ADD_VALUES, Y)); PetscFunctionReturn(0); }; diff --git a/examples/Hdiv-mixed/src/setup-solvers.c b/examples/Hdiv-mixed/src/setup-solvers.c index 3ea1f631f1..3fef27c924 100644 --- a/examples/Hdiv-mixed/src/setup-solvers.c +++ b/examples/Hdiv-mixed/src/setup-solvers.c @@ -1,52 +1,49 @@ #include "../include/setup-solvers.h" -#include "../include/setup-matops.h" + #include "../include/setup-libceed.h" +#include "../include/setup-matops.h" #include "petscvec.h" // ----------------------------------------------------------------------------- // Setup operator context data // ----------------------------------------------------------------------------- -PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, - VecType vec_type, - OperatorApplyContext ctx_jacobian) { +PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_jacobian) { PetscFunctionBeginUser; ctx_jacobian->dm = dm; - PetscCall( DMCreateLocalVector(dm, &ctx_jacobian->X_loc) ); - PetscCall( VecDuplicate(ctx_jacobian->X_loc, &ctx_jacobian->Y_loc) ); - ctx_jacobian->x_ceed = ceed_data->x_ceed; - ctx_jacobian->y_ceed = ceed_data->y_ceed; - ctx_jacobian->ceed = ceed; + PetscCall(DMCreateLocalVector(dm, &ctx_jacobian->X_loc)); + PetscCall(VecDuplicate(ctx_jacobian->X_loc, &ctx_jacobian->Y_loc)); + ctx_jacobian->x_ceed = ceed_data->x_ceed; + ctx_jacobian->y_ceed = ceed_data->y_ceed; + ctx_jacobian->ceed = ceed; ctx_jacobian->op_apply = ceed_data->op_jacobian; ctx_jacobian->vec_type = vec_type; PetscFunctionReturn(0); } -PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, - OperatorApplyContext ctx_residual) { +PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual) { PetscFunctionBeginUser; ctx_residual->dm = dm; - PetscCall( DMCreateLocalVector(dm, &ctx_residual->X_loc) ); - PetscCall( VecDuplicate(ctx_residual->X_loc, &ctx_residual->Y_loc) ); - ctx_residual->x_ceed = ceed_data->x_ceed; - ctx_residual->y_ceed = ceed_data->y_ceed; - ctx_residual->ceed = ceed; + PetscCall(DMCreateLocalVector(dm, &ctx_residual->X_loc)); + PetscCall(VecDuplicate(ctx_residual->X_loc, &ctx_residual->Y_loc)); + ctx_residual->x_ceed = ceed_data->x_ceed; + ctx_residual->y_ceed = ceed_data->y_ceed; + ctx_residual->ceed = ceed; ctx_residual->op_apply = ceed_data->op_residual; PetscFunctionReturn(0); } -PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, - OperatorApplyContext ctx_error) { +PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_error) { PetscFunctionBeginUser; ctx_error->dm = dm; - PetscCall( DMCreateLocalVector(dm, &ctx_error->X_loc) ); - PetscCall( VecDuplicate(ctx_error->X_loc, &ctx_error->Y_loc) ); - ctx_error->x_ceed = ceed_data->x_ceed; - ctx_error->y_ceed = ceed_data->y_ceed; - ctx_error->ceed = ceed; + PetscCall(DMCreateLocalVector(dm, &ctx_error->X_loc)); + PetscCall(VecDuplicate(ctx_error->X_loc, &ctx_error->Y_loc)); + ctx_error->x_ceed = ceed_data->x_ceed; + ctx_error->y_ceed = ceed_data->y_ceed; + ctx_error->ceed = ceed; ctx_error->op_apply = ceed_data->op_error; PetscFunctionReturn(0); @@ -60,10 +57,10 @@ PetscErrorCode ApplyMatOp(Mat A, Vec X, Vec Y) { PetscFunctionBeginUser; - PetscCall( MatShellGetContext(A, &op_apply_ctx) ); + PetscCall(MatShellGetContext(A, &op_apply_ctx)); // libCEED for local action of residual evaluator - PetscCall( ApplyLocalCeedOp(X, Y, op_apply_ctx) ); + PetscCall(ApplyLocalCeedOp(X, Y, op_apply_ctx)); PetscFunctionReturn(0); }; @@ -77,12 +74,12 @@ PetscErrorCode SNESFormResidual(SNES snes, Vec X, Vec Y, void *ctx_residual) { PetscFunctionBeginUser; // Use computed BCs - //PetscCall( DMPlexInsertBoundaryValues(ctx->dm, PETSC_TRUE, + // PetscCall( DMPlexInsertBoundaryValues(ctx->dm, PETSC_TRUE, // ctx->X_loc, // 1.0, NULL, NULL, NULL) ); // libCEED for local action of residual evaluator - PetscCall( ApplyLocalCeedOp(X, Y, ctx) ); + PetscCall(ApplyLocalCeedOp(X, Y, ctx)); PetscFunctionReturn(0); }; @@ -90,8 +87,7 @@ PetscErrorCode SNESFormResidual(SNES snes, Vec X, Vec Y, void *ctx_residual) { // ----------------------------------------------------------------------------- // Jacobian setup // ----------------------------------------------------------------------------- -PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, - void *ctx_jacobian) { +PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, void *ctx_jacobian) { OperatorApplyContext ctx = (OperatorApplyContext)ctx_jacobian; PetscFunctionBeginUser; @@ -99,10 +95,9 @@ PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, PetscCall(DMCreateMatrix(ctx->dm, &A)); // Assemble matrix analytically PetscCount num_entries; - CeedInt *rows, *cols; + CeedInt *rows, *cols; CeedVector coo_values; - CeedOperatorLinearAssembleSymbolic(ctx->op_apply, &num_entries, &rows, - &cols); + CeedOperatorLinearAssembleSymbolic(ctx->op_apply, &num_entries, &rows, &cols); PetscCall(MatSetPreallocationCOO(A, num_entries, rows, cols)); free(rows); free(cols); @@ -113,16 +108,16 @@ PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, PetscCall(MatSetValuesCOO(A, values, ADD_VALUES)); CeedVectorRestoreArrayRead(coo_values, &values); MatView(A, PETSC_VIEWER_STDOUT_WORLD); - //CeedVectorView(coo_values, "%12.8f", stdout); + // CeedVectorView(coo_values, "%12.8f", stdout); CeedVectorDestroy(&coo_values); - PetscCall( MatDestroy(&A) ); + PetscCall(MatDestroy(&A)); // J_pre might be AIJ (e.g., when using coloring), so we need to assemble it - PetscCall( MatAssemblyBegin(J_pre, MAT_FINAL_ASSEMBLY) ); - PetscCall( MatAssemblyEnd(J_pre, MAT_FINAL_ASSEMBLY) ); + PetscCall(MatAssemblyBegin(J_pre, MAT_FINAL_ASSEMBLY)); + PetscCall(MatAssemblyEnd(J_pre, MAT_FINAL_ASSEMBLY)); if (J != J_pre) { - PetscCall( MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY) ); - PetscCall( MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY) ); + PetscCall(MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY)); + PetscCall(MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY)); } PetscFunctionReturn(0); }; @@ -130,57 +125,51 @@ PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, // --------------------------------------------------------------------------- // Setup Solver // --------------------------------------------------------------------------- -PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, - SNES snes, KSP ksp, Vec *U) { - - PetscInt U_l_size, U_g_size; +PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, SNES snes, KSP ksp, Vec *U) { + PetscInt U_l_size, U_g_size; PetscFunctionBeginUser; // Create global unknown solution U - PetscCall( VecGetSize(*U, &U_g_size) ); + PetscCall(VecGetSize(*U, &U_g_size)); // Local size for matShell - PetscCall( VecGetLocalSize(*U, &U_l_size) ); + PetscCall(VecGetLocalSize(*U, &U_l_size)); Vec R; - PetscCall( VecDuplicate(*U, &R) ); + PetscCall(VecDuplicate(*U, &R)); // --------------------------------------------------------------------------- // Setup SNES // --------------------------------------------------------------------------- // Operator Mat mat_jacobian; - PetscCall( SNESSetDM(snes, app_ctx->ctx_jacobian->dm) ); + PetscCall(SNESSetDM(snes, app_ctx->ctx_jacobian->dm)); // -- Form Action of Jacobian on delta_u - PetscCall( MatCreateShell(app_ctx->comm, U_l_size, U_l_size, U_g_size, - U_g_size, app_ctx->ctx_jacobian, &mat_jacobian) ); - PetscCall( MatShellSetOperation(mat_jacobian, MATOP_MULT, - (void (*)(void))ApplyMatOp) ); - PetscCall( MatShellSetVecType(mat_jacobian, app_ctx->ctx_jacobian->vec_type) ); + PetscCall(MatCreateShell(app_ctx->comm, U_l_size, U_l_size, U_g_size, U_g_size, app_ctx->ctx_jacobian, &mat_jacobian)); + PetscCall(MatShellSetOperation(mat_jacobian, MATOP_MULT, (void (*)(void))ApplyMatOp)); + PetscCall(MatShellSetVecType(mat_jacobian, app_ctx->ctx_jacobian->vec_type)); // Set SNES residual evaluation function - PetscCall( SNESSetFunction(snes, R, SNESFormResidual, - app_ctx->ctx_residual) ); + PetscCall(SNESSetFunction(snes, R, SNESFormResidual, app_ctx->ctx_residual)); // -- SNES Jacobian - PetscCall( SNESSetJacobian(snes, mat_jacobian, mat_jacobian, - SNESFormJacobian, app_ctx->ctx_jacobian) ); + PetscCall(SNESSetJacobian(snes, mat_jacobian, mat_jacobian, SNESFormJacobian, app_ctx->ctx_jacobian)); // Setup KSP - PetscCall( KSPSetFromOptions(ksp) ); + PetscCall(KSPSetFromOptions(ksp)); // Default to critical-point (CP) line search (related to Wolfe's curvature condition) SNESLineSearch line_search; - PetscCall( SNESGetLineSearch(snes, &line_search) ); - PetscCall( SNESLineSearchSetType(line_search, SNESLINESEARCHCP) ); - PetscCall( SNESSetFromOptions(snes) ); + PetscCall(SNESGetLineSearch(snes, &line_search)); + PetscCall(SNESLineSearchSetType(line_search, SNESLINESEARCHCP)); + PetscCall(SNESSetFromOptions(snes)); // Solve - PetscCall( VecSet(*U, 0.0)); - PetscCall( SNESSolve(snes, NULL, *U)); + PetscCall(VecSet(*U, 0.0)); + PetscCall(SNESSolve(snes, NULL, *U)); // Free PETSc objects - PetscCall( MatDestroy(&mat_jacobian) ); - PetscCall( VecDestroy(&R) ); + PetscCall(MatDestroy(&mat_jacobian)); + PetscCall(VecDestroy(&R)); PetscFunctionReturn(0); }; @@ -188,65 +177,57 @@ PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, // ----------------------------------------------------------------------------- // This function calculates the L2 error in the final solution // ----------------------------------------------------------------------------- -PetscErrorCode ComputeL2Error(CeedData ceed_data, AppCtx app_ctx, Vec U, - CeedScalar *l2_error_u, CeedScalar *l2_error_p) { +PetscErrorCode ComputeL2Error(CeedData ceed_data, AppCtx app_ctx, Vec U, CeedScalar *l2_error_u, CeedScalar *l2_error_p) { PetscScalar *x; PetscMemType mem_type; - CeedVector collocated_error; + CeedVector collocated_error; PetscFunctionBeginUser; CeedInt dim, num_elem, num_qpts; - PetscCall( DMGetDimension(app_ctx->ctx_error->dm, &dim) ); + PetscCall(DMGetDimension(app_ctx->ctx_error->dm, &dim)); CeedBasisGetNumQuadraturePoints(ceed_data->basis_u, &num_qpts); num_elem = ceed_data->num_elem; - CeedVectorCreate(app_ctx->ctx_error->ceed, num_elem*num_qpts*(dim+1), - &collocated_error); + CeedVectorCreate(app_ctx->ctx_error->ceed, num_elem * num_qpts * (dim + 1), &collocated_error); // Global-to-local - PetscCall( DMGlobalToLocal(app_ctx->ctx_error->dm, U, INSERT_VALUES, - app_ctx->ctx_error->X_loc) ); + PetscCall(DMGlobalToLocal(app_ctx->ctx_error->dm, U, INSERT_VALUES, app_ctx->ctx_error->X_loc)); // Setup CEED vector - PetscCall( VecGetArrayAndMemType(app_ctx->ctx_error->X_loc, &x, &mem_type) ); - CeedVectorSetArray(app_ctx->ctx_error->x_ceed, MemTypeP2C(mem_type), - CEED_USE_POINTER, - x); + PetscCall(VecGetArrayAndMemType(app_ctx->ctx_error->X_loc, &x, &mem_type)); + CeedVectorSetArray(app_ctx->ctx_error->x_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, x); // Apply CEED operator - CeedOperatorApply(app_ctx->ctx_error->op_apply, app_ctx->ctx_error->x_ceed, - collocated_error, - CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(app_ctx->ctx_error->op_apply, app_ctx->ctx_error->x_ceed, collocated_error, CEED_REQUEST_IMMEDIATE); // Restore PETSc vector CeedVectorTakeArray(app_ctx->ctx_error->x_ceed, MemTypeP2C(mem_type), NULL); - PetscCall( VecRestoreArrayReadAndMemType(app_ctx->ctx_error->X_loc, - (const PetscScalar **)&x) ); + PetscCall(VecRestoreArrayReadAndMemType(app_ctx->ctx_error->X_loc, (const PetscScalar **)&x)); // Compute L2 error for each field - CeedInt cent_qpts = num_qpts / 2; - CeedVector collocated_error_u, collocated_error_p; - const CeedScalar *E_U; // to store total error - CeedInt length_u, length_p; + CeedInt cent_qpts = num_qpts / 2; + CeedVector collocated_error_u, collocated_error_p; + const CeedScalar *E_U; // to store total error + CeedInt length_u, length_p; length_p = num_elem; - length_u = num_elem*num_qpts*dim; + length_u = num_elem * num_qpts * dim; CeedScalar e_u[length_u], e_p[length_p]; CeedVectorCreate(app_ctx->ctx_error->ceed, length_p, &collocated_error_p); CeedVectorCreate(app_ctx->ctx_error->ceed, length_u, &collocated_error_u); // E_U is ordered as [p_0,u_0/.../p_n,u_n] for 0 to n elements // For each element p_0 size is num_qpts, and u_0 is dim*num_qpts CeedVectorGetArrayRead(collocated_error, CEED_MEM_HOST, &E_U); - for (CeedInt n=0; n < num_elem; n++) { - for (CeedInt i=0; i < 1; i++) { - CeedInt j = i + n*1; - CeedInt k = cent_qpts + n*num_qpts*(dim+1); - e_p[j] = E_U[k]; + for (CeedInt n = 0; n < num_elem; n++) { + for (CeedInt i = 0; i < 1; i++) { + CeedInt j = i + n * 1; + CeedInt k = cent_qpts + n * num_qpts * (dim + 1); + e_p[j] = E_U[k]; } } - for (CeedInt n=0; n < num_elem; n++) { - for (CeedInt i=0; i < dim*num_qpts; i++) { - CeedInt j = i + n*num_qpts*dim; - CeedInt k = num_qpts + i + n*num_qpts*(dim+1); - e_u[j] = E_U[k]; + for (CeedInt n = 0; n < num_elem; n++) { + for (CeedInt i = 0; i < dim * num_qpts; i++) { + CeedInt j = i + n * num_qpts * dim; + CeedInt k = num_qpts + i + n * num_qpts * (dim + 1); + e_u[j] = E_U[k]; } } diff --git a/examples/Hdiv-mixed/src/setup-ts.c b/examples/Hdiv-mixed/src/setup-ts.c index 24aaec705d..e93bca2495 100644 --- a/examples/Hdiv-mixed/src/setup-ts.c +++ b/examples/Hdiv-mixed/src/setup-ts.c @@ -1,29 +1,28 @@ #include "../include/setup-ts.h" -#include "../include/setup-matops.h" + +#include + +#include "../include/post-processing.h" #include "../include/setup-libceed.h" +#include "../include/setup-matops.h" #include "../include/setup-solvers.h" -#include "../include/post-processing.h" #include "ceed/ceed.h" #include "petscerror.h" #include "petscsystypes.h" -#include - // ----------------------------------------------------------------------------- // Setup operator context data for initial condition, u field // ----------------------------------------------------------------------------- -PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm_u0, Ceed ceed, - CeedData ceed_data, - OperatorApplyContext ctx_initial_u0) { +PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm_u0, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_initial_u0) { PetscFunctionBeginUser; ctx_initial_u0->comm = comm; - ctx_initial_u0->dm = dm_u0; - PetscCall( DMCreateLocalVector(dm_u0, &ctx_initial_u0->X_loc) ); - PetscCall( VecDuplicate(ctx_initial_u0->X_loc, &ctx_initial_u0->Y_loc) ); - ctx_initial_u0->x_ceed = ceed_data->u0_ceed; - ctx_initial_u0->y_ceed = ceed_data->v0_ceed; - ctx_initial_u0->ceed = ceed; + ctx_initial_u0->dm = dm_u0; + PetscCall(DMCreateLocalVector(dm_u0, &ctx_initial_u0->X_loc)); + PetscCall(VecDuplicate(ctx_initial_u0->X_loc, &ctx_initial_u0->Y_loc)); + ctx_initial_u0->x_ceed = ceed_data->u0_ceed; + ctx_initial_u0->y_ceed = ceed_data->v0_ceed; + ctx_initial_u0->ceed = ceed; ctx_initial_u0->op_apply = ceed_data->op_ics_u; PetscFunctionReturn(0); @@ -32,18 +31,16 @@ PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm_u0, Ceed ceed, // ----------------------------------------------------------------------------- // Setup operator context data for initial condition, p field // ----------------------------------------------------------------------------- -PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm_p0, Ceed ceed, - CeedData ceed_data, - OperatorApplyContext ctx_initial_p0) { +PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm_p0, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_initial_p0) { PetscFunctionBeginUser; ctx_initial_p0->comm = comm; - ctx_initial_p0->dm = dm_p0; - PetscCall( DMCreateLocalVector(dm_p0, &ctx_initial_p0->X_loc) ); - PetscCall( VecDuplicate(ctx_initial_p0->X_loc, &ctx_initial_p0->Y_loc) ); - ctx_initial_p0->x_ceed = ceed_data->p0_ceed; - ctx_initial_p0->y_ceed = ceed_data->q0_ceed; - ctx_initial_p0->ceed = ceed; + ctx_initial_p0->dm = dm_p0; + PetscCall(DMCreateLocalVector(dm_p0, &ctx_initial_p0->X_loc)); + PetscCall(VecDuplicate(ctx_initial_p0->X_loc, &ctx_initial_p0->Y_loc)); + ctx_initial_p0->x_ceed = ceed_data->p0_ceed; + ctx_initial_p0->y_ceed = ceed_data->q0_ceed; + ctx_initial_p0->ceed = ceed; ctx_initial_p0->op_apply = ceed_data->op_ics_p; PetscFunctionReturn(0); @@ -52,19 +49,18 @@ PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm_p0, Ceed ceed, // ----------------------------------------------------------------------------- // Setup operator context data for Residual of Richard problem // ----------------------------------------------------------------------------- -PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, - CeedData ceed_data, OperatorApplyContext ctx_residual_ut) { +PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual_ut) { PetscFunctionBeginUser; ctx_residual_ut->comm = comm; - ctx_residual_ut->dm = dm; - PetscCall( DMCreateLocalVector(dm, &ctx_residual_ut->X_loc) ); - PetscCall( VecDuplicate(ctx_residual_ut->X_loc, &ctx_residual_ut->Y_loc) ); - PetscCall( VecDuplicate(ctx_residual_ut->X_loc, &ctx_residual_ut->X_t_loc) ); - ctx_residual_ut->x_ceed = ceed_data->x_ceed; + ctx_residual_ut->dm = dm; + PetscCall(DMCreateLocalVector(dm, &ctx_residual_ut->X_loc)); + PetscCall(VecDuplicate(ctx_residual_ut->X_loc, &ctx_residual_ut->Y_loc)); + PetscCall(VecDuplicate(ctx_residual_ut->X_loc, &ctx_residual_ut->X_t_loc)); + ctx_residual_ut->x_ceed = ceed_data->x_ceed; ctx_residual_ut->x_t_ceed = ceed_data->x_t_ceed; - ctx_residual_ut->y_ceed = ceed_data->y_ceed; - ctx_residual_ut->ceed = ceed; + ctx_residual_ut->y_ceed = ceed_data->y_ceed; + ctx_residual_ut->ceed = ceed; ctx_residual_ut->op_apply = ceed_data->op_residual; PetscFunctionReturn(0); @@ -73,318 +69,279 @@ PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, // ----------------------------------------------------------------------------- // Create global initial conditions vector // ----------------------------------------------------------------------------- -PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, - VecType vec_type, Vec U) { - +PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, VecType vec_type, Vec U) { PetscFunctionBeginUser; // ---------------------------------------------- // Create local rhs for u field // ---------------------------------------------- - Vec rhs_u_loc; + Vec rhs_u_loc; PetscScalar *ru; PetscMemType ru_mem_type; - PetscCall( DMCreateLocalVector(app_ctx->ctx_initial_u0->dm, &rhs_u_loc) ); - PetscCall( VecZeroEntries(rhs_u_loc) ); - PetscCall( VecGetArrayAndMemType(rhs_u_loc, &ru, &ru_mem_type) ); - CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, - &ceed_data->rhs_u0_ceed, - NULL); - CeedVectorSetArray(ceed_data->rhs_u0_ceed, MemTypeP2C(ru_mem_type), - CEED_USE_POINTER, ru); + PetscCall(DMCreateLocalVector(app_ctx->ctx_initial_u0->dm, &rhs_u_loc)); + PetscCall(VecZeroEntries(rhs_u_loc)); + PetscCall(VecGetArrayAndMemType(rhs_u_loc, &ru, &ru_mem_type)); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u0, &ceed_data->rhs_u0_ceed, NULL); + CeedVectorSetArray(ceed_data->rhs_u0_ceed, MemTypeP2C(ru_mem_type), CEED_USE_POINTER, ru); // Apply operator to create RHS for u field - CeedOperatorApply(ceed_data->op_rhs_u0, ceed_data->x_coord, - ceed_data->rhs_u0_ceed, CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(ceed_data->op_rhs_u0, ceed_data->x_coord, ceed_data->rhs_u0_ceed, CEED_REQUEST_IMMEDIATE); // ---------------------------------------------- // Create global rhs for u field // ---------------------------------------------- Vec rhs_u0; CeedVectorTakeArray(ceed_data->rhs_u0_ceed, MemTypeP2C(ru_mem_type), NULL); - PetscCall( VecRestoreArrayAndMemType(rhs_u_loc, &ru) ); - PetscCall( DMCreateGlobalVector(app_ctx->ctx_initial_u0->dm, &rhs_u0) ); - PetscCall( VecZeroEntries(rhs_u0) ); - PetscCall( DMLocalToGlobal(app_ctx->ctx_initial_u0->dm, rhs_u_loc, ADD_VALUES, - rhs_u0) ); + PetscCall(VecRestoreArrayAndMemType(rhs_u_loc, &ru)); + PetscCall(DMCreateGlobalVector(app_ctx->ctx_initial_u0->dm, &rhs_u0)); + PetscCall(VecZeroEntries(rhs_u0)); + PetscCall(DMLocalToGlobal(app_ctx->ctx_initial_u0->dm, rhs_u_loc, ADD_VALUES, rhs_u0)); // ---------------------------------------------- // Solve for U0, M*U0 = rhs_u0 // ---------------------------------------------- Vec U0; - PetscCall( DMCreateGlobalVector(app_ctx->ctx_initial_u0->dm, &U0) ); - PetscCall( VecZeroEntries(U0) ); + PetscCall(DMCreateGlobalVector(app_ctx->ctx_initial_u0->dm, &U0)); + PetscCall(VecZeroEntries(U0)); PetscInt U0_g_size, U0_l_size; - PetscCall( VecGetSize(U0, &U0_g_size) ); + PetscCall(VecGetSize(U0, &U0_g_size)); // Local size for matShell - PetscCall( VecGetLocalSize(U0, &U0_l_size) ); + PetscCall(VecGetLocalSize(U0, &U0_l_size)); // Operator Mat mat_ksp_u0; // -- Form Action of residual on u - PetscCall( MatCreateShell(app_ctx->comm, U0_l_size, U0_l_size, U0_g_size, - U0_g_size, app_ctx->ctx_initial_u0, &mat_ksp_u0) ); - PetscCall( MatShellSetOperation(mat_ksp_u0, MATOP_MULT, - (void (*)(void))ApplyMatOp) ); - PetscCall( MatShellSetVecType(mat_ksp_u0, vec_type) ); + PetscCall(MatCreateShell(app_ctx->comm, U0_l_size, U0_l_size, U0_g_size, U0_g_size, app_ctx->ctx_initial_u0, &mat_ksp_u0)); + PetscCall(MatShellSetOperation(mat_ksp_u0, MATOP_MULT, (void (*)(void))ApplyMatOp)); + PetscCall(MatShellSetVecType(mat_ksp_u0, vec_type)); KSP ksp_u0; - PetscCall( KSPCreate(app_ctx->comm, &ksp_u0) ); - PetscCall( KSPSetOperators(ksp_u0, mat_ksp_u0, mat_ksp_u0) ); - PetscCall( KSPSetFromOptions(ksp_u0) ); - PetscCall( KSPSetUp(ksp_u0) ); - PetscCall( KSPSolve(ksp_u0, rhs_u0, U0) ); + PetscCall(KSPCreate(app_ctx->comm, &ksp_u0)); + PetscCall(KSPSetOperators(ksp_u0, mat_ksp_u0, mat_ksp_u0)); + PetscCall(KSPSetFromOptions(ksp_u0)); + PetscCall(KSPSetUp(ksp_u0)); + PetscCall(KSPSolve(ksp_u0, rhs_u0, U0)); // ---------------------------------------------- // Create local rhs for p field // ---------------------------------------------- - Vec rhs_p_loc; + Vec rhs_p_loc; PetscScalar *rp; PetscMemType rp_mem_type; - PetscCall( DMCreateLocalVector(app_ctx->ctx_initial_p0->dm, &rhs_p_loc) ); - PetscCall( VecZeroEntries(rhs_p_loc) ); - PetscCall( VecGetArrayAndMemType(rhs_p_loc, &rp, &rp_mem_type) ); - CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, - &ceed_data->rhs_p0_ceed, - NULL); - CeedVectorSetArray(ceed_data->rhs_p0_ceed, MemTypeP2C(rp_mem_type), - CEED_USE_POINTER, rp); + PetscCall(DMCreateLocalVector(app_ctx->ctx_initial_p0->dm, &rhs_p_loc)); + PetscCall(VecZeroEntries(rhs_p_loc)); + PetscCall(VecGetArrayAndMemType(rhs_p_loc, &rp, &rp_mem_type)); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_p0, &ceed_data->rhs_p0_ceed, NULL); + CeedVectorSetArray(ceed_data->rhs_p0_ceed, MemTypeP2C(rp_mem_type), CEED_USE_POINTER, rp); // Apply operator to create RHS for p field - CeedOperatorApply(ceed_data->op_rhs_p0, ceed_data->x_coord, - ceed_data->rhs_p0_ceed, CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(ceed_data->op_rhs_p0, ceed_data->x_coord, ceed_data->rhs_p0_ceed, CEED_REQUEST_IMMEDIATE); // ---------------------------------------------- // Create global rhs for p field // ---------------------------------------------- Vec rhs_p0; CeedVectorTakeArray(ceed_data->rhs_p0_ceed, MemTypeP2C(rp_mem_type), NULL); - PetscCall( VecRestoreArrayAndMemType(rhs_p_loc, &rp) ); - PetscCall( DMCreateGlobalVector(app_ctx->ctx_initial_p0->dm, &rhs_p0) ); - PetscCall( VecZeroEntries(rhs_p0) ); - PetscCall( DMLocalToGlobal(app_ctx->ctx_initial_p0->dm, rhs_p_loc, ADD_VALUES, - rhs_p0) ); + PetscCall(VecRestoreArrayAndMemType(rhs_p_loc, &rp)); + PetscCall(DMCreateGlobalVector(app_ctx->ctx_initial_p0->dm, &rhs_p0)); + PetscCall(VecZeroEntries(rhs_p0)); + PetscCall(DMLocalToGlobal(app_ctx->ctx_initial_p0->dm, rhs_p_loc, ADD_VALUES, rhs_p0)); // ---------------------------------------------- // Solve for P0, M*P0 = rhs_p0 // ---------------------------------------------- Vec P0; - PetscCall( DMCreateGlobalVector(app_ctx->ctx_initial_p0->dm, &P0) ); - PetscCall( VecZeroEntries(P0) ); + PetscCall(DMCreateGlobalVector(app_ctx->ctx_initial_p0->dm, &P0)); + PetscCall(VecZeroEntries(P0)); PetscInt P0_g_size, P0_l_size; - PetscCall( VecGetSize(P0, &P0_g_size) ); + PetscCall(VecGetSize(P0, &P0_g_size)); // Local size for matShell - PetscCall( VecGetLocalSize(P0, &P0_l_size) ); + PetscCall(VecGetLocalSize(P0, &P0_l_size)); // Operator Mat mat_ksp_p0; // -- Form Action of residual on u - PetscCall( MatCreateShell(app_ctx->comm, P0_l_size, P0_l_size, P0_g_size, - P0_g_size, app_ctx->ctx_initial_p0, &mat_ksp_p0) ); - PetscCall( MatShellSetOperation(mat_ksp_p0, MATOP_MULT, - (void (*)(void))ApplyMatOp) ); - PetscCall( MatShellSetVecType(mat_ksp_p0, vec_type) ); + PetscCall(MatCreateShell(app_ctx->comm, P0_l_size, P0_l_size, P0_g_size, P0_g_size, app_ctx->ctx_initial_p0, &mat_ksp_p0)); + PetscCall(MatShellSetOperation(mat_ksp_p0, MATOP_MULT, (void (*)(void))ApplyMatOp)); + PetscCall(MatShellSetVecType(mat_ksp_p0, vec_type)); KSP ksp_p0; - PetscCall( KSPCreate(app_ctx->comm, &ksp_p0) ); - PetscCall( KSPSetOperators(ksp_p0, mat_ksp_p0, mat_ksp_p0) ); - PetscCall( KSPSetFromOptions(ksp_p0) ); - PetscCall( KSPSetUp(ksp_p0) ); - PetscCall( KSPSolve(ksp_p0, rhs_p0, P0) ); + PetscCall(KSPCreate(app_ctx->comm, &ksp_p0)); + PetscCall(KSPSetOperators(ksp_p0, mat_ksp_p0, mat_ksp_p0)); + PetscCall(KSPSetFromOptions(ksp_p0)); + PetscCall(KSPSetUp(ksp_p0)); + PetscCall(KSPSolve(ksp_p0, rhs_p0, P0)); // ---------------------------------------------- // Create final initial conditions U // ---------------------------------------------- // Global-to-local for U0, P0 - PetscCall( DMGlobalToLocal(app_ctx->ctx_initial_u0->dm, U0, INSERT_VALUES, - app_ctx->ctx_initial_u0->X_loc) ); - PetscCall( DMGlobalToLocal(app_ctx->ctx_initial_p0->dm, P0, INSERT_VALUES, - app_ctx->ctx_initial_p0->X_loc) ); + PetscCall(DMGlobalToLocal(app_ctx->ctx_initial_u0->dm, U0, INSERT_VALUES, app_ctx->ctx_initial_u0->X_loc)); + PetscCall(DMGlobalToLocal(app_ctx->ctx_initial_p0->dm, P0, INSERT_VALUES, app_ctx->ctx_initial_p0->X_loc)); // Get array u0,p0 const PetscScalar *u0, *p0; - PetscCall( VecGetArrayRead(app_ctx->ctx_initial_u0->X_loc, &u0) ); - PetscCall( VecGetArrayRead(app_ctx->ctx_initial_p0->X_loc, &p0) ); + PetscCall(VecGetArrayRead(app_ctx->ctx_initial_u0->X_loc, &u0)); + PetscCall(VecGetArrayRead(app_ctx->ctx_initial_p0->X_loc, &p0)); // Get array of local vector U = [p,u] PetscScalar *u; - PetscInt U_l_size; - PetscCall( VecGetLocalSize(U, &U_l_size) ); - PetscCall( VecZeroEntries(app_ctx->ctx_residual_ut->X_loc) ); - PetscCall( VecGetArray(app_ctx->ctx_residual_ut->X_loc, &u) ); - for (PetscInt i = 0; inum_elem; i++) { + PetscInt U_l_size; + PetscCall(VecGetLocalSize(U, &U_l_size)); + PetscCall(VecZeroEntries(app_ctx->ctx_residual_ut->X_loc)); + PetscCall(VecGetArray(app_ctx->ctx_residual_ut->X_loc, &u)); + for (PetscInt i = 0; i < ceed_data->num_elem; i++) { u[i] = p0[i]; } - for (PetscInt i = ceed_data->num_elem; inum_elem]; + for (PetscInt i = ceed_data->num_elem; i < U_l_size; i++) { + u[i] = u0[i - ceed_data->num_elem]; } - PetscCall( VecRestoreArray(app_ctx->ctx_residual_ut->X_loc, &u) ); - PetscCall( VecRestoreArrayRead(app_ctx->ctx_initial_p0->X_loc, &p0) ); - PetscCall( VecRestoreArrayRead(app_ctx->ctx_initial_u0->X_loc, &u0) ); - PetscCall( DMLocalToGlobal(app_ctx->ctx_residual_ut->dm, - app_ctx->ctx_residual_ut->X_loc, - ADD_VALUES, U) ); + PetscCall(VecRestoreArray(app_ctx->ctx_residual_ut->X_loc, &u)); + PetscCall(VecRestoreArrayRead(app_ctx->ctx_initial_p0->X_loc, &p0)); + PetscCall(VecRestoreArrayRead(app_ctx->ctx_initial_u0->X_loc, &u0)); + PetscCall(DMLocalToGlobal(app_ctx->ctx_residual_ut->dm, app_ctx->ctx_residual_ut->X_loc, ADD_VALUES, U)); // Clean up - PetscCall( VecDestroy(&rhs_u_loc) ); - PetscCall( VecDestroy(&rhs_u0) ); - PetscCall( VecDestroy(&U0) ); - PetscCall( VecDestroy(&rhs_p_loc) ); - PetscCall( VecDestroy(&rhs_p0) ); - PetscCall( VecDestroy(&P0) ); - PetscCall( MatDestroy(&mat_ksp_p0) ); - PetscCall( MatDestroy(&mat_ksp_u0) ); - PetscCall( KSPDestroy(&ksp_p0) ); - PetscCall( KSPDestroy(&ksp_u0) ); + PetscCall(VecDestroy(&rhs_u_loc)); + PetscCall(VecDestroy(&rhs_u0)); + PetscCall(VecDestroy(&U0)); + PetscCall(VecDestroy(&rhs_p_loc)); + PetscCall(VecDestroy(&rhs_p0)); + PetscCall(VecDestroy(&P0)); + PetscCall(MatDestroy(&mat_ksp_p0)); + PetscCall(MatDestroy(&mat_ksp_u0)); + PetscCall(KSPDestroy(&ksp_p0)); + PetscCall(KSPDestroy(&ksp_u0)); PetscFunctionReturn(0); - } -PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, - void *ctx_residual_ut) { - OperatorApplyContext ctx = (OperatorApplyContext)ctx_residual_ut; - const PetscScalar *x, *x_t; - PetscScalar *y; - PetscMemType x_mem_type, x_t_mem_type, y_mem_type; +PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, void *ctx_residual_ut) { + OperatorApplyContext ctx = (OperatorApplyContext)ctx_residual_ut; + const PetscScalar *x, *x_t; + PetscScalar *y; + PetscMemType x_mem_type, x_t_mem_type, y_mem_type; PetscFunctionBeginUser; // Update time dependent data - if(ctx->t != time) { - CeedOperatorContextSetDouble(ctx->op_apply, - ctx->solution_time_label, &time); + if (ctx->t != time) { + CeedOperatorContextSetDouble(ctx->op_apply, ctx->solution_time_label, &time); ctx->t = time; } - //PetscScalar dt; - //PetscCall( TSGetTimeStep(ts, &dt) ); - //if (ctx->dt != dt) { - // CeedOperatorContextSetDouble(ctx->op_apply, - // ctx->timestep_label, &dt); - // ctx->dt = dt; - //} - // Global-to-local - PetscCall( DMGlobalToLocal(ctx->dm, X, INSERT_VALUES, ctx->X_loc) ); - PetscCall( DMGlobalToLocal(ctx->dm, X_t, INSERT_VALUES, ctx->X_t_loc) ); + // PetscScalar dt; + // PetscCall( TSGetTimeStep(ts, &dt) ); + // if (ctx->dt != dt) { + // CeedOperatorContextSetDouble(ctx->op_apply, + // ctx->timestep_label, &dt); + // ctx->dt = dt; + // } + // Global-to-local + PetscCall(DMGlobalToLocal(ctx->dm, X, INSERT_VALUES, ctx->X_loc)); + PetscCall(DMGlobalToLocal(ctx->dm, X_t, INSERT_VALUES, ctx->X_t_loc)); // Place PETSc vectors in CEED vectors - PetscCall( VecGetArrayReadAndMemType(ctx->X_loc, &x, &x_mem_type) ); - PetscCall( VecGetArrayReadAndMemType(ctx->X_t_loc, &x_t, &x_t_mem_type) ); - PetscCall( VecGetArrayAndMemType(ctx->Y_loc, &y, &y_mem_type) ); - CeedVectorSetArray(ctx->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, - (PetscScalar *)x); - CeedVectorSetArray(ctx->x_t_ceed, MemTypeP2C(x_t_mem_type), - CEED_USE_POINTER, (PetscScalar *)x_t); + PetscCall(VecGetArrayReadAndMemType(ctx->X_loc, &x, &x_mem_type)); + PetscCall(VecGetArrayReadAndMemType(ctx->X_t_loc, &x_t, &x_t_mem_type)); + PetscCall(VecGetArrayAndMemType(ctx->Y_loc, &y, &y_mem_type)); + CeedVectorSetArray(ctx->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, (PetscScalar *)x); + CeedVectorSetArray(ctx->x_t_ceed, MemTypeP2C(x_t_mem_type), CEED_USE_POINTER, (PetscScalar *)x_t); CeedVectorSetArray(ctx->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); // Apply CEED operator - CeedOperatorApply(ctx->op_apply, ctx->x_ceed, ctx->y_ceed, - CEED_REQUEST_IMMEDIATE); + CeedOperatorApply(ctx->op_apply, ctx->x_ceed, ctx->y_ceed, CEED_REQUEST_IMMEDIATE); // Restore vectors CeedVectorTakeArray(ctx->x_ceed, MemTypeP2C(x_mem_type), NULL); CeedVectorTakeArray(ctx->x_t_ceed, MemTypeP2C(x_t_mem_type), NULL); CeedVectorTakeArray(ctx->y_ceed, MemTypeP2C(y_mem_type), NULL); - PetscCall( VecRestoreArrayReadAndMemType(ctx->X_loc, &x) ); - PetscCall( VecRestoreArrayReadAndMemType(ctx->X_t_loc, &x_t) ); - PetscCall( VecRestoreArrayAndMemType(ctx->Y_loc, &y) ); + PetscCall(VecRestoreArrayReadAndMemType(ctx->X_loc, &x)); + PetscCall(VecRestoreArrayReadAndMemType(ctx->X_t_loc, &x_t)); + PetscCall(VecRestoreArrayAndMemType(ctx->Y_loc, &y)); // Local-to-Global - PetscCall( VecZeroEntries(Y) ); - PetscCall( DMLocalToGlobal(ctx->dm, ctx->Y_loc, ADD_VALUES, Y) ); + PetscCall(VecZeroEntries(Y)); + PetscCall(DMLocalToGlobal(ctx->dm, ctx->Y_loc, ADD_VALUES, Y)); PetscFunctionReturn(0); } -PetscErrorCode WriteOutput(Vec U, PetscInt steps, - PetscScalar time, AppCtx app_ctx) { - char output_filename[PETSC_MAX_PATH_LEN]; +PetscErrorCode WriteOutput(Vec U, PetscInt steps, PetscScalar time, AppCtx app_ctx) { + char output_filename[PETSC_MAX_PATH_LEN]; PetscViewer viewer_p, viewer_u; PetscMPIInt rank; PetscFunctionBeginUser; // Create output directory MPI_Comm_rank(app_ctx->comm, &rank); - if (!rank) {PetscCall( PetscMkdir(app_ctx->output_dir) );} + if (!rank) { + PetscCall(PetscMkdir(app_ctx->output_dir)); + } // Build file name - PetscCall( PetscSNPrintf(output_filename, sizeof output_filename, - "%s/richard_pressure-%03" PetscInt_FMT ".vtu", - app_ctx->output_dir, - steps) ); - PetscCall(PetscViewerVTKOpen(app_ctx->comm, output_filename, - FILE_MODE_WRITE, &viewer_p)); + PetscCall(PetscSNPrintf(output_filename, sizeof output_filename, "%s/richard_pressure-%03" PetscInt_FMT ".vtu", app_ctx->output_dir, steps)); + PetscCall(PetscViewerVTKOpen(app_ctx->comm, output_filename, FILE_MODE_WRITE, &viewer_p)); PetscCall(VecView(U, viewer_p)); PetscCall(PetscViewerDestroy(&viewer_p)); // Project velocity to H1 - Vec U_H1; // velocity in H1 space for post-processing - PetscCall( DMCreateGlobalVector(app_ctx->ctx_H1->dm, &U_H1) ); - PetscCall( ProjectVelocity(app_ctx, U, &U_H1) ); + Vec U_H1; // velocity in H1 space for post-processing + PetscCall(DMCreateGlobalVector(app_ctx->ctx_H1->dm, &U_H1)); + PetscCall(ProjectVelocity(app_ctx, U, &U_H1)); // Build file name - PetscCall( PetscSNPrintf(output_filename, sizeof output_filename, - "%s/richard_velocity-%03" PetscInt_FMT ".vtu", - app_ctx->output_dir, - steps) ); - PetscCall(PetscViewerVTKOpen(app_ctx->comm, output_filename, - FILE_MODE_WRITE, &viewer_u)); + PetscCall(PetscSNPrintf(output_filename, sizeof output_filename, "%s/richard_velocity-%03" PetscInt_FMT ".vtu", app_ctx->output_dir, steps)); + PetscCall(PetscViewerVTKOpen(app_ctx->comm, output_filename, FILE_MODE_WRITE, &viewer_u)); PetscCall(VecView(U_H1, viewer_u)); PetscCall(PetscViewerDestroy(&viewer_u)); - PetscCall( VecDestroy(&U_H1) ); + PetscCall(VecDestroy(&U_H1)); PetscFunctionReturn(0); } // User provided TS Monitor -PetscErrorCode TSMonitorRichard(TS ts, PetscInt steps, PetscReal time, - Vec U, void *ctx) { - AppCtx app_ctx = (AppCtx)ctx; +PetscErrorCode TSMonitorRichard(TS ts, PetscInt steps, PetscReal time, Vec U, void *ctx) { + AppCtx app_ctx = (AppCtx)ctx; PetscFunctionBeginUser; // Print every 'output_freq' steps - if (app_ctx->output_freq <= 0 - || steps % app_ctx->output_freq != 0) - PetscFunctionReturn(0); + if (app_ctx->output_freq <= 0 || steps % app_ctx->output_freq != 0) PetscFunctionReturn(0); - PetscCall( WriteOutput(U, steps, time, app_ctx) ); + PetscCall(WriteOutput(U, steps, time, app_ctx)); PetscFunctionReturn(0); } // TS: Create, setup, and solve -PetscErrorCode TSSolveRichard(CeedData ceed_data, AppCtx app_ctx, - TS ts, Vec *U) { - TSAdapt adapt; +PetscErrorCode TSSolveRichard(CeedData ceed_data, AppCtx app_ctx, TS ts, Vec *U) { + TSAdapt adapt; PetscFunctionBeginUser; - PetscCall( TSSetDM(ts, app_ctx->ctx_residual_ut->dm) ); - PetscCall( TSSetType(ts, TSBDF) ); - PetscCall( TSSetIFunction(ts, NULL, TSFormIResidual, - app_ctx->ctx_residual_ut) ); - - PetscCall( TSSetMaxTime(ts, app_ctx->t_final) ); - PetscCall( TSSetExactFinalTime(ts, TS_EXACTFINALTIME_STEPOVER) ); - PetscCall( TSSetTimeStep(ts, 1.e-2) ); - PetscCall( TSGetAdapt(ts, &adapt) ); - PetscCall( TSAdaptSetStepLimits(adapt, 1.e-12, 1.e2) ); - PetscCall( TSSetFromOptions(ts) ); + PetscCall(TSSetDM(ts, app_ctx->ctx_residual_ut->dm)); + PetscCall(TSSetType(ts, TSBDF)); + PetscCall(TSSetIFunction(ts, NULL, TSFormIResidual, app_ctx->ctx_residual_ut)); + + PetscCall(TSSetMaxTime(ts, app_ctx->t_final)); + PetscCall(TSSetExactFinalTime(ts, TS_EXACTFINALTIME_STEPOVER)); + PetscCall(TSSetTimeStep(ts, 1.e-2)); + PetscCall(TSGetAdapt(ts, &adapt)); + PetscCall(TSAdaptSetStepLimits(adapt, 1.e-12, 1.e2)); + PetscCall(TSSetFromOptions(ts)); app_ctx->ctx_residual_ut->t = -1.0; - //ceed_data->ctx_residual_ut->dt = -1.0; + // ceed_data->ctx_residual_ut->dt = -1.0; if (app_ctx->view_solution) { - PetscCall( TSMonitorSet(ts, TSMonitorRichard, app_ctx, NULL) ); + PetscCall(TSMonitorSet(ts, TSMonitorRichard, app_ctx, NULL)); } // Solve PetscScalar start_time; - PetscCall( TSGetTime(ts, &start_time) ); + PetscCall(TSGetTime(ts, &start_time)); PetscCall(TSSetTime(ts, start_time)); PetscCall(TSSetStepNumber(ts, 0)); - PetscCall( PetscBarrier((PetscObject) ts) ); - PetscCall( TSSolve(ts, *U) ); + PetscCall(PetscBarrier((PetscObject)ts)); + PetscCall(TSSolve(ts, *U)); - PetscScalar final_time; - PetscCall( TSGetSolveTime(ts, &final_time) ); + PetscScalar final_time; + PetscCall(TSGetSolveTime(ts, &final_time)); app_ctx->t_final = final_time; PetscFunctionReturn(0); } - // ----------------------------------------------------------------------------- diff --git a/tests/t330-basis.h b/tests/t330-basis.h index 435f8ef8a3..260cfb6ee8 100644 --- a/tests/t330-basis.h +++ b/tests/t330-basis.h @@ -58,7 +58,7 @@ static void BuildHdivQuadrilateral(CeedInt q, CeedScalar *q_ref, CeedScalar *q_w CeedScalar D[8] = {0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25}; // Loop over quadrature points CeedScalar Bx[8], By[8]; - CeedScalar x[2]; + CeedScalar X[2]; for (CeedInt i = 0; i < q; i++) { for (CeedInt j = 0; j < q; j++) { From fb2b95a18035cde52893c7c772fa649538ca987c Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Thu, 1 Dec 2022 11:41:59 -0700 Subject: [PATCH 11/15] some clean-up in Hdiv-mass folder --- examples/Hdiv-mass/Makefile | 2 +- examples/Hdiv-mass/basis/L2-P0.h | 58 ++++ .../Hdiv-mass/{conv_plot.py => conv_rate.py} | 28 +- examples/Hdiv-mass/conv_test.sh | 14 +- examples/Hdiv-mass/conv_test_result.csv | 10 - examples/Hdiv-mass/include/cl-options.h | 7 +- examples/Hdiv-mass/include/matops.h | 13 - examples/Hdiv-mass/include/petsc-macros.h | 17 -- examples/Hdiv-mass/include/post-processing.h | 12 + examples/Hdiv-mass/include/problems.h | 19 -- examples/Hdiv-mass/include/register-problem.h | 18 ++ examples/Hdiv-mass/include/setup-dm.h | 6 +- examples/Hdiv-mass/include/setup-fe.h | 20 ++ examples/Hdiv-mass/include/setup-libceed.h | 15 +- examples/Hdiv-mass/include/setup-matops.h | 13 + examples/Hdiv-mass/include/setup-solvers.h | 16 + examples/Hdiv-mass/include/structs.h | 78 ++--- examples/Hdiv-mass/main.c | 287 +++++------------- examples/Hdiv-mass/main.h | 7 +- examples/Hdiv-mass/problems/mass2d.c | 17 +- examples/Hdiv-mass/problems/mass3d.c | 19 +- .../Hdiv-mass/problems/register-problem.c | 33 ++ .../Hdiv-mass/qfunctions/poisson-error2d.h | 5 +- .../Hdiv-mass/qfunctions/poisson-error3d.h | 7 +- examples/Hdiv-mass/src/cl-options.c | 47 +-- examples/Hdiv-mass/src/matops.c | 106 ------- examples/Hdiv-mass/src/post-processing.c | 77 +++++ examples/Hdiv-mass/src/setup-dm.c | 89 ++---- examples/Hdiv-mass/src/setup-fe.c | 149 +++++++++ examples/Hdiv-mass/src/setup-libceed.c | 184 +++-------- examples/Hdiv-mass/src/setup-matops.c | 51 ++++ examples/Hdiv-mass/src/setup-solvers.c | 123 ++++++++ 32 files changed, 806 insertions(+), 741 deletions(-) create mode 100644 examples/Hdiv-mass/basis/L2-P0.h rename examples/Hdiv-mass/{conv_plot.py => conv_rate.py} (75%) delete mode 100644 examples/Hdiv-mass/conv_test_result.csv delete mode 100644 examples/Hdiv-mass/include/matops.h delete mode 100644 examples/Hdiv-mass/include/petsc-macros.h create mode 100644 examples/Hdiv-mass/include/post-processing.h delete mode 100644 examples/Hdiv-mass/include/problems.h create mode 100644 examples/Hdiv-mass/include/register-problem.h create mode 100644 examples/Hdiv-mass/include/setup-fe.h create mode 100644 examples/Hdiv-mass/include/setup-matops.h create mode 100644 examples/Hdiv-mass/include/setup-solvers.h create mode 100644 examples/Hdiv-mass/problems/register-problem.c delete mode 100644 examples/Hdiv-mass/src/matops.c create mode 100644 examples/Hdiv-mass/src/post-processing.c create mode 100644 examples/Hdiv-mass/src/setup-fe.c create mode 100644 examples/Hdiv-mass/src/setup-matops.c create mode 100644 examples/Hdiv-mass/src/setup-solvers.c diff --git a/examples/Hdiv-mass/Makefile b/examples/Hdiv-mass/Makefile index 6cfc950525..eecd42bcd2 100644 --- a/examples/Hdiv-mass/Makefile +++ b/examples/Hdiv-mass/Makefile @@ -69,7 +69,7 @@ print: $(PETSc.pc) $(ceed.pc) @true clean: - $(RM) -r $(OBJDIR) main *.vtu + $(RM) -r $(OBJDIR) main *.vtu *.csv $(PETSc.pc): $(if $(wildcard $@),,$(error \ diff --git a/examples/Hdiv-mass/basis/L2-P0.h b/examples/Hdiv-mass/basis/L2-P0.h new file mode 100644 index 0000000000..6149d4a34d --- /dev/null +++ b/examples/Hdiv-mass/basis/L2-P0.h @@ -0,0 +1,58 @@ +// Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. +// All Rights reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +// Build L2 constant basis + +static void L2BasisP0(CeedInt dim, CeedInt Q, CeedScalar *q_ref, CeedScalar *q_weights, CeedScalar *interp, CeedQuadMode quad_mode) { + // Get 1D quadrature on [-1,1] + CeedScalar q_ref_1d[Q], q_weight_1d[Q]; + switch (quad_mode) { + case CEED_GAUSS: + CeedGaussQuadrature(Q, q_ref_1d, q_weight_1d); + break; + case CEED_GAUSS_LOBATTO: + CeedLobattoQuadrature(Q, q_ref_1d, q_weight_1d); + break; + } + + // P0 L2 basis is just a constant + CeedScalar P0 = 1.0; + // Loop over quadrature points + if (dim == 2) { + for (CeedInt i = 0; i < Q; i++) { + for (CeedInt j = 0; j < Q; j++) { + CeedInt k1 = Q * i + j; + q_ref[k1] = q_ref_1d[j]; + q_ref[k1 + Q * Q] = q_ref_1d[i]; + q_weights[k1] = q_weight_1d[j] * q_weight_1d[i]; + interp[k1] = P0; + } + } + } else { + for (CeedInt k = 0; k < Q; k++) { + for (CeedInt i = 0; i < Q; i++) { + for (CeedInt j = 0; j < Q; j++) { + CeedInt k1 = Q * Q * k + Q * i + j; + q_ref[k1 + 0 * Q * Q] = q_ref_1d[j]; + q_ref[k1 + 1 * Q * Q] = q_ref_1d[i]; + q_ref[k1 + 2 * Q * Q] = q_ref_1d[k]; + q_weights[k1] = q_weight_1d[j] * q_weight_1d[i] * q_weight_1d[k]; + interp[k1] = P0; + } + } + } + } +} diff --git a/examples/Hdiv-mass/conv_plot.py b/examples/Hdiv-mass/conv_rate.py similarity index 75% rename from examples/Hdiv-mass/conv_plot.py rename to examples/Hdiv-mass/conv_rate.py index 6e9b9e623d..18aff73f3c 100644 --- a/examples/Hdiv-mass/conv_plot.py +++ b/examples/Hdiv-mass/conv_rate.py @@ -16,13 +16,16 @@ # software, applications, hardware, advanced system engineering and early # testbed platforms, in support of the nation's exascale computing imperative. +# After ./conv_test.sh you can get the table of convergence order by +# python conv_rate.py -f conv_test_result.csv + import pandas as pd import argparse from pylab import * from matplotlib import use -def plot(): +def convergence_rate(): # Define argparse for the input variables parser = argparse.ArgumentParser(description='Get input arguments') parser.add_argument('-f', @@ -40,23 +43,18 @@ def plot(): data = data.sort_values('run') E_u = data['error_u'] - #E_hdiv = data['error_hdiv'] h = 1/data['mesh_res'] - H2 = amin(E_u)* (h/amin(h))**2 # H = C h^2 + N = data['mesh_res'] + conv_u = [] + conv_u.append(0) - ax.loglog(h, E_u, 'o', color='black', label = 'Velocity') - #ax.loglog(h, E_hdiv, '*', color='red', label = 'Velocity in H(div)') - ax.loglog(h, H2, '--', color='black', label='O(h$^2$)') + for i in range(1,len(E_u)): + conv_u.append(log10(E_u[i]/E_u[i-1])/log10(h[i]/h[i-1])) - ax.legend(loc='upper left') - ax.set_xlabel('h') - ax.set_ylabel('L2 Error') - ax.set_title('Convergence by h Refinement') - #xlim(.06, .3) - fig.tight_layout() - plt.savefig('convrate_mass.png', bbox_inches='tight') - + result = {'Number of element/direction':N, 'convergence order of u':conv_u} + df = pd.DataFrame(result) + print(df) if __name__ == "__main__": - plot() + convergence_rate() diff --git a/examples/Hdiv-mass/conv_test.sh b/examples/Hdiv-mass/conv_test.sh index cdc2fddd74..3ba642a064 100755 --- a/examples/Hdiv-mass/conv_test.sh +++ b/examples/Hdiv-mass/conv_test.sh @@ -24,17 +24,17 @@ do d) dim=${OPTARG};; esac done -echo "Running convergence test in ${dim}D for mass problem"; +echo "Running convergence test in ${dim}D for Projection problem in H(div) space"; declare -A run_flags - run_flags[pc_type]=svd + #run_flags[pc_type]=svd if [[ $dim -eq 2 ]]; then run_flags[problem]=mass2d run_flags[dm_plex_dim]=$dim run_flags[dm_plex_box_faces]=2,2 run_flags[dm_plex_box_lower]=0,0 - run_flags[dm_plex_box_upper]=1,0.1 + run_flags[dm_plex_box_upper]=1,1 else run_flags[problem]=mass3d run_flags[dm_plex_dim]=$dim @@ -42,9 +42,9 @@ declare -A run_flags fi declare -A test_flags - test_flags[res_start]=2 - test_flags[res_stride]=1 - test_flags[res_end]=10 + test_flags[res_start]=4 + test_flags[res_stride]=2 + test_flags[res_end]=12 file_name=conv_test_result.csv @@ -64,7 +64,7 @@ for ((res=${test_flags[res_start]}; res<=${test_flags[res_end]}; res+=${test_fla args="$args -$arg ${run_flags[$arg]}" fi done - ./main $args | grep "L2 Error" | awk -v i="$i" -v res="$res" '{ printf "%d,%d,%.5f\n", i, res, $4}' >> $file_name + ./main $args | grep "L2 error of u" | awk -v i="$i" -v res="$res" '{ printf "%d,%d,%e\n", i, res, $6}' >> $file_name i=$((i+1)) done diff --git a/examples/Hdiv-mass/conv_test_result.csv b/examples/Hdiv-mass/conv_test_result.csv deleted file mode 100644 index 296e9812fd..0000000000 --- a/examples/Hdiv-mass/conv_test_result.csv +++ /dev/null @@ -1,10 +0,0 @@ -run,mesh_res,error_u -0,2,0.06228 -1,3,0.02836 -2,4,0.01616 -3,5,0.01043 -4,6,0.00728 -5,7,0.00537 -6,8,0.00413 -7,9,0.00327 -8,10,0.00265 diff --git a/examples/Hdiv-mass/include/cl-options.h b/examples/Hdiv-mass/include/cl-options.h index a48941521e..79f4fb51bc 100644 --- a/examples/Hdiv-mass/include/cl-options.h +++ b/examples/Hdiv-mass/include/cl-options.h @@ -1,12 +1,9 @@ #ifndef cloptions_h #define cloptions_h -#include "../include/structs.h" - -// Register problems to be available on the command line -PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx); +#include "structs.h" // Process general command line options -PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx); +PetscErrorCode ProcessCommandLineOptions(AppCtx app_ctx); #endif // cloptions_h diff --git a/examples/Hdiv-mass/include/matops.h b/examples/Hdiv-mass/include/matops.h deleted file mode 100644 index f3f662215b..0000000000 --- a/examples/Hdiv-mass/include/matops.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef matops_h -#define matops_h - -#include -#include - -#include "structs.h" - -PetscErrorCode ApplyLocal_Ceed(User user, Vec X, Vec Y); -PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y); -PetscErrorCode ComputeError(User user, Vec X, CeedVector target, CeedScalar *l2_error); - -#endif // matops_h diff --git a/examples/Hdiv-mass/include/petsc-macros.h b/examples/Hdiv-mass/include/petsc-macros.h deleted file mode 100644 index f8c63ebdc4..0000000000 --- a/examples/Hdiv-mass/include/petsc-macros.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef petsc_macros -#define petsc_macros - -#if PETSC_VERSION_LT(3, 14, 0) -#define DMPlexGetClosureIndices(a, b, c, d, e, f, g, h, i) DMPlexGetClosureIndices(a, b, c, d, f, g, i) -#define DMPlexRestoreClosureIndices(a, b, c, d, e, f, g, h, i) DMPlexRestoreClosureIndices(a, b, c, d, f, g, i) -#endif - -#if PETSC_VERSION_LT(3, 14, 0) -#define DMAddBoundary(a, b, c, d, e, f, g, h, i, j, k, l, m, n) DMAddBoundary(a, b, c, e, h, i, j, k, f, g, m) -#elif PETSC_VERSION_LT(3, 16, 0) -#define DMAddBoundary(a, b, c, d, e, f, g, h, i, j, k, l, m, n) DMAddBoundary(a, b, c, e, h, i, j, k, l, f, g, m) -#else -#define DMAddBoundary(a, b, c, d, e, f, g, h, i, j, k, l, m, n) DMAddBoundary(a, b, c, d, f, g, h, i, j, k, l, m, n) -#endif - -#endif diff --git a/examples/Hdiv-mass/include/post-processing.h b/examples/Hdiv-mass/include/post-processing.h new file mode 100644 index 0000000000..6338a2f2c4 --- /dev/null +++ b/examples/Hdiv-mass/include/post-processing.h @@ -0,0 +1,12 @@ +#ifndef post_processing_h +#define post_processing_h + +#include +#include + +#include "setup-fe.h" +#include "structs.h" + +PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, KSP ksp, Vec X, CeedScalar l2_error_u); + +#endif // post_processing_h \ No newline at end of file diff --git a/examples/Hdiv-mass/include/problems.h b/examples/Hdiv-mass/include/problems.h deleted file mode 100644 index 19e0499bb6..0000000000 --- a/examples/Hdiv-mass/include/problems.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef problems_h -#define problems_h - -#include "../include/structs.h" - -// ----------------------------------------------------------------------------- -// Set up problems function prototype -// ----------------------------------------------------------------------------- -// 1) poisson-quad2d -PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx); - -// 2) poisson-hex3d -PetscErrorCode Hdiv_POISSON_MASS3D(ProblemData *problem_data, void *ctx); - -// 3) poisson-prism3d - -// 4) richard - -#endif // problems_h diff --git a/examples/Hdiv-mass/include/register-problem.h b/examples/Hdiv-mass/include/register-problem.h new file mode 100644 index 0000000000..ab15dfbe50 --- /dev/null +++ b/examples/Hdiv-mass/include/register-problem.h @@ -0,0 +1,18 @@ +#ifndef register_problem_h +#define register_problem_h + +#include "structs.h" + +// Register problems to be available on the command line +PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx); + +// ----------------------------------------------------------------------------- +// Set up problems function prototype +// ----------------------------------------------------------------------------- +// 1) poisson-quad2d +PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData problem_data, void *ctx); + +// 2) poisson-hex3d +PetscErrorCode Hdiv_POISSON_MASS3D(ProblemData problem_data, void *ctx); + +#endif // register_problem_h diff --git a/examples/Hdiv-mass/include/setup-dm.h b/examples/Hdiv-mass/include/setup-dm.h index 53ded5c4ce..192db3d16d 100644 --- a/examples/Hdiv-mass/include/setup-dm.h +++ b/examples/Hdiv-mass/include/setup-dm.h @@ -6,11 +6,11 @@ #include #include -#include "../include/structs.h" +#include "structs.h" // --------------------------------------------------------------------------- -// Set-up DM +// Create DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDistributedDM(MPI_Comm comm, ProblemData *problem_data, DM *dm); +PetscErrorCode CreateDM(MPI_Comm comm, Ceed ceed, DM *dm); #endif // setupdm_h diff --git a/examples/Hdiv-mass/include/setup-fe.h b/examples/Hdiv-mass/include/setup-fe.h new file mode 100644 index 0000000000..f30914931b --- /dev/null +++ b/examples/Hdiv-mass/include/setup-fe.h @@ -0,0 +1,20 @@ +#ifndef setupfe_h +#define setupfe_h + +#include +#include +#include +#include + +#include "structs.h" + +// --------------------------------------------------------------------------- +// Setup H(div) FE space +// --------------------------------------------------------------------------- +CeedMemType MemTypeP2C(PetscMemType mtype); +PetscErrorCode SetupFEHdiv(AppCtx app_ctx, ProblemData problem_data, DM dm); +CeedElemTopology ElemTopologyP2C(DMPolytopeType cell_type); +PetscInt Involute(PetscInt i); +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr); +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, CeedElemRestriction *elem_restr_u, CeedElemRestriction *elem_restr_p); +#endif // setupfe_h diff --git a/examples/Hdiv-mass/include/setup-libceed.h b/examples/Hdiv-mass/include/setup-libceed.h index 7be8d75464..c28c543108 100644 --- a/examples/Hdiv-mass/include/setup-libceed.h +++ b/examples/Hdiv-mass/include/setup-libceed.h @@ -1,19 +1,10 @@ #ifndef setuplibceed_h #define setuplibceed_h -#include "../include/structs.h" +#include "setup-fe.h" +#include "structs.h" -// Convert PETSc MemType to libCEED MemType -CeedMemType MemTypeP2C(PetscMemType mtype); // Destroy libCEED objects PetscErrorCode CeedDataDestroy(CeedData ceed_data); -// Utility function - essential BC dofs are encoded in closure indices as -(i+1) -PetscInt Involute(PetscInt i); -// Utility function to create local CEED restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr); -// Utility function to create local CEED Oriented restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, CeedElemRestriction *elem_restr_oriented); -// Set up libCEED for a given degree -PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *problem_data, PetscInt U_g_size, PetscInt U_loc_size, CeedData ceed_data, - CeedVector rhs_ceed, CeedVector *target); +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData problem_data, CeedData ceed_data, CeedVector rhs_ceed); #endif // setuplibceed_h diff --git a/examples/Hdiv-mass/include/setup-matops.h b/examples/Hdiv-mass/include/setup-matops.h new file mode 100644 index 0000000000..4240f28ef5 --- /dev/null +++ b/examples/Hdiv-mass/include/setup-matops.h @@ -0,0 +1,13 @@ +#ifndef setup_matops_h +#define setup_matops_h + +#include +#include + +#include "setup-fe.h" +#include "structs.h" + +PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx); +PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx); + +#endif // setup_matops_h \ No newline at end of file diff --git a/examples/Hdiv-mass/include/setup-solvers.h b/examples/Hdiv-mass/include/setup-solvers.h new file mode 100644 index 0000000000..71110cb53b --- /dev/null +++ b/examples/Hdiv-mass/include/setup-solvers.h @@ -0,0 +1,16 @@ +#ifndef setup_solvers_h +#define setup_solvers_h + +#include +#include + +#include "petscvec.h" +#include "structs.h" + +PetscErrorCode SetupResidualOperatorCtx(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual); +PetscErrorCode SetupErrorOperatorCtx(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_error_u); +PetscErrorCode ApplyMatOp(Mat A, Vec X, Vec Y); +PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, KSP ksp, Vec rhs, Vec *X); +PetscErrorCode ComputeL2Error(Vec X, PetscScalar *l2_error, OperatorApplyContext op_error_ctx); +PetscErrorCode CtxVecDestroy(ProblemData problem_data, AppCtx app_ctx); +#endif // setup_solvers_h \ No newline at end of file diff --git a/examples/Hdiv-mass/include/structs.h b/examples/Hdiv-mass/include/structs.h index 69f65d916b..a39fb829b0 100644 --- a/examples/Hdiv-mass/include/structs.h +++ b/examples/Hdiv-mass/include/structs.h @@ -4,78 +4,48 @@ #include #include +// PETSc operator contexts +typedef struct OperatorApplyContext_ *OperatorApplyContext; +struct OperatorApplyContext_ { + MPI_Comm comm; + DM dm; + Vec X_loc, Y_loc; + CeedVector x_ceed, y_ceed; + CeedOperator op_apply; + Ceed ceed; +}; + // Application context from user command line options typedef struct AppCtx_ *AppCtx; struct AppCtx_ { + char ceed_resource[PETSC_MAX_PATH_LEN]; // libCEED backend + MPI_Comm comm; // libCEED arguments PetscInt degree; PetscInt q_extra; // Problem type arguments - PetscFunctionList problems; - char problem_name[PETSC_MAX_PATH_LEN]; + PetscFunctionList problems; + char problem_name[PETSC_MAX_PATH_LEN]; + OperatorApplyContext ctx_residual, ctx_error_u; }; // libCEED data struct typedef struct CeedData_ *CeedData; struct CeedData_ { - CeedBasis basis_x, basis_u; - CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_u_i; - CeedQFunction qf_residual, qf_error; - CeedOperator op_residual, op_error; - CeedVector x_ceed, y_ceed; - CeedQFunctionContext pq2d_context; -}; - -// 1) poisson-quad2d -#ifndef PHYSICS_POISSONQUAD2D_STRUCT -#define PHYSICS_POISSONQUAD2D_STRUCT -typedef struct PQ2DContext_ *PQ2DContext; -struct PQ2DContext_ { - CeedScalar kappa; -}; -#endif - -// 2) poisson-hex3d -#ifndef PHYSICS_POISSONHEX3D_STRUCT -#define PHYSICS_POISSONHEX3D_STRUCT -typedef struct PH3DContext_ *PH3DContext; -struct PH3DContext_ { - CeedScalar kappa; -}; -#endif - -// 3) poisson-prism3d - -// 4) richard - -// Struct that contains all enums and structs used for the physics of all problems -typedef struct Physics_ *Physics; -struct Physics_ { - PQ2DContext pq2d_ctx; - PH3DContext ph3d_ctx; -}; - -// PETSc user data -typedef struct User_ *User; -struct User_ { - MPI_Comm comm; - Vec X_loc, Y_loc; - CeedVector x_ceed, y_ceed; - CeedOperator op_apply, op_error; - DM dm; - Ceed ceed; - AppCtx app_ctx; - Physics phys; + CeedBasis basis_x, basis_u, basis_p; + CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_u_i, elem_restr_p; + CeedQFunction qf_residual, qf_error; + CeedOperator op_residual, op_error; + CeedVector x_ceed, y_ceed; }; // Problem specific data -typedef struct { +typedef struct ProblemData_ *ProblemData; +struct ProblemData_ { CeedQFunctionUser setup_rhs, residual, setup_error; const char *setup_rhs_loc, *residual_loc, *setup_error_loc; CeedQuadMode quadrature_mode; CeedInt elem_node, dim; - PetscErrorCode (*setup_ctx)(Ceed, CeedData, Physics); - -} ProblemData; +}; #endif // structs_h \ No newline at end of file diff --git a/examples/Hdiv-mass/main.c b/examples/Hdiv-mass/main.c index 338e745cfb..abc929eef8 100644 --- a/examples/Hdiv-mass/main.c +++ b/examples/Hdiv-mass/main.c @@ -14,10 +14,7 @@ // software, applications, hardware, advanced system engineering and early // testbed platforms, in support of the nation's exascale computing imperative. -// libCEED + PETSc Example: Mixed-Poisson in H(div) space -// -// This example demonstrates a simple usage of libCEED with PETSc to solve -// elasticity problems. +// libCEED + PETSc Example: Projection problem in H(div) space // // The code uses higher level communication protocols in DMPlex. // @@ -27,7 +24,8 @@ // ./main -pc_type svd -problem mass2d -dm_plex_dim 2 -dm_plex_box_faces 4,4 // ./main -pc_type svd -problem mass3d -dm_plex_dim 3 -dm_plex_box_faces 4,4,4 -const char help[] = "Solve H(div)-mixed problem using PETSc and libCEED\n"; +#include +const char help[] = "Solve Projection problem in H(div) space using PETSc and libCEED\n"; #include "main.h" @@ -35,59 +33,34 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // Initialize PETSc // --------------------------------------------------------------------------- - PetscInt ierr; - ierr = PetscInitialize(&argc, &argv, NULL, help); - if (ierr) return ierr; + PetscCall(PetscInitialize(&argc, &argv, NULL, help)); + MPI_Comm comm = PETSC_COMM_WORLD; // --------------------------------------------------------------------------- // Create structs // --------------------------------------------------------------------------- AppCtx app_ctx; - ierr = PetscCalloc1(1, &app_ctx); - CHKERRQ(ierr); + PetscCall(PetscCalloc1(1, &app_ctx)); - ProblemData *problem_data = NULL; - ierr = PetscCalloc1(1, &problem_data); - CHKERRQ(ierr); - - User user; - ierr = PetscCalloc1(1, &user); - CHKERRQ(ierr); + ProblemData problem_data = NULL; + PetscCall(PetscCalloc1(1, &problem_data)); CeedData ceed_data; - ierr = PetscCalloc1(1, &ceed_data); - CHKERRQ(ierr); - - Physics phys_ctx; - ierr = PetscCalloc1(1, &phys_ctx); - CHKERRQ(ierr); + PetscCall(PetscCalloc1(1, &ceed_data)); - user->app_ctx = app_ctx; - user->phys = phys_ctx; + OperatorApplyContext ctx_residual, ctx_error_u; + PetscCall(PetscCalloc1(1, &ctx_residual)); + PetscCall(PetscCalloc1(1, &ctx_error_u)); + // Context for residual + app_ctx->ctx_residual = ctx_residual; + // Context for computing error + app_ctx->ctx_error_u = ctx_error_u; + app_ctx->comm = comm; // --------------------------------------------------------------------------- // Process command line options // --------------------------------------------------------------------------- - // -- Register problems to be available on the command line - ierr = RegisterProblems_Hdiv(app_ctx); - CHKERRQ(ierr); - - // -- Process general command line options - MPI_Comm comm = PETSC_COMM_WORLD; - ierr = ProcessCommandLineOptions(comm, app_ctx); - CHKERRQ(ierr); - - // --------------------------------------------------------------------------- - // Choose the problem from the list of registered problems - // --------------------------------------------------------------------------- - { - PetscErrorCode (*p)(ProblemData *, void *); - ierr = PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p); - CHKERRQ(ierr); - if (!p) SETERRQ(PETSC_COMM_SELF, 1, "Problem '%s' not found", app_ctx->problem_name); - ierr = (*p)(problem_data, &user); - CHKERRQ(ierr); - } + PetscCall(ProcessCommandLineOptions(app_ctx)); // --------------------------------------------------------------------------- // Initialize libCEED @@ -95,202 +68,104 @@ int main(int argc, char **argv) { // -- Initialize backend Ceed ceed; CeedInit("/cpu/self/ref/serial", &ceed); - CeedMemType mem_type_backend; - CeedGetPreferredMemType(ceed, &mem_type_backend); + // CeedInit(app_ctx->ceed_resource, &ceed); + // --------------------------------------------------------------------------- - // Set-up DM + // Choose the problem from the list of registered problems // --------------------------------------------------------------------------- - // PETSc objects - DM dm; - VecType vec_type; - ierr = CreateDistributedDM(comm, problem_data, &dm); - CHKERRQ(ierr); - ierr = DMGetVecType(dm, &vec_type); - CHKERRQ(ierr); - if (!vec_type) { // Not yet set by user -dm_vec_type - switch (mem_type_backend) { - case CEED_MEM_HOST: - vec_type = VECSTANDARD; - break; - case CEED_MEM_DEVICE: { - const char *resolved; - CeedGetResource(ceed, &resolved); - if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; - else if (strstr(resolved, "/gpu/hip/occa")) vec_type = VECSTANDARD; // https://github.com/CEED/libCEED/issues/678 - else if (strstr(resolved, "/gpu/hip")) vec_type = VECHIP; - else vec_type = VECSTANDARD; - } - } - ierr = DMSetVecType(dm, vec_type); - CHKERRQ(ierr); + PetscCall(RegisterProblems_Hdiv(app_ctx)); + { + PetscErrorCode (*p)(ProblemData, void *); + PetscCall(PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p)); + if (!p) SETERRQ(PETSC_COMM_SELF, 1, "Problem '%s' not found", app_ctx->problem_name); + PetscCall((*p)(problem_data, &app_ctx)); } + // --------------------------------------------------------------------------- - // Create global, local solution, local rhs vector + // Create DM and Setup FE space // --------------------------------------------------------------------------- - Vec U_g, U_loc; - PetscInt U_l_size, U_g_size, U_loc_size; - // Create global and local solution vectors - ierr = DMCreateGlobalVector(dm, &U_g); - CHKERRQ(ierr); - ierr = VecGetSize(U_g, &U_g_size); - CHKERRQ(ierr); - // Local size for matShell - ierr = VecGetLocalSize(U_g, &U_l_size); - CHKERRQ(ierr); - // Create local unknown vector U_loc - ierr = DMCreateLocalVector(dm, &U_loc); - CHKERRQ(ierr); - // Local size for libCEED - ierr = VecGetSize(U_loc, &U_loc_size); - CHKERRQ(ierr); + DM dm; + PetscCall(CreateDM(comm, ceed, &dm)); + PetscCall(SetupFEHdiv(app_ctx, problem_data, dm)); - // Get RHS vector + // --------------------------------------------------------------------------- + // Create zero rhs local vector + // --------------------------------------------------------------------------- + CeedVector rhs_ceed; Vec rhs_loc; PetscScalar *r; - CeedVector rhs_ceed, target; PetscMemType mem_type; - ierr = VecDuplicate(U_loc, &rhs_loc); - CHKERRQ(ierr); - ierr = VecZeroEntries(rhs_loc); - CHKERRQ(ierr); - ierr = VecGetArrayAndMemType(rhs_loc, &r, &mem_type); - CHKERRQ(ierr); - CeedVectorCreate(ceed, U_l_size, &rhs_ceed); + PetscInt rhs_l_size; + // Create global and local solution vectors + PetscCall(DMCreateLocalVector(dm, &rhs_loc)); + PetscCall(VecGetSize(rhs_loc, &rhs_l_size)); + PetscCall(VecZeroEntries(rhs_loc)); + PetscCall(VecGetArrayAndMemType(rhs_loc, &r, &mem_type)); + CeedVectorCreate(ceed, rhs_l_size, &rhs_ceed); CeedVectorSetArray(rhs_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, r); // --------------------------------------------------------------------------- - // Setup libCEED + // Setup libCEED qfunctions and operators // --------------------------------------------------------------------------- - // -- Set up libCEED objects - ierr = SetupLibceed(dm, ceed, app_ctx, problem_data, U_g_size, U_loc_size, ceed_data, rhs_ceed, &target); - CHKERRQ(ierr); + PetscCall(SetupLibceed(dm, ceed, app_ctx, problem_data, ceed_data, rhs_ceed)); // --------------------------------------------------------------------------- - // Gather RHS + // Setup rhs global vector entries with the computed rhs_ceed // --------------------------------------------------------------------------- Vec rhs; + PetscCall(DMCreateGlobalVector(dm, &rhs)); CeedVectorTakeArray(rhs_ceed, MemTypeP2C(mem_type), NULL); - ierr = VecRestoreArrayAndMemType(rhs_loc, &r); - CHKERRQ(ierr); - ierr = VecDuplicate(U_g, &rhs); - CHKERRQ(ierr); - ierr = VecZeroEntries(rhs); - CHKERRQ(ierr); - ierr = DMLocalToGlobal(dm, rhs_loc, ADD_VALUES, rhs); - CHKERRQ(ierr); - // VecView(rhs, PETSC_VIEWER_STDOUT_WORLD); + PetscCall(VecRestoreArrayAndMemType(rhs_loc, &r)); + PetscCall(VecZeroEntries(rhs)); + PetscCall(DMLocalToGlobal(dm, rhs_loc, ADD_VALUES, rhs)); + CeedVectorDestroy(&rhs_ceed); // --------------------------------------------------------------------------- - // Setup Mat, KSP + // Solve A*X=rhs; setup-solver.c // --------------------------------------------------------------------------- - user->comm = comm; - user->dm = dm; - user->X_loc = U_loc; - ierr = VecDuplicate(U_loc, &user->Y_loc); - CHKERRQ(ierr); - user->x_ceed = ceed_data->x_ceed; - user->y_ceed = ceed_data->y_ceed; - user->op_apply = ceed_data->op_residual; - user->op_error = ceed_data->op_error; - user->ceed = ceed; - // Operator - Mat mat; - ierr = MatCreateShell(comm, U_l_size, U_l_size, U_g_size, U_g_size, user, &mat); - CHKERRQ(ierr); - ierr = MatShellSetOperation(mat, MATOP_MULT, (void (*)(void))MatMult_Ceed); - CHKERRQ(ierr); - ierr = MatShellSetVecType(mat, vec_type); - CHKERRQ(ierr); - + Vec X; KSP ksp; - ierr = KSPCreate(comm, &ksp); - CHKERRQ(ierr); - ierr = KSPSetOperators(ksp, mat, mat); - CHKERRQ(ierr); - ierr = KSPSetFromOptions(ksp); - CHKERRQ(ierr); - ierr = KSPSetUp(ksp); - CHKERRQ(ierr); - ierr = KSPSolve(ksp, rhs, U_g); - CHKERRQ(ierr); - // printf("U_g\n"); - // VecView(U_g, PETSC_VIEWER_STDOUT_WORLD); - // --------------------------------------------------------------------------- - // Compute pointwise L2 maximum error - // --------------------------------------------------------------------------- - CeedScalar l2_error; - ierr = ComputeError(user, U_g, target, &l2_error); - CHKERRQ(ierr); + PetscCall(SetupResidualOperatorCtx(app_ctx->comm, dm, ceed, ceed_data, app_ctx->ctx_residual)); + // Create global and local solution vectors + PetscCall(DMCreateGlobalVector(dm, &X)); + PetscCall(KSPCreate(app_ctx->comm, &ksp)); + PetscCall(KSPSetDM(ksp, dm)); + PetscCall(KSPSetDMActive(ksp, PETSC_FALSE)); + PetscCall(PDESolver(ceed_data, app_ctx, ksp, rhs, &X)); + + // --------------------------------------------------------------------------- + // Compute L2 error of mms problem; setup-solver.c + // --------------------------------------------------------------------------- + CeedScalar l2_error_u = 0.0; + PetscCall(SetupErrorOperatorCtx(app_ctx->comm, dm, ceed, ceed_data, app_ctx->ctx_error_u)); + PetscCall(ComputeL2Error(X, &l2_error_u, app_ctx->ctx_error_u)); // --------------------------------------------------------------------------- - // Output results + // Print solver iterations and final norms; post-processing // --------------------------------------------------------------------------- - KSPType ksp_type; - KSPConvergedReason reason; - PetscReal rnorm; - PetscInt its; - ierr = KSPGetType(ksp, &ksp_type); - CHKERRQ(ierr); - ierr = KSPGetConvergedReason(ksp, &reason); - CHKERRQ(ierr); - ierr = KSPGetIterationNumber(ksp, &its); - CHKERRQ(ierr); - ierr = KSPGetResidualNorm(ksp, &rnorm); - CHKERRQ(ierr); - ierr = PetscPrintf(comm, - " KSP:\n" - " KSP Type : %s\n" - " KSP Convergence : %s\n" - " Total KSP Iterations : %" PetscInt_FMT - "\n" - " Final rnorm : %e\n" - " L2 Error : %e\n", - ksp_type, KSPConvergedReasons[reason], its, (double)rnorm, (double)l2_error); - CHKERRQ(ierr); + PetscCall(PrintOutput(dm, ceed, app_ctx, ksp, X, l2_error_u)); // --------------------------------------------------------------------------- // Free objects // --------------------------------------------------------------------------- // Free PETSc objects - ierr = DMDestroy(&dm); - CHKERRQ(ierr); - ierr = VecDestroy(&U_g); - CHKERRQ(ierr); - ierr = VecDestroy(&U_loc); - CHKERRQ(ierr); - ierr = VecDestroy(&rhs); - CHKERRQ(ierr); - ierr = VecDestroy(&rhs_loc); - CHKERRQ(ierr); - ierr = VecDestroy(&user->Y_loc); - CHKERRQ(ierr); - ierr = MatDestroy(&mat); - CHKERRQ(ierr); - ierr = KSPDestroy(&ksp); - CHKERRQ(ierr); - + PetscCall(DMDestroy(&dm)); + PetscCall(VecDestroy(&X)); + PetscCall(VecDestroy(&rhs)); + PetscCall(VecDestroy(&rhs_loc)); + PetscCall(KSPDestroy(&ksp)); + PetscCall(CtxVecDestroy(problem_data, app_ctx)); // -- Function list - ierr = PetscFunctionListDestroy(&app_ctx->problems); - CHKERRQ(ierr); - + PetscCall(PetscFunctionListDestroy(&app_ctx->problems)); // -- Structs - ierr = PetscFree(app_ctx); - CHKERRQ(ierr); - ierr = PetscFree(problem_data); - CHKERRQ(ierr); - ierr = PetscFree(user); - CHKERRQ(ierr); - ierr = PetscFree(phys_ctx->pq2d_ctx); - CHKERRQ(ierr); - ierr = PetscFree(phys_ctx); - CHKERRQ(ierr); - + PetscCall(CeedDataDestroy(ceed_data)); + PetscCall(PetscFree(app_ctx)); + PetscCall(PetscFree(ctx_residual)); + PetscCall(PetscFree(ctx_error_u)); + PetscCall(PetscFree(problem_data)); // Free libCEED objects CeedVectorDestroy(&rhs_ceed); - CeedVectorDestroy(&target); - ierr = CeedDataDestroy(ceed_data); - CHKERRQ(ierr); CeedDestroy(&ceed); return PetscFinalize(); diff --git a/examples/Hdiv-mass/main.h b/examples/Hdiv-mass/main.h index c5ecde00d9..f433f47a27 100644 --- a/examples/Hdiv-mass/main.h +++ b/examples/Hdiv-mass/main.h @@ -3,9 +3,12 @@ #define MAIN_H #include "include/cl-options.h" -#include "include/matops.h" -#include "include/problems.h" +#include "include/post-processing.h" +#include "include/register-problem.h" #include "include/setup-dm.h" +#include "include/setup-fe.h" #include "include/setup-libceed.h" +#include "include/setup-matops.h" +#include "include/setup-solvers.h" #endif // MAIN_H \ No newline at end of file diff --git a/examples/Hdiv-mass/problems/mass2d.c b/examples/Hdiv-mass/problems/mass2d.c index 2e58f87045..0cdef98f85 100644 --- a/examples/Hdiv-mass/problems/mass2d.c +++ b/examples/Hdiv-mass/problems/mass2d.c @@ -17,22 +17,16 @@ /// @file /// Utility functions for setting up POISSON_QUAD2D -#include "../include/problems.h" +#include "../include/register-problem.h" #include "../include/setup-libceed.h" #include "../qfunctions/poisson-error2d.h" #include "../qfunctions/poisson-mass2d.h" #include "../qfunctions/poisson-rhs2d.h" // Hdiv_POISSON_MASS2D is registered in cl-option.c -PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx) { - User user = *(User *)ctx; - MPI_Comm comm = PETSC_COMM_WORLD; - PetscInt ierr; +PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData problem_data, void *ctx) { PetscFunctionBeginUser; - ierr = PetscCalloc1(1, &user->phys->pq2d_ctx); - CHKERRQ(ierr); - // ------------------------------------------------------ // SET UP POISSON_QUAD2D // ------------------------------------------------------ @@ -46,12 +40,5 @@ PetscErrorCode Hdiv_POISSON_MASS2D(ProblemData *problem_data, void *ctx) { problem_data->setup_error = SetupError2D; problem_data->setup_error_loc = SetupError2D_loc; - // ------------------------------------------------------ - // Command line Options - // ------------------------------------------------------ - PetscOptionsBegin(comm, NULL, "Options for Hdiv-mass problem", NULL); - - PetscOptionsEnd(); - PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mass/problems/mass3d.c b/examples/Hdiv-mass/problems/mass3d.c index 2997ebb572..b2b847d19f 100644 --- a/examples/Hdiv-mass/problems/mass3d.c +++ b/examples/Hdiv-mass/problems/mass3d.c @@ -15,24 +15,16 @@ // testbed platforms, in support of the nation's exascale computing imperative. /// @file -/// Utility functions for setting up POISSON_QUAD2D -#include "../include/problems.h" +#include "../include/register-problem.h" #include "../include/setup-libceed.h" #include "../qfunctions/poisson-error3d.h" #include "../qfunctions/poisson-mass3d.h" #include "../qfunctions/poisson-rhs3d.h" -// Hdiv_POISSON_MASS2D is registered in cl-option.c -PetscErrorCode Hdiv_POISSON_MASS3D(ProblemData *problem_data, void *ctx) { - User user = *(User *)ctx; - MPI_Comm comm = PETSC_COMM_WORLD; - PetscInt ierr; +PetscErrorCode Hdiv_POISSON_MASS3D(ProblemData problem_data, void *ctx) { PetscFunctionBeginUser; - ierr = PetscCalloc1(1, &user->phys->ph3d_ctx); - CHKERRQ(ierr); - // ------------------------------------------------------ // SET UP POISSON_QUAD3D // ------------------------------------------------------ @@ -46,12 +38,5 @@ PetscErrorCode Hdiv_POISSON_MASS3D(ProblemData *problem_data, void *ctx) { problem_data->setup_error = SetupError3D; problem_data->setup_error_loc = SetupError3D_loc; - // ------------------------------------------------------ - // Command line Options - // ------------------------------------------------------ - PetscOptionsBegin(comm, NULL, "Options for Hdiv-mass problem", NULL); - - PetscOptionsEnd(); - PetscFunctionReturn(0); } diff --git a/examples/Hdiv-mass/problems/register-problem.c b/examples/Hdiv-mass/problems/register-problem.c new file mode 100644 index 0000000000..0e66ad3a15 --- /dev/null +++ b/examples/Hdiv-mass/problems/register-problem.c @@ -0,0 +1,33 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +/// @file +/// Command line option processing for H(div) example using PETSc + +#include "../include/register-problem.h" + +// Register problems to be available on the command line +PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx) { + app_ctx->problems = NULL; + + PetscFunctionBeginUser; + // 1) poisson-quad2d (Hdiv_POISSON_MASS2D is created in poisson-mass2d.c) + PetscCall(PetscFunctionListAdd(&app_ctx->problems, "mass2d", Hdiv_POISSON_MASS2D)); + // 2) poisson-hex3d + PetscCall(PetscFunctionListAdd(&app_ctx->problems, "mass3d", Hdiv_POISSON_MASS3D)); + + PetscFunctionReturn(0); +} diff --git a/examples/Hdiv-mass/qfunctions/poisson-error2d.h b/examples/Hdiv-mass/qfunctions/poisson-error2d.h index f48574f93c..e7f19719dc 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-error2d.h +++ b/examples/Hdiv-mass/qfunctions/poisson-error2d.h @@ -45,9 +45,10 @@ CEED_QFUNCTION(SetupError2D)(void *ctx, const CeedInt Q, const CeedScalar *const CeedScalar u1[2] = {u[0][i], u[1][i]}, uh[2]; AlphaMatVecMult2x2(1 / det_J, J, u1, uh); // Error - error[i + 0 * Q] = (uh[0] - target[i + 0 * Q]) * (uh[0] - target[i + 0 * Q]) * w[i] * det_J; - error[i + 1 * Q] = (uh[1] - target[i + 1 * Q]) * (uh[1] - target[i + 1 * Q]) * w[i] * det_J; + CeedScalar err2_ux = (uh[0] - target[i + 0 * Q]) * (uh[0] - target[i + 0 * Q]); + CeedScalar err2_uy = (uh[1] - target[i + 1 * Q]) * (uh[1] - target[i + 1 * Q]); + error[i + 0 * Q] = (err2_ux + err2_uy) * w[i] * det_J; } // End of Quadrature Point Loop return 0; diff --git a/examples/Hdiv-mass/qfunctions/poisson-error3d.h b/examples/Hdiv-mass/qfunctions/poisson-error3d.h index 2d1c821165..b8b09d5025 100644 --- a/examples/Hdiv-mass/qfunctions/poisson-error3d.h +++ b/examples/Hdiv-mass/qfunctions/poisson-error3d.h @@ -46,10 +46,11 @@ CEED_QFUNCTION(SetupError3D)(void *ctx, const CeedInt Q, const CeedScalar *const CeedScalar u1[3] = {u[0][i], u[1][i], u[2][i]}, uh[3]; AlphaMatVecMult3x3(1 / det_J, J, u1, uh); // Error - error[i + 0 * Q] = (uh[0] - target[i + 0 * Q]) * (uh[0] - target[i + 0 * Q]) * w[i] * det_J; - error[i + 1 * Q] = (uh[1] - target[i + 1 * Q]) * (uh[1] - target[i + 1 * Q]) * w[i] * det_J; - error[i + 2 * Q] = (uh[2] - target[i + 2 * Q]) * (uh[2] - target[i + 2 * Q]) * w[i] * det_J; + CeedScalar err2_ux = (uh[0] - target[i + 0 * Q]) * (uh[0] - target[i + 0 * Q]); + CeedScalar err2_uy = (uh[1] - target[i + 1 * Q]) * (uh[1] - target[i + 1 * Q]); + CeedScalar err2_uz = (uh[2] - target[i + 2 * Q]) * (uh[2] - target[i + 2 * Q]); + error[i + 0 * Q] = (err2_ux + err2_uy + err2_uz) * w[i] * det_J; } // End of Quadrature Point Loop return 0; diff --git a/examples/Hdiv-mass/src/cl-options.c b/examples/Hdiv-mass/src/cl-options.c index 105de000df..4b785b430a 100644 --- a/examples/Hdiv-mass/src/cl-options.c +++ b/examples/Hdiv-mass/src/cl-options.c @@ -19,46 +19,31 @@ #include "../include/cl-options.h" -#include "../include/problems.h" - -// Register problems to be available on the command line -PetscErrorCode RegisterProblems_Hdiv(AppCtx app_ctx) { - app_ctx->problems = NULL; - PetscErrorCode ierr; +// Process general command line options +PetscErrorCode ProcessCommandLineOptions(AppCtx app_ctx) { + PetscBool problem_flag = PETSC_FALSE; + PetscBool ceed_flag = PETSC_FALSE; PetscFunctionBeginUser; - // 1) poisson-quad2d (Hdiv_POISSON_MASS2D is created in poisson-mass2d.c) - ierr = PetscFunctionListAdd(&app_ctx->problems, "mass2d", Hdiv_POISSON_MASS2D); - CHKERRQ(ierr); - // 2) poisson-hex3d - ierr = PetscFunctionListAdd(&app_ctx->problems, "mass3d", Hdiv_POISSON_MASS3D); - CHKERRQ(ierr); - - // 3) poisson-prism3d - // 4) richard + PetscOptionsBegin(app_ctx->comm, NULL, "H(div) examples in PETSc with libCEED", NULL); - PetscFunctionReturn(0); -} + PetscCall(PetscOptionsString("-ceed", "CEED resource specifier", NULL, app_ctx->ceed_resource, app_ctx->ceed_resource, + sizeof(app_ctx->ceed_resource), &ceed_flag)); -// Process general command line options -PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx) { - PetscBool problem_flag = PETSC_FALSE; - PetscErrorCode ierr; - PetscFunctionBeginUser; - - PetscOptionsBegin(comm, NULL, "H(div) examples in PETSc with libCEED", NULL); + // Provide default ceed resource if not specified + if (!ceed_flag) { + const char *ceed_resource = "/cpu/self"; + strncpy(app_ctx->ceed_resource, ceed_resource, 10); + } - ierr = PetscOptionsFList("-problem", "Problem to solve", NULL, app_ctx->problems, app_ctx->problem_name, app_ctx->problem_name, - sizeof(app_ctx->problem_name), &problem_flag); - CHKERRQ(ierr); + PetscCall(PetscOptionsFList("-problem", "Problem to solve", NULL, app_ctx->problems, app_ctx->problem_name, app_ctx->problem_name, + sizeof(app_ctx->problem_name), &problem_flag)); app_ctx->degree = 1; - ierr = PetscOptionsInt("-degree", "Polynomial degree of finite elements", NULL, app_ctx->degree, &app_ctx->degree, NULL); - CHKERRQ(ierr); + PetscCall(PetscOptionsInt("-degree", "Polynomial degree of finite elements", NULL, app_ctx->degree, &app_ctx->degree, NULL)); app_ctx->q_extra = 0; - ierr = PetscOptionsInt("-q_extra", "Number of extra quadrature points", NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL); - CHKERRQ(ierr); + PetscCall(PetscOptionsInt("-q_extra", "Number of extra quadrature points", NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL)); // Provide default problem if not specified if (!problem_flag) { diff --git a/examples/Hdiv-mass/src/matops.c b/examples/Hdiv-mass/src/matops.c deleted file mode 100644 index 45c39f0160..0000000000 --- a/examples/Hdiv-mass/src/matops.c +++ /dev/null @@ -1,106 +0,0 @@ -#include "../include/matops.h" - -#include "../include/setup-libceed.h" - -// ----------------------------------------------------------------------------- -// This function uses libCEED to compute the action of the Laplacian with -// Dirichlet boundary conditions -// ----------------------------------------------------------------------------- -PetscErrorCode ApplyLocal_Ceed(User user, Vec X, Vec Y) { - PetscErrorCode ierr; - PetscScalar *x, *y; - PetscMemType x_mem_type, y_mem_type; - - PetscFunctionBeginUser; - - // Global-to-local - ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); - CHKERRQ(ierr); - - // Setup libCEED vectors - ierr = VecGetArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x, &x_mem_type); - CHKERRQ(ierr); - ierr = VecGetArrayAndMemType(user->Y_loc, &y, &y_mem_type); - CHKERRQ(ierr); - CeedVectorSetArray(user->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, x); - CeedVectorSetArray(user->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); - - // Apply libCEED operator - CeedOperatorApply(user->op_apply, user->x_ceed, user->y_ceed, CEED_REQUEST_IMMEDIATE); - - // Restore PETSc vectors - CeedVectorTakeArray(user->x_ceed, MemTypeP2C(x_mem_type), NULL); - CeedVectorTakeArray(user->y_ceed, MemTypeP2C(y_mem_type), NULL); - ierr = VecRestoreArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x); - CHKERRQ(ierr); - ierr = VecRestoreArrayAndMemType(user->Y_loc, &y); - CHKERRQ(ierr); - - // Local-to-global - ierr = VecZeroEntries(Y); - CHKERRQ(ierr); - ierr = DMLocalToGlobal(user->dm, user->Y_loc, ADD_VALUES, Y); - CHKERRQ(ierr); - - PetscFunctionReturn(0); -}; - -// ----------------------------------------------------------------------------- -// This function wraps the libCEED operator for a MatShell -// ----------------------------------------------------------------------------- -PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y) { - PetscErrorCode ierr; - User user; - - PetscFunctionBeginUser; - - ierr = MatShellGetContext(A, &user); - CHKERRQ(ierr); - - // libCEED for local action of residual evaluator - ierr = ApplyLocal_Ceed(user, X, Y); - CHKERRQ(ierr); - - PetscFunctionReturn(0); -}; - -// ----------------------------------------------------------------------------- -// This function calculates the error in the final solution -// ----------------------------------------------------------------------------- -PetscErrorCode ComputeError(User user, Vec X, CeedVector target, CeedScalar *l2_error) { - PetscErrorCode ierr; - PetscScalar *x; - PetscMemType mem_type; - CeedVector collocated_error; - CeedSize length; - - PetscFunctionBeginUser; - CeedVectorGetLength(target, &length); - CeedVectorCreate(user->ceed, length, &collocated_error); - - // Global-to-local - ierr = DMGlobalToLocal(user->dm, X, INSERT_VALUES, user->X_loc); - CHKERRQ(ierr); - - // Setup CEED vector - ierr = VecGetArrayAndMemType(user->X_loc, &x, &mem_type); - CHKERRQ(ierr); - CeedVectorSetArray(user->x_ceed, MemTypeP2C(mem_type), CEED_USE_POINTER, x); - - // Apply CEED operator - CeedOperatorApply(user->op_error, user->x_ceed, collocated_error, CEED_REQUEST_IMMEDIATE); - // Restore PETSc vector - CeedVectorTakeArray(user->x_ceed, MemTypeP2C(mem_type), NULL); - ierr = VecRestoreArrayReadAndMemType(user->X_loc, (const PetscScalar **)&x); - CHKERRQ(ierr); - - CeedScalar error; - CeedVectorNorm(collocated_error, CEED_NORM_1, &error); - *l2_error = sqrt(error); - // Cleanup - CeedVectorDestroy(&collocated_error); - - PetscFunctionReturn(0); -}; - -// ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mass/src/post-processing.c b/examples/Hdiv-mass/src/post-processing.c new file mode 100644 index 0000000000..c1281a1146 --- /dev/null +++ b/examples/Hdiv-mass/src/post-processing.c @@ -0,0 +1,77 @@ +#include "../include/post-processing.h" + +// ----------------------------------------------------------------------------- +// This function print the output +// ----------------------------------------------------------------------------- +PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, KSP ksp, Vec X, CeedScalar l2_error_u) { + PetscFunctionBeginUser; + + const char *used_resource; + CeedMemType mem_type_backend; + CeedGetResource(ceed, &used_resource); + CeedGetPreferredMemType(ceed, &mem_type_backend); + char hostname[PETSC_MAX_PATH_LEN]; + PetscCall(PetscGetHostName(hostname, sizeof hostname)); + PetscMPIInt comm_size; + PetscCall(MPI_Comm_size(app_ctx->comm, &comm_size)); + PetscCall(PetscPrintf(app_ctx->comm, + "\n-- Mixed-Elasticity Example - libCEED + PETSc --\n" + " MPI:\n" + " Hostname : %s\n" + " Total ranks : %d\n" + " libCEED:\n" + " libCEED Backend : %s\n" + " libCEED Backend MemType : %s\n", + hostname, comm_size, used_resource, CeedMemTypes[mem_type_backend])); + + MatType mat_type; + VecType vec_type; + PetscCall(DMGetMatType(dm, &mat_type)); + PetscCall(DMGetVecType(dm, &vec_type)); + PetscCall(PetscPrintf(app_ctx->comm, + " PETSc:\n" + " DM MatType : %s\n" + " DM VecType : %s\n", + mat_type, vec_type)); + PetscInt X_l_size, X_g_size; + PetscCall(VecGetSize(X, &X_g_size)); + PetscCall(VecGetLocalSize(X, &X_l_size)); + PetscInt c_start, c_end; + PetscCall(DMPlexGetHeightStratum(dm, 0, &c_start, &c_end)); + DMPolytopeType cell_type; + PetscCall(DMPlexGetCellType(dm, c_start, &cell_type)); + CeedElemTopology elem_topo = ElemTopologyP2C(cell_type); + PetscCall(PetscPrintf(app_ctx->comm, + " Problem:\n" + " Problem Name : %s\n" + " Mesh:\n" + " Local Elements : %" PetscInt_FMT "\n" + " Element topology : %s\n", + app_ctx->problem_name, c_end - c_start, CeedElemTopologies[elem_topo])); + PetscInt ksp_its; + KSPType ksp_type; + KSPConvergedReason ksp_reason; + PetscReal ksp_rnorm; + PC pc; + PCType pc_type; + PetscCall(KSPGetPC(ksp, &pc)); + PetscCall(PCGetType(pc, &pc_type)); + PetscCall(KSPGetType(ksp, &ksp_type)); + PetscCall(KSPGetConvergedReason(ksp, &ksp_reason)); + PetscCall(KSPGetIterationNumber(ksp, &ksp_its)); + PetscCall(KSPGetResidualNorm(ksp, &ksp_rnorm)); + PetscCall(PetscPrintf(app_ctx->comm, + " KSP:\n" + " KSP Type : %s\n" + " PC Type : %s\n" + " KSP Convergence : %s\n" + " Total KSP Iterations : %" PetscInt_FMT "\n" + " Final rnorm : %e\n", + ksp_type, pc_type, KSPConvergedReasons[ksp_reason], ksp_its, (double)ksp_rnorm)); + + PetscCall(PetscPrintf(app_ctx->comm, + " L2 Error (MMS):\n" + " L2 error of u : %e\n", + (double)l2_error_u)); + PetscFunctionReturn(0); +}; \ No newline at end of file diff --git a/examples/Hdiv-mass/src/setup-dm.c b/examples/Hdiv-mass/src/setup-dm.c index a48ee93606..ce925c71a8 100644 --- a/examples/Hdiv-mass/src/setup-dm.c +++ b/examples/Hdiv-mass/src/setup-dm.c @@ -1,68 +1,45 @@ #include "../include/setup-dm.h" +#include "petscerror.h" + // --------------------------------------------------------------------------- -// Set-up DM +// Create DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDistributedDM(MPI_Comm comm, ProblemData *problem_data, DM *dm) { - PetscErrorCode ierr; - PetscSection sec; - PetscInt dofs_per_face; - PetscInt p_start, p_end; - PetscInt c_start, c_end; // cells - PetscInt f_start, f_end; // faces - PetscInt v_start, v_end; // vertices - +PetscErrorCode CreateDM(MPI_Comm comm, Ceed ceed, DM *dm) { PetscFunctionBeginUser; + CeedMemType mem_type_backend; + CeedGetPreferredMemType(ceed, &mem_type_backend); + + VecType vec_type = NULL; + MatType mat_type = NULL; + switch (mem_type_backend) { + case CEED_MEM_HOST: + vec_type = VECSTANDARD; + break; + case CEED_MEM_DEVICE: { + const char *resolved; + CeedGetResource(ceed, &resolved); + if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; + else if (strstr(resolved, "/gpu/hip/occa")) vec_type = VECSTANDARD; // https://github.com/CEED/libCEED/issues/678 + else if (strstr(resolved, "/gpu/hip")) vec_type = VECHIP; + else vec_type = VECSTANDARD; + } + } + if (strstr(vec_type, VECCUDA)) mat_type = MATAIJCUSPARSE; + else if (strstr(vec_type, VECKOKKOS)) mat_type = MATAIJKOKKOS; + else mat_type = MATAIJ; + // Create DMPLEX - ierr = DMCreate(comm, dm); - CHKERRQ(ierr); - ierr = DMSetType(*dm, DMPLEX); - CHKERRQ(ierr); + PetscCall(DMCreate(comm, dm)); + PetscCall(DMSetType(*dm, DMPLEX)); + PetscCall(DMSetMatType(*dm, mat_type)); + PetscCall(DMSetVecType(*dm, vec_type)); // Set Tensor elements - ierr = PetscOptionsSetValue(NULL, "-dm_plex_simplex", "0"); - CHKERRQ(ierr); + PetscCall(PetscOptionsSetValue(NULL, "-dm_plex_simplex", "0")); // Set CL options - ierr = DMSetFromOptions(*dm); - CHKERRQ(ierr); - ierr = DMViewFromOptions(*dm, NULL, "-dm_view"); - CHKERRQ(ierr); - - // Get plex limits - ierr = DMPlexGetChart(*dm, &p_start, &p_end); - CHKERRQ(ierr); - ierr = DMPlexGetHeightStratum(*dm, 0, &c_start, &c_end); - CHKERRQ(ierr); - ierr = DMPlexGetHeightStratum(*dm, 1, &f_start, &f_end); - CHKERRQ(ierr); - ierr = DMPlexGetDepthStratum(*dm, 0, &v_start, &v_end); - CHKERRQ(ierr); - // Create section - ierr = PetscSectionCreate(comm, &sec); - CHKERRQ(ierr); - ierr = PetscSectionSetNumFields(sec, 1); - CHKERRQ(ierr); - ierr = PetscSectionSetFieldName(sec, 0, "Velocity"); - CHKERRQ(ierr); - ierr = PetscSectionSetFieldComponents(sec, 0, 1); - CHKERRQ(ierr); - ierr = PetscSectionSetChart(sec, p_start, p_end); - CHKERRQ(ierr); - // Setup dofs per face - for (PetscInt f = f_start; f < f_end; f++) { - ierr = DMPlexGetConeSize(*dm, f, &dofs_per_face); - CHKERRQ(ierr); - ierr = PetscSectionSetFieldDof(sec, f, 0, dofs_per_face); - CHKERRQ(ierr); - ierr = PetscSectionSetDof(sec, f, dofs_per_face); - CHKERRQ(ierr); - } - ierr = PetscSectionSetUp(sec); - CHKERRQ(ierr); - ierr = DMSetSection(*dm, sec); - CHKERRQ(ierr); - ierr = PetscSectionDestroy(&sec); - CHKERRQ(ierr); + PetscCall(DMSetFromOptions(*dm)); + PetscCall(DMViewFromOptions(*dm, NULL, "-dm_view")); PetscFunctionReturn(0); }; \ No newline at end of file diff --git a/examples/Hdiv-mass/src/setup-fe.c b/examples/Hdiv-mass/src/setup-fe.c new file mode 100644 index 0000000000..09bf4d78cd --- /dev/null +++ b/examples/Hdiv-mass/src/setup-fe.c @@ -0,0 +1,149 @@ +#include "../include/setup-fe.h" + +#include "petscerror.h" +// ----------------------------------------------------------------------------- +// Convert PETSc MemType to libCEED MemType +// ----------------------------------------------------------------------------- +CeedMemType MemTypeP2C(PetscMemType mem_type) { return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; } + +// --------------------------------------------------------------------------- +// Setup FE for H(div) space +// --------------------------------------------------------------------------- +PetscErrorCode SetupFEHdiv(AppCtx app_ctx, ProblemData problem_data, DM dm) { + PetscSection sec; + PetscInt dofs_per_face; + PetscInt p_start, p_end; + PetscInt c_start, c_end; // cells + PetscInt f_start, f_end; // faces + PetscInt v_start, v_end; // vertices + + PetscFunctionBeginUser; + + // Get plex limits + PetscCall(DMPlexGetChart(dm, &p_start, &p_end)); + PetscCall(DMPlexGetHeightStratum(dm, 0, &c_start, &c_end)); + PetscCall(DMPlexGetHeightStratum(dm, 1, &f_start, &f_end)); + PetscCall(DMPlexGetDepthStratum(dm, 0, &v_start, &v_end)); + // Create section + PetscCall(PetscSectionCreate(app_ctx->comm, &sec)); + PetscCall(PetscSectionSetNumFields(sec, 1)); + PetscCall(PetscSectionSetFieldName(sec, 0, "Velocity")); + PetscCall(PetscSectionSetFieldComponents(sec, 0, 1)); + PetscCall(PetscSectionSetChart(sec, p_start, p_end)); + // Setup dofs per face + for (PetscInt f = f_start; f < f_end; f++) { + PetscCall(DMPlexGetConeSize(dm, f, &dofs_per_face)); + PetscCall(PetscSectionSetFieldDof(sec, f, 0, dofs_per_face)); + PetscCall(PetscSectionSetDof(sec, f, dofs_per_face)); + } + PetscCall(PetscSectionSetUp(sec)); + PetscCall(DMSetSection(dm, sec)); + PetscCall(PetscSectionDestroy(&sec)); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// Utility function - convert from DMPolytopeType to CeedElemTopology +// ----------------------------------------------------------------------------- +CeedElemTopology ElemTopologyP2C(DMPolytopeType cell_type) { + switch (cell_type) { + case DM_POLYTOPE_TRIANGLE: + return CEED_TOPOLOGY_TRIANGLE; + case DM_POLYTOPE_QUADRILATERAL: + return CEED_TOPOLOGY_QUAD; + case DM_POLYTOPE_TETRAHEDRON: + return CEED_TOPOLOGY_TET; + case DM_POLYTOPE_HEXAHEDRON: + return CEED_TOPOLOGY_HEX; + default: + return 0; + } +}; + +// ----------------------------------------------------------------------------- +// Utility function - essential BC dofs are encoded in closure indices as -(i+1) +// ----------------------------------------------------------------------------- +PetscInt Involute(PetscInt i) { return i >= 0 ? i : -(i + 1); }; + +// ----------------------------------------------------------------------------- +// Get CEED restriction data from DMPlex +// ----------------------------------------------------------------------------- +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { + PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; + + PetscFunctionBeginUser; + + PetscCall(DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, &elem_size, &num_comp, &num_dof, &elem_restr_offsets)); + CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets, elem_restr); + PetscCall(PetscFree(elem_restr_offsets)); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// Get Oriented CEED restriction data from DMPlex +// ----------------------------------------------------------------------------- +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, CeedElemRestriction *elem_restr_u, CeedElemRestriction *elem_restr_p) { + PetscSection section; + PetscInt p, num_elem, num_dof, *restr_indices_u, *restr_indices_p, elem_offset, num_fields, dim, c_start, c_end; + Vec U_loc; + const PetscInt *ornt; // this is for orientation of dof + + PetscFunctionBeginUser; + + PetscCall(DMGetDimension(dm, &dim)); + PetscCall(DMGetLocalSection(dm, §ion)); + PetscCall(PetscSectionGetNumFields(section, &num_fields)); + PetscInt num_comp[num_fields], field_offsets[num_fields + 1]; + field_offsets[0] = 0; + for (PetscInt f = 0; f < num_fields; f++) { + PetscCall(PetscSectionGetFieldComponents(section, f, &num_comp[f])); + field_offsets[f + 1] = field_offsets[f] + num_comp[f]; + } + PetscCall(DMPlexGetHeightStratum(dm, 0, &c_start, &c_end)); + num_elem = c_end - c_start; + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &restr_indices_u)); + PetscCall(PetscMalloc1(num_elem, &restr_indices_p)); + bool *orient_indices; // to flip the dof + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &orient_indices)); + + for (p = 0, elem_offset = 0; p < num_elem; p++) { + restr_indices_p[p] = p; // each cell has on P0 dof + PetscInt num_indices, *indices, faces_per_elem, dofs_per_face; + PetscCall(DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL)); + PetscCall(DMPlexGetConeOrientation(dm, p, &ornt)); + // Get number of faces per element + PetscCall(DMPlexGetConeSize(dm, p, &faces_per_elem)); + dofs_per_face = faces_per_elem - 2; + for (PetscInt f = 0; f < faces_per_elem; f++) { + for (PetscInt i = 0; i < dofs_per_face; i++) { + PetscInt ii = dofs_per_face * f + i; + // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. + PetscInt loc = Involute(indices[ii * num_comp[0]]); + restr_indices_u[elem_offset] = loc; + // Set orientation + orient_indices[elem_offset] = ornt[f] < 0; + elem_offset++; + } + } + PetscCall(DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL)); + } + // if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) + // SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, + // "ElemRestriction of size (%" PetscInt_FMT ",%" PetscInt_FMT ") + // initialized %" PetscInt_FMT "nodes", num_elem, + // dim*PetscPowInt(P, dim),elem_offset); + PetscCall(DMGetLocalVector(dm, &U_loc)); + PetscCall(VecGetLocalSize(U_loc, &num_dof)); + PetscCall(DMRestoreLocalVector(dm, &U_loc)); + // dof per element in Hdiv is dim*P^dim, for linear element P=2 + CeedElemRestrictionCreateOriented(ceed, num_elem, dim * PetscPowInt(P, dim), field_offsets[num_fields], 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, + restr_indices_u, orient_indices, elem_restr_u); + CeedElemRestrictionCreate(ceed, num_elem, 1, 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_p, elem_restr_p); + PetscCall(PetscFree(restr_indices_u)); + PetscCall(PetscFree(orient_indices)); + PetscCall(PetscFree(restr_indices_p)); + + PetscFunctionReturn(0); +}; \ No newline at end of file diff --git a/examples/Hdiv-mass/src/setup-libceed.c b/examples/Hdiv-mass/src/setup-libceed.c index fd5c29b7b6..82ac374751 100644 --- a/examples/Hdiv-mass/src/setup-libceed.c +++ b/examples/Hdiv-mass/src/setup-libceed.c @@ -1,19 +1,15 @@ #include "../include/setup-libceed.h" +#include + #include "../basis/Hdiv-hex.h" #include "../basis/Hdiv-quad.h" -#include "../include/petsc-macros.h" +#include "../basis/L2-P0.h" -// ----------------------------------------------------------------------------- -// Convert PETSc MemType to libCEED MemType -// ----------------------------------------------------------------------------- -CeedMemType MemTypeP2C(PetscMemType mem_type) { return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; } // ----------------------------------------------------------------------------- // Destroy libCEED objects // ----------------------------------------------------------------------------- PetscErrorCode CeedDataDestroy(CeedData ceed_data) { - PetscErrorCode ierr; - PetscFunctionBegin; // Vectors @@ -23,133 +19,30 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data) { CeedElemRestrictionDestroy(&ceed_data->elem_restr_x); CeedElemRestrictionDestroy(&ceed_data->elem_restr_u); CeedElemRestrictionDestroy(&ceed_data->elem_restr_u_i); + CeedElemRestrictionDestroy(&ceed_data->elem_restr_p); // Bases CeedBasisDestroy(&ceed_data->basis_x); CeedBasisDestroy(&ceed_data->basis_u); + CeedBasisDestroy(&ceed_data->basis_p); // QFunctions CeedQFunctionDestroy(&ceed_data->qf_residual); CeedQFunctionDestroy(&ceed_data->qf_error); // Operators CeedOperatorDestroy(&ceed_data->op_residual); CeedOperatorDestroy(&ceed_data->op_error); - ierr = PetscFree(ceed_data); - CHKERRQ(ierr); - - PetscFunctionReturn(0); -}; - -// ----------------------------------------------------------------------------- -// Utility function - essential BC dofs are encoded in closure indices as -(i+1) -// ----------------------------------------------------------------------------- -PetscInt Involute(PetscInt i) { return i >= 0 ? i : -(i + 1); }; - -// ----------------------------------------------------------------------------- -// Get CEED restriction data from DMPlex -// ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { - PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; - PetscErrorCode ierr; - - PetscFunctionBeginUser; - - ierr = DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, &elem_size, &num_comp, &num_dof, &elem_restr_offsets); - CHKERRQ(ierr); + PetscCall(PetscFree(ceed_data)); - CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets, elem_restr); - ierr = PetscFree(elem_restr_offsets); - CHKERRQ(ierr); - - PetscFunctionReturn(0); -}; - -// ----------------------------------------------------------------------------- -// Get Oriented CEED restriction data from DMPlex -// ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, CeedInt P, CeedElemRestriction *elem_restr_oriented) { - PetscSection section; - PetscInt p, num_elem, num_dof, *restr_indices, elem_offset, num_fields, dim, c_start, c_end; - Vec U_loc; - PetscErrorCode ierr; - const PetscInt *ornt; // this is for orientation of dof - PetscFunctionBeginUser; - ierr = DMGetDimension(dm, &dim); - CHKERRQ(ierr); - ierr = DMGetLocalSection(dm, §ion); - CHKERRQ(ierr); - ierr = PetscSectionGetNumFields(section, &num_fields); - CHKERRQ(ierr); - PetscInt num_comp[num_fields], field_offsets[num_fields + 1]; - field_offsets[0] = 0; - for (PetscInt f = 0; f < num_fields; f++) { - ierr = PetscSectionGetFieldComponents(section, f, &num_comp[f]); - CHKERRQ(ierr); - field_offsets[f + 1] = field_offsets[f] + num_comp[f]; - } - ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); - CHKERRQ(ierr); - num_elem = c_end - c_start; - ierr = PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &restr_indices); - CHKERRQ(ierr); - bool *orient_indices; // to flip the dof - ierr = PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &orient_indices); - CHKERRQ(ierr); - for (p = 0, elem_offset = 0; p < num_elem; p++) { - PetscInt num_indices, *indices, faces_per_elem, dofs_per_face; - ierr = DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL); - CHKERRQ(ierr); - - ierr = DMPlexGetConeOrientation(dm, p, &ornt); - CHKERRQ(ierr); - // Get number of faces per element - ierr = DMPlexGetConeSize(dm, p, &faces_per_elem); - CHKERRQ(ierr); - dofs_per_face = faces_per_elem - 2; - for (PetscInt f = 0; f < faces_per_elem; f++) { - for (PetscInt i = 0; i < dofs_per_face; i++) { - PetscInt ii = dofs_per_face * f + i; - // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. - PetscInt loc = Involute(indices[ii * num_comp[0]]); - restr_indices[elem_offset] = loc; - // Set orientation - orient_indices[elem_offset] = ornt[f] < 0; - elem_offset++; - } - } - ierr = DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL); - CHKERRQ(ierr); - } - // if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) - // SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, - // "ElemRestriction of size (%" PetscInt_FMT ",%" PetscInt_FMT ") - // initialized %" PetscInt_FMT "nodes", num_elem, - // dim*PetscPowInt(P, dim),elem_offset); - ierr = DMGetLocalVector(dm, &U_loc); - CHKERRQ(ierr); - ierr = VecGetLocalSize(U_loc, &num_dof); - CHKERRQ(ierr); - ierr = DMRestoreLocalVector(dm, &U_loc); - CHKERRQ(ierr); - // dof per element in Hdiv is dim*P^dim, for linear element P=2 - CeedElemRestrictionCreateOriented(ceed, num_elem, dim * PetscPowInt(P, dim), field_offsets[num_fields], 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, - restr_indices, orient_indices, elem_restr_oriented); - ierr = PetscFree(restr_indices); - CHKERRQ(ierr); - ierr = PetscFree(orient_indices); - CHKERRQ(ierr); PetscFunctionReturn(0); }; // ----------------------------------------------------------------------------- // Set up libCEED on the fine grid for a given degree // ----------------------------------------------------------------------------- -PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *problem_data, PetscInt U_g_size, PetscInt U_loc_size, CeedData ceed_data, - CeedVector rhs_ceed, CeedVector *target) { - int ierr; +PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData problem_data, CeedData ceed_data, CeedVector rhs_ceed) { CeedInt P = app_ctx->degree + 1; // Number of quadratures in 1D, q_extra is set in cl-options.c - CeedInt Q = P + 1 + app_ctx->q_extra; - CeedInt dim, num_comp_x, num_comp_u; - // CeedInt elem_node = problem_data->elem_node; + CeedInt Q = P + 1 + app_ctx->q_extra; + CeedInt dim, num_comp_x, num_comp_u; DM dm_coord; Vec coords; PetscInt c_start, c_end, num_elem; @@ -169,58 +62,52 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *probl CeedInt num_qpts = PetscPowInt(Q, dim); CeedInt P_u = dim * PetscPowInt(P, dim); // dof per element CeedScalar q_ref[dim * num_qpts], q_weights[num_qpts]; - CeedScalar div[P_u * num_qpts], interp[dim * P_u * num_qpts]; + CeedScalar div[P_u * num_qpts], interp[dim * P_u * num_qpts], interp_p[num_qpts], *grad = NULL; if (dim == 2) { HdivBasisQuad(Q, q_ref, q_weights, interp, div, problem_data->quadrature_mode); CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_QUAD, num_comp_u, P_u, num_qpts, interp, div, q_ref, q_weights, &ceed_data->basis_u); + L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); + CeedBasisCreateH1(ceed, CEED_TOPOLOGY_QUAD, 1, 1, num_qpts, interp_p, grad, q_ref, q_weights, &ceed_data->basis_p); } else { HdivBasisHex(Q, q_ref, q_weights, interp, div, problem_data->quadrature_mode); CeedBasisCreateHdiv(ceed, CEED_TOPOLOGY_HEX, num_comp_u, P_u, num_qpts, interp, div, q_ref, q_weights, &ceed_data->basis_u); + L2BasisP0(dim, Q, q_ref, q_weights, interp_p, problem_data->quadrature_mode); + CeedBasisCreateH1(ceed, CEED_TOPOLOGY_HEX, 1, 1, num_qpts, interp_p, grad, q_ref, q_weights, &ceed_data->basis_p); } - CeedBasisCreateTensorH1Lagrange(ceed, dim, num_comp_x, 2, Q, problem_data->quadrature_mode, &ceed_data->basis_x); // --------------------------------------------------------------------------- // libCEED restrictions // --------------------------------------------------------------------------- - ierr = DMGetCoordinateDM(dm, &dm_coord); - CHKERRQ(ierr); - ierr = DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL); - CHKERRQ(ierr); + PetscCall(DMGetCoordinateDM(dm, &dm_coord)); + PetscCall(DMPlexSetClosurePermutationTensor(dm_coord, PETSC_DETERMINE, NULL)); + CeedInt height = 0; // 0 means no boundary conditions DMLabel domain_label = 0; PetscInt value = 0; // -- Coordinate restriction - ierr = CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, value, &ceed_data->elem_restr_x); - CHKERRQ(ierr); - // -- Solution and projected true solution restriction - ierr = CreateRestrictionFromPlexOriented(ceed, dm, P, &ceed_data->elem_restr_u); - CHKERRQ(ierr); - // -- Geometric ceed_data restriction - ierr = DMPlexGetHeightStratum(dm, 0, &c_start, &c_end); - CHKERRQ(ierr); + PetscCall(CreateRestrictionFromPlex(ceed, dm_coord, height, domain_label, value, &ceed_data->elem_restr_x)); + // -- Solution restriction, Error restriction + PetscCall(CreateRestrictionFromPlexOriented(ceed, dm, P, &ceed_data->elem_restr_u, &ceed_data->elem_restr_p)); + PetscCall(DMPlexGetHeightStratum(dm, 0, &c_start, &c_end)); num_elem = c_end - c_start; - + // -- Target restriction for MMS CeedElemRestrictionCreateStrided(ceed, num_elem, num_qpts, dim, dim * num_elem * num_qpts, CEED_STRIDES_BACKEND, &ceed_data->elem_restr_u_i); - // --------------------------------------------------------------------------- // Element coordinates // --------------------------------------------------------------------------- - ierr = DMGetCoordinatesLocal(dm, &coords); - CHKERRQ(ierr); - ierr = VecGetArrayRead(coords, &coordArray); - CHKERRQ(ierr); - + PetscCall(DMGetCoordinatesLocal(dm, &coords)); + PetscCall(VecGetArrayRead(coords, &coordArray)); CeedElemRestrictionCreateVector(ceed_data->elem_restr_x, &x_coord, NULL); CeedVectorSetArray(x_coord, CEED_MEM_HOST, CEED_COPY_VALUES, (PetscScalar *)coordArray); - ierr = VecRestoreArrayRead(coords, &coordArray); - CHKERRQ(ierr); + PetscCall(VecRestoreArrayRead(coords, &coordArray)); // --------------------------------------------------------------------------- // Setup RHS and true solution // --------------------------------------------------------------------------- - CeedVectorCreate(ceed, num_elem * num_qpts * dim, target); + CeedVector target; + CeedVectorCreate(ceed, num_elem * num_qpts * dim, &target); // Create the q-function that sets up the RHS and true solution CeedQFunctionCreateInterior(ceed, 1, problem_data->setup_rhs, problem_data->setup_rhs_loc, &qf_setup_rhs); CeedQFunctionAddInput(qf_setup_rhs, "x", num_comp_x, CEED_EVAL_INTERP); @@ -233,7 +120,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *probl CeedOperatorSetField(op_setup_rhs, "x", ceed_data->elem_restr_x, ceed_data->basis_x, CEED_VECTOR_ACTIVE); CeedOperatorSetField(op_setup_rhs, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); CeedOperatorSetField(op_setup_rhs, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_setup_rhs, "true_soln", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, *target); + CeedOperatorSetField(op_setup_rhs, "true_soln", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, target); CeedOperatorSetField(op_setup_rhs, "rhs", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); // Setup RHS and true solution @@ -242,9 +129,9 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *probl // --------------------------------------------------------------------------- // Persistent libCEED vectors // --------------------------------------------------------------------------- - // -- Operator action variables - CeedVectorCreate(ceed, U_loc_size, &ceed_data->x_ceed); - CeedVectorCreate(ceed, U_loc_size, &ceed_data->y_ceed); + // -- Operator action variables: we use them in setup-solvers.c + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->x_ceed, NULL); + CeedElemRestrictionCreateVector(ceed_data->elem_restr_u, &ceed_data->y_ceed, NULL); // Local residual evaluator // --------------------------------------------------------------------------- @@ -277,14 +164,16 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *probl CeedQFunctionAddInput(qf_error, "u", dim, CEED_EVAL_INTERP); CeedQFunctionAddInput(qf_error, "true_soln", dim, CEED_EVAL_NONE); CeedQFunctionAddInput(qf_error, "weight", 1, CEED_EVAL_WEIGHT); - CeedQFunctionAddOutput(qf_error, "error", dim, CEED_EVAL_NONE); + // CeedQFunctionAddOutput(qf_error, "error", 1, CEED_EVAL_NONE); + CeedQFunctionAddOutput(qf_error, "error", 1, CEED_EVAL_INTERP); // Create the operator that builds the error CeedOperatorCreate(ceed, qf_error, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_error); CeedOperatorSetField(op_error, "dx", ceed_data->elem_restr_x, ceed_data->basis_x, x_coord); CeedOperatorSetField(op_error, "u", ceed_data->elem_restr_u, ceed_data->basis_u, CEED_VECTOR_ACTIVE); - CeedOperatorSetField(op_error, "true_soln", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, *target); + CeedOperatorSetField(op_error, "true_soln", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, target); CeedOperatorSetField(op_error, "weight", CEED_ELEMRESTRICTION_NONE, ceed_data->basis_x, CEED_VECTOR_NONE); - CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_u_i, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + // CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_e_i, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); + CeedOperatorSetField(op_error, "error", ceed_data->elem_restr_p, ceed_data->basis_p, CEED_VECTOR_ACTIVE); // -- Save libCEED data to apply operator in matops.c ceed_data->qf_error = qf_error; ceed_data->op_error = op_error; @@ -292,6 +181,7 @@ PetscErrorCode SetupLibceed(DM dm, Ceed ceed, AppCtx app_ctx, ProblemData *probl CeedQFunctionDestroy(&qf_setup_rhs); CeedOperatorDestroy(&op_setup_rhs); CeedVectorDestroy(&x_coord); + CeedVectorDestroy(&target); PetscFunctionReturn(0); }; diff --git a/examples/Hdiv-mass/src/setup-matops.c b/examples/Hdiv-mass/src/setup-matops.c new file mode 100644 index 0000000000..27fb84f3ba --- /dev/null +++ b/examples/Hdiv-mass/src/setup-matops.c @@ -0,0 +1,51 @@ +#include "../include/setup-matops.h" + +#include + +// ----------------------------------------------------------------------------- +// Apply the local action of a libCEED operator and store result in PETSc vector +// i.e. compute A X = Y +// ----------------------------------------------------------------------------- +PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx) { + PetscFunctionBeginUser; + + // Zero target vector + PetscCall(VecZeroEntries(Y)); + + // Sum into target vector + PetscCall(ApplyAddLocalCeedOp(X, Y, op_apply_ctx)); + + PetscFunctionReturn(0); +} + +PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx) { + PetscScalar *x, *y; + PetscMemType x_mem_type, y_mem_type; + + PetscFunctionBeginUser; + + // Global-to-local + PetscCall(DMGlobalToLocal(op_apply_ctx->dm, X, INSERT_VALUES, op_apply_ctx->X_loc)); + + // Setup libCEED vectors + PetscCall(VecGetArrayReadAndMemType(op_apply_ctx->X_loc, (const PetscScalar **)&x, &x_mem_type)); + PetscCall(VecGetArrayAndMemType(op_apply_ctx->Y_loc, &y, &y_mem_type)); + CeedVectorSetArray(op_apply_ctx->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, x); + CeedVectorSetArray(op_apply_ctx->y_ceed, MemTypeP2C(y_mem_type), CEED_USE_POINTER, y); + + // Apply libCEED operator + CeedOperatorApply(op_apply_ctx->op_apply, op_apply_ctx->x_ceed, op_apply_ctx->y_ceed, CEED_REQUEST_IMMEDIATE); + + // Restore PETSc vectors + CeedVectorTakeArray(op_apply_ctx->x_ceed, MemTypeP2C(x_mem_type), NULL); + CeedVectorTakeArray(op_apply_ctx->y_ceed, MemTypeP2C(y_mem_type), NULL); + PetscCall(VecRestoreArrayReadAndMemType(op_apply_ctx->X_loc, (const PetscScalar **)&x)); + PetscCall(VecRestoreArrayAndMemType(op_apply_ctx->Y_loc, &y)); + + // Local-to-global + PetscCall(DMLocalToGlobal(op_apply_ctx->dm, op_apply_ctx->Y_loc, ADD_VALUES, Y)); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- \ No newline at end of file diff --git a/examples/Hdiv-mass/src/setup-solvers.c b/examples/Hdiv-mass/src/setup-solvers.c new file mode 100644 index 0000000000..fac3e91970 --- /dev/null +++ b/examples/Hdiv-mass/src/setup-solvers.c @@ -0,0 +1,123 @@ +#include "../include/setup-solvers.h" + +#include "../include/setup-libceed.h" +#include "../include/setup-matops.h" +#include "petscvec.h" + +// ----------------------------------------------------------------------------- +// Setup operator context data +// ----------------------------------------------------------------------------- +PetscErrorCode SetupResidualOperatorCtx(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual) { + PetscFunctionBeginUser; + + ctx_residual->comm = comm; + ctx_residual->dm = dm; + PetscCall(DMCreateLocalVector(dm, &ctx_residual->X_loc)); + PetscCall(VecDuplicate(ctx_residual->X_loc, &ctx_residual->Y_loc)); + ctx_residual->x_ceed = ceed_data->x_ceed; + ctx_residual->y_ceed = ceed_data->y_ceed; + ctx_residual->ceed = ceed; + ctx_residual->op_apply = ceed_data->op_residual; + + PetscFunctionReturn(0); +} + +PetscErrorCode SetupErrorOperatorCtx(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_error_u) { + PetscFunctionBeginUser; + + ctx_error_u->comm = comm; + ctx_error_u->dm = dm; + PetscCall(DMCreateLocalVector(dm, &ctx_error_u->X_loc)); + PetscCall(VecDuplicate(ctx_error_u->X_loc, &ctx_error_u->Y_loc)); + ctx_error_u->x_ceed = ceed_data->x_ceed; + ctx_error_u->y_ceed = ceed_data->y_ceed; + ctx_error_u->ceed = ceed; + ctx_error_u->op_apply = ceed_data->op_error; + + PetscFunctionReturn(0); +} + +// ----------------------------------------------------------------------------- +// This function wraps the libCEED operator for a MatShell +// ----------------------------------------------------------------------------- +PetscErrorCode ApplyMatOp(Mat A, Vec X, Vec Y) { + OperatorApplyContext op_apply_ctx; + + PetscFunctionBeginUser; + + PetscCall(MatShellGetContext(A, &op_apply_ctx)); + + // libCEED for local action of residual evaluator + PetscCall(ApplyLocalCeedOp(X, Y, op_apply_ctx)); + + PetscFunctionReturn(0); +}; + +// --------------------------------------------------------------------------- +// Setup Solver +// --------------------------------------------------------------------------- +PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, KSP ksp, Vec rhs, Vec *X) { + PetscInt X_l_size, X_g_size; + + PetscFunctionBeginUser; + + // Create global unknown solution U + PetscCall(VecGetSize(*X, &X_g_size)); + // Local size for matShell + PetscCall(VecGetLocalSize(*X, &X_l_size)); + + // --------------------------------------------------------------------------- + // Setup SNES + // --------------------------------------------------------------------------- + // Operator + Mat mat_op; + VecType vec_type; + PetscCall(DMGetVecType(app_ctx->ctx_residual->dm, &vec_type)); + // -- Form Action of Jacobian on delta_u + PetscCall(MatCreateShell(app_ctx->comm, X_l_size, X_l_size, X_g_size, X_g_size, app_ctx->ctx_residual, &mat_op)); + PetscCall(MatShellSetOperation(mat_op, MATOP_MULT, (void (*)(void))ApplyMatOp)); + PetscCall(MatShellSetVecType(mat_op, vec_type)); + + PC pc; + PetscCall(KSPGetPC(ksp, &pc)); + PetscCall(PCSetType(pc, PCNONE)); + PetscCall(KSPSetType(ksp, KSPCG)); + // PetscCall(KSPSetNormType(ksp, KSP_NORM_NATURAL)); + PetscCall(KSPSetTolerances(ksp, 1e-10, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT)); + + PetscCall(KSPSetOperators(ksp, mat_op, mat_op)); + PetscCall(KSPSetFromOptions(ksp)); + PetscCall(VecZeroEntries(*X)); + PetscCall(KSPSolve(ksp, rhs, *X)); + + PetscCall(MatDestroy(&mat_op)); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// This function calculates the error in the final solution +// ----------------------------------------------------------------------------- +PetscErrorCode ComputeL2Error(Vec X, PetscScalar *l2_error, OperatorApplyContext op_error_ctx) { + Vec E; + PetscFunctionBeginUser; + + PetscCall(VecDuplicate(X, &E)); + PetscCall(ApplyLocalCeedOp(X, E, op_error_ctx)); + PetscScalar error_sq = 1.0; + PetscCall(VecSum(E, &error_sq)); + *l2_error = sqrt(error_sq); + PetscCall(VecDestroy(&E)); + PetscFunctionReturn(0); +}; + +PetscErrorCode CtxVecDestroy(ProblemData problem_data, AppCtx app_ctx) { + PetscFunctionBegin; + + PetscCall(VecDestroy(&app_ctx->ctx_residual->Y_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_residual->X_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_error_u->Y_loc)); + PetscCall(VecDestroy(&app_ctx->ctx_error_u->X_loc)); + + PetscFunctionReturn(0); +} \ No newline at end of file From 52a4361557283288c105baa28c9c718a8b2a525f Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Fri, 16 Dec 2022 09:56:18 -0700 Subject: [PATCH 12/15] some clean-up in Hdiv-mixed folder --- examples/Hdiv-mixed/include/post-processing.h | 6 +- examples/Hdiv-mixed/include/setup-dm.h | 2 +- examples/Hdiv-mixed/include/setup-fe.h | 7 + examples/Hdiv-mixed/include/setup-libceed.h | 12 +- examples/Hdiv-mixed/include/setup-solvers.h | 2 +- examples/Hdiv-mixed/include/setup-ts.h | 2 +- examples/Hdiv-mixed/main.c | 116 ++++++------- examples/Hdiv-mixed/src/cl-options.c | 2 +- examples/Hdiv-mixed/src/post-processing.c | 10 +- examples/Hdiv-mixed/src/setup-dm.c | 26 ++- examples/Hdiv-mixed/src/setup-fe.c | 158 +++++++++++++++++- examples/Hdiv-mixed/src/setup-libceed.c | 137 --------------- examples/Hdiv-mixed/src/setup-solvers.c | 46 ++--- examples/Hdiv-mixed/src/setup-ts.c | 4 +- 14 files changed, 277 insertions(+), 253 deletions(-) diff --git a/examples/Hdiv-mixed/include/post-processing.h b/examples/Hdiv-mixed/include/post-processing.h index 9c3506cfcf..5bd940be39 100644 --- a/examples/Hdiv-mixed/include/post-processing.h +++ b/examples/Hdiv-mixed/include/post-processing.h @@ -6,10 +6,10 @@ #include "../include/setup-libceed.h" #include "structs.h" -PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, CeedMemType mem_type_backend, TS ts, SNES snes, KSP ksp, Vec U, - CeedScalar l2_error_u, CeedScalar l2_error_p); +PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, TS ts, SNES snes, KSP ksp, Vec U, CeedScalar l2_error_u, + CeedScalar l2_error_p); PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_Hdiv); -PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_H1); +PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_H1); PetscErrorCode ProjectVelocity(AppCtx app_ctx, Vec U, Vec *U_H1); PetscErrorCode CtxVecDestroy(AppCtx app_ctx); #endif // post_processing_h diff --git a/examples/Hdiv-mixed/include/setup-dm.h b/examples/Hdiv-mixed/include/setup-dm.h index 71469cbd37..775b685b92 100644 --- a/examples/Hdiv-mixed/include/setup-dm.h +++ b/examples/Hdiv-mixed/include/setup-dm.h @@ -11,7 +11,7 @@ // --------------------------------------------------------------------------- // Setup DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, VecType vec_type, DM *dm); +PetscErrorCode CreateDM(MPI_Comm comm, Ceed ceed, DM *dm); PetscErrorCode PerturbVerticesSmooth(DM dm); PetscErrorCode PerturbVerticesRandom(DM dm); diff --git a/examples/Hdiv-mixed/include/setup-fe.h b/examples/Hdiv-mixed/include/setup-fe.h index 382b047740..8836489f8b 100644 --- a/examples/Hdiv-mixed/include/setup-fe.h +++ b/examples/Hdiv-mixed/include/setup-fe.h @@ -11,6 +11,13 @@ // --------------------------------------------------------------------------- // Setup FE // --------------------------------------------------------------------------- +CeedMemType MemTypeP2C(PetscMemType mtype); PetscErrorCode SetupFEHdiv(MPI_Comm comm, DM dm, DM dm_u0, DM dm_p0); PetscErrorCode SetupFEH1(ProblemData problem_data, AppCtx app_ctx, DM dm_H1); +PetscInt Involute(PetscInt i); +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr); +// Utility function to create local CEED Oriented restriction from DMPlex +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u, + CeedElemRestriction *elem_restr_p, CeedElemRestriction *elem_restr_u0, + CeedElemRestriction *elem_restr_p0); #endif // setupfe_h diff --git a/examples/Hdiv-mixed/include/setup-libceed.h b/examples/Hdiv-mixed/include/setup-libceed.h index 3b509b4a65..d04c3b1fa5 100644 --- a/examples/Hdiv-mixed/include/setup-libceed.h +++ b/examples/Hdiv-mixed/include/setup-libceed.h @@ -1,20 +1,10 @@ #ifndef setuplibceed_h #define setuplibceed_h +#include "setup-fe.h" #include "structs.h" -// Convert PETSc MemType to libCEED MemType -CeedMemType MemTypeP2C(PetscMemType mtype); // Destroy libCEED objects PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data); -// Utility function - essential BC dofs are encoded in closure indices as -(i+1) -PetscInt Involute(PetscInt i); -// Utility function to create local CEED restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr); -// Utility function to create local CEED Oriented restriction from DMPlex -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u, - CeedElemRestriction *elem_restr_p, CeedElemRestriction *elem_restr_u0, - CeedElemRestriction *elem_restr_p0); -// Set up libCEED for a given degree PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, Ceed ceed, AppCtx app_ctx, ProblemData problem_data, CeedData ceed_data); #endif // setuplibceed_h diff --git a/examples/Hdiv-mixed/include/setup-solvers.h b/examples/Hdiv-mixed/include/setup-solvers.h index 5502172408..36d50a210d 100644 --- a/examples/Hdiv-mixed/include/setup-solvers.h +++ b/examples/Hdiv-mixed/include/setup-solvers.h @@ -7,7 +7,7 @@ #include "petscvec.h" #include "structs.h" -PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_jacobian); +PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_jacobian); PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual); PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_error); PetscErrorCode ApplyMatOp(Mat A, Vec X, Vec Y); diff --git a/examples/Hdiv-mixed/include/setup-ts.h b/examples/Hdiv-mixed/include/setup-ts.h index d490ba9bab..9db7f31203 100644 --- a/examples/Hdiv-mixed/include/setup-ts.h +++ b/examples/Hdiv-mixed/include/setup-ts.h @@ -6,7 +6,7 @@ #include "structs.h" -PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, VecType vec_type, Vec U); +PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, Vec U); PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual_ut); PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_initial_u0); PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_initial_p0); diff --git a/examples/Hdiv-mixed/main.c b/examples/Hdiv-mixed/main.c index a0df25b77e..c990c08e8f 100644 --- a/examples/Hdiv-mixed/main.c +++ b/examples/Hdiv-mixed/main.c @@ -41,6 +41,7 @@ int main(int argc, char **argv) { // Initialize PETSc // --------------------------------------------------------------------------- PetscCall(PetscInitialize(&argc, &argv, NULL, help)); + MPI_Comm comm = PETSC_COMM_WORLD; // --------------------------------------------------------------------------- // Create structs @@ -77,6 +78,12 @@ int main(int argc, char **argv) { // Context for post-processing app_ctx->ctx_Hdiv = ctx_Hdiv; app_ctx->ctx_H1 = ctx_H1; + app_ctx->comm = comm; + + // --------------------------------------------------------------------------- + // Process command line options + // --------------------------------------------------------------------------- + PetscCall(ProcessCommandLineOptions(app_ctx)); // --------------------------------------------------------------------------- // Initialize libCEED @@ -85,41 +92,20 @@ int main(int argc, char **argv) { Ceed ceed; CeedInit("/cpu/self/ref/serial", &ceed); // CeedInit(app_ctx->ceed_resource, &ceed); - CeedMemType mem_type_backend; - CeedGetPreferredMemType(ceed, &mem_type_backend); - - VecType vec_type = NULL; - MatType mat_type = NULL; - switch (mem_type_backend) { - case CEED_MEM_HOST: - vec_type = VECSTANDARD; - break; - case CEED_MEM_DEVICE: { - const char *resolved; - CeedGetResource(ceed, &resolved); - if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; - else if (strstr(resolved, "/gpu/hip")) vec_type = VECKOKKOS; - else vec_type = VECSTANDARD; - } - } - if (strstr(vec_type, VECCUDA)) mat_type = MATAIJCUSPARSE; - else if (strstr(vec_type, VECKOKKOS)) mat_type = MATAIJKOKKOS; - else mat_type = MATAIJ; // -- Process general command line options - MPI_Comm comm = PETSC_COMM_WORLD; // --------------------------------------------------------------------------- // Create DM // --------------------------------------------------------------------------- DM dm, dm_u0, dm_p0, dm_H1; // DM for mixed problem - PetscCall(CreateDM(comm, mat_type, vec_type, &dm)); + PetscCall(CreateDM(app_ctx->comm, ceed, &dm)); // DM for projecting initial velocity to Hdiv space - PetscCall(CreateDM(comm, mat_type, vec_type, &dm_u0)); + PetscCall(CreateDM(app_ctx->comm, ceed, &dm_u0)); // DM for projecting initial pressure in L2 - PetscCall(CreateDM(comm, mat_type, vec_type, &dm_p0)); + PetscCall(CreateDM(app_ctx->comm, ceed, &dm_p0)); // DM for projecting solution U into H1 space for PetscViewer - PetscCall(CreateDM(comm, mat_type, vec_type, &dm_H1)); + PetscCall(CreateDM(app_ctx->comm, ceed, &dm_H1)); // TODO: add mesh option // perturb dm to have smooth random mesh // PetscCall( PerturbVerticesSmooth(dm) ); @@ -129,18 +115,10 @@ int main(int argc, char **argv) { // PetscCall(PerturbVerticesRandom(dm) ); // PetscCall(PerturbVerticesRandom(dm_H1) ); - // --------------------------------------------------------------------------- - // Process command line options - // --------------------------------------------------------------------------- - // -- Register problems to be available on the command line - PetscCall(RegisterProblems_Hdiv(app_ctx)); - - app_ctx->comm = comm; - PetscCall(ProcessCommandLineOptions(app_ctx)); - // --------------------------------------------------------------------------- // Choose the problem from the list of registered problems // --------------------------------------------------------------------------- + PetscCall(RegisterProblems_Hdiv(app_ctx)); { PetscErrorCode (*p)(Ceed, ProblemData, DM, void *); PetscCall(PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p)); @@ -151,7 +129,7 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // Setup FE for H(div) mixed-problem and H1 projection in post-processing.c // --------------------------------------------------------------------------- - PetscCall(SetupFEHdiv(comm, dm, dm_u0, dm_p0)); + PetscCall(SetupFEHdiv(app_ctx->comm, dm, dm_u0, dm_p0)); PetscCall(SetupFEH1(problem_data, app_ctx, dm_H1)); // --------------------------------------------------------------------------- @@ -167,36 +145,36 @@ int main(int argc, char **argv) { PetscCall(SetupLibceed(dm, dm_u0, dm_p0, dm_H1, ceed, app_ctx, problem_data, ceed_data)); // --------------------------------------------------------------------------- - // Setup pressure boundary conditions + // Setup pressure boundary conditions (not working) // --------------------------------------------------------------------------- // --Create empty local vector for libCEED - Vec P_loc; - PetscInt P_loc_size; - CeedScalar *p0; - CeedVector P_ceed; - PetscMemType pressure_mem_type; - PetscCall(DMCreateLocalVector(dm, &P_loc)); - PetscCall(VecGetSize(P_loc, &P_loc_size)); - PetscCall(VecZeroEntries(P_loc)); - PetscCall(VecGetArrayAndMemType(P_loc, &p0, &pressure_mem_type)); - CeedVectorCreate(ceed, P_loc_size, &P_ceed); - CeedVectorSetArray(P_ceed, MemTypeP2C(pressure_mem_type), CEED_USE_POINTER, p0); - // -- Apply operator to create local pressure vector on boundary - PetscCall(DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, P_ceed)); - // CeedVectorView(P_ceed, "%12.8f", stdout); - // -- Map local to global - Vec P; - CeedVectorTakeArray(P_ceed, MemTypeP2C(pressure_mem_type), NULL); - PetscCall(VecRestoreArrayAndMemType(P_loc, &p0)); - PetscCall(DMCreateGlobalVector(dm, &P)); - PetscCall(VecZeroEntries(P)); - PetscCall(DMLocalToGlobal(dm, P_loc, ADD_VALUES, P)); + // Vec P_loc; + // PetscInt P_loc_size; + // CeedScalar *p0; + // CeedVector P_ceed; + // PetscMemType pressure_mem_type; + // PetscCall(DMCreateLocalVector(dm, &P_loc)); + // PetscCall(VecGetSize(P_loc, &P_loc_size)); + // PetscCall(VecZeroEntries(P_loc)); + // PetscCall(VecGetArrayAndMemType(P_loc, &p0, &pressure_mem_type)); + // CeedVectorCreate(ceed, P_loc_size, &P_ceed); + // CeedVectorSetArray(P_ceed, MemTypeP2C(pressure_mem_type), CEED_USE_POINTER, p0); + //// -- Apply operator to create local pressure vector on boundary + // PetscCall(DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, P_ceed)); + //// CeedVectorView(P_ceed, "%12.8f", stdout); + //// -- Map local to global + // Vec P; + // CeedVectorTakeArray(P_ceed, MemTypeP2C(pressure_mem_type), NULL); + // PetscCall(VecRestoreArrayAndMemType(P_loc, &p0)); + // PetscCall(DMCreateGlobalVector(dm, &P)); + // PetscCall(VecZeroEntries(P)); + // PetscCall(DMLocalToGlobal(dm, P_loc, ADD_VALUES, P)); // --------------------------------------------------------------------------- // Setup context for projection problem; post-processing.c // --------------------------------------------------------------------------- - PetscCall(SetupProjectVelocityCtx_Hdiv(comm, dm, ceed, ceed_data, app_ctx->ctx_Hdiv)); - PetscCall(SetupProjectVelocityCtx_H1(comm, dm_H1, ceed, ceed_data, vec_type, app_ctx->ctx_H1)); + PetscCall(SetupProjectVelocityCtx_Hdiv(app_ctx->comm, dm, ceed, ceed_data, app_ctx->ctx_Hdiv)); + PetscCall(SetupProjectVelocityCtx_H1(app_ctx->comm, dm_H1, ceed, ceed_data, app_ctx->ctx_H1)); // --------------------------------------------------------------------------- // Setup TSSolve for Richard problem @@ -206,13 +184,13 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // Setup context for initial conditions // --------------------------------------------------------------------------- - PetscCall(SetupResidualOperatorCtx_U0(comm, dm_u0, ceed, ceed_data, app_ctx->ctx_initial_u0)); - PetscCall(SetupResidualOperatorCtx_P0(comm, dm_p0, ceed, ceed_data, app_ctx->ctx_initial_p0)); - PetscCall(SetupResidualOperatorCtx_Ut(comm, dm, ceed, ceed_data, app_ctx->ctx_residual_ut)); - PetscCall(CreateInitialConditions(ceed_data, app_ctx, vec_type, U)); + PetscCall(SetupResidualOperatorCtx_U0(app_ctx->comm, dm_u0, ceed, ceed_data, app_ctx->ctx_initial_u0)); + PetscCall(SetupResidualOperatorCtx_P0(app_ctx->comm, dm_p0, ceed, ceed_data, app_ctx->ctx_initial_p0)); + PetscCall(SetupResidualOperatorCtx_Ut(app_ctx->comm, dm, ceed, ceed_data, app_ctx->ctx_residual_ut)); + PetscCall(CreateInitialConditions(ceed_data, app_ctx, U)); // VecView(U, PETSC_VIEWER_STDOUT_WORLD); // Solve Richards problem - PetscCall(TSCreate(comm, &ts)); + PetscCall(TSCreate(app_ctx->comm, &ts)); PetscCall(VecZeroEntries(app_ctx->ctx_residual_ut->X_loc)); PetscCall(VecZeroEntries(app_ctx->ctx_residual_ut->X_t_loc)); PetscCall(TSSolveRichard(ceed_data, app_ctx, ts, &U)); @@ -225,10 +203,10 @@ int main(int argc, char **argv) { SNES snes; KSP ksp; if (!problem_data->has_ts) { - PetscCall(SetupJacobianOperatorCtx(dm, ceed, ceed_data, vec_type, app_ctx->ctx_jacobian)); + PetscCall(SetupJacobianOperatorCtx(dm, ceed, ceed_data, app_ctx->ctx_jacobian)); PetscCall(SetupResidualOperatorCtx(dm, ceed, ceed_data, app_ctx->ctx_residual)); // Create SNES - PetscCall(SNESCreate(comm, &snes)); + PetscCall(SNESCreate(app_ctx->comm, &snes)); PetscCall(SNESGetKSP(snes, &ksp)); PetscCall(PDESolver(ceed_data, app_ctx, snes, ksp, &U)); // VecView(U, PETSC_VIEWER_STDOUT_WORLD); @@ -244,14 +222,14 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // Print solver iterations and final norms // --------------------------------------------------------------------------- - PetscCall(PrintOutput(dm, ceed, app_ctx, problem_data->has_ts, mem_type_backend, ts, snes, ksp, U, l2_error_u, l2_error_p)); + PetscCall(PrintOutput(dm, ceed, app_ctx, problem_data->has_ts, ts, snes, ksp, U, l2_error_u, l2_error_p)); // --------------------------------------------------------------------------- // Save solution (paraview) // --------------------------------------------------------------------------- if (app_ctx->view_solution) { PetscViewer viewer_p; - PetscCall(PetscViewerVTKOpen(comm, "darcy_pressure.vtu", FILE_MODE_WRITE, &viewer_p)); + PetscCall(PetscViewerVTKOpen(app_ctx->comm, "darcy_pressure.vtu", FILE_MODE_WRITE, &viewer_p)); PetscCall(VecView(U, viewer_p)); PetscCall(PetscViewerDestroy(&viewer_p)); @@ -260,7 +238,7 @@ int main(int argc, char **argv) { PetscCall(ProjectVelocity(app_ctx, U, &U_H1)); PetscViewer viewer_u; - PetscCall(PetscViewerVTKOpen(comm, "darcy_velocity.vtu", FILE_MODE_WRITE, &viewer_u)); + PetscCall(PetscViewerVTKOpen(app_ctx->comm, "darcy_velocity.vtu", FILE_MODE_WRITE, &viewer_u)); PetscCall(VecView(U_H1, viewer_u)); PetscCall(PetscViewerDestroy(&viewer_u)); PetscCall(VecDestroy(&U_H1)); diff --git a/examples/Hdiv-mixed/src/cl-options.c b/examples/Hdiv-mixed/src/cl-options.c index 0d67fedb59..0b8bf4e387 100644 --- a/examples/Hdiv-mixed/src/cl-options.c +++ b/examples/Hdiv-mixed/src/cl-options.c @@ -25,7 +25,7 @@ PetscErrorCode ProcessCommandLineOptions(AppCtx app_ctx) { PetscBool ceed_flag = PETSC_FALSE; PetscFunctionBeginUser; - PetscOptionsBegin(app_ctx->comm, NULL, "H(div) examples in PETSc with libCEED", NULL); + PetscOptionsBegin(app_ctx->comm, NULL, "H(div) mixed-problem in PETSc with libCEED", NULL); PetscCall(PetscOptionsString("-ceed", "CEED resource specifier", NULL, app_ctx->ceed_resource, app_ctx->ceed_resource, sizeof(app_ctx->ceed_resource), &ceed_flag)); diff --git a/examples/Hdiv-mixed/src/post-processing.c b/examples/Hdiv-mixed/src/post-processing.c index 2a99a61cf3..8bc99d0a04 100644 --- a/examples/Hdiv-mixed/src/post-processing.c +++ b/examples/Hdiv-mixed/src/post-processing.c @@ -5,12 +5,14 @@ // ----------------------------------------------------------------------------- // This function print the output // ----------------------------------------------------------------------------- -PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, CeedMemType mem_type_backend, TS ts, SNES snes, KSP ksp, Vec U, - CeedScalar l2_error_u, CeedScalar l2_error_p) { +PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, TS ts, SNES snes, KSP ksp, Vec U, CeedScalar l2_error_u, + CeedScalar l2_error_p) { PetscFunctionBeginUser; const char *used_resource; + CeedMemType mem_type_backend; CeedGetResource(ceed, &used_resource); + CeedGetPreferredMemType(ceed, &mem_type_backend); char hostname[PETSC_MAX_PATH_LEN]; PetscCall(PetscGetHostName(hostname, sizeof hostname)); PetscInt comm_size; @@ -128,9 +130,11 @@ PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, Cee PetscFunctionReturn(0); } -PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_H1) { +PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_H1) { PetscFunctionBeginUser; + VecType vec_type; + PetscCall(DMGetVecType(dm_H1, &vec_type)); ctx_H1->comm = comm; ctx_H1->dm = dm_H1; PetscCall(DMCreateLocalVector(dm_H1, &ctx_H1->X_loc)); diff --git a/examples/Hdiv-mixed/src/setup-dm.c b/examples/Hdiv-mixed/src/setup-dm.c index 3e39ebd7e6..186dd90eca 100644 --- a/examples/Hdiv-mixed/src/setup-dm.c +++ b/examples/Hdiv-mixed/src/setup-dm.c @@ -3,11 +3,33 @@ #include "petscerror.h" // --------------------------------------------------------------------------- -// Setup DM +// Create DM // --------------------------------------------------------------------------- -PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, VecType vec_type, DM *dm) { +PetscErrorCode CreateDM(MPI_Comm comm, Ceed ceed, DM *dm) { PetscFunctionBeginUser; + CeedMemType mem_type_backend; + CeedGetPreferredMemType(ceed, &mem_type_backend); + + VecType vec_type = NULL; + MatType mat_type = NULL; + switch (mem_type_backend) { + case CEED_MEM_HOST: + vec_type = VECSTANDARD; + break; + case CEED_MEM_DEVICE: { + const char *resolved; + CeedGetResource(ceed, &resolved); + if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA; + else if (strstr(resolved, "/gpu/hip/occa")) vec_type = VECSTANDARD; // https://github.com/CEED/libCEED/issues/678 + else if (strstr(resolved, "/gpu/hip")) vec_type = VECHIP; + else vec_type = VECSTANDARD; + } + } + if (strstr(vec_type, VECCUDA)) mat_type = MATAIJCUSPARSE; + else if (strstr(vec_type, VECKOKKOS)) mat_type = MATAIJKOKKOS; + else mat_type = MATAIJ; + // Create DMPLEX PetscCall(DMCreate(comm, dm)); PetscCall(DMSetType(*dm, DMPLEX)); diff --git a/examples/Hdiv-mixed/src/setup-fe.c b/examples/Hdiv-mixed/src/setup-fe.c index e4d059bb03..da4b000055 100644 --- a/examples/Hdiv-mixed/src/setup-fe.c +++ b/examples/Hdiv-mixed/src/setup-fe.c @@ -2,6 +2,11 @@ #include "petscerror.h" +// ----------------------------------------------------------------------------- +// Convert PETSc MemType to libCEED MemType +// ----------------------------------------------------------------------------- +CeedMemType MemTypeP2C(PetscMemType mem_type) { return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; } + // --------------------------------------------------------------------------- // Setup FE // --------------------------------------------------------------------------- @@ -122,4 +127,155 @@ PetscErrorCode SetupFEH1(ProblemData problem_data, AppCtx app_ctx, DM dm_H1) { PetscCall(PetscSectionSetComponentName(section, 0, 2, "Velocity_Z")); } PetscFunctionReturn(0); -}; \ No newline at end of file +}; + +// ----------------------------------------------------------------------------- +// Utility function - convert from DMPolytopeType to CeedElemTopology +// ----------------------------------------------------------------------------- +CeedElemTopology ElemTopologyP2C(DMPolytopeType cell_type) { + switch (cell_type) { + case DM_POLYTOPE_TRIANGLE: + return CEED_TOPOLOGY_TRIANGLE; + case DM_POLYTOPE_QUADRILATERAL: + return CEED_TOPOLOGY_QUAD; + case DM_POLYTOPE_TETRAHEDRON: + return CEED_TOPOLOGY_TET; + case DM_POLYTOPE_HEXAHEDRON: + return CEED_TOPOLOGY_HEX; + default: + return 0; + } +}; + +// ----------------------------------------------------------------------------- +// Utility function - essential BC dofs are encoded in closure indices as -(i+1) +// ----------------------------------------------------------------------------- +PetscInt Involute(PetscInt i) { return i >= 0 ? i : -(i + 1); }; + +// ----------------------------------------------------------------------------- +// Get CEED restriction data from DMPlex +// ----------------------------------------------------------------------------- +PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { + PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; + + PetscFunctionBeginUser; + + PetscCall(DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, &elem_size, &num_comp, &num_dof, &elem_restr_offsets)); + + CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets, elem_restr); + PetscCall(PetscFree(elem_restr_offsets)); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- +// Get Oriented CEED restriction data from DMPlex +// ----------------------------------------------------------------------------- +PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u, + CeedElemRestriction *elem_restr_p, CeedElemRestriction *elem_restr_u0, + CeedElemRestriction *elem_restr_p0) { + PetscSection section, section_u0, section_p0; + PetscInt p, num_elem, num_dof, num_dof_u0, num_dof_p0, *restr_indices_u, *restr_indices_p, *restr_indices_u0, *restr_indices_p0, elem_offset, + num_fields, num_fields_u0, num_fields_p0, dim, c_start, c_end; + Vec U_loc; + const PetscInt *ornt; // this is for orientation of dof + PetscFunctionBeginUser; + // Section for mixed problem + PetscCall(DMGetDimension(dm, &dim)); + PetscCall(DMGetLocalSection(dm, §ion)); + PetscCall(PetscSectionGetNumFields(section, &num_fields)); + PetscInt num_comp[num_fields], field_offsets[num_fields + 1]; + field_offsets[0] = 0; + for (PetscInt f = 0; f < num_fields; f++) { + PetscCall(PetscSectionGetFieldComponents(section, f, &num_comp[f])); + field_offsets[f + 1] = field_offsets[f] + num_comp[f]; + } + // Section for initial conditions u0 + PetscCall(DMGetLocalSection(dm_u0, §ion_u0)); + PetscCall(PetscSectionGetNumFields(section_u0, &num_fields_u0)); + PetscInt num_comp_u0[num_fields_u0], field_offsets_u0[num_fields_u0 + 1]; + field_offsets_u0[0] = 0; + for (PetscInt f = 0; f < num_fields_u0; f++) { + PetscCall(PetscSectionGetFieldComponents(section_u0, f, &num_comp_u0[f])); + field_offsets_u0[f + 1] = field_offsets_u0[f] + num_comp_u0[f]; + } + // Section for initial conditions p0 + PetscCall(DMGetLocalSection(dm_p0, §ion_p0)); + PetscCall(PetscSectionGetNumFields(section_p0, &num_fields_p0)); + PetscInt num_comp_p0[num_fields_p0], field_offsets_p0[num_fields_p0 + 1]; + field_offsets_p0[0] = 0; + for (PetscInt f = 0; f < num_fields_p0; f++) { + PetscCall(PetscSectionGetFieldComponents(section_p0, f, &num_comp_p0[f])); + field_offsets_p0[f + 1] = field_offsets_p0[f] + num_comp_p0[f]; + } + + PetscCall(DMPlexGetHeightStratum(dm, 0, &c_start, &c_end)); + num_elem = c_end - c_start; + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &restr_indices_u)); + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &restr_indices_u0)); + PetscCall(PetscMalloc1(num_elem, &restr_indices_p)); + PetscCall(PetscMalloc1(num_elem, &restr_indices_p0)); + bool *orient_indices_u, *orient_indices_u0; // to flip the dof + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &orient_indices_u)); + PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &orient_indices_u0)); + for (p = 0, elem_offset = 0; p < num_elem; p++) { + PetscInt num_indices, *indices, faces_per_elem, dofs_per_face, num_indices_u0, *indices_u0, num_indices_p0, *indices_p0; + PetscCall(DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL)); + PetscCall(DMPlexGetClosureIndices(dm_u0, section_u0, section_u0, p, PETSC_TRUE, &num_indices_u0, &indices_u0, NULL, NULL)); + PetscCall(DMPlexGetClosureIndices(dm_p0, section_p0, section_p0, p, PETSC_TRUE, &num_indices_p0, &indices_p0, NULL, NULL)); + restr_indices_p[p] = indices[num_indices - 1]; + restr_indices_p0[p] = indices_p0[0]; + PetscCall(DMPlexGetConeOrientation(dm, p, &ornt)); + // Get number of faces per element + PetscCall(DMPlexGetConeSize(dm, p, &faces_per_elem)); + dofs_per_face = faces_per_elem - 2; + for (PetscInt f = 0; f < faces_per_elem; f++) { + for (PetscInt i = 0; i < dofs_per_face; i++) { + PetscInt ii = dofs_per_face * f + i; + // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. + PetscInt loc = Involute(indices[ii * num_comp[0]]); + restr_indices_u[elem_offset] = loc; + // Set orientation + orient_indices_u[elem_offset] = ornt[f] < 0; + PetscInt loc_u0 = Involute(indices_u0[ii * num_comp_u0[0]]); + restr_indices_u0[elem_offset] = loc_u0; + // Set orientation + orient_indices_u0[elem_offset] = ornt[f] < 0; + elem_offset++; + } + } + PetscCall(DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL)); + PetscCall(DMPlexRestoreClosureIndices(dm_u0, section_u0, section_u0, p, PETSC_TRUE, &num_indices_u0, &indices_u0, NULL, NULL)); + PetscCall(DMPlexRestoreClosureIndices(dm_p0, section_p0, section_p0, p, PETSC_TRUE, &num_indices_p0, &indices_p0, NULL, NULL)); + } + // if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) + // SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, + // "ElemRestriction of size (%" PetscInt_FMT ", %" PetscInt_FMT" ) + // initialized %" PetscInt_FMT " nodes", num_elem, + // dim*PetscPowInt(P, dim),elem_offset); + + PetscCall(DMGetLocalVector(dm, &U_loc)); + PetscCall(VecGetLocalSize(U_loc, &num_dof)); + PetscCall(DMRestoreLocalVector(dm, &U_loc)); + // dof per element in Hdiv is dim*P^dim, for linear element P=2 + CeedElemRestrictionCreateOriented(ceed, num_elem, dim * PetscPowInt(P, dim), 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_u, + orient_indices_u, elem_restr_u); + CeedElemRestrictionCreate(ceed, num_elem, 1, 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_p, elem_restr_p); + PetscCall(DMGetLocalVector(dm_u0, &U_loc)); + PetscCall(VecGetLocalSize(U_loc, &num_dof_u0)); + PetscCall(DMRestoreLocalVector(dm_u0, &U_loc)); + // dof per element in Hdiv is dim*P^dim, for linear element P=2 + CeedElemRestrictionCreateOriented(ceed, num_elem, dim * PetscPowInt(P, dim), 1, 1, num_dof_u0, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_u0, + orient_indices_u0, elem_restr_u0); + PetscCall(DMGetLocalVector(dm_p0, &U_loc)); + PetscCall(VecGetLocalSize(U_loc, &num_dof_p0)); + PetscCall(DMRestoreLocalVector(dm_p0, &U_loc)); + CeedElemRestrictionCreate(ceed, num_elem, 1, 1, 1, num_dof_p0, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_p0, elem_restr_p0); + PetscCall(PetscFree(restr_indices_p)); + PetscCall(PetscFree(restr_indices_u)); + PetscCall(PetscFree(orient_indices_u)); + PetscCall(PetscFree(restr_indices_u0)); + PetscCall(PetscFree(orient_indices_u0)); + PetscCall(PetscFree(restr_indices_p0)); + PetscFunctionReturn(0); +}; diff --git a/examples/Hdiv-mixed/src/setup-libceed.c b/examples/Hdiv-mixed/src/setup-libceed.c index dc091f35e0..e147300890 100644 --- a/examples/Hdiv-mixed/src/setup-libceed.c +++ b/examples/Hdiv-mixed/src/setup-libceed.c @@ -9,10 +9,6 @@ #include "../include/setup-boundary.h" #include "ceed/ceed.h" -// ----------------------------------------------------------------------------- -// Convert PETSc MemType to libCEED MemType -// ----------------------------------------------------------------------------- -CeedMemType MemTypeP2C(PetscMemType mem_type) { return PetscMemTypeDevice(mem_type) ? CEED_MEM_DEVICE : CEED_MEM_HOST; } // ----------------------------------------------------------------------------- // Destroy libCEED objects // ----------------------------------------------------------------------------- @@ -82,139 +78,6 @@ PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data) { PetscFunctionReturn(0); }; -// ----------------------------------------------------------------------------- -// Utility function - essential BC dofs are encoded in closure indices as -(i+1) -// ----------------------------------------------------------------------------- -PetscInt Involute(PetscInt i) { return i >= 0 ? i : -(i + 1); }; - -// ----------------------------------------------------------------------------- -// Get CEED restriction data from DMPlex -// ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr) { - PetscInt num_elem, elem_size, num_dof, num_comp, *elem_restr_offsets; - - PetscFunctionBeginUser; - - PetscCall(DMPlexGetLocalOffsets(dm, domain_label, value, height, 0, &num_elem, &elem_size, &num_comp, &num_dof, &elem_restr_offsets)); - - CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, elem_restr_offsets, elem_restr); - PetscCall(PetscFree(elem_restr_offsets)); - - PetscFunctionReturn(0); -}; - -// ----------------------------------------------------------------------------- -// Get Oriented CEED restriction data from DMPlex -// ----------------------------------------------------------------------------- -PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u, - CeedElemRestriction *elem_restr_p, CeedElemRestriction *elem_restr_u0, - CeedElemRestriction *elem_restr_p0) { - PetscSection section, section_u0, section_p0; - PetscInt p, num_elem, num_dof, num_dof_u0, num_dof_p0, *restr_indices_u, *restr_indices_p, *restr_indices_u0, *restr_indices_p0, elem_offset, - num_fields, num_fields_u0, num_fields_p0, dim, c_start, c_end; - Vec U_loc; - const PetscInt *ornt; // this is for orientation of dof - PetscFunctionBeginUser; - // Section for mixed problem - PetscCall(DMGetDimension(dm, &dim)); - PetscCall(DMGetLocalSection(dm, §ion)); - PetscCall(PetscSectionGetNumFields(section, &num_fields)); - PetscInt num_comp[num_fields], field_offsets[num_fields + 1]; - field_offsets[0] = 0; - for (PetscInt f = 0; f < num_fields; f++) { - PetscCall(PetscSectionGetFieldComponents(section, f, &num_comp[f])); - field_offsets[f + 1] = field_offsets[f] + num_comp[f]; - } - // Section for initial conditions u0 - PetscCall(DMGetLocalSection(dm_u0, §ion_u0)); - PetscCall(PetscSectionGetNumFields(section_u0, &num_fields_u0)); - PetscInt num_comp_u0[num_fields_u0], field_offsets_u0[num_fields_u0 + 1]; - field_offsets_u0[0] = 0; - for (PetscInt f = 0; f < num_fields_u0; f++) { - PetscCall(PetscSectionGetFieldComponents(section_u0, f, &num_comp_u0[f])); - field_offsets_u0[f + 1] = field_offsets_u0[f] + num_comp_u0[f]; - } - // Section for initial conditions p0 - PetscCall(DMGetLocalSection(dm_p0, §ion_p0)); - PetscCall(PetscSectionGetNumFields(section_p0, &num_fields_p0)); - PetscInt num_comp_p0[num_fields_p0], field_offsets_p0[num_fields_p0 + 1]; - field_offsets_p0[0] = 0; - for (PetscInt f = 0; f < num_fields_p0; f++) { - PetscCall(PetscSectionGetFieldComponents(section_p0, f, &num_comp_p0[f])); - field_offsets_p0[f + 1] = field_offsets_p0[f] + num_comp_p0[f]; - } - - PetscCall(DMPlexGetHeightStratum(dm, 0, &c_start, &c_end)); - num_elem = c_end - c_start; - PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &restr_indices_u)); - PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &restr_indices_u0)); - PetscCall(PetscMalloc1(num_elem, &restr_indices_p)); - PetscCall(PetscMalloc1(num_elem, &restr_indices_p0)); - bool *orient_indices_u, *orient_indices_u0; // to flip the dof - PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &orient_indices_u)); - PetscCall(PetscMalloc1(num_elem * dim * PetscPowInt(P, dim), &orient_indices_u0)); - for (p = 0, elem_offset = 0; p < num_elem; p++) { - PetscInt num_indices, *indices, faces_per_elem, dofs_per_face, num_indices_u0, *indices_u0, num_indices_p0, *indices_p0; - PetscCall(DMPlexGetClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL)); - PetscCall(DMPlexGetClosureIndices(dm_u0, section_u0, section_u0, p, PETSC_TRUE, &num_indices_u0, &indices_u0, NULL, NULL)); - PetscCall(DMPlexGetClosureIndices(dm_p0, section_p0, section_p0, p, PETSC_TRUE, &num_indices_p0, &indices_p0, NULL, NULL)); - restr_indices_p[p] = indices[num_indices - 1]; - restr_indices_p0[p] = indices_p0[0]; - PetscCall(DMPlexGetConeOrientation(dm, p, &ornt)); - // Get number of faces per element - PetscCall(DMPlexGetConeSize(dm, p, &faces_per_elem)); - dofs_per_face = faces_per_elem - 2; - for (PetscInt f = 0; f < faces_per_elem; f++) { - for (PetscInt i = 0; i < dofs_per_face; i++) { - PetscInt ii = dofs_per_face * f + i; - // Essential boundary conditions are encoded as -(loc+1), but we don't care so we decode. - PetscInt loc = Involute(indices[ii * num_comp[0]]); - restr_indices_u[elem_offset] = loc; - // Set orientation - orient_indices_u[elem_offset] = ornt[f] < 0; - PetscInt loc_u0 = Involute(indices_u0[ii * num_comp_u0[0]]); - restr_indices_u0[elem_offset] = loc_u0; - // Set orientation - orient_indices_u0[elem_offset] = ornt[f] < 0; - elem_offset++; - } - } - PetscCall(DMPlexRestoreClosureIndices(dm, section, section, p, PETSC_TRUE, &num_indices, &indices, NULL, NULL)); - PetscCall(DMPlexRestoreClosureIndices(dm_u0, section_u0, section_u0, p, PETSC_TRUE, &num_indices_u0, &indices_u0, NULL, NULL)); - PetscCall(DMPlexRestoreClosureIndices(dm_p0, section_p0, section_p0, p, PETSC_TRUE, &num_indices_p0, &indices_p0, NULL, NULL)); - } - // if (elem_offset != num_elem*dim*PetscPowInt(P, dim)) - // SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, - // "ElemRestriction of size (%" PetscInt_FMT ", %" PetscInt_FMT" ) - // initialized %" PetscInt_FMT " nodes", num_elem, - // dim*PetscPowInt(P, dim),elem_offset); - - PetscCall(DMGetLocalVector(dm, &U_loc)); - PetscCall(VecGetLocalSize(U_loc, &num_dof)); - PetscCall(DMRestoreLocalVector(dm, &U_loc)); - // dof per element in Hdiv is dim*P^dim, for linear element P=2 - CeedElemRestrictionCreateOriented(ceed, num_elem, dim * PetscPowInt(P, dim), 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_u, - orient_indices_u, elem_restr_u); - CeedElemRestrictionCreate(ceed, num_elem, 1, 1, 1, num_dof, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_p, elem_restr_p); - PetscCall(DMGetLocalVector(dm_u0, &U_loc)); - PetscCall(VecGetLocalSize(U_loc, &num_dof_u0)); - PetscCall(DMRestoreLocalVector(dm_u0, &U_loc)); - // dof per element in Hdiv is dim*P^dim, for linear element P=2 - CeedElemRestrictionCreateOriented(ceed, num_elem, dim * PetscPowInt(P, dim), 1, 1, num_dof_u0, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_u0, - orient_indices_u0, elem_restr_u0); - PetscCall(DMGetLocalVector(dm_p0, &U_loc)); - PetscCall(VecGetLocalSize(U_loc, &num_dof_p0)); - PetscCall(DMRestoreLocalVector(dm_p0, &U_loc)); - CeedElemRestrictionCreate(ceed, num_elem, 1, 1, 1, num_dof_p0, CEED_MEM_HOST, CEED_COPY_VALUES, restr_indices_p0, elem_restr_p0); - PetscCall(PetscFree(restr_indices_p)); - PetscCall(PetscFree(restr_indices_u)); - PetscCall(PetscFree(orient_indices_u)); - PetscCall(PetscFree(restr_indices_u0)); - PetscCall(PetscFree(orient_indices_u0)); - PetscCall(PetscFree(restr_indices_p0)); - PetscFunctionReturn(0); -}; - // ----------------------------------------------------------------------------- // Set up libCEED on the fine grid for a given degree // ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/src/setup-solvers.c b/examples/Hdiv-mixed/src/setup-solvers.c index 3fef27c924..72038d53a8 100644 --- a/examples/Hdiv-mixed/src/setup-solvers.c +++ b/examples/Hdiv-mixed/src/setup-solvers.c @@ -7,9 +7,11 @@ // ----------------------------------------------------------------------------- // Setup operator context data // ----------------------------------------------------------------------------- -PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_jacobian) { +PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_jacobian) { PetscFunctionBeginUser; + VecType vec_type; + PetscCall(DMGetVecType(dm, &vec_type)); ctx_jacobian->dm = dm; PetscCall(DMCreateLocalVector(dm, &ctx_jacobian->X_loc)); PetscCall(VecDuplicate(ctx_jacobian->X_loc, &ctx_jacobian->Y_loc)); @@ -88,29 +90,29 @@ PetscErrorCode SNESFormResidual(SNES snes, Vec X, Vec Y, void *ctx_residual) { // Jacobian setup // ----------------------------------------------------------------------------- PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, void *ctx_jacobian) { - OperatorApplyContext ctx = (OperatorApplyContext)ctx_jacobian; + // OperatorApplyContext ctx = (OperatorApplyContext)ctx_jacobian; PetscFunctionBeginUser; - Mat A; - PetscCall(DMCreateMatrix(ctx->dm, &A)); - // Assemble matrix analytically - PetscCount num_entries; - CeedInt *rows, *cols; - CeedVector coo_values; - CeedOperatorLinearAssembleSymbolic(ctx->op_apply, &num_entries, &rows, &cols); - PetscCall(MatSetPreallocationCOO(A, num_entries, rows, cols)); - free(rows); - free(cols); - CeedVectorCreate(ctx->ceed, num_entries, &coo_values); - CeedOperatorLinearAssemble(ctx->op_apply, coo_values); - const CeedScalar *values; - CeedVectorGetArrayRead(coo_values, CEED_MEM_HOST, &values); - PetscCall(MatSetValuesCOO(A, values, ADD_VALUES)); - CeedVectorRestoreArrayRead(coo_values, &values); - MatView(A, PETSC_VIEWER_STDOUT_WORLD); - // CeedVectorView(coo_values, "%12.8f", stdout); - CeedVectorDestroy(&coo_values); - PetscCall(MatDestroy(&A)); + // Mat A; + // PetscCall(DMCreateMatrix(ctx->dm, &A)); + //// Assemble matrix analytically + // PetscCount num_entries; + // CeedInt *rows, *cols; + // CeedVector coo_values; + // CeedOperatorLinearAssembleSymbolic(ctx->op_apply, &num_entries, &rows, &cols); + // PetscCall(MatSetPreallocationCOO(A, num_entries, rows, cols)); + // free(rows); + // free(cols); + // CeedVectorCreate(ctx->ceed, num_entries, &coo_values); + // CeedOperatorLinearAssemble(ctx->op_apply, coo_values); + // const CeedScalar *values; + // CeedVectorGetArrayRead(coo_values, CEED_MEM_HOST, &values); + // PetscCall(MatSetValuesCOO(A, values, ADD_VALUES)); + // CeedVectorRestoreArrayRead(coo_values, &values); + // MatView(A, PETSC_VIEWER_STDOUT_WORLD); + //// CeedVectorView(coo_values, "%12.8f", stdout); + // CeedVectorDestroy(&coo_values); + // PetscCall(MatDestroy(&A)); // J_pre might be AIJ (e.g., when using coloring), so we need to assemble it PetscCall(MatAssemblyBegin(J_pre, MAT_FINAL_ASSEMBLY)); diff --git a/examples/Hdiv-mixed/src/setup-ts.c b/examples/Hdiv-mixed/src/setup-ts.c index e93bca2495..8600967c00 100644 --- a/examples/Hdiv-mixed/src/setup-ts.c +++ b/examples/Hdiv-mixed/src/setup-ts.c @@ -69,7 +69,7 @@ PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, Ceed // ----------------------------------------------------------------------------- // Create global initial conditions vector // ----------------------------------------------------------------------------- -PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, VecType vec_type, Vec U) { +PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, Vec U) { PetscFunctionBeginUser; // ---------------------------------------------- // Create local rhs for u field @@ -77,6 +77,8 @@ PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, VecTy Vec rhs_u_loc; PetscScalar *ru; PetscMemType ru_mem_type; + VecType vec_type; + PetscCall(DMGetVecType(app_ctx->ctx_initial_u0->dm, &vec_type)); PetscCall(DMCreateLocalVector(app_ctx->ctx_initial_u0->dm, &rhs_u_loc)); PetscCall(VecZeroEntries(rhs_u_loc)); PetscCall(VecGetArrayAndMemType(rhs_u_loc, &ru, &ru_mem_type)); From 0b6fa7179ff0202dd403e99e9b55d9f008e5b557 Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Tue, 27 Dec 2022 11:04:24 -0700 Subject: [PATCH 13/15] Only ref/serial backend gives correct convergence order --- examples/Hdiv-mass/conv_test.sh | 2 ++ examples/Hdiv-mass/main.c | 3 +-- examples/Hdiv-mixed/conv_test.sh | 4 +++- examples/Hdiv-mixed/main.c | 3 +-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/Hdiv-mass/conv_test.sh b/examples/Hdiv-mass/conv_test.sh index 3ba642a064..e13f7161b4 100755 --- a/examples/Hdiv-mass/conv_test.sh +++ b/examples/Hdiv-mass/conv_test.sh @@ -28,6 +28,7 @@ echo "Running convergence test in ${dim}D for Projection problem in H(div) space declare -A run_flags #run_flags[pc_type]=svd + run_flags[ceed]=/cpu/self/ref/serial if [[ $dim -eq 2 ]]; then run_flags[problem]=mass2d @@ -68,3 +69,4 @@ for ((res=${test_flags[res_start]}; res<=${test_flags[res_end]}; res+=${test_fla i=$((i+1)) done +python conv_rate.py -f conv_test_result.csv \ No newline at end of file diff --git a/examples/Hdiv-mass/main.c b/examples/Hdiv-mass/main.c index abc929eef8..65a1cb75a8 100644 --- a/examples/Hdiv-mass/main.c +++ b/examples/Hdiv-mass/main.c @@ -67,8 +67,7 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // -- Initialize backend Ceed ceed; - CeedInit("/cpu/self/ref/serial", &ceed); - // CeedInit(app_ctx->ceed_resource, &ceed); + CeedInit(app_ctx->ceed_resource, &ceed); // --------------------------------------------------------------------------- // Choose the problem from the list of registered problems diff --git a/examples/Hdiv-mixed/conv_test.sh b/examples/Hdiv-mixed/conv_test.sh index 33ca308c8a..f44774abce 100755 --- a/examples/Hdiv-mixed/conv_test.sh +++ b/examples/Hdiv-mixed/conv_test.sh @@ -30,6 +30,7 @@ echo "Running convergence test in ${dim}D for Darcy problem"; declare -A run_flags run_flags[pc_type]=svd + run_flags[ceed]=/cpu/self/ref/serial if [[ $dim -eq 2 ]]; then run_flags[problem]=darcy2d @@ -68,7 +69,8 @@ for ((res=${test_flags[res_start]}; res<=${test_flags[res_end]}; res+=${test_fla args="$args -$arg ${run_flags[$arg]}" fi done - ./main -view_solution $args | grep "L2 error of u and p" | awk -v i="$i" -v res="$res" '{ printf "%d,%d,%.5f,%.5f\n", i, res, $8, $9}' >> $file_name + ./main $args | grep "L2 error of u and p" | awk -v i="$i" -v res="$res" '{ printf "%d,%d,%.5f,%.5f\n", i, res, $8, $9}' >> $file_name i=$((i+1)) done +python conv_plot.py -f conv_test_result.csv \ No newline at end of file diff --git a/examples/Hdiv-mixed/main.c b/examples/Hdiv-mixed/main.c index c990c08e8f..e701a25381 100644 --- a/examples/Hdiv-mixed/main.c +++ b/examples/Hdiv-mixed/main.c @@ -90,8 +90,7 @@ int main(int argc, char **argv) { // --------------------------------------------------------------------------- // -- Initialize backend Ceed ceed; - CeedInit("/cpu/self/ref/serial", &ceed); - // CeedInit(app_ctx->ceed_resource, &ceed); + CeedInit(app_ctx->ceed_resource, &ceed); // -- Process general command line options // --------------------------------------------------------------------------- From bd1b2c3a81b763417a783274627e69c6dd35777b Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Wed, 22 Feb 2023 10:38:07 -0700 Subject: [PATCH 14/15] Added GetDiagonal --- examples/Hdiv-mixed/include/setup-matops.h | 1 + examples/Hdiv-mixed/src/setup-matops.c | 33 ++++++++++++++++++++++ examples/Hdiv-mixed/src/setup-solvers.c | 33 ++++++++-------------- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/examples/Hdiv-mixed/include/setup-matops.h b/examples/Hdiv-mixed/include/setup-matops.h index dad7737099..51bb686bcf 100644 --- a/examples/Hdiv-mixed/include/setup-matops.h +++ b/examples/Hdiv-mixed/include/setup-matops.h @@ -8,5 +8,6 @@ PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx); PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_ctx); +PetscErrorCode GetDiagonal(Mat A, Vec D); #endif // setup_matops_h diff --git a/examples/Hdiv-mixed/src/setup-matops.c b/examples/Hdiv-mixed/src/setup-matops.c index d0ea7662aa..d223ddae48 100644 --- a/examples/Hdiv-mixed/src/setup-matops.c +++ b/examples/Hdiv-mixed/src/setup-matops.c @@ -1,6 +1,8 @@ #include "../include/setup-matops.h" +#include #include "../include/setup-libceed.h" +#include "ceed/ceed.h" // ----------------------------------------------------------------------------- // Apply the local action of a libCEED operator and store result in PETSc vector @@ -49,3 +51,34 @@ PetscErrorCode ApplyAddLocalCeedOp(Vec X, Vec Y, OperatorApplyContext op_apply_c }; // ----------------------------------------------------------------------------- +// This function returns the computed diagonal of the operator +// ----------------------------------------------------------------------------- +PetscErrorCode GetDiagonal(Mat A, Vec D) { + OperatorApplyContext op_apply_ctx; + PetscScalar *x; + PetscMemType x_mem_type; + + PetscFunctionBeginUser; + + PetscCall(MatShellGetContext(A, &op_apply_ctx)); + + // -- Place PETSc vector in libCEED vector + PetscCall(VecGetArrayAndMemType(op_apply_ctx->X_loc, &x, &x_mem_type)); + CeedVectorSetArray(op_apply_ctx->x_ceed, MemTypeP2C(x_mem_type), CEED_USE_POINTER, x); + + // -- Compute Diagonal + CeedOperatorLinearAssembleDiagonal(op_apply_ctx->op_apply, op_apply_ctx->x_ceed, CEED_REQUEST_IMMEDIATE); + CeedVectorView(op_apply_ctx->x_ceed, "%12.8f", stdout); + // -- Local-to-Global + CeedVectorTakeArray(op_apply_ctx->x_ceed, MemTypeP2C(x_mem_type), NULL); + PetscCall(VecRestoreArrayAndMemType(op_apply_ctx->X_loc, &x)); + PetscCall(VecZeroEntries(D)); + PetscCall(DMLocalToGlobal(op_apply_ctx->dm, op_apply_ctx->X_loc, ADD_VALUES, D)); + + // Cleanup + PetscCall(VecZeroEntries(op_apply_ctx->X_loc)); + + PetscFunctionReturn(0); +}; + +// ----------------------------------------------------------------------------- diff --git a/examples/Hdiv-mixed/src/setup-solvers.c b/examples/Hdiv-mixed/src/setup-solvers.c index 72038d53a8..3c0d773489 100644 --- a/examples/Hdiv-mixed/src/setup-solvers.c +++ b/examples/Hdiv-mixed/src/setup-solvers.c @@ -93,27 +93,6 @@ PetscErrorCode SNESFormJacobian(SNES snes, Vec U, Mat J, Mat J_pre, void *ctx_ja // OperatorApplyContext ctx = (OperatorApplyContext)ctx_jacobian; PetscFunctionBeginUser; - // Mat A; - // PetscCall(DMCreateMatrix(ctx->dm, &A)); - //// Assemble matrix analytically - // PetscCount num_entries; - // CeedInt *rows, *cols; - // CeedVector coo_values; - // CeedOperatorLinearAssembleSymbolic(ctx->op_apply, &num_entries, &rows, &cols); - // PetscCall(MatSetPreallocationCOO(A, num_entries, rows, cols)); - // free(rows); - // free(cols); - // CeedVectorCreate(ctx->ceed, num_entries, &coo_values); - // CeedOperatorLinearAssemble(ctx->op_apply, coo_values); - // const CeedScalar *values; - // CeedVectorGetArrayRead(coo_values, CEED_MEM_HOST, &values); - // PetscCall(MatSetValuesCOO(A, values, ADD_VALUES)); - // CeedVectorRestoreArrayRead(coo_values, &values); - // MatView(A, PETSC_VIEWER_STDOUT_WORLD); - //// CeedVectorView(coo_values, "%12.8f", stdout); - // CeedVectorDestroy(&coo_values); - // PetscCall(MatDestroy(&A)); - // J_pre might be AIJ (e.g., when using coloring), so we need to assemble it PetscCall(MatAssemblyBegin(J_pre, MAT_FINAL_ASSEMBLY)); PetscCall(MatAssemblyEnd(J_pre, MAT_FINAL_ASSEMBLY)); @@ -148,6 +127,7 @@ PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, SNES snes, KSP ksp, // -- Form Action of Jacobian on delta_u PetscCall(MatCreateShell(app_ctx->comm, U_l_size, U_l_size, U_g_size, U_g_size, app_ctx->ctx_jacobian, &mat_jacobian)); PetscCall(MatShellSetOperation(mat_jacobian, MATOP_MULT, (void (*)(void))ApplyMatOp)); + PetscCall(MatShellSetOperation(mat_jacobian, MATOP_GET_DIAGONAL, (void (*)(void))GetDiagonal)); PetscCall(MatShellSetVecType(mat_jacobian, app_ctx->ctx_jacobian->vec_type)); // Set SNES residual evaluation function @@ -156,8 +136,17 @@ PetscErrorCode PDESolver(CeedData ceed_data, AppCtx app_ctx, SNES snes, KSP ksp, PetscCall(SNESSetJacobian(snes, mat_jacobian, mat_jacobian, SNESFormJacobian, app_ctx->ctx_jacobian)); // Setup KSP + PetscCall(KSPSetType(ksp, KSPGMRES)); + PetscCall(KSPSetNormType(ksp, KSP_NORM_PRECONDITIONED)); + // PC setup + PC pc; + PetscCall(KSPGetPC(ksp, &pc)); + PetscCall(PCSetType(pc, PCJACOBI)); + PetscCall(PCJacobiSetType(pc, PC_JACOBI_DIAGONAL)); + // Set user options and view PetscCall(KSPSetFromOptions(ksp)); - + PetscCall(KSPViewFromOptions(ksp, NULL, "-ksp_view")); + PetscCall(PCViewFromOptions(pc, NULL, "-pc_view")); // Default to critical-point (CP) line search (related to Wolfe's curvature condition) SNESLineSearch line_search; From fab729da2ef3ca33cd600e0e3684834d6ad07a9e Mon Sep 17 00:00:00 2001 From: rezgarshakeri Date: Thu, 23 Feb 2023 18:26:16 -0700 Subject: [PATCH 15/15] make format --- examples/Hdiv-mixed/problems/darcy2d.c | 2 +- examples/Hdiv-mixed/problems/darcy3d.c | 2 +- examples/Hdiv-mixed/problems/richard2d.c | 2 +- examples/Hdiv-mixed/problems/richard3d.c | 2 +- examples/Hdiv-mixed/src/setup-libceed.c | 6 +++--- examples/Hdiv-mixed/src/setup-matops.c | 1 + examples/Hdiv-mixed/src/setup-ts.c | 2 +- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/examples/Hdiv-mixed/problems/darcy2d.c b/examples/Hdiv-mixed/problems/darcy2d.c index 4864993661..e5e535c117 100644 --- a/examples/Hdiv-mixed/problems/darcy2d.c +++ b/examples/Hdiv-mixed/problems/darcy2d.c @@ -24,7 +24,7 @@ #include "../qfunctions/darcy-true-quartic2d.h" #include "../qfunctions/darcy-true2d.h" #include "../qfunctions/post-processing2d.h" -//#include "../qfunctions/pressure-boundary2d.h" +// #include "../qfunctions/pressure-boundary2d.h" PetscErrorCode Hdiv_DARCY2D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; diff --git a/examples/Hdiv-mixed/problems/darcy3d.c b/examples/Hdiv-mixed/problems/darcy3d.c index 4cdbc75a71..52e101931c 100644 --- a/examples/Hdiv-mixed/problems/darcy3d.c +++ b/examples/Hdiv-mixed/problems/darcy3d.c @@ -22,7 +22,7 @@ #include "../qfunctions/darcy-system3d.h" #include "../qfunctions/darcy-true3d.h" #include "../qfunctions/post-processing3d.h" -//#include "../qfunctions/pressure-boundary3d.h" +// #include "../qfunctions/pressure-boundary3d.h" PetscErrorCode Hdiv_DARCY3D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx) { AppCtx app_ctx = *(AppCtx *)ctx; diff --git a/examples/Hdiv-mixed/problems/richard2d.c b/examples/Hdiv-mixed/problems/richard2d.c index a2b4cad337..27b88300f0 100644 --- a/examples/Hdiv-mixed/problems/richard2d.c +++ b/examples/Hdiv-mixed/problems/richard2d.c @@ -23,7 +23,7 @@ #include "../qfunctions/richard-ics2d.h" #include "../qfunctions/richard-system2d.h" #include "../qfunctions/richard-true2d.h" -//#include "../qfunctions/pressure-boundary2d.h" +// #include "../qfunctions/pressure-boundary2d.h" #include "petscsystypes.h" PetscErrorCode Hdiv_RICHARD2D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx) { diff --git a/examples/Hdiv-mixed/problems/richard3d.c b/examples/Hdiv-mixed/problems/richard3d.c index 92663aa2c3..438723d934 100644 --- a/examples/Hdiv-mixed/problems/richard3d.c +++ b/examples/Hdiv-mixed/problems/richard3d.c @@ -23,7 +23,7 @@ #include "../qfunctions/richard-ics3d.h" #include "../qfunctions/richard-system3d.h" #include "../qfunctions/richard-true3d.h" -//#include "../qfunctions/pressure-boundary2d.h" +// #include "../qfunctions/pressure-boundary2d.h" #include "petscsystypes.h" PetscErrorCode Hdiv_RICHARD3D(Ceed ceed, ProblemData problem_data, DM dm, void *ctx) { diff --git a/examples/Hdiv-mixed/src/setup-libceed.c b/examples/Hdiv-mixed/src/setup-libceed.c index e147300890..bdb0f238d3 100644 --- a/examples/Hdiv-mixed/src/setup-libceed.c +++ b/examples/Hdiv-mixed/src/setup-libceed.c @@ -172,8 +172,8 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, Ceed ceed, AppC CeedOperatorCreate(ceed, qf_true, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_true); if (problem_data->has_ts) { double final_time = app_ctx->t_final; - CeedOperatorContextGetFieldLabel(op_true, "final_time", &app_ctx->ctx_residual_ut->final_time_label); - CeedOperatorContextSetDouble(op_true, app_ctx->ctx_residual_ut->final_time_label, &final_time); + CeedOperatorGetContextFieldLabel(op_true, "final_time", &app_ctx->ctx_residual_ut->final_time_label); + CeedOperatorSetContextDouble(op_true, app_ctx->ctx_residual_ut->final_time_label, &final_time); } CeedOperatorSetField(op_true, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord); CeedOperatorSetField(op_true, "true force", ceed_data->elem_restr_p_i, CEED_BASIS_COLLOCATED, true_force); @@ -309,7 +309,7 @@ PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, Ceed ceed, AppC CeedOperatorCreate(ceed, qf_residual, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_residual); if (problem_data->has_ts) { // double t = app_ctx->ctx_residual_ut->t; - CeedOperatorContextGetFieldLabel(op_residual, "time", &app_ctx->ctx_residual_ut->solution_time_label); + CeedOperatorGetContextFieldLabel(op_residual, "time", &app_ctx->ctx_residual_ut->solution_time_label); // CeedOperatorContextGetFieldLabel(op_residual, "time_step", // &app_ctx->ctx_residual_ut->timestep_label); // CeedOperatorContextSetDouble(op_residual, diff --git a/examples/Hdiv-mixed/src/setup-matops.c b/examples/Hdiv-mixed/src/setup-matops.c index d223ddae48..d90a24e525 100644 --- a/examples/Hdiv-mixed/src/setup-matops.c +++ b/examples/Hdiv-mixed/src/setup-matops.c @@ -1,4 +1,5 @@ #include "../include/setup-matops.h" + #include #include "../include/setup-libceed.h" diff --git a/examples/Hdiv-mixed/src/setup-ts.c b/examples/Hdiv-mixed/src/setup-ts.c index 8600967c00..47208750c7 100644 --- a/examples/Hdiv-mixed/src/setup-ts.c +++ b/examples/Hdiv-mixed/src/setup-ts.c @@ -224,7 +224,7 @@ PetscErrorCode TSFormIResidual(TS ts, PetscReal time, Vec X, Vec X_t, Vec Y, voi // Update time dependent data if (ctx->t != time) { - CeedOperatorContextSetDouble(ctx->op_apply, ctx->solution_time_label, &time); + CeedOperatorSetContextDouble(ctx->op_apply, ctx->solution_time_label, &time); ctx->t = time; } // PetscScalar dt;