Skip to content

Commit

Permalink
some clean-up in Hdiv-mixed folder
Browse files Browse the repository at this point in the history
  • Loading branch information
@rezgarshakeri
rezgarshakeri committed Dec 16, 2022
1 parent 038dc82 commit c54385e
Show file tree
Hide file tree
Showing 14 changed files with 277 additions and 253 deletions.
6 changes: 3 additions & 3 deletions examples/Hdiv-mixed/include/post-processing.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@

#include "../include/setup-libceed.h"
#include "structs.h"
PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, CeedMemType mem_type_backend, TS ts, SNES snes, KSP ksp, Vec U,
CeedScalar l2_error_u, CeedScalar l2_error_p);
PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, TS ts, SNES snes, KSP ksp, Vec U, CeedScalar l2_error_u,
CeedScalar l2_error_p);
PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_Hdiv);
PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_H1);
PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_H1);
PetscErrorCode ProjectVelocity(AppCtx app_ctx, Vec U, Vec *U_H1);
PetscErrorCode CtxVecDestroy(AppCtx app_ctx);
#endif // post_processing_h
2 changes: 1 addition & 1 deletion examples/Hdiv-mixed/include/setup-dm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
// ---------------------------------------------------------------------------
// Setup DM
// ---------------------------------------------------------------------------
PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, VecType vec_type, DM *dm);
PetscErrorCode CreateDM(MPI_Comm comm, Ceed ceed, DM *dm);
PetscErrorCode PerturbVerticesSmooth(DM dm);
PetscErrorCode PerturbVerticesRandom(DM dm);

Expand Down
7 changes: 7 additions & 0 deletions examples/Hdiv-mixed/include/setup-fe.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@
// ---------------------------------------------------------------------------
// Setup FE
// ---------------------------------------------------------------------------
CeedMemType MemTypeP2C(PetscMemType mtype);
PetscErrorCode SetupFEHdiv(MPI_Comm comm, DM dm, DM dm_u0, DM dm_p0);
PetscErrorCode SetupFEH1(ProblemData problem_data, AppCtx app_ctx, DM dm_H1);
PetscInt Involute(PetscInt i);
PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr);
// Utility function to create local CEED Oriented restriction from DMPlex
PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u,
CeedElemRestriction *elem_restr_p, CeedElemRestriction *elem_restr_u0,
CeedElemRestriction *elem_restr_p0);
#endif // setupfe_h
12 changes: 1 addition & 11 deletions examples/Hdiv-mixed/include/setup-libceed.h
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,10 @@
#ifndef setuplibceed_h
#define setuplibceed_h

#include "setup-fe.h"
#include "structs.h"

// Convert PETSc MemType to libCEED MemType
CeedMemType MemTypeP2C(PetscMemType mtype);
// Destroy libCEED objects
PetscErrorCode CeedDataDestroy(CeedData ceed_data, ProblemData problem_data);
// Utility function - essential BC dofs are encoded in closure indices as -(i+1)
PetscInt Involute(PetscInt i);
// Utility function to create local CEED restriction from DMPlex
PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt height, DMLabel domain_label, CeedInt value, CeedElemRestriction *elem_restr);
// Utility function to create local CEED Oriented restriction from DMPlex
PetscErrorCode CreateRestrictionFromPlexOriented(Ceed ceed, DM dm, DM dm_u0, DM dm_p0, CeedInt P, CeedElemRestriction *elem_restr_u,
CeedElemRestriction *elem_restr_p, CeedElemRestriction *elem_restr_u0,
CeedElemRestriction *elem_restr_p0);
// Set up libCEED for a given degree
PetscErrorCode SetupLibceed(DM dm, DM dm_u0, DM dm_p0, DM dm_H1, Ceed ceed, AppCtx app_ctx, ProblemData problem_data, CeedData ceed_data);
#endif // setuplibceed_h
2 changes: 1 addition & 1 deletion examples/Hdiv-mixed/include/setup-solvers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
#include "petscvec.h"
#include "structs.h"

PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_jacobian);
PetscErrorCode SetupJacobianOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_jacobian);
PetscErrorCode SetupResidualOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual);
PetscErrorCode SetupErrorOperatorCtx(DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_error);
PetscErrorCode ApplyMatOp(Mat A, Vec X, Vec Y);
Expand Down
2 changes: 1 addition & 1 deletion examples/Hdiv-mixed/include/setup-ts.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

#include "structs.h"

PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, VecType vec_type, Vec U);
PetscErrorCode CreateInitialConditions(CeedData ceed_data, AppCtx app_ctx, Vec U);
PetscErrorCode SetupResidualOperatorCtx_Ut(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_residual_ut);
PetscErrorCode SetupResidualOperatorCtx_U0(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_initial_u0);
PetscErrorCode SetupResidualOperatorCtx_P0(MPI_Comm comm, DM dm, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_initial_p0);
Expand Down
116 changes: 47 additions & 69 deletions examples/Hdiv-mixed/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ int main(int argc, char **argv) {
// Initialize PETSc
// ---------------------------------------------------------------------------
PetscCall(PetscInitialize(&argc, &argv, NULL, help));
MPI_Comm comm = PETSC_COMM_WORLD;

// ---------------------------------------------------------------------------
// Create structs
Expand Down Expand Up @@ -77,6 +78,12 @@ int main(int argc, char **argv) {
// Context for post-processing
app_ctx->ctx_Hdiv = ctx_Hdiv;
app_ctx->ctx_H1 = ctx_H1;
app_ctx->comm = comm;

// ---------------------------------------------------------------------------
// Process command line options
// ---------------------------------------------------------------------------
PetscCall(ProcessCommandLineOptions(app_ctx));

// ---------------------------------------------------------------------------
// Initialize libCEED
Expand All @@ -85,41 +92,20 @@ int main(int argc, char **argv) {
Ceed ceed;
CeedInit("/cpu/self/ref/serial", &ceed);
// CeedInit(app_ctx->ceed_resource, &ceed);
CeedMemType mem_type_backend;
CeedGetPreferredMemType(ceed, &mem_type_backend);

VecType vec_type = NULL;
MatType mat_type = NULL;
switch (mem_type_backend) {
case CEED_MEM_HOST:
vec_type = VECSTANDARD;
break;
case CEED_MEM_DEVICE: {
const char *resolved;
CeedGetResource(ceed, &resolved);
if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA;
else if (strstr(resolved, "/gpu/hip")) vec_type = VECKOKKOS;
else vec_type = VECSTANDARD;
}
}
if (strstr(vec_type, VECCUDA)) mat_type = MATAIJCUSPARSE;
else if (strstr(vec_type, VECKOKKOS)) mat_type = MATAIJKOKKOS;
else mat_type = MATAIJ;

// -- Process general command line options
MPI_Comm comm = PETSC_COMM_WORLD;
// ---------------------------------------------------------------------------
// Create DM
// ---------------------------------------------------------------------------
DM dm, dm_u0, dm_p0, dm_H1;
// DM for mixed problem
PetscCall(CreateDM(comm, mat_type, vec_type, &dm));
PetscCall(CreateDM(app_ctx->comm, ceed, &dm));
// DM for projecting initial velocity to Hdiv space
PetscCall(CreateDM(comm, mat_type, vec_type, &dm_u0));
PetscCall(CreateDM(app_ctx->comm, ceed, &dm_u0));
// DM for projecting initial pressure in L2
PetscCall(CreateDM(comm, mat_type, vec_type, &dm_p0));
PetscCall(CreateDM(app_ctx->comm, ceed, &dm_p0));
// DM for projecting solution U into H1 space for PetscViewer
PetscCall(CreateDM(comm, mat_type, vec_type, &dm_H1));
PetscCall(CreateDM(app_ctx->comm, ceed, &dm_H1));
// TODO: add mesh option
// perturb dm to have smooth random mesh
// PetscCall( PerturbVerticesSmooth(dm) );
Expand All @@ -129,18 +115,10 @@ int main(int argc, char **argv) {
// PetscCall(PerturbVerticesRandom(dm) );
// PetscCall(PerturbVerticesRandom(dm_H1) );

// ---------------------------------------------------------------------------
// Process command line options
// ---------------------------------------------------------------------------
// -- Register problems to be available on the command line
PetscCall(RegisterProblems_Hdiv(app_ctx));

app_ctx->comm = comm;
PetscCall(ProcessCommandLineOptions(app_ctx));

// ---------------------------------------------------------------------------
// Choose the problem from the list of registered problems
// ---------------------------------------------------------------------------
PetscCall(RegisterProblems_Hdiv(app_ctx));
{
PetscErrorCode (*p)(Ceed, ProblemData, DM, void *);
PetscCall(PetscFunctionListFind(app_ctx->problems, app_ctx->problem_name, &p));
Expand All @@ -151,7 +129,7 @@ int main(int argc, char **argv) {
// ---------------------------------------------------------------------------
// Setup FE for H(div) mixed-problem and H1 projection in post-processing.c
// ---------------------------------------------------------------------------
PetscCall(SetupFEHdiv(comm, dm, dm_u0, dm_p0));
PetscCall(SetupFEHdiv(app_ctx->comm, dm, dm_u0, dm_p0));
PetscCall(SetupFEH1(problem_data, app_ctx, dm_H1));

// ---------------------------------------------------------------------------
Expand All @@ -167,36 +145,36 @@ int main(int argc, char **argv) {
PetscCall(SetupLibceed(dm, dm_u0, dm_p0, dm_H1, ceed, app_ctx, problem_data, ceed_data));

// ---------------------------------------------------------------------------
// Setup pressure boundary conditions
// Setup pressure boundary conditions (not working)
// ---------------------------------------------------------------------------
// --Create empty local vector for libCEED
Vec P_loc;
PetscInt P_loc_size;
CeedScalar *p0;
CeedVector P_ceed;
PetscMemType pressure_mem_type;
PetscCall(DMCreateLocalVector(dm, &P_loc));
PetscCall(VecGetSize(P_loc, &P_loc_size));
PetscCall(VecZeroEntries(P_loc));
PetscCall(VecGetArrayAndMemType(P_loc, &p0, &pressure_mem_type));
CeedVectorCreate(ceed, P_loc_size, &P_ceed);
CeedVectorSetArray(P_ceed, MemTypeP2C(pressure_mem_type), CEED_USE_POINTER, p0);
// -- Apply operator to create local pressure vector on boundary
PetscCall(DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, P_ceed));
// CeedVectorView(P_ceed, "%12.8f", stdout);
// -- Map local to global
Vec P;
CeedVectorTakeArray(P_ceed, MemTypeP2C(pressure_mem_type), NULL);
PetscCall(VecRestoreArrayAndMemType(P_loc, &p0));
PetscCall(DMCreateGlobalVector(dm, &P));
PetscCall(VecZeroEntries(P));
PetscCall(DMLocalToGlobal(dm, P_loc, ADD_VALUES, P));
// Vec P_loc;
// PetscInt P_loc_size;
// CeedScalar *p0;
// CeedVector P_ceed;
// PetscMemType pressure_mem_type;
// PetscCall(DMCreateLocalVector(dm, &P_loc));
// PetscCall(VecGetSize(P_loc, &P_loc_size));
// PetscCall(VecZeroEntries(P_loc));
// PetscCall(VecGetArrayAndMemType(P_loc, &p0, &pressure_mem_type));
// CeedVectorCreate(ceed, P_loc_size, &P_ceed);
// CeedVectorSetArray(P_ceed, MemTypeP2C(pressure_mem_type), CEED_USE_POINTER, p0);
//// -- Apply operator to create local pressure vector on boundary
// PetscCall(DMAddBoundariesPressure(ceed, ceed_data, app_ctx, problem_data, dm, P_ceed));
//// CeedVectorView(P_ceed, "%12.8f", stdout);
//// -- Map local to global
// Vec P;
// CeedVectorTakeArray(P_ceed, MemTypeP2C(pressure_mem_type), NULL);
// PetscCall(VecRestoreArrayAndMemType(P_loc, &p0));
// PetscCall(DMCreateGlobalVector(dm, &P));
// PetscCall(VecZeroEntries(P));
// PetscCall(DMLocalToGlobal(dm, P_loc, ADD_VALUES, P));

// ---------------------------------------------------------------------------
// Setup context for projection problem; post-processing.c
// ---------------------------------------------------------------------------
PetscCall(SetupProjectVelocityCtx_Hdiv(comm, dm, ceed, ceed_data, app_ctx->ctx_Hdiv));
PetscCall(SetupProjectVelocityCtx_H1(comm, dm_H1, ceed, ceed_data, vec_type, app_ctx->ctx_H1));
PetscCall(SetupProjectVelocityCtx_Hdiv(app_ctx->comm, dm, ceed, ceed_data, app_ctx->ctx_Hdiv));
PetscCall(SetupProjectVelocityCtx_H1(app_ctx->comm, dm_H1, ceed, ceed_data, app_ctx->ctx_H1));

// ---------------------------------------------------------------------------
// Setup TSSolve for Richard problem
Expand All @@ -206,13 +184,13 @@ int main(int argc, char **argv) {
// ---------------------------------------------------------------------------
// Setup context for initial conditions
// ---------------------------------------------------------------------------
PetscCall(SetupResidualOperatorCtx_U0(comm, dm_u0, ceed, ceed_data, app_ctx->ctx_initial_u0));
PetscCall(SetupResidualOperatorCtx_P0(comm, dm_p0, ceed, ceed_data, app_ctx->ctx_initial_p0));
PetscCall(SetupResidualOperatorCtx_Ut(comm, dm, ceed, ceed_data, app_ctx->ctx_residual_ut));
PetscCall(CreateInitialConditions(ceed_data, app_ctx, vec_type, U));
PetscCall(SetupResidualOperatorCtx_U0(app_ctx->comm, dm_u0, ceed, ceed_data, app_ctx->ctx_initial_u0));
PetscCall(SetupResidualOperatorCtx_P0(app_ctx->comm, dm_p0, ceed, ceed_data, app_ctx->ctx_initial_p0));
PetscCall(SetupResidualOperatorCtx_Ut(app_ctx->comm, dm, ceed, ceed_data, app_ctx->ctx_residual_ut));
PetscCall(CreateInitialConditions(ceed_data, app_ctx, U));
// VecView(U, PETSC_VIEWER_STDOUT_WORLD);
// Solve Richards problem
PetscCall(TSCreate(comm, &ts));
PetscCall(TSCreate(app_ctx->comm, &ts));
PetscCall(VecZeroEntries(app_ctx->ctx_residual_ut->X_loc));
PetscCall(VecZeroEntries(app_ctx->ctx_residual_ut->X_t_loc));
PetscCall(TSSolveRichard(ceed_data, app_ctx, ts, &U));
Expand All @@ -225,10 +203,10 @@ int main(int argc, char **argv) {
SNES snes;
KSP ksp;
if (!problem_data->has_ts) {
PetscCall(SetupJacobianOperatorCtx(dm, ceed, ceed_data, vec_type, app_ctx->ctx_jacobian));
PetscCall(SetupJacobianOperatorCtx(dm, ceed, ceed_data, app_ctx->ctx_jacobian));
PetscCall(SetupResidualOperatorCtx(dm, ceed, ceed_data, app_ctx->ctx_residual));
// Create SNES
PetscCall(SNESCreate(comm, &snes));
PetscCall(SNESCreate(app_ctx->comm, &snes));
PetscCall(SNESGetKSP(snes, &ksp));
PetscCall(PDESolver(ceed_data, app_ctx, snes, ksp, &U));
// VecView(U, PETSC_VIEWER_STDOUT_WORLD);
Expand All @@ -244,14 +222,14 @@ int main(int argc, char **argv) {
// ---------------------------------------------------------------------------
// Print solver iterations and final norms
// ---------------------------------------------------------------------------
PetscCall(PrintOutput(dm, ceed, app_ctx, problem_data->has_ts, mem_type_backend, ts, snes, ksp, U, l2_error_u, l2_error_p));
PetscCall(PrintOutput(dm, ceed, app_ctx, problem_data->has_ts, ts, snes, ksp, U, l2_error_u, l2_error_p));

// ---------------------------------------------------------------------------
// Save solution (paraview)
// ---------------------------------------------------------------------------
if (app_ctx->view_solution) {
PetscViewer viewer_p;
PetscCall(PetscViewerVTKOpen(comm, "darcy_pressure.vtu", FILE_MODE_WRITE, &viewer_p));
PetscCall(PetscViewerVTKOpen(app_ctx->comm, "darcy_pressure.vtu", FILE_MODE_WRITE, &viewer_p));
PetscCall(VecView(U, viewer_p));
PetscCall(PetscViewerDestroy(&viewer_p));

Expand All @@ -260,7 +238,7 @@ int main(int argc, char **argv) {
PetscCall(ProjectVelocity(app_ctx, U, &U_H1));

PetscViewer viewer_u;
PetscCall(PetscViewerVTKOpen(comm, "darcy_velocity.vtu", FILE_MODE_WRITE, &viewer_u));
PetscCall(PetscViewerVTKOpen(app_ctx->comm, "darcy_velocity.vtu", FILE_MODE_WRITE, &viewer_u));
PetscCall(VecView(U_H1, viewer_u));
PetscCall(PetscViewerDestroy(&viewer_u));
PetscCall(VecDestroy(&U_H1));
Expand Down
2 changes: 1 addition & 1 deletion examples/Hdiv-mixed/src/cl-options.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ PetscErrorCode ProcessCommandLineOptions(AppCtx app_ctx) {
PetscBool ceed_flag = PETSC_FALSE;
PetscFunctionBeginUser;

PetscOptionsBegin(app_ctx->comm, NULL, "H(div) examples in PETSc with libCEED", NULL);
PetscOptionsBegin(app_ctx->comm, NULL, "H(div) mixed-problem in PETSc with libCEED", NULL);

PetscCall(PetscOptionsString("-ceed", "CEED resource specifier", NULL, app_ctx->ceed_resource, app_ctx->ceed_resource,
sizeof(app_ctx->ceed_resource), &ceed_flag));
Expand Down
10 changes: 7 additions & 3 deletions examples/Hdiv-mixed/src/post-processing.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
// -----------------------------------------------------------------------------
// This function print the output
// -----------------------------------------------------------------------------
PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, CeedMemType mem_type_backend, TS ts, SNES snes, KSP ksp, Vec U,
CeedScalar l2_error_u, CeedScalar l2_error_p) {
PetscErrorCode PrintOutput(DM dm, Ceed ceed, AppCtx app_ctx, PetscBool has_ts, TS ts, SNES snes, KSP ksp, Vec U, CeedScalar l2_error_u,
CeedScalar l2_error_p) {
PetscFunctionBeginUser;

const char *used_resource;
CeedMemType mem_type_backend;
CeedGetResource(ceed, &used_resource);
CeedGetPreferredMemType(ceed, &mem_type_backend);
char hostname[PETSC_MAX_PATH_LEN];
PetscCall(PetscGetHostName(hostname, sizeof hostname));
PetscInt comm_size;
Expand Down Expand Up @@ -128,9 +130,11 @@ PetscErrorCode SetupProjectVelocityCtx_Hdiv(MPI_Comm comm, DM dm, Ceed ceed, Cee
PetscFunctionReturn(0);
}

PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, VecType vec_type, OperatorApplyContext ctx_H1) {
PetscErrorCode SetupProjectVelocityCtx_H1(MPI_Comm comm, DM dm_H1, Ceed ceed, CeedData ceed_data, OperatorApplyContext ctx_H1) {
PetscFunctionBeginUser;

VecType vec_type;
PetscCall(DMGetVecType(dm_H1, &vec_type));
ctx_H1->comm = comm;
ctx_H1->dm = dm_H1;
PetscCall(DMCreateLocalVector(dm_H1, &ctx_H1->X_loc));
Expand Down
26 changes: 24 additions & 2 deletions examples/Hdiv-mixed/src/setup-dm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,33 @@
#include "petscerror.h"

// ---------------------------------------------------------------------------
// Setup DM
// Create DM
// ---------------------------------------------------------------------------
PetscErrorCode CreateDM(MPI_Comm comm, MatType mat_type, VecType vec_type, DM *dm) {
PetscErrorCode CreateDM(MPI_Comm comm, Ceed ceed, DM *dm) {
PetscFunctionBeginUser;

CeedMemType mem_type_backend;
CeedGetPreferredMemType(ceed, &mem_type_backend);

VecType vec_type = NULL;
MatType mat_type = NULL;
switch (mem_type_backend) {
case CEED_MEM_HOST:
vec_type = VECSTANDARD;
break;
case CEED_MEM_DEVICE: {
const char *resolved;
CeedGetResource(ceed, &resolved);
if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA;
else if (strstr(resolved, "/gpu/hip/occa")) vec_type = VECSTANDARD; // https://github.com/CEED/libCEED/issues/678
else if (strstr(resolved, "/gpu/hip")) vec_type = VECHIP;
else vec_type = VECSTANDARD;
}
}
if (strstr(vec_type, VECCUDA)) mat_type = MATAIJCUSPARSE;
else if (strstr(vec_type, VECKOKKOS)) mat_type = MATAIJKOKKOS;
else mat_type = MATAIJ;

// Create DMPLEX
PetscCall(DMCreate(comm, dm));
PetscCall(DMSetType(*dm, DMPLEX));
Expand Down
Loading

0 comments on commit c54385e

Please sign in to comment.