Merge branch 'develop' into feature/update-to-dev

MmgTools · Aug 13, 2024 · c7549f1 · c7549f1
2 parents 57dfbe3 + beb4147
commit c7549f1
Show file tree

Hide file tree

Showing 8 changed files with 120 additions and 38 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,4 +1,4 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.5)
 
 INCLUDE(CMakeDependentOption)
 INCLUDE(cmake/modules/macros.cmake)

diff --git a/cmake/testing/pmmg_tests.cmake b/cmake/testing/pmmg_tests.cmake
@@ -170,6 +170,31 @@ IF( BUILD_TESTING )
       -out ${CI_DIR_RESULTS}/opnbdy-island.o.mesh
       )
 
+    ### test -m option
+    #### The 2 tests are intentionnaly failing (inside an assert in debug mode, with an error
+    #### message otherwise) due to lack of memory when computing the hash table
+    add_test ( NAME memory-pmmg_sphere-2
+      COMMAND ${MPIEXEC} ${MPI_ARGS} ${MPIEXEC_NUMPROC_FLAG} 2 $<TARGET_FILE:${PROJECT_NAME}>
+      -mmg-v 5 -v 5 -m 15
+      ${CI_DIR}/Sphere/sphere
+      -out ${CI_DIR_RESULTS}/memory-sphere.o.mesh
+      )
+    set_property(TEST memory-pmmg_sphere-2
+      PROPERTY
+      PASS_REGULAR_EXPRESSION "MAXIMUM MEMORY AUTHORIZED PER PROCESS \\(MB\\)    15"
+      )
+
+    add_test ( NAME memory-mmg_sphere-2
+      COMMAND ${MPIEXEC} ${MPI_ARGS} ${MPIEXEC_NUMPROC_FLAG} 2 $<TARGET_FILE:${PROJECT_NAME}>
+      -mmg-v 5 -v 5 -m 15
+      ${CI_DIR}/Sphere/sphere
+      -out ${CI_DIR_RESULTS}/memory-sphere.o.mesh
+      )
+    set_property(TEST memory-mmg_sphere-2
+      PROPERTY
+      PASS_REGULAR_EXPRESSION "MAXIMUM MEMORY AUTHORIZED \\(MB\\)    15"
+      )
+
     ###############################################################################
     #####
     #####        Test centralized/distributed I/O (on multidomain and openbdy tests)

diff --git a/src/mergemesh_pmmg.c b/src/mergemesh_pmmg.c
@@ -1097,11 +1097,10 @@ int PMMG_gather_parmesh( PMMG_pParMesh parmesh,
                          PMMG_pInt_comm *rcv_int_node_comm,
                          int **rcv_next_node_comm,
                          PMMG_pExt_comm **rcv_ext_node_comm ) {
-
-  size_t     pack_size_tot,next_disp;
-  int        *rcv_pack_size,ier,ier_glob,k,*displs,ier_pack;
+  size_t     pack_size_tot,next_disp,*displs,buf_idx;
+  int        *rcv_pack_size,ier,ier_glob,k,ier_pack;
   int        nprocs,root,pack_size;
-  char       *rcv_buffer,*buffer,*ptr;
+  char       *rcv_buffer,*ptr_to_free,*buffer;
 
   nprocs        = parmesh->nprocs;
   root          = parmesh->info.root;
@@ -1120,7 +1119,7 @@ int PMMG_gather_parmesh( PMMG_pParMesh parmesh,
   /** 1: Memory alloc */
   if ( parmesh->myrank == root ) {
     PMMG_MALLOC( parmesh, rcv_pack_size        ,nprocs,int,"rcv_pack_size",ier=0);
-    PMMG_MALLOC( parmesh, displs               ,nprocs,int,"displs for gatherv",ier=0);
+    PMMG_MALLOC( parmesh, displs               ,nprocs,size_t,"displs for gatherv",ier=0);
     PMMG_CALLOC( parmesh, (*rcv_grps)          ,nprocs,PMMG_Grp,"rcv_grps",ier=0);
     PMMG_MALLOC( parmesh, (*rcv_int_node_comm) ,nprocs,PMMG_Int_comm,"rcv_int_comm" ,ier=0);
     PMMG_MALLOC( parmesh, (*rcv_next_node_comm),nprocs,int,"rcv_next_comm" ,ier=0);
@@ -1144,26 +1143,33 @@ int PMMG_gather_parmesh( PMMG_pParMesh parmesh,
   if ( parmesh->myrank == root ) {
     displs[0] = 0;
     for ( k=1; k<nprocs; ++k ) {
-      assert ( displs[k-1] <= INT_MAX - rcv_pack_size[k-1] && "INT_MAX overflow");
       next_disp = displs[k-1] + rcv_pack_size[k-1];
-      if(next_disp>INT_MAX){
-        /* The displacements argument to MPI_Gatherv() is an array of int
-         * (signed) so the number of elements must be smaller than 2^31.
-         * To get around this we must pack more data in a single element
-         * or use multiple messages.
-         */
-        fprintf(stderr, "  ## Error: too many elements for MPI_Gatherv()\n");
-        MPI_Abort(parmesh->comm, 1);      /* error detected only on root */
-      }
       displs[k] = next_disp;
     }
+
+    /* On root, we will gather all the meshes in rcv_buffer so we have to
+     * compute the total pack size */
     pack_size_tot        = (size_t)(displs[nprocs-1])+(size_t)(rcv_pack_size[nprocs-1]);
     assert ( pack_size_tot < SIZE_MAX && "SIZE_MAX overflow" );
-    PMMG_MALLOC( parmesh,rcv_buffer,pack_size_tot,char,"rcv_buffer",ier=0);
+
+    /* root will write directly in the suitable position of rcv_buffer */
+    buf_idx = displs[root];
   }
+  else {
+    /* on ranks other than root we just need to store the local mesh so buffer
+     * will be of size pack_size */
+    pack_size_tot = pack_size;
+    /* we will write the mesh at the starting position */
+    buf_idx = 0;
+  }
+
+  PMMG_MALLOC( parmesh,rcv_buffer,pack_size_tot,char,"rcv_buffer",ier=0);
 
   /* Parmesh compression */
-  PMMG_MALLOC ( parmesh,buffer,pack_size,char,"buffer to send",ier=0 );
+  buffer = &rcv_buffer[buf_idx];
+
+  /* Save input allocated address to avoid arrors at unalloc */
+  ptr_to_free = rcv_buffer;
 
 #ifndef NDEBUG
   /* Remark: in release mode, a non allocated buffer used in gatherv creates a
@@ -1174,17 +1180,40 @@ int PMMG_gather_parmesh( PMMG_pParMesh parmesh,
   }
 #endif
 
-  ptr = buffer;
+  /* /!\ mpipack_parmesh and mpiunpack_parmesh are modifying the buffer pointer
+   * making it not valid for realloc / unalloc */
+
+  /* Save adress of buffer because it will be set the the end of the char array
+  by the \a PMMG_mpipack_parmesh function */
+  char *buffer_to_send = buffer;
   ier_pack = PMMG_mpipack_parmesh ( parmesh ,&buffer );
+
+  /* Do not use \a buffer pointer after this call: it points toward the end of
+   * the packed array which is useless */
+  buffer = NULL;
+
   assert ( ier_pack );
 
   /* Gather the packed parmeshes */
   ier = MG_MIN ( ier, ier_pack );
-  MPI_CHECK( MPI_Gatherv ( ptr,pack_size,MPI_CHAR,
-                           rcv_buffer,rcv_pack_size,
-                           displs,MPI_CHAR,root,parmesh->comm ),ier=0 );
 
-  PMMG_DEL_MEM(parmesh,ptr,char,"buffer to send");
+  /* Here the gatherv call has been replaced by a send/recv to avoid errors when
+   * displacements overflow the INT_MAX value */
+  if (parmesh->myrank == root) {
+    int i;
+    for ( i = 0; i < nprocs; ++i ) {
+      if ( i != root ) {
+        MPI_CHECK(
+          MPI_Recv(rcv_buffer + displs[i], rcv_pack_size[i], MPI_CHAR, i,
+                   MPI_MERGEMESH_TAG, parmesh->comm, MPI_STATUS_IGNORE),
+          ier = 0);
+      }
+    }
+  } else {
+    MPI_CHECK(
+      MPI_Send(buffer_to_send, pack_size, MPI_CHAR, root, MPI_MERGEMESH_TAG,parmesh->comm),
+      ier = 0);
+  }
 
   /** 4: Unpack parmeshes */
 #ifndef NDEBUG
@@ -1195,7 +1224,6 @@ int PMMG_gather_parmesh( PMMG_pParMesh parmesh,
 #endif
 
   if ( parmesh->myrank == root ) {
-    ptr = rcv_buffer;
     for ( k=0; k<nprocs; ++k ) {
       ier_pack = PMMG_mpiunpack_parmesh ( parmesh,(*rcv_grps),k,(*rcv_int_node_comm)+k,
                                           (*rcv_next_node_comm)+k,(*rcv_ext_node_comm)+k,
@@ -1208,7 +1236,9 @@ int PMMG_gather_parmesh( PMMG_pParMesh parmesh,
   /* Free temporary arrays */
   PMMG_DEL_MEM(parmesh,rcv_pack_size,int,"rcv_pack_size");
   PMMG_DEL_MEM(parmesh,displs,int,"displs");
-  PMMG_DEL_MEM(parmesh,ptr ,char,"rcv_buffer");
+  /* the address of rcv_buffer is modified by packing/unpacking so it is needed
+   * to send the initially allocated address stored in to the unalloc macro */
+  PMMG_DEL_MEM(parmesh,ptr_to_free,char,"rcv_buffer");
 
   return ier;
 }

diff --git a/src/mpi_pmmg.h b/src/mpi_pmmg.h
@@ -47,7 +47,7 @@
 #define MPI_TRANSFER_GRP_TAG            8000
 #define MPI_COMMUNICATORS_REF_TAG       9000
 #define MPI_ANALYS_TAG                 10000
-
+#define MPI_MERGEMESH_TAG              11000
 
 #define MPI_CHECK(func_call,on_failure) do {                            \
     int mpi_ret_val;                                                    \

diff --git a/src/mpipack_pmmg.c b/src/mpipack_pmmg.c
@@ -1132,6 +1132,8 @@ int PMMG_mpipack_grp ( PMMG_pGrp grp,char **buffer ) {
  * buffer pointer at the end of the written area. The parmesh groups must have
  * been merged before entering this function.
  *
+ * \remark the \a buffer pointer is modified (shifted) thus, after this
+ * function, it cannot be used for deallocation anymore
  */
 int PMMG_mpipack_parmesh ( PMMG_pParMesh parmesh ,char **buffer ) {
   PMMG_pGrp grp;

diff --git a/src/mpiunpack_pmmg.c b/src/mpiunpack_pmmg.c
@@ -1112,6 +1112,9 @@ int PMMG_mpiunpack_grp ( PMMG_pParMesh parmesh,PMMG_pGrp listgrp,int igrp,char *
  * pointer toward a buffer of type "x". Then we can get the variable value by
  * dereferencing the adress of the buffer.
  *
+ * \remark the \a buffer pointer is modified (shifted) thus, after this
+ * function, it cannot be used for deallocation anymore
+ *
  */
 int PMMG_mpiunpack_parmesh ( PMMG_pParMesh parmesh,PMMG_pGrp listgrp,int igrp,
                              PMMG_pInt_comm int_node_comm,

diff --git a/src/parmmg.c b/src/parmmg.c
@@ -24,7 +24,7 @@
 /**
  * \file parmmg.c
  * \brief main file for the parmmg application
- * \author Cécile Dobrzynski (Bx INP/Inria)
+ * \author Cecile Dobrzynski (Bx INP/Inria)
  * \author Algiane Froehly (Inria)
  * \version 5
  * \copyright GNU Lesser General Public License.

diff --git a/src/zaldy_pmmg.c b/src/zaldy_pmmg.c
@@ -58,7 +58,7 @@ void PMMG_parmesh_SetMemGloMax( PMMG_pParMesh parmesh )
 
   assert ( (parmesh != NULL) && "trying to set glo max mem in empty parmesh" );
 
-  /** Step 1: Get the numper of processes per node */
+  /** Step 1: Get the number of processes per node */
   MPI_Initialized( &flag );
 
   if ( flag ) {
@@ -71,30 +71,48 @@ void PMMG_parmesh_SetMemGloMax( PMMG_pParMesh parmesh )
     parmesh->size_shm = 1;
   }
 
-  /** Step 2: Set maximal memory per process depending on the -m option setting */
+  /** Step 2: Set maximal memory per process depending on the -m option setting:
+  - if the user doesn't provides a memory value or provides an invalid value: we equirepartite the memory over the MPI processes of the node. Functions that consumes different amounts of memory depending on the process have to manage internally the memory repartition (for example the \a PMMG_loadMesh_centralized function).
+  - if the user provides a valid memory value (under or equal to the physical memory), it is used as is guessing that the user know what he is asking (it may be useful during the parallel calls of Mmg to not have a memory equirepartition as some process may use a smaller amount of memory than others but we are not able to predict it.
+ */
   maxAvail = MMG5_memSize();
 
   if ( parmesh->info.mem <= 0 ) {
-    /* Nos users specifications */
+    /* Nos users specifications: equirepartition of */
     if ( !maxAvail ) {
       /* default value when not able to compute the available memory = 800 MB */
       printf("  Maximum memory per process set to default value: %d MB.\n",MMG5_MEMMAX);
-      parmesh->memGloMax = MMG5_MEMMAX << 20;
+      parmesh->memGloMax = (MMG5_MEMMAX/parmesh->size_shm) << 20;
     }
     else {
-      /* maximal memory = total physical memory */
-      parmesh->memGloMax = maxAvail;
+      /* maximal memory = equirepartition of total physical memory over the MPI processes of the node. */
+      parmesh->memGloMax = maxAvail/parmesh->size_shm;
     }
   }
   else {
-    /* memory asked by user if possible, otherwise total physical memory */
+    int memOverflow = 0;
+    /* Memory asked by user if possible (authorized to ask the entire memory nod per process, independently of the number of process per node). */
     if ( maxAvail && (size_t)parmesh->info.mem*MMG5_MILLION > maxAvail ) {
-      fprintf(stderr,"\n  ## Warning: %s: asking for %d MB of memory per process ",
+      /* User asks for more than the memory of the node */
+      fprintf(stdout,"\n  ## Warning: %s: asking for %d MB of memory per process ",
               __func__,parmesh->info.mem);
-      fprintf(stderr,"when only %zu available.\n",maxAvail/MMG5_MILLION);
+      fprintf(stdout,"when only %zu available on the node.\n",maxAvail/MMG5_MILLION);
+      memOverflow = 1;
     }
     else {
-      parmesh->memGloMax= (size_t)parmesh->info.mem*MMG5_MILLION;
+      if ( (size_t)parmesh->info.mem*MMG5_MILLION > maxAvail/parmesh->size_shm ) {
+      /* User asks for more than the equirepartition of the node memory across the MPI processes */
+        fprintf(stdout,"\n  ## Warning: %s: asking for %d MB per MPI process with %d process per node and %zu MB available on the node.\n",
+                __func__,parmesh->info.mem,parmesh->size_shm,maxAvail/MMG5_MILLION);
+        memOverflow = 1;
+      }
+    }
+
+    /* In all cases, impose what the user ask */
+    parmesh->memGloMax= (size_t)parmesh->info.mem*MMG5_MILLION;
+
+    if ( memOverflow ) {
+      fprintf(stdout,"              The program may run out of memory and be killed (Signal 9 or SIGKILL error).\n\n");
     }
   }
 
@@ -122,6 +140,10 @@ int PMMG_parmesh_SetMemMax( PMMG_pParMesh parmesh ) {
   for( i = 0; i < parmesh->ngrp; ++i ) {
     mesh = parmesh->listgrp[i].mesh;
     mesh->memMax = parmesh->memGloMax;
+
+    /* Hack to not let Mmg recomputes the available memory by itself (it has no
+     * knowledge that it is called in parallel) */
+    mesh->info.mem = mesh->memMax/MMG5_MILLION;
   }
 
   return 1;
@@ -317,7 +339,7 @@ int PMMG_setMeshSize_realloc( MMG5_pMesh mesh,int npmax_old,int xpmax_old,
                   "adja array", return 0);
   }
 
-  if ( !nemax_old )
+  if ( !xtmax_old )
     PMMG_CALLOC(mesh, mesh->xtetra, mesh->xtmax+1, MMG5_xTetra,
                 "boundary tetra array", return 0);
   else