Fix wrong pointer passed to Fortran MPI in FFTE

.github/CI/spack.yaml

@@ -1,6 +1,7 @@
 spack:
   definitions:
   - pkgs:
+    - openmpi
     - fftw+mpi
     - heffte+fftw+cuda cuda_arch=70
     - accfft

.github/workflows/main.yml

@@ -36,13 +36,10 @@ jobs:
           spack repo add `pwd`/spack/ || true
           cp .github/CI/spack.yaml .
           spack env activate -d .
-          spack uninstall -a -y fiber ${{ matrix.library }} || true
           spack add fiber fft=${{ matrix.library }} %[email protected]
+          spack concretize
+          spack uninstall -a -y --dependents ${{ matrix.library }} || true
           spack install
-
-      - name: Test with ${{ matrix.library }}
-        run: |
-          source .github/CI/spack_setup.sh
-          spack env activate -d .
           spack load fiber fft=${{ matrix.library }}
           mpirun -n 2 test3D_C2C -lib ${{ matrix.library }} -backend fftw -size 4 4 4 -pgrid 1 2 
+

external/ffte/tools/CMakeLists.txt

@@ -0,0 +1,50 @@
+cmake_minimum_required(VERSION 3.10)
+
+project("FFTE" VERSION 7.0 LANGUAGES Fortran)
+
+find_package(MPI REQUIRED)
+
+if(CMAKE_Fortran_COMPILER_ID STREQUAL XL)
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qfixed=72 -qxlf77=intarg")
+endif()
+
+add_library(ffte zdfft2d.f fft235.f kernel.f dzfft2d.f factor.f zdfft3d.f fftever.f dzfft3d.f mfft235.f zfft1d.f zfft2d.f zfft3d.f)
+# CUDA (cuzfft1d.f cuzfft3d.f cuztrans.f cuzfft2d.f)
+
+add_library(fftempi mpi/pdzfft3dv.f mpi/pzdfft3d.f mpi/pdzfft2d.f mpi/pzdfft3dv.f mpi/pzdfft2d.f mpi/pfftever.f mpi/pdzfft3d.f mpi/pfactor.f mpi/pzfft1d.f mpi/pzfft2d.f mpi/pzfft3d.f mpi/pzfft3dv.f)
+# CUDA (mpi/pcuzfft1d.f mpi/pcuzfft3d.f mpi/pcuzfft3dv.f mpi/pcuzfft2d.f)
+target_include_directories(fftempi PUBLIC ${MPI_Fortran_INCLUDE_DIRS})
+target_link_libraries(fftempi ${MPI_Fortran_LIBRARIES})
+
+foreach(par "" p)
+    if ("${par}" STREQUAL p)
+        set(dir mpi/tests/)
+        set(lib fftempi ffte)
+    else()
+        set(dir tests/)
+        set(lib ffte)
+    endif()
+    foreach(knd test rtest speed)
+        if (${knd} STREQUAL rtest)
+            set(strt 2)
+        else()
+            set(strt 1)
+        endif()
+        foreach(dim RANGE ${strt} 3)
+            set(target "${par}${knd}${dim}d")
+            add_executable(${target} "${dir}${par}${knd}${dim}d.f")
+            set_property(TARGET ${target} PROPERTY INSTALL_RPATH_USE_LINK_PATH True)
+            target_link_libraries(${target} ${lib})
+            install(TARGETS ${target} DESTINATION bin)
+            if ("${par}" STREQUAL p AND ${dim} EQUAL 3)
+                set(target "${par}${knd}${dim}dv")
+                add_executable(${target} "${dir}${par}${knd}${dim}dv.f")
+                set_property(TARGET ${target} PROPERTY INSTALL_RPATH_USE_LINK_PATH True)
+                target_link_libraries(${target} ${lib})
+                install(TARGETS ${target} DESTINATION bin)
+            endif()
+        endforeach()
+    endforeach()
+endforeach()
+
+install(TARGETS ffte fftempi DESTINATION lib)

external/ffte/tools/fftever.f

@@ -0,0 +1,6 @@
+      subroutine fftever(i,j,k)
+      integer i,j,k
+      i=7
+      j=0
+      k=0
+      end

external/ffte/tools/fiberfft.sh

@@ -0,0 +1,73 @@
+#! /usr/bin/env bash
+
+ver=7.0
+bffte=buildffte
+bfiber=buildfiber
+
+date
+
+for mod in xl/16.1.1-10 spectrum-mpi/10.4.0.3-20210112 cmake/3.18.4
+do
+  module load $mod
+done
+
+mkdir /tmp/$USER || true
+cd /tmp/$USER
+
+if test -f ffte-${ver}.tgz ; then
+  echo :FIBER: Already downloaded
+else
+  echo :FIBER: Downloadig FFTE
+  curl -OL http://ffte.jp/ffte-${ver}.tgz
+fi
+
+if test -d ffte-${ver} ; then
+  echo :FIBER: Already unpacked
+else
+  echo :FIBER: Unpacking FFTE
+  tar -xzf ffte-${ver}.tgz
+fi
+
+if test -d fiber ; then
+  echo :FIBER: Already cloned
+else
+  echo :FIBER: Clonining FIBER branch
+  git clone https://github.com/luszczek/fiber
+fi
+# must be on backend/ffte branch
+cd fiber ; git pull --rebase ; git checkout backend/ffte ; cd ..
+
+#
+# Setup and build FFTE
+#
+cp fiber/external/ffte/tools/CMakeLists.txt ffte-${ver} # FFTE doesnt use CMake
+cp fiber/external/ffte/tools/fftever.f ffte-${ver} # FFTE version
+cp fiber/external/ffte/tools/pfftever.f ffte-${ver}/mpi # parallel FFTE version
+cp ffte-${ver}/param.h ffte-${ver}/mpi # FFTE parameters for scratch buffers
+
+if test -d ${bffte} ; then
+  echo :FIBER: Build dir already created, cleaning
+  rm -r ${bffte}/*
+else
+  mkdir ${bffte}
+fi
+echo :FIBER: Building FFTE
+cd ${bffte} ; cmake -D CMAKE_INSTALL_PREFIX=/tmp/$USER/usr ../ffte-${ver} ; make -j ; make install ; cd ..
+
+#
+# Setup and build FFTE
+#
+if test -d ${bfiber} ; then
+  echo :FIBER: Build dir already created, cleaning
+  rm -r ${bfiber}/*
+else
+  mkdir ${bfiber}
+fi
+echo :FIBER: Building FIBER
+cd ${bfiber} ; cmake -D CMAKE_INSTALL_PREFIX=/tmp/$USER/usr -D FIBER_FFT_LIB_DIRS=/tmp/$USER/usr/lib -D FIBER_ENABLE_FFTE=ON ../fiber ; make -j ; make install ; cd ..
+
+echo :FIBER: Benchmarks installed in /tmp/$USER/${bfiber}/benchmarks
+
+date
+
+exit 1

external/ffte/tools/pfftever.f

@@ -0,0 +1,6 @@
+      subroutine pfftever(i,j,k)
+      integer i,j,k
+      i=7
+      j=0
+      k=0
+      end

include/fiber_backend_accfft.h

@@ -122,6 +122,7 @@ void compute_z2z_accfft( int const inbox_low[3], int const inbox_high[3],
     //     accfft_destroy_plan_gpu_c(plan);
 
     printf("done kia \n");
+
 }    
 
 
@@ -229,4 +230,4 @@ void compute_z2d_accfft( int const inbox_low[3], int const inbox_high[3],
 {}
 
 #endif
-#endif
+#endif

include/fiber_backend_ffte.h

@@ -18,9 +18,9 @@ int init_ffte(int option){
 }
 
 extern void fiber_pzfft3d(void const *in, void *out, int *nx, int *ny, int *nz,
-    MPI_Fint fcomm, int *comm_size, int *opt);
+    MPI_Fint *fcomm, int *comm_size, int *opt);
 extern void fiber_pdzfft3d(double const *in, void *out, int *nx, int *ny, int *nz,
-    MPI_Fint fcomm, int *comm_rank, int *comm_size, int *opt);
+    MPI_Fint *fcomm, int *comm_rank, int *comm_size, int *opt);
 
 /* =====================  Complex-to-Complex transform ========================= */
 
@@ -48,7 +48,7 @@ void compute_z2z_ffte(int const inbox_low[3], int const inbox_high[3],
 
     MPI_Comm_size(comm, &comm_size);
     opt = 0;
-    fiber_pzfft3d(in, out, nd+0, nd+1, nd+2, fcomm, &comm_size, &opt);
+    fiber_pzfft3d(in, out, nd+0, nd+1, nd+2, &fcomm, &comm_size, &opt);
 
     MPI_Barrier(comm);
     timer[0] += MPI_Wtime();
@@ -63,7 +63,7 @@ void compute_z2z_ffte(int const inbox_low[3], int const inbox_high[3],
     compute FFT
     */
     opt = 1;
-    fiber_pzfft3d(in, out, nd+0, nd+1, nd+2, fcomm, &comm_size, &opt);
+    fiber_pzfft3d(in, out, nd+0, nd+1, nd+2, &fcomm, &comm_size, &opt);
 
     MPI_Barrier(comm);
     timer[1] = +MPI_Wtime();
@@ -103,7 +103,7 @@ void compute_d2z_ffte(int const inbox_low[3], int const inbox_high[3],
     MPI_Comm_rank(comm, &comm_rank);
     MPI_Comm_size(comm, &comm_size);
     opt = 0;
-    fiber_pdzfft3d(in, out, nd+0, nd+1, nd+2, fcomm, &comm_rank, &comm_size, &opt);
+    fiber_pdzfft3d(in, out, nd+0, nd+1, nd+2, &fcomm, &comm_rank, &comm_size, &opt);
 
     MPI_Barrier(comm);
     timer[0] += MPI_Wtime();
@@ -118,7 +118,7 @@ void compute_d2z_ffte(int const inbox_low[3], int const inbox_high[3],
     compute FFT
     */
     opt = 1;
-    fiber_pdzfft3d(in, out, nd+0, nd+1, nd+2, fcomm, &comm_rank, &comm_size, &opt);
+    fiber_pdzfft3d(in, out, nd+0, nd+1, nd+2, &fcomm, &comm_rank, &comm_size, &opt);
 
     MPI_Barrier(comm);
     timer[1] = +MPI_Wtime();