diff --git a/.github/CI/spack.yaml b/.github/CI/spack.yaml index 52f8d71..1082287 100644 --- a/.github/CI/spack.yaml +++ b/.github/CI/spack.yaml @@ -1,6 +1,7 @@ spack: definitions: - pkgs: + - openmpi - fftw+mpi - heffte+fftw+cuda cuda_arch=70 - accfft diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b2bbcc..933b2d3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,13 +36,10 @@ jobs: spack repo add `pwd`/spack/ || true cp .github/CI/spack.yaml . spack env activate -d . - spack uninstall -a -y fiber ${{ matrix.library }} || true spack add fiber fft=${{ matrix.library }} %gcc@7.3.0 + spack concretize + spack uninstall -a -y --dependents ${{ matrix.library }} || true spack install - - - name: Test with ${{ matrix.library }} - run: | - source .github/CI/spack_setup.sh - spack env activate -d . spack load fiber fft=${{ matrix.library }} mpirun -n 2 test3D_C2C -lib ${{ matrix.library }} -backend fftw -size 4 4 4 -pgrid 1 2 + diff --git a/external/ffte/tools/CMakeLists.txt b/external/ffte/tools/CMakeLists.txt new file mode 100644 index 0000000..7764666 --- /dev/null +++ b/external/ffte/tools/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required(VERSION 3.10) + +project("FFTE" VERSION 7.0 LANGUAGES Fortran) + +find_package(MPI REQUIRED) + +if(CMAKE_Fortran_COMPILER_ID STREQUAL XL) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qfixed=72 -qxlf77=intarg") +endif() + +add_library(ffte zdfft2d.f fft235.f kernel.f dzfft2d.f factor.f zdfft3d.f fftever.f dzfft3d.f mfft235.f zfft1d.f zfft2d.f zfft3d.f) +# CUDA (cuzfft1d.f cuzfft3d.f cuztrans.f cuzfft2d.f) + +add_library(fftempi mpi/pdzfft3dv.f mpi/pzdfft3d.f mpi/pdzfft2d.f mpi/pzdfft3dv.f mpi/pzdfft2d.f mpi/pfftever.f mpi/pdzfft3d.f mpi/pfactor.f mpi/pzfft1d.f mpi/pzfft2d.f mpi/pzfft3d.f mpi/pzfft3dv.f) +# CUDA (mpi/pcuzfft1d.f mpi/pcuzfft3d.f mpi/pcuzfft3dv.f mpi/pcuzfft2d.f) +target_include_directories(fftempi PUBLIC ${MPI_Fortran_INCLUDE_DIRS}) +target_link_libraries(fftempi ${MPI_Fortran_LIBRARIES}) + +foreach(par "" p) + if ("${par}" STREQUAL p) + set(dir mpi/tests/) + set(lib fftempi ffte) + else() + set(dir tests/) + set(lib ffte) + endif() + foreach(knd test rtest speed) + if (${knd} STREQUAL rtest) + set(strt 2) + else() + set(strt 1) + endif() + foreach(dim RANGE ${strt} 3) + set(target "${par}${knd}${dim}d") + add_executable(${target} "${dir}${par}${knd}${dim}d.f") + set_property(TARGET ${target} PROPERTY INSTALL_RPATH_USE_LINK_PATH True) + target_link_libraries(${target} ${lib}) + install(TARGETS ${target} DESTINATION bin) + if ("${par}" STREQUAL p AND ${dim} EQUAL 3) + set(target "${par}${knd}${dim}dv") + add_executable(${target} "${dir}${par}${knd}${dim}dv.f") + set_property(TARGET ${target} PROPERTY INSTALL_RPATH_USE_LINK_PATH True) + target_link_libraries(${target} ${lib}) + install(TARGETS ${target} DESTINATION bin) + endif() + endforeach() + endforeach() +endforeach() + +install(TARGETS ffte fftempi DESTINATION lib) diff --git a/external/ffte/tools/fftever.f b/external/ffte/tools/fftever.f new file mode 100644 index 0000000..f997a79 --- /dev/null +++ b/external/ffte/tools/fftever.f @@ -0,0 +1,6 @@ + subroutine fftever(i,j,k) + integer i,j,k + i=7 + j=0 + k=0 + end diff --git a/external/ffte/tools/fiberfft.sh b/external/ffte/tools/fiberfft.sh new file mode 100644 index 0000000..a8b2af1 --- /dev/null +++ b/external/ffte/tools/fiberfft.sh @@ -0,0 +1,73 @@ +#! /usr/bin/env bash + +ver=7.0 +bffte=buildffte +bfiber=buildfiber + +date + +for mod in xl/16.1.1-10 spectrum-mpi/10.4.0.3-20210112 cmake/3.18.4 +do + module load $mod +done + +mkdir /tmp/$USER || true +cd /tmp/$USER + +if test -f ffte-${ver}.tgz ; then + echo :FIBER: Already downloaded +else + echo :FIBER: Downloadig FFTE + curl -OL http://ffte.jp/ffte-${ver}.tgz +fi + +if test -d ffte-${ver} ; then + echo :FIBER: Already unpacked +else + echo :FIBER: Unpacking FFTE + tar -xzf ffte-${ver}.tgz +fi + +if test -d fiber ; then + echo :FIBER: Already cloned +else + echo :FIBER: Clonining FIBER branch + git clone https://github.com/luszczek/fiber +fi +# must be on backend/ffte branch +cd fiber ; git pull --rebase ; git checkout backend/ffte ; cd .. + +# +# Setup and build FFTE +# +cp fiber/external/ffte/tools/CMakeLists.txt ffte-${ver} # FFTE doesnt use CMake +cp fiber/external/ffte/tools/fftever.f ffte-${ver} # FFTE version +cp fiber/external/ffte/tools/pfftever.f ffte-${ver}/mpi # parallel FFTE version +cp ffte-${ver}/param.h ffte-${ver}/mpi # FFTE parameters for scratch buffers + +if test -d ${bffte} ; then + echo :FIBER: Build dir already created, cleaning + rm -r ${bffte}/* +else + mkdir ${bffte} +fi +echo :FIBER: Building FFTE +cd ${bffte} ; cmake -D CMAKE_INSTALL_PREFIX=/tmp/$USER/usr ../ffte-${ver} ; make -j ; make install ; cd .. + +# +# Setup and build FFTE +# +if test -d ${bfiber} ; then + echo :FIBER: Build dir already created, cleaning + rm -r ${bfiber}/* +else + mkdir ${bfiber} +fi +echo :FIBER: Building FIBER +cd ${bfiber} ; cmake -D CMAKE_INSTALL_PREFIX=/tmp/$USER/usr -D FIBER_FFT_LIB_DIRS=/tmp/$USER/usr/lib -D FIBER_ENABLE_FFTE=ON ../fiber ; make -j ; make install ; cd .. + +echo :FIBER: Benchmarks installed in /tmp/$USER/${bfiber}/benchmarks + +date + +exit 1 diff --git a/external/ffte/tools/pfftever.f b/external/ffte/tools/pfftever.f new file mode 100644 index 0000000..6a3373b --- /dev/null +++ b/external/ffte/tools/pfftever.f @@ -0,0 +1,6 @@ + subroutine pfftever(i,j,k) + integer i,j,k + i=7 + j=0 + k=0 + end diff --git a/include/fiber_backend_accfft.h b/include/fiber_backend_accfft.h index a5a7e4d..e09a40f 100644 --- a/include/fiber_backend_accfft.h +++ b/include/fiber_backend_accfft.h @@ -122,6 +122,7 @@ void compute_z2z_accfft( int const inbox_low[3], int const inbox_high[3], // accfft_destroy_plan_gpu_c(plan); printf("done kia \n"); + } @@ -229,4 +230,4 @@ void compute_z2d_accfft( int const inbox_low[3], int const inbox_high[3], {} #endif -#endif \ No newline at end of file +#endif diff --git a/include/fiber_backend_ffte.h b/include/fiber_backend_ffte.h index 3d9b1a0..59624aa 100644 --- a/include/fiber_backend_ffte.h +++ b/include/fiber_backend_ffte.h @@ -18,9 +18,9 @@ int init_ffte(int option){ } extern void fiber_pzfft3d(void const *in, void *out, int *nx, int *ny, int *nz, - MPI_Fint fcomm, int *comm_size, int *opt); + MPI_Fint *fcomm, int *comm_size, int *opt); extern void fiber_pdzfft3d(double const *in, void *out, int *nx, int *ny, int *nz, - MPI_Fint fcomm, int *comm_rank, int *comm_size, int *opt); + MPI_Fint *fcomm, int *comm_rank, int *comm_size, int *opt); /* ===================== Complex-to-Complex transform ========================= */ @@ -48,7 +48,7 @@ void compute_z2z_ffte(int const inbox_low[3], int const inbox_high[3], MPI_Comm_size(comm, &comm_size); opt = 0; - fiber_pzfft3d(in, out, nd+0, nd+1, nd+2, fcomm, &comm_size, &opt); + fiber_pzfft3d(in, out, nd+0, nd+1, nd+2, &fcomm, &comm_size, &opt); MPI_Barrier(comm); timer[0] += MPI_Wtime(); @@ -63,7 +63,7 @@ void compute_z2z_ffte(int const inbox_low[3], int const inbox_high[3], compute FFT */ opt = 1; - fiber_pzfft3d(in, out, nd+0, nd+1, nd+2, fcomm, &comm_size, &opt); + fiber_pzfft3d(in, out, nd+0, nd+1, nd+2, &fcomm, &comm_size, &opt); MPI_Barrier(comm); timer[1] = +MPI_Wtime(); @@ -103,7 +103,7 @@ void compute_d2z_ffte(int const inbox_low[3], int const inbox_high[3], MPI_Comm_rank(comm, &comm_rank); MPI_Comm_size(comm, &comm_size); opt = 0; - fiber_pdzfft3d(in, out, nd+0, nd+1, nd+2, fcomm, &comm_rank, &comm_size, &opt); + fiber_pdzfft3d(in, out, nd+0, nd+1, nd+2, &fcomm, &comm_rank, &comm_size, &opt); MPI_Barrier(comm); timer[0] += MPI_Wtime(); @@ -118,7 +118,7 @@ void compute_d2z_ffte(int const inbox_low[3], int const inbox_high[3], compute FFT */ opt = 1; - fiber_pdzfft3d(in, out, nd+0, nd+1, nd+2, fcomm, &comm_rank, &comm_size, &opt); + fiber_pdzfft3d(in, out, nd+0, nd+1, nd+2, &fcomm, &comm_rank, &comm_size, &opt); MPI_Barrier(comm); timer[1] = +MPI_Wtime(); diff --git a/spack/packages/2decomp-fft/__pycache__/package.cpython-36.pyc b/spack/packages/2decomp-fft/__pycache__/package.cpython-36.pyc new file mode 100644 index 0000000..ff2fa06 Binary files /dev/null and b/spack/packages/2decomp-fft/__pycache__/package.cpython-36.pyc differ