From 6cac69239292d1b2822b76e288407d27174b67a2 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 24 Aug 2021 09:14:02 -0400 Subject: [PATCH 1/3] Updates container files for use with 2021.03 --- container/Dockerfile | 133 ++++++++++++++++++++++++++--- container/Dockerfile.gnu | 6 +- container/Dockerfile.intel_sources | 6 +- container/Singularity.gnu | 8 +- container/Singularity.intel_am4 | 42 +++++---- 5 files changed, 149 insertions(+), 46 deletions(-) diff --git a/container/Dockerfile b/container/Dockerfile index 6d85a2c..7a60a98 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -1,18 +1,123 @@ -FROM thomasrobinson/centos7-netcdff:4.5.3-c4.7.4-gcc-mpich-slurm -## Dockerfile used to create AM4 +FROM intel/oneapi-hpckit:2021.2-devel-centos8 as builder +LABEL maintainer "Tom Robinson" -## Set up spack -RUN . /opt/spack/share/spack/setup-env.sh -## Make the AM4 directory +#------------------------------------------------------------- +## Set up packages needed +RUN yum update -y +RUN yum install -y git +RUN yum install -y patch +RUN yum install -y zlib +RUN yum install -y wget +RUN yum install -y curl +RUN yum install -y m4 + +## Set compilers +ENV FC=ifort +ENV CC=icc +ENV build=/opt +ENV IO_LIBS=${build}/io_libs +## Build zlib and szip and curl +RUN cd $build \ +&& zlib="zlib-1.2.11" \ +&& rm -rf zlib* \ +&& wget http://www.zlib.net/zlib-1.2.11.tar.gz \ +&& tar xzf zlib-1.2.11.tar.gz \ +&& cd $zlib \ +&& ./configure --prefix=${IO_LIBS} \ +&& make \ +&& make -j 20 install +ENV CC "icc -fPIC" +RUN cd $build \ +&& szip="szip-2.1.1" \ +&& rm -rf szip* \ +&& wget https://support.hdfgroup.org/ftp/lib-external/szip/2.1.1/src/szip-2.1.1.tar.gz \ +&& tar xzf szip-2.1.1.tar.gz \ +&& cd $szip \ +&& ./configure FC=ifort CC=icc --prefix=${IO_LIBS} CPPDEFS="-fPIC" \ +&& make \ +&& make -j 20 install +RUN cd $build \ +&& curl="curl-7.74.0" \ +&& rm -rf curl* \ +&& wget https://curl.haxx.se/download/${curl}.tar.gz \ +&& tar xzf ${curl}.tar.gz \ +&& cd $curl \ +&& ./configure FC=ifort CC=icc --prefix=${IO_LIBS} \ +&& make \ +&& make -j 20 install + +ENV LD_LIBRARY_PATH=${IO_LIBS}/lib:${LD_LIBRARY_PATH}:/opt/io_libs/lib + +## Set compilers +ENV FC=ifort +ENV CC=icc +## Install HDF5 +RUN cd /opt \ +&& hdf5="hdf5-1.12.0" \ +&& wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/${hdf5}/src/${hdf5}.tar.gz \ +&& tar xzf ${hdf5}.tar.gz \ +&& cd $hdf5 \ +&& hdf5_opts="FC=ifort CC=icc --prefix=/opt/hdf5 --enable-fortran --enable-hl" \ +&& ./configure $hdf5_opts \ +&& make -j 20 install \ +&& echo "HDF5 finished building" + +#********************* +## Install NetCDF-C +#********************* +ENV LD_LIBRARY_PATH=/opt/hdf5/lib:${LD_LIBRARY_PATH} +RUN cd /opt \ +&& version="4.7.4" \ +&& netcdfc="netcdf-c-"${version} \ +&& rm -rf netcdf \ +&& wget -O ${netcdfc}.tar.gz https://github.com/Unidata/netcdf-c/archive/v${version}.tar.gz \ +&& tar xzf ${netcdfc}.tar.gz \ +&& cd $netcdfc \ +&& ./configure --prefix=/opt/netcdf-c CPPFLAGS='-I/opt/hdf5/include -I${IO_LIBS}/include' LDFLAGS='-L/opt/hdf5/lib -L${IO_LIBS}/lib' --disable-dap \ +&& make \ +&& make -j 20 install \ +&& echo " NetCDF-C finished building" + +ENV LD_LIBRARY_PATH=/opt/netcdf-c/lib:${LD_LIBRARY_PATH} +ENV PATH=/opt/netcdf-c/bin:${PATH} + +## Install netcdf fortran +ENV LDFLAGS="-L/opt/netcdf-c/lib -lnetcdf" +RUN cd /opt \ +&& nfversion=4.5.3 \ +&& netcdff="netcdf-fortran-${nfversion}" \ +&& rm -rf $netcdff \ +&& wget -O ${netcdff}.tar.gz https://github.com/Unidata/netcdf-fortran/archive/v${nfversion}.tar.gz \ +&& tar xzf ${netcdff}.tar.gz \ +&& cd $netcdff \ +&& ./configure CPPFLAGS="-I/opt/netcdf-c/include -I/opt/hdf5/include/" --prefix=/opt/netcdf-fortran \ +&& make \ +&& make -j20 install + +ENV PATH=/opt/netcdf-fortran/bin:${PATH} + +ENV LD_LIBRARY_PATH=/opt/netcdf-c/lib:/opt/hdf5/lib:/opt/netcdf-fortran/lib:${LD_LIBRARY_PATH} +ENV LIBRARY_PATH=${LD_LIBRARY_PATH} + +## Build the model RUN mkdir -p /opt/AM4 -## Build the AM4 from github -RUN git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b 2021.02 \ - && cd AM4/exec \ - && make gcc=on HDF_INCLUDE=-I/opt/hdf5/include SH=sh CLUBB=off \ - && cp am4_xanadu_2021.02.x /opt/AM4 \ - && make clean_all -## Add the AM4 executable to the path -ENV PATH=/opt/AM4/:${PATH} +RUN git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b 2021.03 \ + && cd AM4/exec \ + && make HDF_INCLUDE=-I/opt/hdf5/include \ + && cp am4_xanadu_2021.03.x /opt/AM4 \ + && make clean_all + +############################################################################################################## +# Stage 2 with the minimum +FROM intel/oneapi-runtime:centos8 +RUN ls +COPY --from=builder /opt/netcdf-c /opt/netcdf-c +COPY --from=builder /opt/netcdf-fortran /opt/netcdf-fortran +COPY --from=builder /opt/hdf5 /opt/hdf5 +COPY --from=builder /opt/AM4 /opt/AM4 +ENV PATH=/opt/AM4:/opt/netcdf-fortran/bin:/opt/netcdf-c/bin:${PATH} +ENV LD_LIBRARY_PATH=/opt/netcdf-c/lib:/opt/hdf5/lib:/opt/netcdf-fortran/lib:/opt/io_libs/lib${LD_LIBRARY_PATH} +ENV LIBRARY_PATH=${LD_LIBRARY_PATH} ## Add permissions to the AM4 -RUN chmod 777 /opt/AM4/am4_xanadu_2021.02.x +RUN chmod 777 /opt/AM4/am4_xanadu_2021.03.x diff --git a/container/Dockerfile.gnu b/container/Dockerfile.gnu index 6d85a2c..402a4a6 100644 --- a/container/Dockerfile.gnu +++ b/container/Dockerfile.gnu @@ -6,13 +6,13 @@ RUN . /opt/spack/share/spack/setup-env.sh ## Make the AM4 directory RUN mkdir -p /opt/AM4 ## Build the AM4 from github -RUN git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b 2021.02 \ +RUN git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b 2021.03 \ && cd AM4/exec \ && make gcc=on HDF_INCLUDE=-I/opt/hdf5/include SH=sh CLUBB=off \ - && cp am4_xanadu_2021.02.x /opt/AM4 \ + && cp am4_xanadu_2021.03.x /opt/AM4 \ && make clean_all ## Add the AM4 executable to the path ENV PATH=/opt/AM4/:${PATH} ## Add permissions to the AM4 -RUN chmod 777 /opt/AM4/am4_xanadu_2021.02.x +RUN chmod 777 /opt/AM4/am4_xanadu_2021.03.x diff --git a/container/Dockerfile.intel_sources b/container/Dockerfile.intel_sources index 3284189..7a60a98 100644 --- a/container/Dockerfile.intel_sources +++ b/container/Dockerfile.intel_sources @@ -101,10 +101,10 @@ ENV LIBRARY_PATH=${LD_LIBRARY_PATH} ## Build the model RUN mkdir -p /opt/AM4 -RUN git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b 2021.02 \ +RUN git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b 2021.03 \ && cd AM4/exec \ && make HDF_INCLUDE=-I/opt/hdf5/include \ - && cp am4_xanadu_2021.02.x /opt/AM4 \ + && cp am4_xanadu_2021.03.x /opt/AM4 \ && make clean_all ############################################################################################################## @@ -119,5 +119,5 @@ ENV PATH=/opt/AM4:/opt/netcdf-fortran/bin:/opt/netcdf-c/bin:${PATH} ENV LD_LIBRARY_PATH=/opt/netcdf-c/lib:/opt/hdf5/lib:/opt/netcdf-fortran/lib:/opt/io_libs/lib${LD_LIBRARY_PATH} ENV LIBRARY_PATH=${LD_LIBRARY_PATH} ## Add permissions to the AM4 -RUN chmod 777 /opt/AM4/am4_xanadu_2021.02.x +RUN chmod 777 /opt/AM4/am4_xanadu_2021.03.x diff --git a/container/Singularity.gnu b/container/Singularity.gnu index 3b82f63..ec53523 100644 --- a/container/Singularity.gnu +++ b/container/Singularity.gnu @@ -10,12 +10,12 @@ Stage: build mkdir -p /opt/AM4 cd /opt ## Build the AM4 from github - git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b 2021.02 + git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b 2021.03 cd AM4/exec make -j 20 gcc=on HDF_INCLUDE=-I/opt/hdf5/include SH=sh CLUBB=off - cp am4_xanadu_2021.02.x /opt/AM4 + cp am4_xanadu_2021.03.x /opt/AM4 make clean_all - chmod 777 /opt/AM4/am4_xanadu_2021.02.x + chmod 777 /opt/AM4/am4_xanadu_2021.03.x ## Add the AM4 executable to the path %environment @@ -24,5 +24,5 @@ ENV PATH=/opt/AM4/:${PATH} ## Run AM4 %runscript ulimit -s unlimited - /opt/AM4/am4_xanadu_2021.02.x + /opt/AM4/am4_xanadu_2021.03.x diff --git a/container/Singularity.intel_am4 b/container/Singularity.intel_am4 index dad7cb6..8540a95 100644 --- a/container/Singularity.intel_am4 +++ b/container/Singularity.intel_am4 @@ -8,35 +8,33 @@ LD_LIBRARY_PATH=/opt/netcdf-c/lib:/opt/netcdf-fortran/lib:/opt/hdf5/lib:/opt/int export PATH=${PATH}:/opt/netcdf-c/bin:/opt/netcdf-fortran/bin cd /opt ## Build the AM4 from github - git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b main + git clone --recursive https://github.com/NOAA-GFDL/AM4.git -b 2021.03 cd AM4/exec make -j 20 HDF_INCLUDE=-I/opt/hdf5/include HDF_LIBS="-L/opt/hdf5/lib -lhdf5 -lhdf5_fortran -lhdf5_hl -lhdf5hl_fortran" SH=sh cp am4_xanadu_2021.03.x /opt/AM4 - make clean_all chmod 777 /opt/AM4/am4_xanadu_2021.03.x - -Bootstrap: docker -From: intel/oneapi-runtime:ubuntu18.04 -Stage: final - -%files from build -/opt/hdf5 -/opt/netcdf-c -/opt/netcdf-fortran -/opt/AM4/am4_xanadu_2021.03.x -/opt/AM4/exec/fms/build/libFMS/.libs/ -## Add the AM4 executable to the path -%environment -PATH=/opt/AM4:/opt/netcdf-c/bin:/opt/netcdf-fortran/bin:${PATH} -LD_LIBRARY_PATH=/opt/AM4/exec/fms/build/libFMS/.libs:/opt/netcdf-c/lib:/opt/netcdf-fortran/lib:/opt/hdf5/lib:/opt/intel/oneapi/lib:/opt/intel/oneapi/lib/intel64/:/opt/intel/oneapi/lib/intel64/lib:/opt/intel/oneapi/lib/intel64/libfabric:${LD_LIBRARY_PATH} -export LIBRARY_PATH=/opt/AM4/exec/fms/build/libFMS/.libs:/opt/netcdf-c/lib:/opt/netcdf-fortran/lib:/opt/hdf5/lib:/opt/intel/oneapi/lib:/opt/intel/oneapi/lib/intel64/:/opt/intel/oneapi/lib/intel64/lib:/opt/intel/oneapi/lib/intel64/libfabric -export KMP_STACKSIZE=512m -export NC_BLKSZ=1M -export F_UFMTENDIAN=big +### The two stage build with intel runtime libs is currently not working. +#Bootstrap: docker +#From: intel/oneapi-runtime:ubuntu18.04 +#Stage: final +# +#%files from build +#/opt/hdf5 +#/opt/netcdf-c +#/opt/netcdf-fortran +#/opt/AM4/am4_xanadu_2021.03.x +#/opt/AM4/exec/fms/build/libFMS/.libs +### Add the AM4 executable to the path +#%environment +#PATH=/opt/AM4:/opt/netcdf-c/bin:/opt/netcdf-fortran/bin:${PATH} +#LD_LIBRARY_PATH=/opt/AM4/exec/fms/build/libFMS/.libs:/opt/netcdf-c/lib:/opt/netcdf-fortran/lib:/opt/hdf5/lib:/opt/intel/oneapi/lib:/opt/intel/oneapi/lib/intel64/:/opt/intel/oneapi/lib/intel64/lib:/opt/intel/oneapi/lib/intel64/libfabric:/opt/intel/oneapi/mkl/2021.2.0/lib/intel64:${LD_LIBRARY_PATH} +#export LIBRARY_PATH=/opt/AM4/exec/fms/build/libFMS/.libs:/opt/netcdf-c/lib:/opt/netcdf-fortran/lib:/opt/hdf5/lib:/opt/intel/oneapi/lib:/opt/intel/oneapi/lib/intel64/:/opt/intel/oneapi/lib/intel64/lib:/opt/intel/oneapi/lib/intel64/libfabric:/opt/intel/oneapi/mkl/2021.2.0/lib/intel64 +#export KMP_STACKSIZE=512m +#export NC_BLKSZ=1M +#export F_UFMTENDIAN=big ## Run AM4 %runscript ulimit -s unlimited /opt/AM4/am4_xanadu_2021.03.x - From 32bf74c7778b40a6d46f79ca43e1f56324183605 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 24 Aug 2021 09:18:06 -0400 Subject: [PATCH 2/3] Switches FMS to 2021.03 --- src/FMS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FMS b/src/FMS index 45559a4..9d25a1e 160000 --- a/src/FMS +++ b/src/FMS @@ -1 +1 @@ -Subproject commit 45559a4a10026a234e624669017d16eb7e0e25fb +Subproject commit 9d25a1e4f5e4b4040b513b5d40b09d7b6f904fbf From 3955cd9722d69cea8b14230107bf7b51328f9e4e Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 24 Aug 2021 09:24:32 -0400 Subject: [PATCH 3/3] Updates README for information about containers --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index dc065f9..9f7cda6 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,14 @@ The following components are available in the ## Building AM4 +###Containers +The [container folder](container) provides example Dockerfiles and Signularity +definition files to use to build AM4 containers using either GCC/GFORTAN or +Intel oneAPI. There is a script that can be used to build the intel +singularity containers, and the first step of this script can be used with the +other GFDL climate models. + +### From source The [exec](exec) directory contains Makefiles that can be used to build the AM4 executable. These Makefiles were generated using the [Make Makefile (mkmf)](https://github.com/NOAA-GFDL/mkmf) program.