From 82a4a3a11577bb71f809c18ad6f35ea7959065f6 Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Thu, 20 Jun 2024 16:31:08 -0400 Subject: [PATCH 01/10] Started the documentation for Joule3 --- docs/source_docs/references/hpc.rst | 1 + docs/source_docs/references/hpc/Joule3.rst | 209 +++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 docs/source_docs/references/hpc/Joule3.rst diff --git a/docs/source_docs/references/hpc.rst b/docs/source_docs/references/hpc.rst index 2fc47c7..dc52b4d 100644 --- a/docs/source_docs/references/hpc.rst +++ b/docs/source_docs/references/hpc.rst @@ -30,4 +30,5 @@ Follow the guide here instead of the generic instructions. :maxdepth: 0 hpc/Joule2 + hpc/Joule3 diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst new file mode 100644 index 0000000..44540f1 --- /dev/null +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -0,0 +1,209 @@ +Joule 3.0 +========== + +Congratulations on getting the access to Joule3.0. You have accomplished +the first step to use the platform for exascale computing. +Follow these notes to build MFIX-Exa code: + +Source code +----------- + +Request access and download the source code following the instructions on the +`MFIX-Exa website. `_ + +Essentials +---------- + +The code require some essential modules such as cmake, gnu, openmpi and cuda +which are readily available on Joule3.0. Use the following set of commands to load in your environment: + +.. code:: bash + + module purge + module load cmake/3.29.3 + module load gnu/13.2.0 + module load openmpi/4.1.5/gnu/13.2.0 + module load cuda/12.4 # If you want the code to be GPU compatible + +NOTE: Installation of all dependencies and code is tested with the version of modules shown above. + +Dependencies +------------ + +Dependencies are only required to be installed once on Joule3.0. If already installed, skip to the next section. + +You need to set certain environment variables before building the dependencies. + +.. code:: bash + + export CC=$(which mpicc) + export CXX=$(which mpic++) + export F77=$(which mpif77) + export FC=$(which mpifort) + export F90=$(which mpif90) + +You can create a 'dep_installed' directory for compiled packages and a 'dep_source_code' for installed packages to keep everything organized, it is optional but recommended. + +.. code:: bash + + cd $HOME//dep_source_code + +HYPRE +~~~~~ + + .. code:: bash + + git clone https://github.com/hypre-space/hypre.git + pushd hypre/src/ + git checkout v2.26.0 + +We specify the environment variable for the HYPRE installation path + .. code:: bash + + export HYPRE_INSTALL_DIR=$HOME//dep_installed/HYPRE/2_26_0 + +Now, configure and install the package + .. code:: bash + + ./configure --prefix=$HYPRE_INSTALL_DIR --with-MPI + make -j8 install + popd + +You have successfully installed HYPRE, and now you can remove the HYPRE source code: + +CATCH2 +~~~~~~ + + .. code:: bash + + git clone --depth 1 --branch v2.13.7 https://github.com/catchorg/Catch2 + pushd Catch2/ + +We specify the environment variable for the HYPRE installation path + .. code:: bash + + export CATCH2_INSTALL_DIR=$HOME//dep_installed/Catch2/2_13_7 + +Now, configure and install the package + .. code:: bash + + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$CATCH2_INSTALL_DIR + cd build + make -j8 install + popd + +You have successfully installed Catch2, and now you can remove the Catch2 source code: + +GMP +~~~~~~ + + .. code:: bash + + wget --no-check-certificate https://ftp.gnu.org/gnu/gmp/gmp-6.2.1.tar.xz + tar -xf gmp-6.2.1.tar.xz + pushd gmp-6.2.1 + export GMP_INSTALL_DIR=$HOME//dep_installed/gmp/6_2_1 + ./configure --prefix=$GMP_INSTALL_DIR + make -j8 install + popd + + +MPFR +~~~~~~ + + .. code:: bash + + wget --no-check-certificate https://ftp.gnu.org/gnu/mpfr/mpfr-4.1.0.tar.xz + tar -xf mpfr-4.1.0.tar.xz + pushd mpfr-4.1.0/ + export MPFR_INSTALL_DIR=$HOME//dep_installed/mpfr/4_1_0 + ./configure --with-gmp=$GMP_INSTALL_DIR --prefix=$MPFR_INSTALL_DIR + make -j8 install + popd + +CGAL +~~~~~~ + + .. code:: bash + + git clone --depth 1 --branch v5.3 https://github.com/CGAL/cgal + pushd cgal/ + export CGAL_INSTALL_DIR=$HOME//dep_installed/cgal/5_3_0 + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$CGAL_INSTALL_DIR + cd build/ + make -j8 install + popd + +PEGTL +~~~~~ + + .. code:: bash + + git clone --branch 3.2.2 https://github.com/taocpp/pegtl + pushd pegtl/ + export PEGTL_INSTALL_DIR=$HOME//dep_installed/pegtl/3_2_2 + cmake -s . -b build -dcmake_install_prefix=$PEGTL_INSTALL_DIR + cd build/ + make -j8 install + popd + +Boost +~~~~~ + + .. code:: bash + + wget --no-check-certificate https://archives.boost.io/release/1.85.0/source/boost_1_85_0.tar.gz + tar -xzvf boost_1_85_0.tar.gz + pushd boost_1_85_0/ + export BOOST_INSTALL_DIR=$HOME//dep_installed/boost/1_85_0 + ./bootstrap.sh --prefix=$BOOST_INSTALL_DIR + ./b2 install + popd + + +CSG EB +~~~~~~ + + .. code:: bash + + git clone https://mfix.netl.doe.gov/gitlab/exa/csg-eb + pushd csg-eb + + CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$CATCH2_INSTALL_DIR + CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$GMP_INSTALL_DIR + CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$MPFR_INSTALL_DIR + CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$CGAL_INSTALL_DIR + CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$PEGTL_INSTALL_DIR/share/pegtl/cmake + + export CMAKE_PREFIX_PATH + + export CSGEB_INSTALL_DIR=$HOME//dep_installed/csgeb + + cmake -Bbuild-csg-eb-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$CSGEB_INSTALL_DIR \ + -DBoost_INCLUDE_DIR=$BOOST_INSTALL_DIR/include + + cmake --build build-csg-eb-release --target install + + popd + +Conduit +~~~~~~~ + + .. code:: bash + + git clone --recursive https://github.com/LLNL/conduit.git + pushd conduit + git checkout v0.8.6 + + export CONDUIT_INSTALL_DIR=$HOME//dep_installed/conduit + + cmake -B build -DCMAKE_INSTALL_PREFIX=$CONDUIT_INSTALL_DIR \ + -DENABLE_OPENMP=OFF \ + -DENABLE_MPI=ON \ + -DENABLE_CUDA=OFF \ + -DCMAKE_BUILD_TYPE=Release + + +If you want to change the version of XYZABC dependency in future to test the exa code, follow the same installation process and change thevariable XYZABC_INSTALL_DIR to a different path to ensure you do not break your previous installation, so you can always go back. -- GitLab From 83b11a22d8646a23de8d85511dc1556acb5a9195 Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Fri, 21 Jun 2024 09:38:02 -0400 Subject: [PATCH 02/10] Added set of installation commands --- docs/source_docs/references/hpc/Joule3.rst | 158 ++++++++++++++++----- 1 file changed, 119 insertions(+), 39 deletions(-) diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst index 44540f1..ee0d20c 100644 --- a/docs/source_docs/references/hpc/Joule3.rst +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -48,54 +48,32 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour cd $HOME//dep_source_code -HYPRE -~~~~~ +#. HYPRE .. code:: bash git clone https://github.com/hypre-space/hypre.git pushd hypre/src/ git checkout v2.26.0 - -We specify the environment variable for the HYPRE installation path - .. code:: bash - export HYPRE_INSTALL_DIR=$HOME//dep_installed/HYPRE/2_26_0 - -Now, configure and install the package - .. code:: bash - ./configure --prefix=$HYPRE_INSTALL_DIR --with-MPI make -j8 install popd -You have successfully installed HYPRE, and now you can remove the HYPRE source code: - -CATCH2 -~~~~~~ +#. CATCH2 .. code:: bash git clone --depth 1 --branch v2.13.7 https://github.com/catchorg/Catch2 pushd Catch2/ - -We specify the environment variable for the HYPRE installation path - .. code:: bash - export CATCH2_INSTALL_DIR=$HOME//dep_installed/Catch2/2_13_7 - -Now, configure and install the package - .. code:: bash - cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$CATCH2_INSTALL_DIR cd build make -j8 install popd -You have successfully installed Catch2, and now you can remove the Catch2 source code: -GMP -~~~~~~ +#. GMP .. code:: bash @@ -108,8 +86,7 @@ GMP popd -MPFR -~~~~~~ +#. MPFR .. code:: bash @@ -121,8 +98,7 @@ MPFR make -j8 install popd -CGAL -~~~~~~ +#. CGAL .. code:: bash @@ -134,8 +110,7 @@ CGAL make -j8 install popd -PEGTL -~~~~~ +#. PEGTL .. code:: bash @@ -147,8 +122,7 @@ PEGTL make -j8 install popd -Boost -~~~~~ +#. Boost .. code:: bash @@ -161,8 +135,7 @@ Boost popd -CSG EB -~~~~~~ +#. CSG EB .. code:: bash @@ -188,8 +161,7 @@ CSG EB popd -Conduit -~~~~~~~ +#. Conduit .. code:: bash @@ -197,13 +169,121 @@ Conduit pushd conduit git checkout v0.8.6 - export CONDUIT_INSTALL_DIR=$HOME//dep_installed/conduit + export CONDUIT_INSTALL_DIR=$HOME//dep_installed/conduit_0_8_6 - cmake -B build -DCMAKE_INSTALL_PREFIX=$CONDUIT_INSTALL_DIR \ + cmake -S ./src -B build -DCMAKE_INSTALL_PREFIX=$CONDUIT_INSTALL_DIR \ -DENABLE_OPENMP=OFF \ -DENABLE_MPI=ON \ -DENABLE_CUDA=OFF \ -DCMAKE_BUILD_TYPE=Release + cd build + make -j8 install + + popd + + +#. VTK-m + + .. code:: bash + + git clone --branch master https://gitlab.kitware.com/vtk/vtk-m.git + pushd vtk-m + git checkout v1.9.0 + + export VTK_INSTALL_DIR=$HOME//dep_installed/vtk-m_1_9_0 + + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$VTK_INSTALL_DIR \ + -DVTKm_ENABLE_OPENMP=OFF \ + -DVTKm_ENABLE_MPI=ON \ + -DVTKm_ENABLE_CUDA=OFF \ + -DVTKm_USE_64BIT_IDS=OFF \ + -DVTKm_USE_DOUBLE_PRECISION=ON \ + -DVTKm_USE_DEFAULT_TYPES_FOR_ASCENT=ON \ + -DVTKm_NO_DEPRECATED_VIRTUAL=ON \ + -DCMAKE_BUILD_TYPE=Release + cd build + make -j8 install + + popd + + +#. Ascent + + .. code:: bash + + git clone --recursive https://github.com/Alpine-DAV/ascent.git + pushd ascent + git checkout v0.9.0 + export ASCENT_INSTALL_DIR=$HOME//dep_installed/ascent_0_9_0 + cmake -S ./src -B build -DCMAKE_INSTALL_PREFIX=$ASCENT_INSTALL_DIR \ + -DCONDUIT_DIR=$CONDUIT_INSTALL_DIR \ + -DVTKM_DIR=$VTK_INSTALL_DIR \ + -DENABLE_VTKH=ON \ + -DENABLE_FORTRAN=OFF \ + -DENABLE_PYTHON=OFF \ + -DENABLE_DOCS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_GTEST=OFF \ + -DENABLE_TESTS=OFF + cd build + make -j8 install + popd If you want to change the version of XYZABC dependency in future to test the exa code, follow the same installation process and change thevariable XYZABC_INSTALL_DIR to a different path to ensure you do not break your previous installation, so you can always go back. + + +You have successfully installed all the libraries required for the installation of MFIX-Exa code. + +Building MFIX-Exa +----------------- + +We build the MFIX-Exa in superbuild mode, i.e., AMReX is build as part of the MFIX-Exa build process. + +.. code:: bash + + export HYPRE_DIR=$HYPRE_INSTALL_DIR + export HYPRE_ROOT=$HYPRE_DIR + export HYPRE_LIBRARIES=$HYPRE_DIR/lib + export HYPRE_INCLUDE_DIRS=$HYPRE_DIR/include + + export ASCENT_DIR=$ASCENT_INSTALL_DIR + export CONDUIT_DIR=$CONDUIT_INSTALL_DIR + export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$ASCENT_DIR/lib/cmake/ascent + export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$CONDUIT_DIR/lib/cmake/conduit + + export CSG_DIR=$CSG_INSTALL_DIR + export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$CSG_DIR + + export Boost_INCLUDE_DIR=$BOOST_INSTALL_DIR/include + + + export MFIX_SOURCE_DIR=$HOME/ + export MFIX_BUILD_DIR=$HOME/ + + cd $MFIX_SOURCE_DIR + + git clone https://mfix.netl.doe.gov/gitlab/exa/mfix.git + + cd mfix/src + + cmake -B $MFIX_BUILD_DIR -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DMFIX_MPI=yes \ + -DMFIX_OMP=no \ + -DMFIX_GPU_BACKEND=NONE \ + -DAMReX_TINY_PROFILE=no \ + -DCMAKE_BUILD_TYPE=Release \ + -DBoost_INCLUDE_DIR=$BOOST_INSTALL_DIR/include \ + -DMFIX_CSG=yes \ + -DMFIX_HYPRE=yes \ + -DAMReX_ASCENT=yes \ + -DAMReX_CONDUIT=yes \ + ../ + + cd $MFIX_BUILD_DIR + + make -j8 + +The MFIX-Exa build is now complete! -- GitLab From eb56b5aacd3dff922cda8b85a84792edc7fe87c0 Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Fri, 21 Jun 2024 09:54:19 -0400 Subject: [PATCH 03/10] Cleaned the structure of commands --- docs/source_docs/references/hpc/Joule3.rst | 64 +++++++++++++++------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst index ee0d20c..ead2c0b 100644 --- a/docs/source_docs/references/hpc/Joule3.rst +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -42,11 +42,16 @@ You need to set certain environment variables before building the dependencies. export FC=$(which mpifort) export F90=$(which mpif90) -You can create a 'dep_installed' directory for compiled packages and a 'dep_source_code' for installed packages to keep everything organized, it is optional but recommended. +You should create a 'dep_installed' directory for the compiled packages and a 'dep_source_code' for the source code of each package to keep everything organized, it is optional but recommended. You only need to copy-paste-execute most of the instructions given below, except replacing few path marked as 'CHANGE_HERE' .. code:: bash - cd $HOME//dep_source_code + export $DEP_SOURCE_DIR=$HOME//dep_source_code # CHANGE_HERE + export $DEP_INSTALL_DIR=$HOME//dep_installed # CHANGE_HERE + export MFIX_SOURCE_DIR=$HOME/ # CHANGE_HERE + export MFIX_BUILD_DIR=$HOME/ # CHANGE_HERE + + cd $DEP_SOURCE_DIR #. HYPRE @@ -55,9 +60,12 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour git clone https://github.com/hypre-space/hypre.git pushd hypre/src/ git checkout v2.26.0 - export HYPRE_INSTALL_DIR=$HOME//dep_installed/HYPRE/2_26_0 + + export HYPRE_INSTALL_DIR=$DEP_INSTALL_DIR/HYPRE/2_26_0 + ./configure --prefix=$HYPRE_INSTALL_DIR --with-MPI make -j8 install + popd #. CATCH2 @@ -66,10 +74,13 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour git clone --depth 1 --branch v2.13.7 https://github.com/catchorg/Catch2 pushd Catch2/ - export CATCH2_INSTALL_DIR=$HOME//dep_installed/Catch2/2_13_7 + + export CATCH2_INSTALL_DIR=$DEP_INSTALL_DIR/Catch2/2_13_7 + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$CATCH2_INSTALL_DIR cd build make -j8 install + popd @@ -80,9 +91,12 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour wget --no-check-certificate https://ftp.gnu.org/gnu/gmp/gmp-6.2.1.tar.xz tar -xf gmp-6.2.1.tar.xz pushd gmp-6.2.1 - export GMP_INSTALL_DIR=$HOME//dep_installed/gmp/6_2_1 + + export GMP_INSTALL_DIR=$DEP_INSTALL_DIR/gmp/6_2_1 + ./configure --prefix=$GMP_INSTALL_DIR make -j8 install + popd @@ -93,9 +107,12 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour wget --no-check-certificate https://ftp.gnu.org/gnu/mpfr/mpfr-4.1.0.tar.xz tar -xf mpfr-4.1.0.tar.xz pushd mpfr-4.1.0/ - export MPFR_INSTALL_DIR=$HOME//dep_installed/mpfr/4_1_0 + + export MPFR_INSTALL_DIR=$DEP_INSTALL_DIR/mpfr/4_1_0 + ./configure --with-gmp=$GMP_INSTALL_DIR --prefix=$MPFR_INSTALL_DIR make -j8 install + popd #. CGAL @@ -104,10 +121,13 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour git clone --depth 1 --branch v5.3 https://github.com/CGAL/cgal pushd cgal/ - export CGAL_INSTALL_DIR=$HOME//dep_installed/cgal/5_3_0 + + export CGAL_INSTALL_DIR=$DEP_INSTALL_DIR/cgal/5_3_0 + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$CGAL_INSTALL_DIR cd build/ make -j8 install + popd #. PEGTL @@ -116,10 +136,13 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour git clone --branch 3.2.2 https://github.com/taocpp/pegtl pushd pegtl/ - export PEGTL_INSTALL_DIR=$HOME//dep_installed/pegtl/3_2_2 + + export PEGTL_INSTALL_DIR=$DEP_INSTALL_DIR/pegtl/3_2_2 + cmake -s . -b build -dcmake_install_prefix=$PEGTL_INSTALL_DIR cd build/ make -j8 install + popd #. Boost @@ -129,9 +152,12 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour wget --no-check-certificate https://archives.boost.io/release/1.85.0/source/boost_1_85_0.tar.gz tar -xzvf boost_1_85_0.tar.gz pushd boost_1_85_0/ - export BOOST_INSTALL_DIR=$HOME//dep_installed/boost/1_85_0 + + export BOOST_INSTALL_DIR=$DEP_INSTALL_DIR/boost/1_85_0 + ./bootstrap.sh --prefix=$BOOST_INSTALL_DIR ./b2 install + popd @@ -147,16 +173,13 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$MPFR_INSTALL_DIR CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$CGAL_INSTALL_DIR CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$PEGTL_INSTALL_DIR/share/pegtl/cmake - export CMAKE_PREFIX_PATH - - export CSGEB_INSTALL_DIR=$HOME//dep_installed/csgeb + export CSGEB_INSTALL_DIR=$DEP_INSTALL_DIR/csgeb cmake -Bbuild-csg-eb-release \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$CSGEB_INSTALL_DIR \ - -DBoost_INCLUDE_DIR=$BOOST_INSTALL_DIR/include - + -DBoost_INCLUDE_DIR=$BOOST_INSTALL_DIR/include cmake --build build-csg-eb-release --target install popd @@ -169,7 +192,7 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour pushd conduit git checkout v0.8.6 - export CONDUIT_INSTALL_DIR=$HOME//dep_installed/conduit_0_8_6 + export CONDUIT_INSTALL_DIR=$DEP_INSTALL_DIR/conduit_0_8_6 cmake -S ./src -B build -DCMAKE_INSTALL_PREFIX=$CONDUIT_INSTALL_DIR \ -DENABLE_OPENMP=OFF \ @@ -191,7 +214,7 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour pushd vtk-m git checkout v1.9.0 - export VTK_INSTALL_DIR=$HOME//dep_installed/vtk-m_1_9_0 + export VTK_INSTALL_DIR=$DEP_INSTALL_DIR/vtk-m_1_9_0 cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$VTK_INSTALL_DIR \ -DVTKm_ENABLE_OPENMP=OFF \ @@ -215,7 +238,9 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour git clone --recursive https://github.com/Alpine-DAV/ascent.git pushd ascent git checkout v0.9.0 - export ASCENT_INSTALL_DIR=$HOME//dep_installed/ascent_0_9_0 + + export ASCENT_INSTALL_DIR=$DEP_INSTALL_DIR/ascent_0_9_0 + cmake -S ./src -B build -DCMAKE_INSTALL_PREFIX=$ASCENT_INSTALL_DIR \ -DCONDUIT_DIR=$CONDUIT_INSTALL_DIR \ -DVTKM_DIR=$VTK_INSTALL_DIR \ @@ -229,6 +254,7 @@ You can create a 'dep_installed' directory for compiled packages and a 'dep_sour -DENABLE_TESTS=OFF cd build make -j8 install + popd If you want to change the version of XYZABC dependency in future to test the exa code, follow the same installation process and change thevariable XYZABC_INSTALL_DIR to a different path to ensure you do not break your previous installation, so you can always go back. @@ -258,10 +284,6 @@ We build the MFIX-Exa in superbuild mode, i.e., AMReX is build as part of the MF export Boost_INCLUDE_DIR=$BOOST_INSTALL_DIR/include - - export MFIX_SOURCE_DIR=$HOME/ - export MFIX_BUILD_DIR=$HOME/ - cd $MFIX_SOURCE_DIR git clone https://mfix.netl.doe.gov/gitlab/exa/mfix.git -- GitLab From bd225253e75aa8ab9711998c094a9701c9cf0370 Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Fri, 21 Jun 2024 09:57:12 -0400 Subject: [PATCH 04/10] Fixed the version change comment --- docs/source_docs/references/hpc/Joule3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst index ead2c0b..6d0aa10 100644 --- a/docs/source_docs/references/hpc/Joule3.rst +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -257,7 +257,7 @@ You should create a 'dep_installed' directory for the compiled packages and a 'd popd -If you want to change the version of XYZABC dependency in future to test the exa code, follow the same installation process and change thevariable XYZABC_INSTALL_DIR to a different path to ensure you do not break your previous installation, so you can always go back. +If you want to change the version of XYZABC dependency in future to test the exa code, follow the same installation process and only change the version in the variable 'XYZABC_INSTALL_DIR' to ensure you do not break your previous installation, so you can always go back. You have successfully installed all the libraries required for the installation of MFIX-Exa code. -- GitLab From d412efbe0e6726066c05847cf86df4b701040ccc Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Fri, 21 Jun 2024 10:14:59 -0400 Subject: [PATCH 05/10] Copied run instructions from Joule2 --- docs/source_docs/references/hpc/Joule3.rst | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst index 6d0aa10..1b849fa 100644 --- a/docs/source_docs/references/hpc/Joule3.rst +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -309,3 +309,77 @@ We build the MFIX-Exa in superbuild mode, i.e., AMReX is build as part of the MF make -j8 The MFIX-Exa build is now complete! + +The build creates an executable (mfix.exe) in the '$MFIX_BUILD_DIR' that you can use to run MFIX-Exa cases. + +Running Jobs +------------ + +Common Slurm commands: + +* ``sinfo`` see available/allocated resources +* ``sbatch runit_cpu.sh`` submit a cpu job to the queue +* ``squeue -u USER`` check job status of user USER +* ``squeue -p PARTITION`` check job status of partition PARTITION +* ``scancel JOBID`` kill a job with id JOBID +* ``salloc -N 1 -p gpu`` grab a GPU node interactively (for up to 48 hrs) +* ``salloc -N 2 -p dev -q dev`` grab two development nodes (for up to 2 hrs) + +Example run scripts: + +.. tabs:: + + .. tab:: CPU + + .. code:: bash + + #!/bin/bash -l + + ##Accounting + #SBATCH --partition=general #bigmem, dev + #SBATCH --qos=normal #long, dev + + ##Submission + #SBATCH --nodes=1 + #SBATCH --job-name="mfix-exa-run" + #SBATCH --output=job.out + #SBATCH --mail-user=first.last@netl.doe.gov + #SBATCH --mail-type=ALL + + ##Load Modules + module purge + module load gnu/13.2.0 + module load openmpi/4.1.5/gnu/13.2.0 + + ##Run the program + mpirun -np 36 ./mfix inputs > screen.txt + + + .. tab:: GPU + + .. code:: bash + + #!/bin/bash -l + + ##Accounting + #SBATCH --partition=gpu + #SBATCH --qos=normal #long + + ##Submission + #SBATCH --nodes=2 + #SBATCH --ntasks-per-node=2 + #SBATCH --ntasks-per-socket=1 + #SBATCH --job-name="mfix-exa-run" + #SBATCH --output=job.out + #SBATCH --mail-user=first.last@netl.doe.gov + #SBATCH --mail-type=ALL + + ##Load Modules + module purge + module load cuda/12.4 + module load gnu/13.2.0 + module load openmpi/4.1.5/gnu/13.2.0 + + ##Run the program + mpirun -np 4 -npersocket 1 ./mfix inputs > screen.txt + -- GitLab From 00360cb90dfb0bee2a56021be7040fe2bb6efdd1 Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Fri, 21 Jun 2024 14:41:27 +0000 Subject: [PATCH 06/10] Fixed typo in variable name --- docs/source_docs/references/hpc/Joule3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst index 1b849fa..1d9ec48 100644 --- a/docs/source_docs/references/hpc/Joule3.rst +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -46,8 +46,8 @@ You should create a 'dep_installed' directory for the compiled packages and a 'd .. code:: bash - export $DEP_SOURCE_DIR=$HOME//dep_source_code # CHANGE_HERE - export $DEP_INSTALL_DIR=$HOME//dep_installed # CHANGE_HERE + export DEP_SOURCE_DIR=$HOME//dep_source_code # CHANGE_HERE + export DEP_INSTALL_DIR=$HOME//dep_installed # CHANGE_HERE export MFIX_SOURCE_DIR=$HOME/ # CHANGE_HERE export MFIX_BUILD_DIR=$HOME/ # CHANGE_HERE -- GitLab From d854ae4756a844fea249c4a9c68498b639ee44dd Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Fri, 21 Jun 2024 14:47:18 +0000 Subject: [PATCH 07/10] Fixed typo in PEGTL installation --- docs/source_docs/references/hpc/Joule3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst index 1d9ec48..ddda3fc 100644 --- a/docs/source_docs/references/hpc/Joule3.rst +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -139,7 +139,7 @@ You should create a 'dep_installed' directory for the compiled packages and a 'd export PEGTL_INSTALL_DIR=$DEP_INSTALL_DIR/pegtl/3_2_2 - cmake -s . -b build -dcmake_install_prefix=$PEGTL_INSTALL_DIR + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$PEGTL_INSTALL_DIR cd build/ make -j8 install -- GitLab From 17481f7647a079a1163ccc2674f0d2b36258eca1 Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Fri, 21 Jun 2024 15:15:13 +0000 Subject: [PATCH 08/10] Update Joule3.rst --- docs/source_docs/references/hpc/Joule3.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst index ddda3fc..91d1183 100644 --- a/docs/source_docs/references/hpc/Joule3.rst +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -25,7 +25,11 @@ which are readily available on Joule3.0. Use the following set of commands to lo module load openmpi/4.1.5/gnu/13.2.0 module load cuda/12.4 # If you want the code to be GPU compatible -NOTE: Installation of all dependencies and code is tested with the version of modules shown above. +Important NOTES: +---------------- + +- Installation of all dependencies and code is tested with the version of modules shown above. +- We are using 8 cores using '-j8' for all make commands. User can change the number of cores based on the core availability on the login node. Dependencies ------------ @@ -310,7 +314,11 @@ We build the MFIX-Exa in superbuild mode, i.e., AMReX is build as part of the MF The MFIX-Exa build is now complete! -The build creates an executable (mfix.exe) in the '$MFIX_BUILD_DIR' that you can use to run MFIX-Exa cases. +The build creates an executable (mfix.exe) in the '$MFIX_BUILD_DIR' that you can use to run MFIX-Exa cases. You can create an alias in ~/.bashrc as shown below to access MFIX-Exa executable from any path. + +.. code:: bash + + echo "alias mfixexa='$MFIX_BUILD_DIR/mfix'" > ~/.bashrc Running Jobs ------------ -- GitLab From e25c9116eecb34b2e07dc9e9b82f96186fb327f9 Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Fri, 21 Jun 2024 15:16:05 +0000 Subject: [PATCH 09/10] Update Joule3.rst --- docs/source_docs/references/hpc/Joule3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst index 91d1183..d05e2de 100644 --- a/docs/source_docs/references/hpc/Joule3.rst +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -314,7 +314,7 @@ We build the MFIX-Exa in superbuild mode, i.e., AMReX is build as part of the MF The MFIX-Exa build is now complete! -The build creates an executable (mfix.exe) in the '$MFIX_BUILD_DIR' that you can use to run MFIX-Exa cases. You can create an alias in ~/.bashrc as shown below to access MFIX-Exa executable from any path. +The build creates an executable (mfix.exe) in the '$MFIX_BUILD_DIR' that you can use to run MFIX-Exa cases. (OPTIONAL) You can create an alias in ~/.bashrc as shown below to access MFIX-Exa executable from any path. .. code:: bash -- GitLab From 6fbf7b078a59b125cd901b59da05198d228d242e Mon Sep 17 00:00:00 2001 From: Aashish Goyal Date: Fri, 21 Jun 2024 15:17:36 +0000 Subject: [PATCH 10/10] Update Joule3.rst --- docs/source_docs/references/hpc/Joule3.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source_docs/references/hpc/Joule3.rst b/docs/source_docs/references/hpc/Joule3.rst index d05e2de..2a9d350 100644 --- a/docs/source_docs/references/hpc/Joule3.rst +++ b/docs/source_docs/references/hpc/Joule3.rst @@ -30,6 +30,7 @@ Important NOTES: - Installation of all dependencies and code is tested with the version of modules shown above. - We are using 8 cores using '-j8' for all make commands. User can change the number of cores based on the core availability on the login node. +- The following instructions of installation are not test for GPU compatibility, and would require modifications. Dependencies ------------ -- GitLab