MPAS-Dev · xylar · May 17, 2022 · Apr 26, 2022 · May 17, 2022 · Apr 26, 2022
diff --git a/compass/version.py b/compass/version.py
@@ -1 +1 @@
-__version__ = '1.1.0-alpha.3'
+__version__ = '1.1.0-alpha.4'
diff --git a/conda/bootstrap.py b/conda/bootstrap.py
@@ -146,7 +146,7 @@ def get_compilers_mpis(config, machine, compilers, mpis, source_path):
 
 
 def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
-                  conda_base, env_name, compass_version):
+                  conda_base, env_name, compass_version, logger):
 
     if args.python is not None:
         python = args.python
@@ -171,6 +171,27 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
         env_suffix = activ_suffix
         conda_mpi = mpi
 
+    lib_suffix = ''
+    if args.with_albany:
+        lib_suffix = f'{lib_suffix}_albany'
+    else:
+        config.set('deploy', 'albany', 'None')
+
+    if args.with_netlib_lapack:
+        lib_suffix = f'{lib_suffix}_netlib_lapack'
+    else:
+        config.set('deploy', 'lapack', 'None')
+
+    if args.with_petsc:
+        lib_suffix = f'{lib_suffix}_petsc'
+        logger.info('Turning off OpenMP because it doesn\'t work well '
+                    'with  PETSc')
+        args.without_openmp = True
+    else:
+        config.set('deploy', 'petsc', 'None')
+
+    activ_suffix = f'{activ_suffix}{lib_suffix}'
+
     if env_type == 'dev':
         activ_path = source_path
     else:
@@ -190,7 +211,7 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
         env_name = spack_env
 
     # add the compiler and MPI library to the spack env name
-    spack_env = '{}_{}_{}'.format(spack_env, compiler, mpi)
+    spack_env = f'{spack_env}_{compiler}_{mpi}{lib_suffix}'
     # spack doesn't like dots
     spack_env = spack_env.replace('.', '_')
 
@@ -333,9 +354,11 @@ def get_env_vars(machine, compiler, mpilib):
 def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
                     spack_base, spack_template_path, env_vars, tmpdir):
 
+    albany = config.get('deploy', 'albany')
     esmf = config.get('deploy', 'esmf')
+    lapack = config.get('deploy', 'lapack')
+    petsc = config.get('deploy', 'petsc')
     scorpio = config.get('deploy', 'scorpio')
-    albany = config.get('deploy', 'albany')
 
     spack_branch_base = f'{spack_base}/spack_for_mache_{mache_version}'
 
@@ -355,6 +378,14 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
 
     if esmf != 'None':
         specs.append(f'esmf@{esmf}+mpi+netcdf~pio+pnetcdf')
+    if lapack != 'None':
+        specs.append(f'netlib-lapack@{lapack}')
+        include_e3sm_lapack = False
+    else:
+        include_e3sm_lapack = True
+    if petsc != 'None':
+        specs.append(f'petsc@{petsc}+mpi+batch')
+
     if scorpio != 'None':
         specs.append(f'scorpio@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc')
 
@@ -368,6 +399,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
         make_spack_env(spack_path=spack_branch_base, env_name=spack_env,
                        spack_specs=specs, compiler=compiler, mpi=mpi,
                        machine=machine,
+                       include_e3sm_lapack=include_e3sm_lapack,
                        include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf,
                        yaml_template=yaml_template, tmpdir=tmpdir)
 
@@ -382,6 +414,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
     spack_script = get_spack_script(
         spack_path=spack_branch_base, env_name=spack_env, compiler=compiler,
         mpi=mpi, shell='sh', machine=machine,
+        include_e3sm_lapack=include_e3sm_lapack,
         include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf,
         yaml_template=yaml_template)
 
@@ -412,6 +445,16 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
             f'export MPAS_EXTERNAL_LIBS="${{MPAS_EXTERNAL_LIBS}} ' \
             f'${{ALBANY_LINK_LIBS}} {stdcxx} {mpicxx}"\n'
 
+    if lapack != 'None':
+        env_vars = f'{env_vars}' \
+                   f'export LAPACK={spack_view}\n' \
+                   f'export USE_LAPACK=true\n'
+
+    if petsc != 'None':
+        env_vars = f'{env_vars}' \
+                   f'export PETSC={spack_view}\n' \
+                   f'export USE_PETSC=true\n'
+
     return spack_branch_base, spack_script, env_vars
 
 
@@ -770,7 +813,7 @@ def main():
             activ_path, conda_env_path, conda_env_name, activate_env, \
             spack_env = get_env_setup(args, config, machine, compiler, mpi,
                                       env_type, source_path, conda_base,
-                                      args.env_name, compass_version)
+                                      args.env_name, compass_version, logger)
 
         build_dir = f'conda/build{activ_suffix}'
 
@@ -808,9 +851,6 @@ def main():
             if env_type != 'dev':
                 permissions_dirs.append(conda_base)
 
-        if not args.with_albany:
-            config.set('deploy', 'albany', 'None')
-
         spack_script = ''
         if compiler is not None:
             env_vars = get_env_vars(machine, compiler, mpi)

diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template
@@ -15,7 +15,7 @@ jigsaw=0.9.14
 jigsawpy=0.3.3
 jupyter
 lxml
-mache=1.3.2
+mache=1.4.1
 matplotlib-base
 metis
 mpas_tools=0.14.0

diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py
@@ -53,7 +53,7 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate,
         channels = '--use-local'
     else:
         channels = ''
-    packages = 'progressbar2 jinja2 "mache=1.3.2"'
+    packages = 'progressbar2 jinja2 "mache=1.4.1"'
     if recreate or not os.path.exists(env_path):
         print('Setting up a conda environment for installing compass\n')
         commands = '{}; ' \

diff --git a/conda/default.cfg b/conda/default.cfg
@@ -20,10 +20,12 @@ python = 3.9
 mpi = nompi
 
 # the version of various packages to include if using spack
+albany = develop
 esmf = 8.2.0
 hdf5 = 1.12.1
+lapack = 3.9.1
 netcdf_c = 4.8.1
 netcdf_fortran = 4.5.3
+petsc = 3.16.1
 pnetcdf = 1.12.2
 scorpio = 1.3.2
-albany = develop
diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml
@@ -1,5 +1,5 @@
 {% set name = "compass" %}
-{% set version = "1.1.0alpha.3" %}
+{% set version = "1.1.0alpha.4" %}
 {% set build = 0 %}
 
 {% if mpi == "nompi" %}
@@ -52,7 +52,7 @@ requirements:
     - jigsawpy 0.3.3
     - jupyter
     - lxml
-    - mache 1.3.2
+    - mache 1.4.1
     - matplotlib-base
     - metis
     - mpas_tools 0.14.0

diff --git a/conda/shared.py b/conda/shared.py
@@ -57,11 +57,19 @@ def parse_args(bootstrap):
                         action='store_true',
                         help="Whether to include albany in the spack "
                              "environment")
+    parser.add_argument("--with_netlib_lapack", dest="with_netlib_lapack",
+                        action='store_true',
+                        help="Whether to include Netlib-LAPACK in the spack "
+                             "environment")
+    parser.add_argument("--with_petsc", dest="with_petsc",
+                        action='store_true',
+                        help="Whether to include PETSc in the spack "
+                             "environment")
     parser.add_argument("--without_openmp", dest="without_openmp",
                         action='store_true',
-                        help="If this flag is included, OPENMP=true will not "
+                        help="If this flag is included, OPENMP=false will "
                              "be added to the load script.  By default, MPAS "
-                             "builds will be with OpenMP.")
+                             "builds will be with OpenMP (OPENMP=true).")
     if bootstrap:
         parser.add_argument("--local_conda_build", dest="local_conda_build",
                             type=str,

diff --git a/conda/spack/badger_gnu_mvapich.yaml b/conda/spack/badger_gnu_mvapich.yaml
@@ -3,15 +3,19 @@ spack:
   - cmake
   - gcc
   - mvapich2
+{% if e3sm_lapack %}
   - intel-mkl
+{% endif %}
 {{ specs }}
   concretization: together
   packages:
     all:
       compiler: [gcc@9.3.0]
       providers:
         mpi: [mvapich2@2.3]
+{% if e3sm_lapack %}
         lapack: [intel-mkl@2020.0.4]
+{% endif %}
     bzip2:
       externals:
       - spec: bzip2@1.0.6

diff --git a/conda/spack/badger_gnu_openmpi.yaml b/conda/spack/badger_gnu_openmpi.yaml
@@ -3,15 +3,19 @@ spack:
   - cmake
   - gcc
   - openmpi
+{% if e3sm_lapack %}
   - intel-mkl
+{% endif %}
 {{ specs }}
   concretization: together
   packages:
     all:
       compiler: [gcc@9.3.0]
       providers:
         mpi: [openmpi@3.1.6]
+{% if e3sm_lapack %}
         lapack: [intel-mkl@2020.0.4]
+{% endif %}
     bzip2:
       externals:
       - spec: bzip2@1.0.6

diff --git a/docs/developers_guide/quick_start.rst b/docs/developers_guide/quick_start.rst
@@ -82,7 +82,8 @@ If you are on one of the :ref:`dev_supported_machines`, run:
 .. code-block:: bash
 
     ./conda/configure_compass_env.py --conda <base_path_to_install_or_update_conda> \
-        -c <compiler> [--mpi <mpi>] [-m <machine>] [--with_albany]
+        -c <compiler> [--mpi <mpi>] [-m <machine>] [--with_albany] \
+        [--with_netlib_lapack] [--with_petsc]
 
 The ``<base_path_to_install_or_update_conda>`` is typically ``~/miniconda3``.
 This is the location where you would like to install Miniconda3 or where it is
@@ -102,6 +103,9 @@ machine you are on.  You can supply the machine name with ``-m <machine>`` if
 you run into trouble with the automatic recognition (e.g. if you're setting
 up the environment on a compute node, which is not recommended).
 
+Environments with Albany
+~~~~~~~~~~~~~~~~~~~~~~~~
+
 If you are working with MALI, you should specify ``--with_albany``.  This will
 ensure that the Albany and Trilinos libraries are included among those built
 with system compilers and MPI libraries, a requirement for many MAlI test
@@ -112,6 +116,15 @@ recommended unless a user wants to be able to run both models with the same
 conda/spack environment.  The main downside is simply that unneeded libraries
 will be linked in to MPAS-Ocean.
 
+Environments with PETSc and Netlib-LAPACK
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you are working with MPAS-Ocean test cases that need PETSC and
+Netlib-LAPACK, you should specify ``--with_petsc --with_netlib_lapack`` to
+point to Spack environments where these libraries are included.  Appropriate
+environment variables for pointing to these libraries will be build into the
+resulting load script (see below).
+
 Unknown machines
 ~~~~~~~~~~~~~~~~
 
@@ -157,18 +170,27 @@ this script will also:
   so changes you make to the repo are immediately reflected in the conda
   environment.
 
-* uses Spack to build several libraries with system compilers and MPI library,
-  including: `SCORPIO <https://github.com/E3SM-Project/scorpio>`_ (parallel
-  i/o for MPAS components) and `ESMF <https://earthsystemmodeling.org/>`_
-  (making mapping files in parallel).
+* with the ``--update_speck`` flag on supported machines, installs or
+  reinstalls a spack environment with various system libraries.  The
+  ``--spack`` flag can be used to point to a location for the spack repo to be
+  checked out.  Without this flag, a default location is used. Spack is used to
+  build several libraries with system compilers and MPI library, including:
+  `SCORPIO <https://github.com/E3SM-Project/scorpio>`_ (parallel i/o for MPAS
+  components) `ESMF <https://earthsystemmodeling.org/>`_ (making mapping files
+  in parallel), `Trilinos <https://trilinos.github.io/>`_,
+  `Albany <https://github.com/sandialabs/Albany>`_,
+  `Netlib-LAPACK <http://www.netlib.org/lapack/>`_ and
+  `PETSc <https://petsc.org/>`_.
+
+* with the ``--with_albany`` flag, creates or uses an existing Spack
+  environment that includes Albany and Trilinos.
 
-* optionally (with the ``--with_albany`` flag) use Spack to install
-  `Trilinos <https://trilinos.github.io/>`_ and
-  `Albany <https://github.com/sandialabs/Albany>`_ libraries.
+* with the ``--with_petsc --with_netlib_lapack`` flags, creates or uses an
+  existing Spack environment that includes PETSc and Netlib-LAPACK.
 
 * make an activation script called ``load_*.sh``, where the details of the
-  name encode the conda environment name, the machine, compilers and MPI
-  libraries, e.g.
+  name encode the conda environment name, the machine, compilers, MPI
+  libraries, and optional libraries,  e.g.
   ``load_dev_compass_<version>_<machine>_<compiler>_<mpi>.sh`` (``<version>``
   is the compass version, ``<machine>`` is the name of the
   machine, ``<compiler>`` is the compiler name, and ``mpi`` is the MPI flavor).
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		__version__ = '1.1.0-alpha.3'
		__version__ = '1.1.0-alpha.4'