From dd17023f952e13f7b7eca3e8521fb9c60f94c486 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 2 Feb 2023 23:04:48 -0700 Subject: [PATCH 1/4] Enable easier mache testing The `configure_compass_env.py` script now takes two flags, `--mache_fork` and `--mache_branch` that are used to clone the fork and branch locally, then install mache from it. This saves the trouble of developers cloning it themselves and building the mache conda package. --- conda/bootstrap.py | 23 ++++++++++++++++++----- conda/compass_env/spec-file.template | 2 ++ conda/configure_compass_env.py | 26 +++++++++++++++++++++++--- conda/shared.py | 8 ++++++++ 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 73a29f5118..31a7d0f934 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -238,10 +238,10 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, activ_path, env_path, env_name, activate_env, spack_env -def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version, +def build_conda_env(env_type, recreate, mpi, conda_mpi, version, python, source_path, conda_template_path, conda_base, env_name, env_path, activate_base, use_local, - local_conda_build, logger): + local_conda_build, logger, local_mache): if env_type != 'dev': install_miniconda(conda_base, activate_base, logger) @@ -284,7 +284,8 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version, conda_openmp = '' spec_file = template.render(supports_otps=supports_otps, mpi=conda_mpi, openmp=conda_openmp, - mpi_prefix=mpi_prefix) + mpi_prefix=mpi_prefix, + include_mache=not local_mache) spec_filename = f'spec-file-{conda_mpi}.txt' with open(spec_filename, 'w') as handle: @@ -802,6 +803,8 @@ def main(): # noqa: C901 compass_version = get_version() + local_mache = args.mache_fork is not None and args.mache_branch is not None + machine = None if not args.env_only: if args.machine is None: @@ -889,10 +892,20 @@ def main(): # noqa: C901 if previous_conda_env != conda_env_name: build_conda_env( - env_type, recreate, machine, mpi, conda_mpi, compass_version, + env_type, recreate, mpi, conda_mpi, compass_version, python, source_path, conda_template_path, conda_base, conda_env_name, conda_env_path, activate_base, args.use_local, - args.local_conda_build, logger) + args.local_conda_build, logger, local_mache) + + if local_mache: + print('Install local mache\n') + commands = f'source {conda_base}/etc/profile.d/conda.sh; ' \ + f'source {conda_base}/etc/profile.d/mamba.sh; ' \ + f'conda activate {conda_env_name}; ' \ + 'cd ../build_mache/mache; ' \ + 'python -m pip install .' + check_call(commands, logger=logger) + previous_conda_env = conda_env_name if env_type != 'dev': diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index af50f17b66..81c40742d3 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -15,7 +15,9 @@ jigsaw=0.9.14 jigsawpy=0.3.3 jupyter lxml +{% if include_mache %} mache=1.10.0 +{% endif %} matplotlib-base metis mpas_tools=0.17.0 diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index 8e782cba89..661a14f26d 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -52,13 +52,13 @@ def bootstrap(activate_install_env, source_path, local_conda_build): def setup_install_env(env_name, activate_base, use_local, logger, recreate, - conda_base): + conda_base, mache): env_path = os.path.join(conda_base, 'envs', env_name) if use_local: channels = '--use-local' else: channels = '' - packages = 'progressbar2 jinja2 "mache=1.10.0"' + packages = 'progressbar2 jinja2 {}'.format(mache) if recreate or not os.path.exists(env_path): print('Setting up a conda environment for installing compass\n') commands = '{} && ' \ @@ -106,8 +106,28 @@ def main(): # install miniconda if needed install_miniconda(conda_base, activate_base, logger) + local_mache = args.mache_fork is not None and args.mache_branch is not None + if local_mache: + mache = '' + else: + mache = '"mache=1.10.0"' + setup_install_env(env_name, activate_base, args.use_local, logger, - args.recreate, conda_base) + args.recreate, conda_base, mache) + + if local_mache: + print('Clone and install local mache\n') + commands = '{}; ' \ + 'rm -rf conda/build_mache; ' \ + 'mkdir -p conda/build_mache; ' \ + 'cd conda/build_mache; ' \ + 'git clone -b {} git@github.com:{}.git mache; ' \ + 'cd mache; ' \ + 'python -m pip install .'.format(activate_install_env, + args.mache_branch, + args.mache_fork) + + check_call(commands, logger=logger) env_type = config.get('deploy', 'env_type') if env_type not in ['dev', 'test_release', 'release']: diff --git a/conda/shared.py b/conda/shared.py index a20cfd8476..be68f03109 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -47,6 +47,10 @@ def parse_args(bootstrap): "packages") parser.add_argument("--use_local", dest="use_local", action='store_true', help="Use locally built conda packages (for testing).") + parser.add_argument("--mache_fork", dest="mache_fork", + help="Point to a mache fork (and branch) for testing") + parser.add_argument("--mache_branch", dest="mache_branch", + help="Point to a mache branch (and fork) for testing") parser.add_argument("--update_spack", dest="update_spack", action='store_true', help="If the shared spack environment should be " @@ -78,6 +82,10 @@ def parse_args(bootstrap): args = parser.parse_args(sys.argv[1:]) + if (args.mache_fork is None) != (args.mache_branch is None): + raise ValueError('You must supply both or neither of ' + '--mache_fork and --mache_branch') + return args From 0576d0dbb2578aac0f3548134337b13bc2e8c8c0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 2 Feb 2023 23:18:27 -0700 Subject: [PATCH 2/4] Require python 3 for deployment I believe all systems have python3 available. This allows us to remove some awkward imports and to use f-strings instead of format commands. --- conda/bootstrap.py | 12 +++--- conda/configure_compass_env.py | 71 +++++++++++++--------------------- conda/shared.py | 47 ++++++++++------------ 3 files changed, 51 insertions(+), 79 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 31a7d0f934..44f6112481 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python - -from __future__ import print_function +#!/usr/bin/env python3 import glob import grp @@ -899,10 +897,10 @@ def main(): # noqa: C901 if local_mache: print('Install local mache\n') - commands = f'source {conda_base}/etc/profile.d/conda.sh; ' \ - f'source {conda_base}/etc/profile.d/mamba.sh; ' \ - f'conda activate {conda_env_name}; ' \ - 'cd ../build_mache/mache; ' \ + commands = f'source {conda_base}/etc/profile.d/conda.sh && ' \ + f'source {conda_base}/etc/profile.d/mamba.sh && ' \ + f'conda activate {conda_env_name} && ' \ + 'cd ../build_mache/mache && ' \ 'python -m pip install .' check_call(commands, logger=logger) diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index 661a14f26d..5ac1cdd70c 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -1,20 +1,8 @@ -#!/usr/bin/env python - -from __future__ import print_function +#!/usr/bin/env python3 import os import sys - -try: - from configparser import ConfigParser -except ImportError: - import six - from six.moves import configparser - - if six.PY2: - ConfigParser = configparser.SafeConfigParser - else: - ConfigParser = configparser.ConfigParser +from configparser import ConfigParser from shared import ( check_call, @@ -41,13 +29,11 @@ def get_config(config_file): def bootstrap(activate_install_env, source_path, local_conda_build): print('Creating the compass conda environment\n') - bootstrap_command = '{}/conda/bootstrap.py'.format(source_path) - command = '{} && ' \ - '{} {}'.format(activate_install_env, bootstrap_command, - ' '.join(sys.argv[1:])) + bootstrap_command = f'{source_path}/conda/bootstrap.py' + command = f'{activate_install_env} && ' \ + f'{bootstrap_command} {" ".join(sys.argv[1:])}' if local_conda_build is not None: - command = '{} --local_conda_build {}'.format(command, - local_conda_build) + command = f'{command} --local_conda_build {local_conda_build}' check_call(command) @@ -58,19 +44,15 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate, channels = '--use-local' else: channels = '' - packages = 'progressbar2 jinja2 {}'.format(mache) + packages = f'progressbar2 jinja2 {mache}' if recreate or not os.path.exists(env_path): print('Setting up a conda environment for installing compass\n') - commands = '{} && ' \ - 'mamba create -y -n {} {} {}'.format(activate_base, - env_name, channels, - packages) + commands = f'{activate_base} && ' \ + f'mamba create -y -n {env_name} {channels} {packages}' else: print('Updating conda environment for installing compass\n') - commands = '{} && ' \ - 'mamba install -y -n {} {} {}'.format(activate_base, - env_name, channels, - packages) + commands = f'{activate_base} && ' \ + f'mamba install -y -n {env_name} {channels} {packages}' check_call(commands, logger=logger) @@ -87,14 +69,14 @@ def main(): env_name = 'compass_bootstrap' source_activation_scripts = \ - 'source {}/etc/profile.d/conda.sh && ' \ - 'source {}/etc/profile.d/mamba.sh'.format(conda_base, conda_base) + f'source {conda_base}/etc/profile.d/conda.sh && ' \ + f'source {conda_base}/etc/profile.d/mamba.sh' - activate_base = '{} && mamba activate'.format(source_activation_scripts) + activate_base = f'{source_activation_scripts} && conda activate' activate_install_env = \ - '{} && ' \ - 'mamba activate {}'.format(source_activation_scripts, env_name) + f'{source_activation_scripts} && ' \ + f'conda activate {env_name}' try: os.makedirs('conda/logs') except OSError: @@ -117,24 +99,23 @@ def main(): if local_mache: print('Clone and install local mache\n') - commands = '{}; ' \ - 'rm -rf conda/build_mache; ' \ - 'mkdir -p conda/build_mache; ' \ - 'cd conda/build_mache; ' \ - 'git clone -b {} git@github.com:{}.git mache; ' \ - 'cd mache; ' \ - 'python -m pip install .'.format(activate_install_env, - args.mache_branch, - args.mache_fork) + commands = f'{activate_install_env} && ' \ + f'rm -rf conda/build_mache && ' \ + f'mkdir -p conda/build_mache && ' \ + f'cd conda/build_mache && ' \ + f'git clone -b {args.mache_branch} ' \ + f'git@github.com:{args.mache_fork}.git mache && ' \ + f'cd mache && ' \ + f'python -m pip install .' check_call(commands, logger=logger) env_type = config.get('deploy', 'env_type') if env_type not in ['dev', 'test_release', 'release']: - raise ValueError('Unexpected env_type: {}'.format(env_type)) + raise ValueError(f'Unexpected env_type: {env_type}') if env_type == 'test_release' and args.use_local: - local_conda_build = os.path.abspath('{}/conda-bld'.format(conda_base)) + local_conda_build = os.path.abspath(f'{conda_base}/conda-bld') else: local_conda_build = None diff --git a/conda/shared.py b/conda/shared.py index be68f03109..28d7b2d7b1 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import argparse import logging import os @@ -7,11 +5,7 @@ import shutil import subprocess import sys - -try: - from urllib.request import Request, urlopen -except ImportError: - from urllib2 import Request, urlopen +from urllib.request import Request, urlopen def parse_args(bootstrap): @@ -100,9 +94,9 @@ def get_conda_base(conda_base, config, shared=False, warn=False): conda_base = os.path.abspath( os.path.join(conda_exe, '..', '..')) if warn: - print('\nWarning: --conda path not supplied. Using conda ' - 'installed at:\n' - ' {}\n'.format(conda_base)) + print(f'\nWarning: --conda path not supplied. Using conda ' + f'installed at:\n' + f' {conda_base}\n') else: raise ValueError('No conda base provided with --conda and ' 'none could be inferred.') @@ -126,9 +120,9 @@ def get_spack_base(spack_base, config): def check_call(commands, env=None, logger=None): print_command = '\n '.join(commands.split(' && ')) if logger is None: - print('\n Running:\n {}\n'.format(print_command)) + print(f'\n Running:\n {print_command}\n') else: - logger.info('\nrunning:\n {}\n'.format(print_command)) + logger.info(f'\nrunning:\n {print_command}\n') if logger is None: process = subprocess.Popen(commands, env=env, executable='/bin/bash', @@ -162,8 +156,8 @@ def install_miniconda(conda_base, activate_base, logger): system = 'MacOSX' else: system = 'Linux' - miniconda = 'Mambaforge-{}-x86_64.sh'.format(system) - url = 'https://github.com/conda-forge/miniforge/releases/latest/download/{}'.format(miniconda) # noqa: E501 + miniconda = f'Mambaforge-{system}-x86_64.sh' + url = f'https://github.com/conda-forge/miniforge/releases/latest/download/{miniconda}' # noqa: E501 print(url) req = Request(url, headers={'User-Agent': 'Mozilla/5.0'}) f = urlopen(req) @@ -172,7 +166,7 @@ def install_miniconda(conda_base, activate_base, logger): outfile.write(html) f.close() - command = '/bin/bash {} -b -p {}'.format(miniconda, conda_base) + command = f'/bin/bash {miniconda} -b -p {conda_base}' check_call(command, logger=logger) os.remove(miniconda) @@ -180,23 +174,22 @@ def install_miniconda(conda_base, activate_base, logger): print('Doing initial setup\n') - commands = '{} && ' \ - 'conda config --add channels conda-forge && ' \ - 'conda config --set channel_priority strict' \ - ''.format(activate_base) + commands = f'{activate_base} && ' \ + f'conda config --add channels conda-forge && ' \ + f'conda config --set channel_priority strict' check_call(commands, logger=logger) - commands = '{} && ' \ - 'conda remove -y boa'.format(activate_base) + commands = f'{activate_base} && ' \ + f'conda remove -y boa' try: check_call(commands, logger=logger) except subprocess.CalledProcessError: pass - commands = '{} && ' \ - 'mamba update -y --all && ' \ - 'mamba init'.format(activate_base) + commands = f'{activate_base} && ' \ + f'mamba update -y --all && ' \ + f'mamba init' check_call(commands, logger=logger) @@ -208,7 +201,7 @@ def backup_bashrc(): files = ['.bashrc', '.bash_profile'] for filename in files: src = os.path.join(home_dir, filename) - dst = os.path.join(home_dir, '{}.conda_bak'.format(filename)) + dst = os.path.join(home_dir, f'{filename}.conda_bak') if os.path.exists(src): shutil.copyfile(src, dst) @@ -217,14 +210,14 @@ def restore_bashrc(): home_dir = os.path.expanduser('~') files = ['.bashrc', '.bash_profile'] for filename in files: - src = os.path.join(home_dir, '{}.conda_bak'.format(filename)) + src = os.path.join(home_dir, f'{filename}.conda_bak') dst = os.path.join(home_dir, filename) if os.path.exists(src): shutil.move(src, dst) def get_logger(name, log_filename): - print('Logging to: {}\n'.format(log_filename)) + print(f'Logging to: {log_filename}\n') try: os.remove(log_filename) except OSError: From 289503a2d34fcf0089b627af6f19312bdaf0da1e Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 9 Feb 2023 12:40:49 -0700 Subject: [PATCH 3/4] Make the tmpdir if it doesn't exist --- conda/configure_compass_env.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index 5ac1cdd70c..a9d86536dc 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -61,6 +61,12 @@ def main(): args = parse_args(bootstrap=False) source_path = os.getcwd() + if args.tmpdir is not None: + try: + os.makedirs(args.tmpdir) + except FileExistsError: + pass + config = get_config(args.config_file) conda_base = get_conda_base(args.conda_base, config, warn=True) From 0f0ab0f8ac12374211331797cf83dfb8783d8284 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 9 Feb 2023 13:15:34 -0800 Subject: [PATCH 4/4] Remove unsupported cxx17 albany variant --- conda/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 44f6112481..5656a6521e 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -422,7 +422,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, f'scorpio@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc') if albany != 'None': - specs.append(f'albany@{albany}+mpas+cxx17') + specs.append(f'albany@{albany}+mpas') yaml_template = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml' if not os.path.exists(yaml_template):