From 6d5870a6d80bd435d2c9afda33acf39b029c77d8 Mon Sep 17 00:00:00 2001 From: Kevin Dalton Date: Thu, 31 Oct 2024 17:44:11 -0400 Subject: [PATCH 1/5] actually use the requested tf version --- install-tfp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-tfp.sh b/install-tfp.sh index eb31977..f763c38 100644 --- a/install-tfp.sh +++ b/install-tfp.sh @@ -4,7 +4,7 @@ TF_VERSION=2.16.1 pip install --upgrade pip pip install tensorflow[and-cuda]==$TF_VERSION -pip install tensorflow-probability[tf]==$TFP_VERSION +pip install tensorflow-probability[tf]==$TFP_VERSION tensorflow[and-cuda]==$TF_VERSION # The following is a workaround for a bug in tensorflow cuda installation # https://github.com/tensorflow/tensorflow/issues/63362#issuecomment-2134680575 From 46ffb65fab195999dea95457c8b27385de0c1226 Mon Sep 17 00:00:00 2001 From: Kevin Dalton Date: Fri, 1 Nov 2024 10:08:18 -0400 Subject: [PATCH 2/5] set tf max version to 2.17 (#176) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0d0843e..b5d0bdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ requires-python = ">= 3.9,<3.12" dependencies = [ "reciprocalspaceship>=0.9.16", "tqdm", - "tensorflow", + "tensorflow<2.18", "tf_keras", "tensorflow-probability[tf]", "matplotlib", From 848f0c1389c170ca8eb4e6019a2a0cb4312f59a8 Mon Sep 17 00:00:00 2001 From: Kevin Dalton Date: Tue, 12 Nov 2024 05:55:54 -0500 Subject: [PATCH 3/5] update pre-install script to tfp 0.25 (#177) * update pre-install script to tfp 0.25 * fix dtype compatibility * simplify install with optional deps * add python 3.12 to matrix build * remove pkg_resources dep --- .github/workflows/build.yml | 2 +- careless/__init__.py | 12 ++++++--- careless/models/likelihoods/laue.py | 3 ++- careless/models/likelihoods/mono.py | 3 ++- careless/models/priors/empirical.py | 3 ++- install-cuda.sh | 5 ++-- install-tfp.sh | 36 +++------------------------ pyproject.toml | 17 ++++++++----- tests/models/likelihoods/test_laue.py | 3 ++- tests/models/likelihoods/test_mono.py | 3 ++- tests/models/priors/test_empirical.py | 3 ++- tests/utils/test_distributions.py | 3 ++- 12 files changed, 42 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfd5afa..0a566a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] # Skip CI if 'skip ci' is contained in latest commit message if: "!contains(github.event.head_commit.message, 'skip ci')" diff --git a/careless/__init__.py b/careless/__init__.py index 707b650..8f0044f 100644 --- a/careless/__init__.py +++ b/careless/__init__.py @@ -1,9 +1,15 @@ # Version number for careless def getVersionNumber(): - import pkg_resources + version = None + try: + from setuptools.version import metadata - version = pkg_resources.require("careless")[0].version - return version + version = metadata.version("careless") + except ImportError: + from setuptools.version import pkg_resources + + version = pkg_resources.require("careless")[0].version + return version __version__ = getVersionNumber() diff --git a/careless/models/likelihoods/laue.py b/careless/models/likelihoods/laue.py index dcf038e..f76e516 100644 --- a/careless/models/likelihoods/laue.py +++ b/careless/models/likelihoods/laue.py @@ -3,6 +3,7 @@ from tensorflow_probability import bijectors as tfb import tensorflow_probability as tfp import tensorflow as tf +import math import numpy as np class ConvolvedLikelihood(): @@ -78,7 +79,7 @@ def dist(self, inputs): scale = self.get_uncertainties(inputs) loc = tf.squeeze(loc) scale = tf.squeeze(scale) - return tfd.Laplace(loc, scale/np.sqrt(2.)) + return tfd.Laplace(loc, scale / math.sqrt(2.)) class StudentTLikelihood(LaueBase): def __init__(self, dof): diff --git a/careless/models/likelihoods/mono.py b/careless/models/likelihoods/mono.py index b95a1aa..d35f02c 100644 --- a/careless/models/likelihoods/mono.py +++ b/careless/models/likelihoods/mono.py @@ -4,6 +4,7 @@ from tensorflow_probability import distributions as tfd from tensorflow_probability import util as tfu from tensorflow_probability import bijectors as tfb +import math import numpy as np class LocationScaleLikelihood(Likelihood): @@ -19,7 +20,7 @@ def call(self, inputs): class LaplaceLikelihood(LocationScaleLikelihood): def call(self, inputs): loc, scale = self.get_loc_and_scale(inputs) - return tfd.Laplace(loc, scale/np.sqrt(2.)) + return tfd.Laplace(loc, scale/math.sqrt(2.)) class StudentTLikelihood(LocationScaleLikelihood): def __init__(self, dof): diff --git a/careless/models/priors/empirical.py b/careless/models/priors/empirical.py index acda244..26dfd6a 100644 --- a/careless/models/priors/empirical.py +++ b/careless/models/priors/empirical.py @@ -3,6 +3,7 @@ import numpy as np from careless.models.priors.base import Prior from careless.models.merging.surrogate_posteriors import RiceWoolfson +import math class ReferencePrior(): @@ -57,7 +58,7 @@ def __init__(self, Fobs, SigFobs, observed=None): """ super().__init__(observed) loc = np.array(Fobs, dtype=np.float32) - scale = np.array(SigFobs, dtype=np.float32)/np.sqrt(2.) + scale = np.array(SigFobs, dtype=np.float32)/math.sqrt(2.) self.base_dist = tfd.Laplace(loc, scale) class NormalReferencePrior(ReferencePrior): diff --git a/install-cuda.sh b/install-cuda.sh index aad6f27..0638753 100644 --- a/install-cuda.sh +++ b/install-cuda.sh @@ -14,13 +14,14 @@ fi conda activate $ENVNAME # Install TensorFlow Probability -source <(curl -s https://raw.githubusercontent.com/rs-station/careless/main/install-tfp.sh) +# Currently not necessary +#source <(curl -s https://raw.githubusercontent.com/rs-station/careless/main/install-tfp.sh) # Reactivate to update cuda paths conda activate $ENVNAME # Install careless -pip install --upgrade careless +pip install careless[cuda] # Run careless devices careless devices diff --git a/install-tfp.sh b/install-tfp.sh index f763c38..302e660 100644 --- a/install-tfp.sh +++ b/install-tfp.sh @@ -1,36 +1,8 @@ -TFP_VERSION=0.24.0 -TF_VERSION=2.16.1 +TFP_VERSION=0.25.0 +TF_VERSION=2.18.0 pip install --upgrade pip -pip install tensorflow[and-cuda]==$TF_VERSION -pip install tensorflow-probability[tf]==$TFP_VERSION tensorflow[and-cuda]==$TF_VERSION - -# The following is a workaround for a bug in tensorflow cuda installation -# https://github.com/tensorflow/tensorflow/issues/63362#issuecomment-2134680575 -mkdir -p $CONDA_PREFIX/etc/conda/activate.d -echo '# Store original LD_LIBRARY_PATH -export ORIGINAL_LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" - -# Get the CUDNN directory -CUDNN_DIR=$(dirname $(dirname $(python -c "import nvidia.cudnn; print(nvidia.cudnn.__file__)"))) - -# Set LD_LIBRARY_PATH to include CUDNN directory -export LD_LIBRARY_PATH=$(find ${CUDNN_DIR}/*/lib/ -type d -printf "%p:")${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - -# Get the ptxas directory -PTXAS_DIR=$(dirname $(dirname $(python -c "import nvidia.cuda_nvcc; print(nvidia.cuda_nvcc.__file__)"))) - -# Set PATH to include the directory containing ptxas -export PATH=$(find ${PTXAS_DIR}/*/bin/ -type d -printf "%p:")${PATH:+:${PATH}} - -'>> $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh - -mkdir -p $CONDA_PREFIX/etc/conda/deactivate.d -echo '# Restore original LD_LIBRARY_PATH -export LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}" - -# Unset environment variables -unset CUDNN_DIR -unset PTXAS_DIR' >> $CONDA_PREFIX/etc/conda/deactivate.d/env_vars.sh +pip install tensorflow[and-cuda]==$TF_VERSION tf_keras +pip install tensorflow-probability[tf]==$TFP_VERSION tensorflow[and-cuda]==$TF_VERSION tf_keras diff --git a/pyproject.toml b/pyproject.toml index b5d0bdb..31f1f41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,23 +10,28 @@ authors=[ description = "Merging crystallography data without much physics." readme = "README.md" dynamic = ["version"] -requires-python = ">= 3.9,<3.12" +requires-python = ">= 3.9,<3.13" dependencies = [ "reciprocalspaceship>=0.9.16", "tqdm", - "tensorflow<2.18", - "tf_keras", - "tensorflow-probability[tf]", + "tensorflow-probability[tf]==0.25", + "tensorflow==2.18.0", "matplotlib", "seaborn", "ray", ] -optional-dependencies.dev = [ + +license={text = "MIT"} + +[project.optional-dependencies] +dev = [ "pytest", "pytest-cov", "pytest-xdist", ] -license={text = "MIT"} +cuda = [ + "tensorflow[and-cuda]==2.18.0", +] [project.urls] Repository = "https://github.com/rs-station/careless" diff --git a/tests/models/likelihoods/test_laue.py b/tests/models/likelihoods/test_laue.py index 64f7d8a..2a40128 100644 --- a/tests/models/likelihoods/test_laue.py +++ b/tests/models/likelihoods/test_laue.py @@ -2,6 +2,7 @@ from careless.models.likelihoods.laue import * from careless.models.base import BaseModel from tensorflow_probability import distributions as tfd +import math from careless.utils.device import disable_gpu status = disable_gpu() @@ -40,7 +41,7 @@ def test_laue_LaplaceLikelihood(laue_inputs): sigiobs = BaseModel.get_uncertainties(laue_inputs) ipred = fake_ipred(laue_inputs) - l_true = tfd.Laplace(iobs, sigiobs/np.sqrt(2.)) + l_true = tfd.Laplace(iobs, sigiobs/math.sqrt(2.)) iconv = likelihood.convolve(ipred) diff --git a/tests/models/likelihoods/test_mono.py b/tests/models/likelihoods/test_mono.py index db11104..5568546 100644 --- a/tests/models/likelihoods/test_mono.py +++ b/tests/models/likelihoods/test_mono.py @@ -2,6 +2,7 @@ from careless.models.likelihoods.mono import * from careless.models.base import BaseModel from tensorflow_probability import distributions as tfd +import math from careless.utils.device import disable_gpu status = disable_gpu() @@ -28,7 +29,7 @@ def test_mono_LaplaceLikelihood(mono_inputs): l_true = tfd.Laplace( tf.squeeze(iobs), - tf.squeeze(sigiobs)/np.sqrt(2.), + tf.squeeze(sigiobs)/math.sqrt(2.), ) z = l_true.sample() diff --git a/tests/models/priors/test_empirical.py b/tests/models/priors/test_empirical.py index b36874c..884d964 100644 --- a/tests/models/priors/test_empirical.py +++ b/tests/models/priors/test_empirical.py @@ -4,6 +4,7 @@ from tensorflow_probability import distributions as tfd from careless.models.merging.surrogate_posteriors import RiceWoolfson from careless.models.priors.empirical import * +import math import numpy as np from careless.utils.device import disable_gpu @@ -48,7 +49,7 @@ def ReferencePrior_test(p, ref, mc_samples): @pytest.mark.parametrize('mc_samples', [(), 3, 1]) def test_LaplaceReferencePrior(mc_samples): p = LaplaceReferencePrior(Fobs[observed], SigFobs[observed], observed) - q = tfd.Laplace(Fobs, SigFobs/np.sqrt(2.)) + q = tfd.Laplace(Fobs, SigFobs/math.sqrt(2.)) ReferencePrior_test(p, q, mc_samples) diff --git a/tests/utils/test_distributions.py b/tests/utils/test_distributions.py index 53da270..fafbda5 100644 --- a/tests/utils/test_distributions.py +++ b/tests/utils/test_distributions.py @@ -1,6 +1,7 @@ from careless.utils.distributions import Stacy,Amoroso from tensorflow_probability import distributions as tfd import numpy as np +import math from careless.utils.device import disable_gpu @@ -76,7 +77,7 @@ def test_halfnormal(xmin=1e-3, xmax=1e1, npoints=1000): ref_dist_class = tfd.HalfNormal scale = np.linspace(0.1, 100, 1000).astype(np.float32) ref_params = (scale,) - amoroso_params = (0., np.sqrt(2.) * scale, 0.5, 2.) + amoroso_params = (0., math.sqrt(2.) * scale, 0.5, 2.) # Construct distributions ref_dist = ref_dist_class(*ref_params) From e6740a97c5c41155217ae94dc05d3ab4d82553b5 Mon Sep 17 00:00:00 2001 From: Kevin Dalton Date: Tue, 12 Nov 2024 06:14:03 -0500 Subject: [PATCH 4/5] [skip ci] update installation --- README.md | 59 +++++++++++-------------------------------------------- 1 file changed, 12 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index f16276e..0385d3f 100644 --- a/README.md +++ b/README.md @@ -10,63 +10,28 @@ Merging crystallography data without much physics. ## Installation As described in the [TensorFlow docs](https://www.tensorflow.org/install/pip#step-by-step_instructions), it is best practice to install `careless` in a fresh [anaconda](https://www.anaconda.com/products/distribution) environment to avoid conflicts with previously installed dependencies. -Create a new environment and install `careless` using the following commands. +Create a new environment using the following commands. ```bash conda create -yn careless python conda activate careless pip install --upgrade pip +``` + +Now install `careless` for CPU, +```bash pip install careless ``` +or for NVIDIA GPUs +```bash +pip install careless[cuda] +``` +You may run `careless devices` to check whether GPU support was successfully installed. If you run into issues please [File an issue](https://github.com/rs-station/careless/issues/new/choose). + + ## Installation with GPU Support Careless supports GPU acceleration on NVIDIA GPUs through the CUDA library. We strongly encourage users to take advantage of this feature. To streamline installation, we maintain a script which installs careless with CUDA support. The following section will guide you through installing careless for the GPU. -1) **Install the NVIDIA driver** for your accelerator card. In most high performance computing scenarios, this driver should be pre-installed. If it is not, we suggest you contact your system administrator as installation will require elevated privileges. - - You may check if the driver is functional by typing `nvidia-smi`. If it is working properly you will see output like the following, - - Thu Jun 13 13:01:32 2024 - +-----------------------------------------------------------------------------------------+ - | NVIDIA-SMI 550.54.15 Driver Version: 550.54.15 CUDA Version: 12.4 | - |-----------------------------------------+------------------------+----------------------+ - | GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | - | Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | - | | | MIG M. | - |=========================================+========================+======================| - | 0 Tesla V100S-PCIE-32GB On | 00000000:86:00.0 Off | 0 | - | N/A 32C P0 25W / 250W | 0MiB / 32768MiB | 0% Default | - | | | N/A | - +-----------------------------------------+------------------------+----------------------+ - - +-----------------------------------------------------------------------------------------+ - | Processes: | - | GPU GI CI PID Type Process name GPU Memory | - | ID ID Usage | - |=========================================================================================| - | No running processes found | - +-----------------------------------------------------------------------------------------+ - A faulty driver will give an error message: - - NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver. Make sure that the latest NVIDIA driver is installed and running. - If the driver isn't installed, you will see: - - nvidia-smi: command not found -2) **Install Anaconda**. Once you have confirmed the NIVIDIA driver is available, proceed to install the Anaconda python distribution by following the instructions [here](https://docs.anaconda.com/free/anaconda/install/). Before proceeding, make sure you activate your conda base environment. For typical installations, this should normally happen by opening a new login shell. Alternatively, you may directly source the `conda.sh` in your Anaconda install directory. -3) **Install careless** and associated dependencies by running the following from your base conda environment: - - source <(curl -s https://raw.githubusercontent.com/rs-station/careless/main/install-cuda.sh) - This will automatically create a new conda environment named careless. - -Careless is now installed in its own environment. Whenever you want to run careless, you must first activate the careless conda environment by issuing `conda activate careless`. After the installation is finished, the script will test GPU availability by running `careless devices`. If the installation was successful and your node has a GPU, it will appare in the output: - - (careless) user@computer:~$ careless devices - Careless version 0.4.3 - ############################################### - # TensorFlow can access the following devices # - ############################################### - - CPU: /physical_device:CPU:0 - - GPU: /physical_device:GPU:0 - ## Dependencies From ed2434116ab04a56b7db5bef2f7480da1feea01b Mon Sep 17 00:00:00 2001 From: Kevin Dalton Date: Tue, 12 Nov 2024 06:15:02 -0500 Subject: [PATCH 5/5] [skip ci] Update VERSION --- careless/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/careless/VERSION b/careless/VERSION index cb498ab..76914dd 100644 --- a/careless/VERSION +++ b/careless/VERSION @@ -1 +1 @@ -0.4.8 +0.4.9