From 40aee2aa84ed5de4f82beb41e2cf061c222d51a9 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 13:37:20 +0100 Subject: [PATCH 01/15] Avoid cubelists of cubelists. --- improver/calibration/__init__.py | 4 +- improver_tests/calibration/test_init.py | 96 +++++++++++-------------- 2 files changed, 43 insertions(+), 57 deletions(-) diff --git a/improver/calibration/__init__.py b/improver/calibration/__init__.py index db23e43628..fb62ccd7c9 100644 --- a/improver/calibration/__init__.py +++ b/improver/calibration/__init__.py @@ -94,9 +94,7 @@ def split_forecasts_and_truth( return forecast, truth, land_sea_mask -def split_forecasts_and_coeffs( - cubes: CubeList, land_sea_mask_name: Optional[str] = None -): +def split_forecasts_and_coeffs(cubes: List, land_sea_mask_name: Optional[str] = None): """Split the input forecast, coefficients, static additional predictors, land sea-mask and probability template, if provided. The coefficients cubes and land-sea mask are identified based on their name. The diff --git a/improver_tests/calibration/test_init.py b/improver_tests/calibration/test_init.py index 82747bdbde..e65b3392e0 100644 --- a/improver_tests/calibration/test_init.py +++ b/improver_tests/calibration/test_init.py @@ -298,7 +298,7 @@ def test_realization_forecast_and_coefficients(self): """Test a realization forecast input.""" (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( split_forecasts_and_coeffs( - CubeList([self.realization_forecast, self.coefficient_cubelist]), + [self.realization_forecast, self.coefficient_cubelist], self.land_sea_mask_name, ) ) @@ -313,7 +313,7 @@ def test_percentile_forecast_and_coefficients(self): """Test a percentile forecast input.""" (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( split_forecasts_and_coeffs( - CubeList([self.percentile_forecast, self.coefficient_cubelist]), + [self.percentile_forecast, self.coefficient_cubelist], self.land_sea_mask_name, ) ) @@ -327,9 +327,9 @@ def test_probability_forecast_and_coefficients(self): """Test a probability forecast input.""" (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( split_forecasts_and_coeffs( - CubeList([self.probability_forecast, self.coefficient_cubelist]), - self.land_sea_mask_name, - ) + [self.probability_forecast, self.coefficient_cubelist] + ), + self.land_sea_mask_name, ) self.assertCubeEqual(forecast, self.probability_forecast[0]) self.assertCubeListEqual(coeffs, self.coefficient_cubelist) @@ -341,13 +341,11 @@ def test_forecast_coefficients_additional_predictor(self): """Test the addition of a static additional predictor.""" (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( split_forecasts_and_coeffs( - CubeList( - [ - self.realization_forecast, - self.coefficient_cubelist, - self.additional_predictors, - ] - ), + [ + self.realization_forecast, + self.coefficient_cubelist, + self.additional_predictors, + ], self.land_sea_mask_name, ) ) @@ -361,13 +359,11 @@ def test_forecast_coefficients_and_land_sea_mask(self): """Test the addition of a land-sea mask.""" (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( split_forecasts_and_coeffs( - CubeList( - [ - self.realization_forecast, - self.coefficient_cubelist, - self.land_sea_mask, - ] - ), + [ + self.realization_forecast, + self.coefficient_cubelist, + self.land_sea_mask, + ], self.land_sea_mask_name, ) ) @@ -382,7 +378,7 @@ def test_no_land_sea_mask_name(self): """Test when not providing the land_sea_mask_name option.""" (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( split_forecasts_and_coeffs( - CubeList([self.realization_forecast, self.coefficient_cubelist]) + [self.realization_forecast, self.coefficient_cubelist] ) ) @@ -396,13 +392,11 @@ def test_forecast_coefficients_prob_template(self): """Test the addition of a probability template cube.""" (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( split_forecasts_and_coeffs( - CubeList( - [ - self.realization_forecast, - self.coefficient_cubelist, - self.probability_forecast, - ] - ), + [ + self.realization_forecast, + self.coefficient_cubelist, + self.probability_forecast, + ], self.land_sea_mask_name, ) ) @@ -417,15 +411,13 @@ def test_all_options(self): land-sea mask and a probability template.""" (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( split_forecasts_and_coeffs( - CubeList( - [ - self.realization_forecast, - self.coefficient_cubelist, - self.additional_predictors, - self.land_sea_mask, - self.probability_forecast, - ] - ), + [ + self.realization_forecast, + self.coefficient_cubelist, + self.additional_predictors, + self.land_sea_mask, + self.probability_forecast, + ], self.land_sea_mask_name, ) ) @@ -440,20 +432,18 @@ def test_probability_forecast_coefficients_prob_template(self): msg = "Providing multiple probability cubes" with self.assertRaisesRegex(ValueError, msg): split_forecasts_and_coeffs( - CubeList( - [ - self.probability_forecast, - self.coefficient_cubelist, - self.probability_forecast, - ] - ), + [ + self.probability_forecast, + self.coefficient_cubelist, + self.probability_forecast, + ], self.land_sea_mask_name, ) def test_no_coefficients(self): """Test if no EMOS coefficients are provided.""" _, coeffs, _, _, _ = split_forecasts_and_coeffs( - CubeList([self.percentile_forecast]), self.land_sea_mask_name + [self.percentile_forecast], self.land_sea_mask_name ) self.assertIsNone(coeffs) @@ -462,7 +452,7 @@ def test_no_forecast(self): msg = "No forecast is present" with self.assertRaisesRegex(ValueError, msg): split_forecasts_and_coeffs( - CubeList([self.coefficient_cubelist]), self.land_sea_mask_name + [self.coefficient_cubelist], self.land_sea_mask_name ) def test_duplicate_forecasts(self): @@ -470,15 +460,13 @@ def test_duplicate_forecasts(self): msg = "Multiple items have been provided" with self.assertRaisesRegex(ValueError, msg): split_forecasts_and_coeffs( - CubeList( - [ - self.percentile_forecast, - self.coefficient_cubelist, - self.land_sea_mask, - self.probability_forecast, - self.percentile_forecast, - ] - ), + [ + self.percentile_forecast, + self.coefficient_cubelist, + self.land_sea_mask, + self.probability_forecast, + self.percentile_forecast, + ], self.land_sea_mask_name, ) From 9eff3be448075e897a9a74aeb3d46cb6bd73d767 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 13:48:23 +0100 Subject: [PATCH 02/15] Replace np.product with np.prod. --- improver_tests/categorical/modal_code/test_ModalCategory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/improver_tests/categorical/modal_code/test_ModalCategory.py b/improver_tests/categorical/modal_code/test_ModalCategory.py index dc9baf6490..029b1b2939 100644 --- a/improver_tests/categorical/modal_code/test_ModalCategory.py +++ b/improver_tests/categorical/modal_code/test_ModalCategory.py @@ -58,7 +58,7 @@ def wxcode_series_fixture( wxfrt = time - timedelta(hours=42) wxdata = np.ones((2, 2), dtype=np.int8) - if len(data[i].shape) > 0 and np.product(wxdata.shape) == data[i].shape[0]: + if len(data[i].shape) > 0 and np.prod(wxdata.shape) == data[i].shape[0]: wxdata = np.reshape(data[i], wxdata.shape) else: if len(data[i].shape) == 0: From de8361f04a9226b54846ea19a72fc3725b239c4b Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 15:14:07 +0100 Subject: [PATCH 03/15] Replace np.int. --- .../calibration/rainforests_calibration/conftest.py | 2 +- .../test_OrographicSmoothingCoefficients.py | 2 +- improver_tests/regrid/test_RegridWithLandSeaMask.py | 8 ++++---- improver_tests/threshold/conftest.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/improver_tests/calibration/rainforests_calibration/conftest.py b/improver_tests/calibration/rainforests_calibration/conftest.py index 879bf66baa..3be9cccc5d 100644 --- a/improver_tests/calibration/rainforests_calibration/conftest.py +++ b/improver_tests/calibration/rainforests_calibration/conftest.py @@ -42,7 +42,7 @@ def thresholds(): @pytest.fixture def lead_times(): - return np.array([24, 48], dtype=np.int) + return np.array([24, 48], dtype=np.int32) @pytest.fixture diff --git a/improver_tests/generate_ancillaries/test_OrographicSmoothingCoefficients.py b/improver_tests/generate_ancillaries/test_OrographicSmoothingCoefficients.py index 11b3987724..c7ddf2ee22 100644 --- a/improver_tests/generate_ancillaries/test_OrographicSmoothingCoefficients.py +++ b/improver_tests/generate_ancillaries/test_OrographicSmoothingCoefficients.py @@ -71,7 +71,7 @@ def smoothing_coefficients_fixture() -> CubeList: def mask_fixture() -> Cube: """Returns an example mask.""" - data = np.zeros((6, 6), dtype=np.int) + data = np.zeros((6, 6), dtype=np.int32) data[2:-2, 2:-2] = 1 mask = set_up_variable_cube( data, diff --git a/improver_tests/regrid/test_RegridWithLandSeaMask.py b/improver_tests/regrid/test_RegridWithLandSeaMask.py index 2c737ed509..3a7f256e2a 100644 --- a/improver_tests/regrid/test_RegridWithLandSeaMask.py +++ b/improver_tests/regrid/test_RegridWithLandSeaMask.py @@ -44,14 +44,14 @@ def define_source_target_grid_data(): data = np.arange(20).reshape(4, 5).astype(np.float32) # input grid mask info - in_mask = np.empty((4, 5), dtype=np.int) + in_mask = np.empty((4, 5), dtype=np.int32) in_mask[:, :] = 1 in_mask[0, 2] = 0 in_mask[2, 2:4] = 0 in_mask[3, 2:4] = 0 # output grid mask info - out_mask = np.empty((8, 11), dtype=np.int) + out_mask = np.empty((8, 11), dtype=np.int32) out_mask[:, :] = 1 out_mask[0, 4:7] = 0 out_mask[1, 5] = 0 @@ -88,14 +88,14 @@ def define_source_target_grid_data_same_domain(): data = np.arange(20).reshape(4, 5).astype(np.float32) # input grid mask info - in_mask = np.empty((4, 5), dtype=np.int) + in_mask = np.empty((4, 5), dtype=np.int32) in_mask[:, :] = 1 in_mask[0, 2] = 0 in_mask[2, 2:4] = 0 in_mask[3, 2:4] = 0 # output grid mask info - out_mask = np.empty((7, 9), dtype=np.int) + out_mask = np.empty((7, 9), dtype=np.int32) out_mask[:, :] = 1 out_mask[0, 3:6] = 0 out_mask[1, 4] = 0 diff --git a/improver_tests/threshold/conftest.py b/improver_tests/threshold/conftest.py index 7d88273454..86b2754009 100644 --- a/improver_tests/threshold/conftest.py +++ b/improver_tests/threshold/conftest.py @@ -80,7 +80,7 @@ def deterministic_cube() -> Cube: def single_realization_cube() -> Cube: """Return the diagnostic cube with a scalar realization coordinate.""" cube = diagnostic_cube() - cube = add_coordinate(cube, [0], "realization", dtype=np.int) + cube = add_coordinate(cube, [0], "realization", dtype=np.int32) return cube From 39076eeb6e0a6ee4e42af3129d398e4ce17adc51 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 15:19:46 +0100 Subject: [PATCH 04/15] Replace np.NAN with np.nan. --- .../nbhood/nbhood/test_BaseNeighbourhoodProcessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/improver_tests/nbhood/nbhood/test_BaseNeighbourhoodProcessing.py b/improver_tests/nbhood/nbhood/test_BaseNeighbourhoodProcessing.py index 06fc322651..7f26e73276 100644 --- a/improver_tests/nbhood/nbhood/test_BaseNeighbourhoodProcessing.py +++ b/improver_tests/nbhood/nbhood/test_BaseNeighbourhoodProcessing.py @@ -66,7 +66,7 @@ def setUp(self): def test_single_point_nan(self): """Test behaviour for a single NaN grid cell.""" - self.cube.data[6][7] = np.NAN + self.cube.data[6][7] = np.nan msg = "NaN detected in input cube data" with self.assertRaisesRegex(ValueError, msg): NBHood(self.RADIUS)(self.cube) From 0206dab4be3af3a2f8f6df578b9036ff6e431a22 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 15:21:54 +0100 Subject: [PATCH 05/15] Replace np.NAN with np.nan. --- improver_tests/threshold/test_LatitudeDependentThreshold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/improver_tests/threshold/test_LatitudeDependentThreshold.py b/improver_tests/threshold/test_LatitudeDependentThreshold.py index 318c5a2c79..21dff3fafe 100644 --- a/improver_tests/threshold/test_LatitudeDependentThreshold.py +++ b/improver_tests/threshold/test_LatitudeDependentThreshold.py @@ -267,7 +267,7 @@ def test_threshold_unit_conversion(self): def test_threshold_point_nan(self): """Test behaviour for a single NaN grid cell.""" - self.cube.data[2][2] = np.NAN + self.cube.data[2][2] = np.nan msg = "NaN detected in input cube data" with self.assertRaisesRegex(ValueError, msg): self.plugin(self.cube) From 7c27c18b0b687cf92aa66519aef21de21c6d619c Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 15:23:23 +0100 Subject: [PATCH 06/15] Replace np.NaN with np.nan. --- .../nowcasting/utilities/test_ApplyOrographicEnhancement.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/improver_tests/nowcasting/utilities/test_ApplyOrographicEnhancement.py b/improver_tests/nowcasting/utilities/test_ApplyOrographicEnhancement.py index a009cd2819..d90269b3e7 100644 --- a/improver_tests/nowcasting/utilities/test_ApplyOrographicEnhancement.py +++ b/improver_tests/nowcasting/utilities/test_ApplyOrographicEnhancement.py @@ -379,7 +379,7 @@ def test_NaN_values(self): [ [ [0.0, 1.0, 2.0], - [np.NaN, np.NaN, np.NaN], + [np.nan, np.nan, np.nan], [0.0, 1.0, MIN_PRECIP_RATE_MMH], ] ] @@ -388,7 +388,7 @@ def test_NaN_values(self): subtracted_cube = self.subtracted_cube.copy() subtracted_cube.convert_units("mm/hr") subtracted_cube.data = np.array( - [[[0.0, 1.0, 2.0], [np.NaN, np.NaN, np.NaN], [0.0, 1.0, 0.0]]] + [[[0.0, 1.0, 2.0], [np.nan, np.nan, np.nan], [0.0, 1.0, 0.0]]] ) subtracted_cube.convert_units("m/s") plugin = ApplyOrographicEnhancement("subtract") From 054586b154916e3e3600f524853e1d5cd7d05131 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 15:29:01 +0100 Subject: [PATCH 07/15] Replace np.product with np.prod. --- .../precipitation_duration/test_PrecipitationDuration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/improver_tests/precipitation/precipitation_duration/test_PrecipitationDuration.py b/improver_tests/precipitation/precipitation_duration/test_PrecipitationDuration.py index c66f049047..045ba15571 100644 --- a/improver_tests/precipitation/precipitation_duration/test_PrecipitationDuration.py +++ b/improver_tests/precipitation/precipitation_duration/test_PrecipitationDuration.py @@ -119,9 +119,9 @@ def precip_cubes( function = request.param if function == set_up_spot_probability_cube: - acc_sites = np.product(acc_data.shape[-2:]) + acc_sites = np.prod(acc_data.shape[-2:]) acc_data = acc_data.reshape(acc_data.shape[:-2] + (acc_sites,)) - rate_sites = np.product(rate_data.shape[-2:]) + rate_sites = np.prod(rate_data.shape[-2:]) rate_data = rate_data.reshape(rate_data.shape[:-2] + (rate_sites,)) frt, times, bounds = data_times(start_time, end_time, period) @@ -667,7 +667,7 @@ def test_process( result = plugin.process(precip_cubes) if result.coords("wmo_id"): - n_sites = np.product(acc_data.shape[-2:]) + n_sites = np.prod(acc_data.shape[-2:]) expected = expected.reshape(expected.shape[:-2] + (n_sites,)) assert_array_equal(result.data, expected) From 21121f0ffb39c3e4eee7ff6572ea77331ac5252e Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 15:58:01 +0100 Subject: [PATCH 08/15] Replace assertRaisesRegexp with assertRaisesRegex and use collections.abc.Callable instead of collections.Callable. --- .../nowcasting/optical_flow/test_utilities.py | 4 ++-- .../temperature/lapse_rate/test_LapseRate.py | 16 ++++++++-------- .../cube_extraction/test_cube_extraction.py | 3 ++- improver_tests/utilities/test_load.py | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/improver_tests/nowcasting/optical_flow/test_utilities.py b/improver_tests/nowcasting/optical_flow/test_utilities.py index 611beb80df..4787e88610 100644 --- a/improver_tests/nowcasting/optical_flow/test_utilities.py +++ b/improver_tests/nowcasting/optical_flow/test_utilities.py @@ -49,7 +49,7 @@ def test_additional_nonscalar_dimension(self): vel2.add_aux_coord(DimCoord(2, standard_name="realization")) (invalid_3d,) = (iris.cube.CubeList([vel1, vel2])).merge() msg = "Cube has 3" - with self.assertRaisesRegexp(InvalidCubeError, msg): + with self.assertRaisesRegex(InvalidCubeError, msg): check_input_coords(invalid_3d) def test_time(self): @@ -57,7 +57,7 @@ def test_time(self): cube = self.valid.copy() cube.remove_coord("time") msg = "Input cube has no time coordinate" - with self.assertRaisesRegexp(InvalidCubeError, msg): + with self.assertRaisesRegex(InvalidCubeError, msg): check_input_coords(cube, require_time=True) diff --git a/improver_tests/temperature/lapse_rate/test_LapseRate.py b/improver_tests/temperature/lapse_rate/test_LapseRate.py index 966f4a6ad9..3eb0cea249 100644 --- a/improver_tests/temperature/lapse_rate/test_LapseRate.py +++ b/improver_tests/temperature/lapse_rate/test_LapseRate.py @@ -179,7 +179,7 @@ def test_fails_if_temperature_is_not_cube(self): not a cube.""" incorrect_input = 50.0 msg = "Temperature input is not a cube, but {0}".format(type(incorrect_input)) - with self.assertRaisesRegexp(TypeError, msg): + with self.assertRaisesRegex(TypeError, msg): LapseRate(nbhood_radius=1).process( incorrect_input, self.orography, self.land_sea_mask ) @@ -189,7 +189,7 @@ def test_fails_if_orography_is_not_cube(self): not a cube.""" incorrect_input = 50.0 msg = "Orography input is not a cube, but {0}".format(type(incorrect_input)) - with self.assertRaisesRegexp(TypeError, msg): + with self.assertRaisesRegex(TypeError, msg): LapseRate(nbhood_radius=1).process( self.temperature, incorrect_input, self.land_sea_mask ) @@ -199,7 +199,7 @@ def test_fails_if_land_sea_mask_is_not_cube(self): not a cube.""" incorrect_input = 50.0 msg = "Land/Sea mask input is not a cube, but {0}".format(type(incorrect_input)) - with self.assertRaisesRegexp(TypeError, msg): + with self.assertRaisesRegex(TypeError, msg): LapseRate(nbhood_radius=1).process( self.temperature, self.orography, incorrect_input ) @@ -209,7 +209,7 @@ def test_fails_if_temperature_wrong_units(self): wrong unit.""" # Swap cubes around so have wrong units. msg = r"Unable to convert from 'Unit\('m'\)' to 'Unit\('K'\)'." - with self.assertRaisesRegexp(ValueError, msg): + with self.assertRaisesRegex(ValueError, msg): LapseRate(nbhood_radius=1).process( self.orography, self.orography, self.land_sea_mask ) @@ -218,7 +218,7 @@ def test_fails_if_orography_wrong_units(self): """Test code raises a Value Error if the orography cube is the wrong unit.""" msg = r"Unable to convert from 'Unit\('K'\)' to 'Unit\('metres'\)'." - with self.assertRaisesRegexp(ValueError, msg): + with self.assertRaisesRegex(ValueError, msg): LapseRate(nbhood_radius=1).process( self.temperature, self.temperature, self.land_sea_mask ) @@ -268,7 +268,7 @@ def test_fails_if_max_less_min_lapse_rate(self): less than input minimum lapse rate""" msg = "Maximum lapse rate is less than minimum lapse rate" - with self.assertRaisesRegexp(ValueError, msg): + with self.assertRaisesRegex(ValueError, msg): LapseRate(max_lapse_rate=-1, min_lapse_rate=1).process( self.temperature, self.orography, self.land_sea_mask ) @@ -278,7 +278,7 @@ def test_fails_if_nbhood_radius_less_than_zero(self): is less than zero""" msg = "Neighbourhood radius is less than zero" - with self.assertRaisesRegexp(ValueError, msg): + with self.assertRaisesRegex(ValueError, msg): LapseRate(nbhood_radius=-1).process( self.temperature, self.orography, self.land_sea_mask ) @@ -288,7 +288,7 @@ def test_fails_if_max_height_diff_less_than_zero(self): is less than zero""" msg = "Maximum height difference is less than zero" - with self.assertRaisesRegexp(ValueError, msg): + with self.assertRaisesRegex(ValueError, msg): LapseRate(max_height_diff=-1).process( self.temperature, self.orography, self.land_sea_mask ) diff --git a/improver_tests/utilities/cube_extraction/test_cube_extraction.py b/improver_tests/utilities/cube_extraction/test_cube_extraction.py index 0712b96f08..20dbbc5dd0 100644 --- a/improver_tests/utilities/cube_extraction/test_cube_extraction.py +++ b/improver_tests/utilities/cube_extraction/test_cube_extraction.py @@ -38,7 +38,8 @@ def islambda(function): True if the input object is a lambda function, False if not. """ return ( - isinstance(function, collections.Callable) and function.__name__ == "" + isinstance(function, collections.abc.Callable) + and function.__name__ == "" ) diff --git a/improver_tests/utilities/test_load.py b/improver_tests/utilities/test_load.py index 7474dcf43b..a1737f1493 100644 --- a/improver_tests/utilities/test_load.py +++ b/improver_tests/utilities/test_load.py @@ -198,7 +198,7 @@ def test_attributes(self): def test_prefix_cube_removed(self): """Test metadata prefix cube is discarded during load""" msg = "No cubes found" - with self.assertRaisesRegexp(ValueError, msg): + with self.assertRaisesRegex(ValueError, msg): load_cube(self.filepath, "prefixes") def test_no_lazy_load(self): From 9c174f4168711244279b27717d4e12a5b4ef7513 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 17:25:51 +0100 Subject: [PATCH 09/15] Simplify test in test_flatten.py to avoid cubelist within a cubelist. --- improver_tests/utilities/test_flatten.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/improver_tests/utilities/test_flatten.py b/improver_tests/utilities/test_flatten.py index 8caa7d9bcb..81f8fbb532 100644 --- a/improver_tests/utilities/test_flatten.py +++ b/improver_tests/utilities/test_flatten.py @@ -23,10 +23,8 @@ def get_cube(name) -> Cube: ([1, [2, [3, 4], 5], [6, [7, 8]], 9], [1, 2, 3, 4, 5, 6, 7, 8, 9]), # Cubes and CubeLists ( - CubeList( - [get_cube("0"), get_cube("1"), CubeList([get_cube("2"), get_cube("3")])] - ), - [get_cube("0"), get_cube("1"), get_cube("2"), get_cube("3")], + CubeList([get_cube("0"), get_cube("1"), get_cube("2"), get_cube("3")]), + CubeList([get_cube("0"), get_cube("1"), get_cube("2"), get_cube("3")]), ), # Numpy arrays ( From 3391b04f41e7587548d3ed1d175256713ed55320 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Thu, 15 May 2025 17:51:01 +0100 Subject: [PATCH 10/15] Replace Cube(None) with an alternative. --- .../psychrometric_calculations/cloud_top_temperature.py | 6 +++--- improver/wind_calculations/vertical_updraught.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/improver/psychrometric_calculations/cloud_top_temperature.py b/improver/psychrometric_calculations/cloud_top_temperature.py index 20e47ef22c..166bb3cbde 100644 --- a/improver/psychrometric_calculations/cloud_top_temperature.py +++ b/improver/psychrometric_calculations/cloud_top_temperature.py @@ -43,9 +43,9 @@ def __init__(self, model_id_attr: str = None): Name of model ID attribute to be copied from source cubes to output cube """ self.model_id_attr = model_id_attr - self.t_at_ccl = Cube(None) - self.p_at_ccl = Cube(None) - self.temperature = Cube(None) + self.t_at_ccl = Cube(shape=(0,)) + self.p_at_ccl = Cube(shape=(0,)) + self.temperature = Cube(shape=(0,)) self.minimum_t_diff = 4 def _calculate_cct(self) -> ndarray: diff --git a/improver/wind_calculations/vertical_updraught.py b/improver/wind_calculations/vertical_updraught.py index 0d23ca3a61..18576e8dcc 100644 --- a/improver/wind_calculations/vertical_updraught.py +++ b/improver/wind_calculations/vertical_updraught.py @@ -39,8 +39,8 @@ def __init__(self, model_id_attr: str = None): Name of model ID attribute to be copied from source cubes to output cube """ self.model_id_attr = model_id_attr - self.cape = Cube(None) - self.precip = Cube(None) + self.cape = Cube(shape=(0,)) + self.precip = Cube(shape=(0,)) self.cube_names = [ "atmosphere_convective_available_potential_energy", "lwe_precipitation_rate_max", From c17dbb7eac198392dd98376b90e4416fc436cfb4 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Fri, 16 May 2025 10:49:12 +0100 Subject: [PATCH 11/15] Unpin environments for testing. --- envs/conda-forge.yml | 19 ++++++++----------- envs/environment_a.yml | 38 +++++++++++++++++++------------------- envs/environment_b.yml | 38 ++++++++++++++++++-------------------- envs/latest.yml | 15 ++++++--------- pyproject.toml | 2 +- setup.cfg | 2 +- 6 files changed, 53 insertions(+), 61 deletions(-) diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml index 2b43060cfe..ba23cc5989 100644 --- a/envs/conda-forge.yml +++ b/envs/conda-forge.yml @@ -8,19 +8,19 @@ name: improver_conda_forge channels: - conda-forge dependencies: - - python>=3.6 + - python=3 # Included in improver-feedstock requirements - - cartopy<0.20 - - cftime<1.5 - - cf-units=2.1.5 + - cartopy + - cftime + - cf-units - clize - dask - - iris>=3.0,<3.1 + - iris - netCDF4 - - numpy<1.21 - - scipy<1.7 + - numpy + - scipy - sigtools - - sphinx=5.3.0 + - sphinx # Additional libraries to run tests, not included in improver-feedstock - bandit - filelock @@ -29,6 +29,3 @@ dependencies: - sphinx-autodoc-typehints - sphinx_rtd_theme - threadpoolctl - # Pinned dependencies of dependencies - - pillow<=10.0.1 # https://github.com/metoppv/improver/issues/2010 - - pandas<=2.0.0 # https://github.com/metoppv/improver/issues/2010 \ No newline at end of file diff --git a/envs/environment_a.yml b/envs/environment_a.yml index be084e5548..0fbab7afe9 100644 --- a/envs/environment_a.yml +++ b/envs/environment_a.yml @@ -4,28 +4,28 @@ channels: - conda-forge - main dependencies: - - python=3.7 + - python=3 # Required - - cartopy=0.19 - - cftime=1.2.1 - - cf-units=2.1.5 - - clize=4.1.1 - - dask=2021.8.1 - - iris=3.0.3 - - netCDF4=1.4.1 - - numpy=1.20.2 - - pandas=1.2.4 - - pytz=2020.5 - - scipy=1.6.2 - - sigtools=2.0.2 - - sphinx=4.0.1 + - cartopy + - cftime + - cf-units + - clize + - dask + - iris + - netCDF4 + - numpy + - pandas + - pytz + - scipy + - sigtools + - sphinx # Optional - - fastparquet=0.7.1 - - statsmodels=0.12.2 - - numba=0.53.1 + - fastparquet + - statsmodels + - numba - pygam=0.8.0 - - pysteps=1.4.1 - - python-stratify=0.1.1 + - pysteps + - python-stratify - python-utils=3.5.2 # Development - astroid diff --git a/envs/environment_b.yml b/envs/environment_b.yml index 05e88d9929..53d9ea0373 100644 --- a/envs/environment_b.yml +++ b/envs/environment_b.yml @@ -7,26 +7,26 @@ name: improver_b channels: - conda-forge dependencies: - - python=3.8 + - python=3 # Required - - cartopy=0.19 - - cftime=1.2.1 - - cf-units=2.1.5 - - clize=4.2.0 - - dask=2021.8.1 - - iris=3.0.3 - - netCDF4=1.5.7 - - numpy=1.20 - - pandas=1.3.4 - - pytz=2020.5 - - scipy=1.6.2 - - sigtools=2.0.3 - - sphinx=4.0.2 + - cartopy + - cftime + - cf-units + - clize + - dask + - iris + - netCDF4 + - numpy + - pandas + - pytz + - scipy + - sigtools + - sphinx # Optional - - lightgbm=3.2.1 - - numba=0.53.1 - - python-stratify=0.2.post0 - - treelite=2.3.0 + - lightgbm + - numba + - python-stratify + - treelite # Development - astroid - bandit @@ -40,5 +40,3 @@ dependencies: - sphinx-autodoc-typehints - sphinx_rtd_theme - threadpoolctl - # pinned dependencies of dependencies - - pyparsing=3.1.2 diff --git a/envs/latest.yml b/envs/latest.yml index 3ee47a7dd0..5c957ae115 100644 --- a/envs/latest.yml +++ b/envs/latest.yml @@ -9,14 +9,14 @@ channels: dependencies: - python=3 # Required - - cartopy<0.20 - - cftime<1.5 - - cf-units=2.1.5 + - cartopy + - cftime + - cf-units - clize - dask - - iris>=3.0 + - iris - netCDF4 - - numpy<1.21 + - numpy - pytz - scipy - sigtools @@ -29,7 +29,7 @@ dependencies: - pygam=0.8.0 - pysteps - python-utils=3.5.2 - - treelite=2.3.0 + - treelite # Development - astroid - bandit @@ -44,6 +44,3 @@ dependencies: - sphinx-autodoc-typehints - sphinx_rtd_theme - threadpoolctl - # Pinned dependencies of dependencies - - pillow<=10.0.1 # https://github.com/metoppv/improver/issues/2010 - - pandas<=2.0.0 # https://github.com/metoppv/improver/issues/2010 diff --git a/pyproject.toml b/pyproject.toml index d79eef4746..bcd661f389 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.ruff] -target-version = "py37" +target-version = "py312" [tool.ruff.lint] extend-select = ["E", "F", "W", "I"] # add C90 later diff --git a/setup.cfg b/setup.cfg index c5d7d18c1a..886ce511e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ classifiers = Operating System :: OS Independent [options] -python_requires = >= 3.6 +python_requires = >= 3.12 packages = find: setup_requires = setuptools >= 38.3.0 From 978e8fca5acffb771708ec0117df82cee20c8ba3 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Fri, 16 May 2025 11:32:10 +0100 Subject: [PATCH 12/15] Remove pygam version. --- envs/environment_a.yml | 3 +-- envs/latest.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/envs/environment_a.yml b/envs/environment_a.yml index 0fbab7afe9..1e59ed1d3e 100644 --- a/envs/environment_a.yml +++ b/envs/environment_a.yml @@ -23,10 +23,9 @@ dependencies: - fastparquet - statsmodels - numba - - pygam=0.8.0 + - pygam - pysteps - python-stratify - - python-utils=3.5.2 # Development - astroid - bandit diff --git a/envs/latest.yml b/envs/latest.yml index 5c957ae115..f889568034 100644 --- a/envs/latest.yml +++ b/envs/latest.yml @@ -26,9 +26,8 @@ dependencies: - statsmodels - lightgbm - numba - - pygam=0.8.0 + - pygam - pysteps - - python-utils=3.5.2 - treelite # Development - astroid From caf563e267fffd2b5ff68ca78bc8e14bce380108 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Fri, 16 May 2025 12:19:59 +0100 Subject: [PATCH 13/15] Update improver_a.yml --- envs/environment_a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/envs/environment_a.yml b/envs/environment_a.yml index 1e59ed1d3e..e4a7679a2a 100644 --- a/envs/environment_a.yml +++ b/envs/environment_a.yml @@ -2,7 +2,6 @@ name: improver_a channels: - conda-forge - - main dependencies: - python=3 # Required From 33b171fb89909c2a1a7086fdd863fa3fc74d1394 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Mon, 19 May 2025 11:39:02 +0100 Subject: [PATCH 14/15] Remove non-essential dependencies from yml files and add pins. --- envs/conda-forge.yml | 13 ++++--------- envs/environment_a.yml | 13 ++++--------- envs/environment_b.yml | 13 ++++--------- envs/latest.yml | 7 +------ 4 files changed, 13 insertions(+), 33 deletions(-) diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml index ba23cc5989..6eca32f676 100644 --- a/envs/conda-forge.yml +++ b/envs/conda-forge.yml @@ -8,18 +8,13 @@ name: improver_conda_forge channels: - conda-forge dependencies: - - python=3 + - python=3.12 # Included in improver-feedstock requirements - cartopy - - cftime - - cf-units - clize - - dask - - iris - - netCDF4 - - numpy - - scipy - - sigtools + - iris=3.12 + - numpy=2.2 + - scipy=1.15 - sphinx # Additional libraries to run tests, not included in improver-feedstock - bandit diff --git a/envs/environment_a.yml b/envs/environment_a.yml index e4a7679a2a..b4decf377a 100644 --- a/envs/environment_a.yml +++ b/envs/environment_a.yml @@ -3,20 +3,15 @@ name: improver_a channels: - conda-forge dependencies: - - python=3 + - python=3.12 # Required - cartopy - - cftime - - cf-units - clize - - dask - - iris - - netCDF4 - - numpy + - iris=3.12 + - numpy=2.2 - pandas - pytz - - scipy - - sigtools + - scipy=1.15 - sphinx # Optional - fastparquet diff --git a/envs/environment_b.yml b/envs/environment_b.yml index 53d9ea0373..c33e3227ba 100644 --- a/envs/environment_b.yml +++ b/envs/environment_b.yml @@ -7,20 +7,15 @@ name: improver_b channels: - conda-forge dependencies: - - python=3 + - python=3.12 # Required - cartopy - - cftime - - cf-units - clize - - dask - - iris - - netCDF4 - - numpy + - iris=3.12 + - numpy=2.2 - pandas - pytz - - scipy - - sigtools + - scipy=1.15 - sphinx # Optional - lightgbm diff --git a/envs/latest.yml b/envs/latest.yml index f889568034..252fef578d 100644 --- a/envs/latest.yml +++ b/envs/latest.yml @@ -7,19 +7,14 @@ name: improver_latest channels: - conda-forge dependencies: - - python=3 + - python # Required - cartopy - - cftime - - cf-units - clize - - dask - iris - - netCDF4 - numpy - pytz - scipy - - sigtools - sphinx # Optional - python-stratify From 19581c94eb0fff85c44fa15df4cc1cd7fcf1d5a6 Mon Sep 17 00:00:00 2001 From: Gavin Evans Date: Tue, 20 May 2025 16:53:21 +0100 Subject: [PATCH 15/15] Minor edits following review comments. --- improver/calibration/__init__.py | 7 +++++-- improver_tests/utilities/test_flatten.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/improver/calibration/__init__.py b/improver/calibration/__init__.py index fb62ccd7c9..42339a75ed 100644 --- a/improver/calibration/__init__.py +++ b/improver/calibration/__init__.py @@ -7,7 +7,7 @@ """ from collections import OrderedDict -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Union from iris.cube import Cube, CubeList @@ -94,7 +94,10 @@ def split_forecasts_and_truth( return forecast, truth, land_sea_mask -def split_forecasts_and_coeffs(cubes: List, land_sea_mask_name: Optional[str] = None): +def split_forecasts_and_coeffs( + cubes: Union[List[CubeList[Cube]], List[List[Cube]]], + land_sea_mask_name: Optional[str] = None, +): """Split the input forecast, coefficients, static additional predictors, land sea-mask and probability template, if provided. The coefficients cubes and land-sea mask are identified based on their name. The diff --git a/improver_tests/utilities/test_flatten.py b/improver_tests/utilities/test_flatten.py index 81f8fbb532..c7a2dfbfc8 100644 --- a/improver_tests/utilities/test_flatten.py +++ b/improver_tests/utilities/test_flatten.py @@ -23,8 +23,8 @@ def get_cube(name) -> Cube: ([1, [2, [3, 4], 5], [6, [7, 8]], 9], [1, 2, 3, 4, 5, 6, 7, 8, 9]), # Cubes and CubeLists ( - CubeList([get_cube("0"), get_cube("1"), get_cube("2"), get_cube("3")]), - CubeList([get_cube("0"), get_cube("1"), get_cube("2"), get_cube("3")]), + [get_cube("0"), get_cube("1"), CubeList([get_cube("2")]), get_cube("3")], + [get_cube("0"), get_cube("1"), get_cube("2"), get_cube("3")], ), # Numpy arrays (