From ba60b058aa473795b20926ac3e5700839e847853 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 19 Jul 2022 08:54:24 +0100 Subject: [PATCH 1/2] libcnb-test: Overhaul the README and crate docs Whilst the individual libcnb-test APIs already had rustdocs and examples, discovering what features existed and where in the modules/types hierarchy to look for these was hard for users new to the crate. Now, the README covers: - what the test framework does - prerequisites for the host machine - examples for all common use-cases - tips/best practices Some rustdocs for individual APIs have been updated to match the style used in the README for consistency. Lastly, the "experimental" label has been removed, since the crate is now mostly complete. GUS-W-11468003. --- examples/execd/tests/integration_test.rs | 10 +- libcnb-test/CHANGELOG.md | 1 + libcnb-test/README.md | 224 +++++++++++++++++++---- libcnb-test/src/build_config.rs | 10 +- libcnb-test/src/test_context.rs | 8 +- libcnb-test/src/test_runner.rs | 40 +--- libcnb-test/tests/integration_test.rs | 7 +- 7 files changed, 219 insertions(+), 81 deletions(-) diff --git a/examples/execd/tests/integration_test.rs b/examples/execd/tests/integration_test.rs index 0252e61b..f6e94fed 100644 --- a/examples/execd/tests/integration_test.rs +++ b/examples/execd/tests/integration_test.rs @@ -15,11 +15,11 @@ fn basic() { TestRunner::default().build( BuildConfig::new("heroku/builder:22", "test-fixtures/empty-app"), |context| { - let log_output = context.run_shell_command("env"); - assert_empty!(log_output.stderr); - assert_contains!(log_output.stdout, "ROLL_1D6="); - assert_contains!(log_output.stdout, "ROLL_4D6="); - assert_contains!(log_output.stdout, "ROLL_1D20="); + let command_output = context.run_shell_command("env"); + assert_empty!(command_output.stderr); + assert_contains!(command_output.stdout, "ROLL_1D6="); + assert_contains!(command_output.stdout, "ROLL_4D6="); + assert_contains!(command_output.stdout, "ROLL_1D20="); }, ); } diff --git a/libcnb-test/CHANGELOG.md b/libcnb-test/CHANGELOG.md index b791e645..a3ce5a4c 100644 --- a/libcnb-test/CHANGELOG.md +++ b/libcnb-test/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Overhaul the crate README/docs, to improve the learning/onboarding UX. ([#478](https://github.com/heroku/libcnb.rs/pull/478)) - Rename `TestRunner::run_test` to `TestRunner::build`, `TestConfig` to `BuildConfig` and `TestContext::run_test` to `TestContext::rebuild`. ([#470](https://github.com/heroku/libcnb.rs/pull/470)) - Add `TestContext::start_container`, `TestContext::run_shell_command` and `ContainerConfig`. ([#469](https://github.com/heroku/libcnb.rs/pull/469)) - Remove `TestContext::prepare_container` and `PrepareContainerContext`. To start a container use `TestContext::start_container` combined with `ContainerConfig` (or else the convenience function `TestContext::run_shell_command`) instead. ([#469](https://github.com/heroku/libcnb.rs/pull/469)) diff --git a/libcnb-test/README.md b/libcnb-test/README.md index 68a466f3..45d97f65 100644 --- a/libcnb-test/README.md +++ b/libcnb-test/README.md @@ -1,62 +1,224 @@ # libcnb-test   [![Docs]][docs.rs] [![Latest Version]][crates.io] [![MSRV]][install-rust] -An experimental integration testing framework for Cloud Native Buildpacks written in Rust with libcnb.rs. +An integration testing framework for Cloud Native Buildpacks written in Rust with [libcnb.rs](https://github.com/heroku/libcnb.rs). -## Experimental +The framework: +- Automatically cross-compiles and packages the buildpack under test +- Performs a build with specified configuration using `pack build` +- Supports starting containers using the resultant application image +- Supports concurrent test execution +- Handles cleanup of the test containers and images +- Provides additional test assertion macros to simplify common test scenarios (for example, `assert_contains!`) -This crate is marked as experimental. It currently implements the most basic building blocks for writing -integration tests with libcnb.rs. Its feature set is deliberately cut down to get ball rolling and get a better feel -which features are required. See [issues tagged with `libcnb-test`][libcnb-test-label] for possible future improvements. -Please use the same tag for feature requests. +## Dependencies -[libcnb-test-label]: https://github.com/heroku/libcnb.rs/labels/libcnb-test +Integration tests require the following to be available on the host: -## Example +- [Docker](https://docs.docker.com/engine/install/) +- [Pack CLI](https://buildpacks.io/docs/install-pack/) +- [Cross-compilation prerequisites](https://docs.rs/libcnb/latest/libcnb/#cross-compilation-prerequisites) (however `libcnb-cargo` itself is not required) + +Only local Docker daemons are fully supported. As such, if you are using Circle CI you must use the +[`machine` executor](https://circleci.com/docs/2.0/executor-types/#using-machine) rather than the +[remote docker](https://circleci.com/docs/2.0/building-docker-images/) feature. + +## Examples + +A basic test that performs a build with the specified builder image and app source fixture, +and then asserts against the resultant `pack build` log output: ```rust,no_run // In $CRATE_ROOT/tests/integration_test.rs -use libcnb_test::{assert_contains, BuildConfig, ContainerConfig, TestRunner}; +use libcnb_test::{assert_contains, assert_empty, BuildConfig, TestRunner}; -// In your code you'll want to mark your function as a test with `#[test]`. -// It is removed here for compatibility with doctest so this code in the readme -// tests for compilation. -fn test() { +// Note: In your code you'll want to uncomment the `#[test]` annotation here. +// It's commented out in these examples so that this documentation can be +// run as a `doctest` and so checked for correctness in CI. +// #[test] +fn basic() { TestRunner::default().build( BuildConfig::new("heroku/builder:22", "test-fixtures/app"), |context| { - assert_contains!(context.pack_stdout, "---> Maven Buildpack"); - assert_contains!(context.pack_stdout, "---> Installing Maven"); - assert_contains!(context.pack_stdout, "---> Running mvn package"); + assert_empty!(context.pack_stderr); + assert_contains!(context.pack_stdout, "Expected build output"); + }, + ); +} +``` + +Performing a second build of the same image to test cache handling, using [`TestContext::rebuild`]: + +```rust,no_run +use libcnb_test::{assert_contains, BuildConfig, TestRunner}; + +// #[test] +fn rebuild() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/app"), + |context| { + assert_contains!(context.pack_stdout, "Installing dependencies"); + + let config = context.config.clone(); + context.rebuild(config, |rebuild_context| { + assert_contains!(rebuild_context.pack_stdout, "Using cached dependencies"); + }); + }, + ); +} +``` + +Testing expected buildpack failures, using [`BuildConfig::expected_pack_result`]: + +```rust,no_run +use libcnb_test::{assert_contains, BuildConfig, PackResult, TestRunner}; + +// #[test] +fn expected_pack_failure() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/invalid-app") + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!(context.pack_stderr, "ERROR: Invalid Procfile!"); + }, + ); +} +``` + +Running a shell command against the built image, using [`TestContext::run_shell_command`]: + +```rust,no_run +use libcnb_test::{assert_empty, BuildConfig, TestRunner}; + +// #[test] +fn starting_web_server_container() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/app"), + |context| { + // ... + let command_output = context.run_shell_command("python --version"); + assert_empty!(command_output.stderr); + assert_eq!(command_output.stdout, "Python 3.10.4\n"); + }, + ); +} +``` +Starting a container using the default process with an exposed port to test a web server, using [`TestContext::start_container`]: + +```rust,no_run +use libcnb_test::{assert_contains, assert_empty, BuildConfig, ContainerConfig, TestRunner}; +use std::thread; +use std::time::Duration; + +const TEST_PORT: u16 = 12345; + +// #[test] +fn starting_web_server_container() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/app"), + |context| { + // ... context.start_container( ContainerConfig::new() - .env("PORT", "12345") - .expose_port(12345), + .env("PORT", TEST_PORT.to_string()) + .expose_port(TEST_PORT), |container| { - assert_eq!( - call_test_fixture_service( - container.address_for_port(12345).unwrap(), - "Hagbard Celine" - ) - .unwrap(), - "enileC drabgaH" + let address_on_host = container.address_for_port(TEST_PORT).unwrap(); + let url = format!("http://{}:{}", address_on_host.ip(), address_on_host.port()); + + // Give the server time to start. + thread::sleep(Duration::from_secs(2)); + + let server_log_output = container.logs_now(); + assert_empty!(server_log_output.stderr); + assert_contains!( + server_log_output.stdout, + &format!("Listening on port {TEST_PORT}") ); + + let response = ureq::get(&url).call().unwrap(); + let body = response.into_string().unwrap(); + assert_contains!(body, "Expected response substring"); }, ); }, ); } +``` + +Inspecting an already running container using Docker Exec, using [`ContainerContext::shell_exec`]: + +```rust,no_run +use libcnb_test::{assert_contains, BuildConfig, ContainerConfig, TestRunner}; + +// #[test] +fn shell_exec() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/app"), + |context| { + // ... + context.start_container(ContainerConfig::new(), |container| { + // ... + let exec_log_output = container.shell_exec("ps"); + assert_contains!(exec_log_output.stdout, "nginx"); + }); + }, + ); +} +``` + +Dynamically modifying test fixtures during test setup, using [`BuildConfig::app_dir_preprocessor`]: + +```rust,no_run +use libcnb_test::{BuildConfig, TestRunner}; +use std::fs; + +// #[test] +fn dynamic_fixture() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/app").app_dir_preprocessor( + |app_dir| { + fs::write(app_dir.join("runtime.txt"), "python-3.10").unwrap(); + }, + ), + |context| { + // ... + }, + ); +} +``` + +Building with multiple buildpacks, using [`BuildConfig::buildpacks`]: + +```rust,no_run +use libcnb_test::{BuildConfig, BuildpackReference, TestRunner}; -fn call_test_fixture_service(addr: std::net::SocketAddr, payload: &str) -> Result { - unimplemented!() +// #[test] +fn additional_buildpacks() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/app").buildpacks(vec![ + BuildpackReference::Crate, + BuildpackReference::Other(String::from("heroku/another-buildpack")), + ]), + |context| { + // ... + }, + ); } ``` -## Known issues +## Tips -- Only local Docker daemons are fully supported. If using Circle CI you must use the - [`machine` executor](https://circleci.com/docs/2.0/executor-types/#using-machine) rather - than the [remote docker](https://circleci.com/docs/2.0/building-docker-images/) feature. +- Rust tests are automatically run in parallel, however only if they are in the same crate. + For integration tests Rust compiles each file as a separate crate. As such, make sure to + include all integration tests in a single file (either inlined or by including additional + test modules) to ensure they run in parallel. +- If you would like to be able to more easily run your unit tests and integration tests + separately, annotate each integration test with `#[ignore = "integration test"]`, which + causes `cargo test` to skip them (running unit/doc tests only). The integration tests + can then be run using `cargo test -- --ignored`, or all tests can be run at once using + `cargo test -- --include-ignored`. +- If you wish to assert against multi-line log output, see the [indoc](https://crates.io/crates/indoc) crate. [Docs]: https://img.shields.io/docsrs/libcnb-test [docs.rs]: https://docs.rs/libcnb-test/latest/libcnb_test/ diff --git a/libcnb-test/src/build_config.rs b/libcnb-test/src/build_config.rs index 5c31f663..644aa4f5 100644 --- a/libcnb-test/src/build_config.rs +++ b/libcnb-test/src/build_config.rs @@ -18,7 +18,7 @@ pub struct BuildConfig { } impl BuildConfig { - /// Creates a new test configuration. + /// Creates a new build configuration. /// /// If the `app_dir` parameter is a relative path, it is treated as relative to the Cargo /// manifest directory ([`CARGO_MANIFEST_DIR`](https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates)), @@ -48,7 +48,7 @@ impl BuildConfig { } } - /// Sets the buildpacks order. + /// Sets the buildpacks (and their ordering) to use when building the app. /// /// Defaults to [`BuildpackReference::Crate`]. /// @@ -220,7 +220,7 @@ impl BuildConfig { /// Set the expected `pack` command result. /// - /// In some cases, users might want to explicitly test that a build fails and asserting against + /// In some cases, users might want to explicitly test that a build fails and assert against /// error output. When passed [`PackResult::Failure`], the test will fail if the pack build /// succeeds and vice-versa. /// @@ -228,13 +228,13 @@ impl BuildConfig { /// /// # Example /// ```no_run - /// use libcnb_test::{BuildConfig, PackResult, TestRunner}; + /// use libcnb_test::{assert_contains, BuildConfig, PackResult, TestRunner}; /// /// TestRunner::default().build( /// BuildConfig::new("heroku/builder:22", "test-fixtures/app") /// .expected_pack_result(PackResult::Failure), /// |context| { - /// // ... + /// assert_contains!(context.pack_stderr, "ERROR: Invalid Procfile!"); /// }, /// ); /// ``` diff --git a/libcnb-test/src/test_context.rs b/libcnb-test/src/test_context.rs index d6bbe881..d85a2dcb 100644 --- a/libcnb-test/src/test_context.rs +++ b/libcnb-test/src/test_context.rs @@ -125,8 +125,8 @@ impl<'a> TestContext<'a> { /// BuildConfig::new("heroku/builder:22", "test-fixtures/app"), /// |context| { /// // ... - /// let log_output = context.run_shell_command("for i in {1..3}; do echo \"${i}\"; done"); - /// assert_eq!(log_output.stdout, "1\n2\n3\n"); + /// let command_output = context.run_shell_command("for i in {1..3}; do echo \"${i}\"; done"); + /// assert_eq!(command_output.stdout, "1\n2\n3\n"); /// }, /// ); /// ``` @@ -184,11 +184,11 @@ impl<'a> TestContext<'a> { /// TestRunner::default().build( /// BuildConfig::new("heroku/builder:22", "test-fixtures/app"), /// |context| { - /// assert_contains!(context.pack_stdout, "---> Installing gems"); + /// assert_contains!(context.pack_stdout, "---> Installing dependencies"); /// /// let config = context.config.clone(); /// context.rebuild(config, |context| { - /// assert_contains!(context.pack_stdout, "---> Using cached gems"); + /// assert_contains!(context.pack_stdout, "---> Using cached dependencies"); /// }); /// }, /// ); diff --git a/libcnb-test/src/test_runner.rs b/libcnb-test/src/test_runner.rs index 913f2a9a..f1c81e44 100644 --- a/libcnb-test/src/test_runner.rs +++ b/libcnb-test/src/test_runner.rs @@ -9,42 +9,17 @@ use std::{env, io}; /// Runner for libcnb integration tests. /// -/// # Dependencies -/// Integration tests require external tools to be available on the host to run: -/// - [pack](https://buildpacks.io/docs/tools/pack/) -/// - [Docker](https://www.docker.com/) -/// /// # Example /// ```no_run -/// use libcnb_test::{assert_contains, BuildConfig, ContainerConfig, TestRunner}; +/// use libcnb_test::{assert_contains, assert_empty, BuildConfig, TestRunner}; /// -/// # fn call_test_fixture_service(addr: std::net::SocketAddr, payload: &str) -> Result { -/// # unimplemented!() -/// # } /// TestRunner::default().build( /// BuildConfig::new("heroku/builder:22", "test-fixtures/app"), /// |context| { -/// assert_contains!(context.pack_stdout, "---> Maven Buildpack"); -/// assert_contains!(context.pack_stdout, "---> Installing Maven"); -/// assert_contains!(context.pack_stdout, "---> Running mvn package"); -/// -/// context.start_container( -/// ContainerConfig::new() -/// .env("PORT", "12345") -/// .expose_port(12345), -/// |container| { -/// assert_eq!( -/// call_test_fixture_service( -/// container.address_for_port(12345).unwrap(), -/// "Hagbard Celine" -/// ) -/// .unwrap(), -/// "enileC drabgaH" -/// ); -/// }, -/// ); +/// assert_empty!(context.pack_stderr); +/// assert_contains!(context.pack_stdout, "Expected build output"); /// }, -/// ); +/// ) /// ``` pub struct TestRunner { pub(crate) docker: Docker, @@ -105,14 +80,13 @@ impl TestRunner { /// /// # Example /// ```no_run - /// use libcnb_test::{assert_contains, BuildConfig, TestRunner}; + /// use libcnb_test::{assert_contains, assert_empty, BuildConfig, TestRunner}; /// /// TestRunner::default().build( /// BuildConfig::new("heroku/builder:22", "test-fixtures/app"), /// |context| { - /// assert_contains!(context.pack_stdout, "---> Ruby Buildpack"); - /// assert_contains!(context.pack_stdout, "---> Installing bundler"); - /// assert_contains!(context.pack_stdout, "---> Installing gems"); + /// assert_empty!(context.pack_stderr); + /// assert_contains!(context.pack_stdout, "Expected build output"); /// }, /// ) /// ``` diff --git a/libcnb-test/tests/integration_test.rs b/libcnb-test/tests/integration_test.rs index cbc24405..8deeb5f2 100644 --- a/libcnb-test/tests/integration_test.rs +++ b/libcnb-test/tests/integration_test.rs @@ -125,9 +125,10 @@ fn starting_containers() { }, ); - let log_output = context.run_shell_command("for i in {1..3}; do echo \"${i}\"; done"); - assert_empty!(log_output.stderr); - assert_eq!(log_output.stdout, "1\n2\n3\n"); + let command_output = + context.run_shell_command("for i in {1..3}; do echo \"${i}\"; done"); + assert_empty!(command_output.stderr); + assert_eq!(command_output.stdout, "1\n2\n3\n"); }, ); } From a9cb16bcbcaf286e11f0dab8a83f95313ac6d21e Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 21 Jul 2022 19:16:30 +0100 Subject: [PATCH 2/2] Correct example test name --- libcnb-test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcnb-test/README.md b/libcnb-test/README.md index 45d97f65..7b17aba7 100644 --- a/libcnb-test/README.md +++ b/libcnb-test/README.md @@ -90,7 +90,7 @@ Running a shell command against the built image, using [`TestContext::run_shell_ use libcnb_test::{assert_empty, BuildConfig, TestRunner}; // #[test] -fn starting_web_server_container() { +fn run_shell_command() { TestRunner::default().build( BuildConfig::new("heroku/builder:22", "test-fixtures/app"), |context| {