diff --git a/.env b/.env index e14bc3897..adc90a62d 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -FAABRIC_VERSION=0.3.4 -FAABRIC_CLI_IMAGE=faasm/faabric:0.3.4 +FAABRIC_VERSION=0.3.5 +FAABRIC_CLI_IMAGE=faasm/faabric:0.3.5 COMPOSE_PROJECT_NAME=faabric-dev CONAN_CACHE_MOUNT_SOURCE=./conan-cache/ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b2cc99531..78ab32090 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm/faabric:0.3.4 + image: faasm/faabric:0.3.5 defaults: run: working-directory: /code/faabric @@ -25,13 +25,13 @@ jobs: with: directory: /code/faabric - name: "Build dependencies to be shared by all runs" - run: inv dev.cmake -b Debug + run: ./bin/inv_wrapper.sh dev.cmake -b Debug docs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm/faabric:0.3.4 + image: faasm/faabric:0.3.5 defaults: run: working-directory: /code/faabric @@ -41,13 +41,13 @@ jobs: - name: "Check out branch" run: git checkout --force ci-branch - name: "Build docs" - run: inv docs + run: ./bin/inv_wrapper.sh docs formatting: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm/faabric:0.3.4 + image: faasm/faabric:0.3.5 defaults: run: working-directory: /code/faabric @@ -81,7 +81,7 @@ jobs: TSAN_OPTIONS: "verbosity=1:halt_on_error=1:suppressions=/code/faabric/thread-sanitizer-ignorelist.txt:history_size=7" UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" container: - image: faasm/faabric:0.3.4 + image: faasm/faabric:0.3.5 options: --privileged defaults: run: @@ -96,9 +96,9 @@ jobs: - name: "Ping redis" run: redis-cli -h redis ping - name: "Run cmake for tests" - run: inv dev.cmake --build=Debug --sanitiser ${{ matrix.sanitiser }} + run: ./bin/inv_wrapper.sh dev.cmake --build=Debug --sanitiser ${{ matrix.sanitiser }} - name: "Build tests" - run: inv dev.cc faabric_tests + run: ./bin/inv_wrapper.sh dev.cc faabric_tests - name: "Run tests" run: ./bin/faabric_tests working-directory: /build/faabric/static @@ -131,7 +131,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: faasm/faabric:0.3.4 + image: faasm/faabric:0.3.5 defaults: run: working-directory: /code/faabric @@ -144,12 +144,12 @@ jobs: build-type: "release" directory: /code/faabric - name: "Run cmake shared" - run: inv dev.cmake --shared --build=Release + run: ./bin/inv_wrapper.sh dev.cmake --shared --build=Release - name: "Build Faabric shared library" - run: inv dev.cc faabric --shared + run: ./bin/inv_wrapper.sh dev.cc faabric --shared - name: "Install Faabric shared library" - run: inv dev.install faabric --shared + run: ./bin/inv_wrapper.sh dev.install faabric --shared - name: "Build examples" - run: inv examples + run: ./bin/inv_wrapper.sh examples - name: "Run example to check" - run: inv examples.execute check + run: ./bin/inv_wrapper.sh examples.execute check diff --git a/.gitignore b/.gitignore index 4d1d564ba..9766f84bd 100644 --- a/.gitignore +++ b/.gitignore @@ -108,4 +108,5 @@ dkms.conf __pycache__/ *.pyc venv/ +venv-bm/ *.egg-info/ diff --git a/VERSION b/VERSION index 42045acae..c2c0004f0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.4 +0.3.5 diff --git a/bin/create_venv.sh b/bin/create_venv.sh new file mode 100755 index 000000000..a877bbeca --- /dev/null +++ b/bin/create_venv.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +THIS_DIR=$(dirname $(readlink -f $0)) +PROJ_ROOT=${THIS_DIR}/.. + +# Set different virtual environment paths so that these don't clash when +# mounting the code in a development container +VENV_PATH="undetected" +if [[ -z "$FAABRIC_DOCKER" ]]; then + VENV_PATH="${PROJ_ROOT}/venv-bm" +else + VENV_PATH="/code/faabric/venv" +fi + +PIP=${VENV_PATH}/bin/pip3 + +function pip_cmd { + source ${VENV_PATH}/bin/activate && ${PIP} "$@" +} + +pushd ${PROJ_ROOT} >> /dev/null + +if [ ! -d ${VENV_PATH} ]; then + python3 -m venv ${VENV_PATH} +fi + +pip_cmd install -U pip +pip_cmd install -U setuptools wheel +pip_cmd install -r requirements.txt + +popd >> /dev/null diff --git a/bin/inv_wrapper.sh b/bin/inv_wrapper.sh new file mode 100755 index 000000000..497d6a2c6 --- /dev/null +++ b/bin/inv_wrapper.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +# Wrapper script for running invoke in virtual env + +source bin/workon.sh && inv $@ diff --git a/bin/workon.sh b/bin/workon.sh index ec0d79d13..51eb596fc 100755 --- a/bin/workon.sh +++ b/bin/workon.sh @@ -7,11 +7,9 @@ MODE="undetected" if [[ -z "$FAABRIC_DOCKER" ]]; then - THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - if [ "$(ps -o comm= -p $$)" = "zsh" ]; then - THIS_DIR="$( cd "$( dirname "${ZSH_ARGZERO}" )" >/dev/null 2>&1 && pwd )" - fi + THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )" PROJ_ROOT="${THIS_DIR}/.." + VENV_PATH="${PROJ_ROOT}/venv-bm" export FAABRIC_BUILD_DIR="${PROJ_ROOT}/build" # Normal terminal @@ -19,6 +17,7 @@ if [[ -z "$FAABRIC_DOCKER" ]]; then else # Running inside the container, we know the project root PROJ_ROOT="/code/faabric" + VENV_PATH="${PROJ_ROOT}/venv" export FAABRIC_BUILD_DIR="/build/faabric" # Use containerised redis @@ -33,19 +32,12 @@ pushd ${PROJ_ROOT}>>/dev/null # Virtualenv # ---------------------------- -if [ ! -d "venv" ]; then - python3 -m venv venv - ( - source venv/bin/activate - pip install -U pip - pip install -U setuptools - pip install -U wheel - pip install -r requirements.txt - ) +if [ ! -d ${VENV_PATH} ]; then + ${PROJ_ROOT}/bin/create_venv.sh fi export VIRTUAL_ENV_DISABLE_PROMPT=1 -source venv/bin/activate +source ${VENV_PATH}/bin/activate # ---------------------------- # Invoke tab-completion diff --git a/docker/faabric-base.dockerfile b/docker/faabric-base.dockerfile index d757da399..afcbbef58 100644 --- a/docker/faabric-base.dockerfile +++ b/docker/faabric-base.dockerfile @@ -1,15 +1,22 @@ FROM ubuntu:20.04 -RUN apt-get update -RUN apt-get install -y software-properties-common gpg wget curl -RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add - -RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add - -RUN add-apt-repository -y -n "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" -RUN add-apt-repository -y -n "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" -RUN add-apt-repository -y -n "deb https://apt.kitware.com/ubuntu/ focal main" -RUN add-apt-repository -y -n ppa:ubuntu-toolchain-r/test +# Configure APT repositories +RUN apt update \ + && apt upgrade -y \ + && apt install -y \ + curl \ + gpg \ + software-properties-common \ + wget \ + && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add - \ + && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add - \ + && add-apt-repository -y -n "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" \ + && add-apt-repository -y -n "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" \ + && add-apt-repository -y -n "deb https://apt.kitware.com/ubuntu/ focal main" \ + && add-apt-repository -y -n ppa:ubuntu-toolchain-r/test -RUN apt update -y && apt install -y \ +# Install APT packages +RUN apt update && apt install -y \ autoconf \ automake \ build-essential \ @@ -49,11 +56,12 @@ RUN apt update -y && apt install -y \ sudo \ unzip -RUN curl -s -L -o /tmp/conan-latest.deb https://github.com/conan-io/conan/releases/download/1.52.0/conan-ubuntu-64.deb && sudo dpkg -i /tmp/conan-latest.deb && rm -f /tmp/conan-latest.deb - -# Update pip -RUN pip install -U pip +# Install Conan +RUN curl -s -L -o \ + /tmp/conan-latest.deb https://github.com/conan-io/conan/releases/download/1.52.0/conan-ubuntu-64.deb \ + && sudo dpkg -i /tmp/conan-latest.deb \ + && rm -f /tmp/conan-latest.deb # Tidy up -RUN apt-get clean autoclean -RUN apt-get autoremove +RUN apt clean autoclean -y \ + && apt autoremove -y diff --git a/docker/faabric.dockerfile b/docker/faabric.dockerfile index f7784afcc..8464ebf71 100644 --- a/docker/faabric.dockerfile +++ b/docker/faabric.dockerfile @@ -1,7 +1,8 @@ -FROM faasm/faabric-base:0.3.4 +FROM faasm/faabric-base:0.3.5 ARG FAABRIC_VERSION # faabic-base image is not re-built often, so tag may be behind +SHELL ["/bin/bash", "-c"] # Flag to say we're in a container ENV FAABRIC_DOCKER="on" @@ -11,25 +12,24 @@ WORKDIR /code RUN git clone -b v${FAABRIC_VERSION} https://github.com/faasm/faabric WORKDIR /code/faabric -RUN pip3 install -r requirements.txt - -# Static build -RUN inv dev.cmake --build=Release -RUN inv dev.cc faabric -RUN inv dev.cc faabric_tests -# Shared build -RUN inv dev.cmake --shared --build=Release -RUN inv dev.cc faabric --shared -RUN inv dev.install faabric --shared +# Python set-up and code builds +RUN ./bin/create_venv.sh \ + && source venv/bin/activate \ + # Static build + && inv dev.cmake --build=Release \ + && inv dev.cc faabric \ + && inv dev.cc faabric_tests \ + # Shared build + && inv dev.cmake --shared --build=Release \ + && inv dev.cc faabric --shared \ + && inv dev.install faabric --shared # GDB config, allow loading repo-specific config -RUN touch /root/.gdbinit RUN echo "set auto-load safe-path /" > /root/.gdbinit # CLI setup ENV TERM xterm-256color -SHELL ["/bin/bash", "-c"] RUN echo ". /code/faabric/bin/workon.sh" >> ~/.bashrc CMD ["/bin/bash", "-l"] diff --git a/docs/source/development.md b/docs/source/development.md index 52b4bf8e1..131bfe2e2 100644 --- a/docs/source/development.md +++ b/docs/source/development.md @@ -129,12 +129,18 @@ docker compose stop ## Creating a new tag -Create a new branch, then find and replace the current version with the relevant -bumped version. It should appear in: +Create a new branch, then bump the code version: -- `VERSION` -- `.env` -- `.github/workflows/tests.yml`. +```bash +inv git.bump +``` + +This will increment the minor version, to bump the code to an arbitrary version +you can run: + +```bash +inv git.bump --ver= +``` Once done, commit and push, then run: @@ -171,13 +177,17 @@ To build the main container, run: source bin/workon.sh # Build -inv container.build +inv docker.build -c faabric [--push] +``` + +The base container is not re-built often, and not re-built as part of Github +Actions. If you ever need to add an APT dependency, or update the Conan +version, run: -# Push -inv container.push +```bash +source bin/workon.sh -# Build and push -inv container.build --push +inv docker.build -c faabric-base [--push] ``` ## Publishing a release diff --git a/tasks/__init__.py b/tasks/__init__.py index f125572d2..49b5cc4ec 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -1,16 +1,16 @@ from invoke import Collection from . import call -from . import container from . import dev +from . import docker from . import docs from . import examples from . import git ns = Collection( call, - container, dev, + docker, docs, examples, git, diff --git a/tasks/container.py b/tasks/docker.py similarity index 54% rename from tasks/container.py rename to tasks/docker.py index 7b788c569..6f090593e 100644 --- a/tasks/container.py +++ b/tasks/docker.py @@ -49,33 +49,35 @@ def _do_push(name): run(cmd, shell=True, check=True) -@task(default=True) -def build(ctx, nocache=False, push=False): +@task(iterable=["c"]) +def build(ctx, c, nocache=False, push=False): """ - Build current version of faabric container + Build containers for faabric. Targets are: `faabric`, and `faabric-base` """ - _do_container_build(FAABRIC_IMAGE_NAME, nocache=nocache, push=push) + for ctr in c: + if ctr == "faabric": + img_name = FAABRIC_IMAGE_NAME + elif ctr == "faabric-base": + img_name = FAABRIC_BASE_IMAGE_NAME + else: + print("Unrecognised container name: {}".format(ctr)) + raise RuntimeError("Unrecognised container name") - -@task -def build_base(ctx, nocache=False, push=False): - """ - Build faabric's base container - """ - _do_container_build(FAABRIC_BASE_IMAGE_NAME, nocache=nocache, push=push) - - -@task -def push(ctx): - """ - Push current version of faabric container - """ - _do_push(FAABRIC_IMAGE_NAME) + _do_container_build(img_name, nocache=nocache, push=push) -@task -def push_base(ctx): +@task(iterable=["c"]) +def push(ctx, c): """ - Push faabric's base container + Push containers for faabric. Targets are: `faabric`, and `faabric-base` """ - _do_push(FAABRIC_BASE_IMAGE_NAME) + for ctr in c: + if ctr == "faabric": + img_name = FAABRIC_IMAGE_NAME + elif ctr == "faabric-base": + img_name = FAABRIC_BASE_IMAGE_NAME + else: + print("Unrecognised container name: {}".format(ctr)) + raise RuntimeError("Unrecognised container name") + + _do_push(img_name) diff --git a/tasks/git.py b/tasks/git.py index 3f2eed174..81746e0ee 100644 --- a/tasks/git.py +++ b/tasks/git.py @@ -3,11 +3,39 @@ from tasks.util.env import get_faabric_config, get_version, PROJ_ROOT from subprocess import run, PIPE, STDOUT +VERSIONED_FILES = [ + ".env", + ".github/workflows/tests.yml", + "VERSION", +] + def get_tag_name(version): return "v{}".format(version) +@task +def bump(ctx, ver=None): + """ + Bump code version + """ + old_ver = get_version() + + if ver: + new_ver = ver + else: + # Just bump the last minor version part + new_ver_parts = old_ver.split(".") + new_ver_minor = int(new_ver_parts[-1]) + 1 + new_ver_parts[-1] = str(new_ver_minor) + new_ver = ".".join(new_ver_parts) + + # Replace version in all files + for f in VERSIONED_FILES: + sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f) + run(sed_cmd, shell=True, check=True) + + @task def tag(ctx, force=False): """