faasm · csegarragonz · Oct 3, 2022 · Sep 28, 2022 · Sep 28, 2022 · Sep 28, 2022
diff --git a/.env b/.env
@@ -1,4 +1,4 @@
-FAABRIC_VERSION=0.3.4
-FAABRIC_CLI_IMAGE=faasm/faabric:0.3.4
+FAABRIC_VERSION=0.3.5
+FAABRIC_CLI_IMAGE=faasm/faabric:0.3.5
 COMPOSE_PROJECT_NAME=faabric-dev
 CONAN_CACHE_MOUNT_SOURCE=./conan-cache/
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -16,7 +16,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/faabric:0.3.4
+      image: faasm/faabric:0.3.5
     defaults:
       run:
         working-directory: /code/faabric
@@ -25,13 +25,13 @@ jobs:
         with:
            directory: /code/faabric
       - name: "Build dependencies to be shared by all runs"
-        run: inv dev.cmake -b Debug
+        run: ./bin/inv_wrapper.sh dev.cmake -b Debug
 
   docs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/faabric:0.3.4
+      image: faasm/faabric:0.3.5
     defaults:
       run:
         working-directory: /code/faabric
@@ -41,13 +41,13 @@ jobs:
       - name: "Check out branch"
         run: git checkout --force ci-branch
       - name: "Build docs"
-        run: inv docs
+        run: ./bin/inv_wrapper.sh docs
 
   formatting:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/faabric:0.3.4
+      image: faasm/faabric:0.3.5
     defaults:
       run:
         working-directory: /code/faabric
@@ -81,7 +81,7 @@ jobs:
       TSAN_OPTIONS: "verbosity=1:halt_on_error=1:suppressions=/code/faabric/thread-sanitizer-ignorelist.txt:history_size=7"
       UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
     container:
-      image: faasm/faabric:0.3.4
+      image: faasm/faabric:0.3.5
       options: --privileged
     defaults:
       run:
@@ -96,9 +96,9 @@ jobs:
       - name: "Ping redis"
         run: redis-cli -h redis ping
       - name: "Run cmake for tests"
-        run: inv dev.cmake --build=Debug --sanitiser ${{ matrix.sanitiser }}
+        run: ./bin/inv_wrapper.sh dev.cmake --build=Debug --sanitiser ${{ matrix.sanitiser }}
       - name: "Build tests"
-        run: inv dev.cc faabric_tests
+        run: ./bin/inv_wrapper.sh dev.cc faabric_tests
       - name: "Run tests"
         run: ./bin/faabric_tests
         working-directory: /build/faabric/static
@@ -131,7 +131,7 @@ jobs:
       REDIS_QUEUE_HOST: redis
       REDIS_STATE_HOST: redis
     container:
-      image: faasm/faabric:0.3.4
+      image: faasm/faabric:0.3.5
     defaults:
       run:
         working-directory: /code/faabric
@@ -144,12 +144,12 @@ jobs:
           build-type: "release"
           directory: /code/faabric
       - name: "Run cmake shared"
-        run: inv dev.cmake --shared --build=Release
+        run: ./bin/inv_wrapper.sh dev.cmake --shared --build=Release
       - name: "Build Faabric shared library"
-        run: inv dev.cc faabric --shared
+        run: ./bin/inv_wrapper.sh dev.cc faabric --shared
       - name: "Install Faabric shared library"
-        run: inv dev.install faabric --shared
+        run: ./bin/inv_wrapper.sh dev.install faabric --shared
       - name: "Build examples"
-        run: inv examples
+        run: ./bin/inv_wrapper.sh examples
       - name: "Run example to check"
-        run: inv examples.execute check
+        run: ./bin/inv_wrapper.sh examples.execute check
diff --git a/.gitignore b/.gitignore
@@ -108,4 +108,5 @@ dkms.conf
 __pycache__/
 *.pyc
 venv/
+venv-bm/
 *.egg-info/
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.3.4
+0.3.5
diff --git a/bin/create_venv.sh b/bin/create_venv.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+THIS_DIR=$(dirname $(readlink -f $0))
+PROJ_ROOT=${THIS_DIR}/..
+
+# Set different virtual environment paths so that these don't clash when
+# mounting the code in a development container
+VENV_PATH="undetected"
+if [[ -z "$FAABRIC_DOCKER" ]]; then
+    VENV_PATH="${PROJ_ROOT}/venv-bm"
+else
+    VENV_PATH="/code/faabric/venv"
+fi
+
+PIP=${VENV_PATH}/bin/pip3
+
+function pip_cmd {
+    source ${VENV_PATH}/bin/activate && ${PIP} "$@"
+}
+
+pushd ${PROJ_ROOT} >> /dev/null
+
+if [ ! -d ${VENV_PATH} ]; then
+    python3 -m venv ${VENV_PATH}
+fi
+
+pip_cmd install -U pip
+pip_cmd install -U setuptools wheel
+pip_cmd install -r requirements.txt
+
+popd >> /dev/null
diff --git a/bin/inv_wrapper.sh b/bin/inv_wrapper.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+# Wrapper script for running invoke in virtual env
+
+source bin/workon.sh && inv $@
diff --git a/bin/workon.sh b/bin/workon.sh
@@ -7,18 +7,17 @@
 MODE="undetected"
 if [[ -z "$FAABRIC_DOCKER" ]]; then
 
-    THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-    if [ "$(ps -o comm= -p $$)" = "zsh" ]; then
-        THIS_DIR="$( cd "$( dirname "${ZSH_ARGZERO}" )" >/dev/null 2>&1 && pwd )"
-    fi
+    THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )"
     PROJ_ROOT="${THIS_DIR}/.."
+    VENV_PATH="${PROJ_ROOT}/venv-bm"
     export FAABRIC_BUILD_DIR="${PROJ_ROOT}/build"
 
     # Normal terminal
     MODE="terminal"
 else
     # Running inside the container, we know the project root
     PROJ_ROOT="/code/faabric"
+    VENV_PATH="${PROJ_ROOT}/venv"
     export FAABRIC_BUILD_DIR="/build/faabric"
 
     # Use containerised redis
@@ -33,19 +32,12 @@ pushd ${PROJ_ROOT}>>/dev/null
 # Virtualenv
 # ----------------------------
 
-if [ ! -d "venv" ]; then
-    python3 -m venv venv
-    (
-        source venv/bin/activate
-        pip install -U pip
-        pip install -U setuptools
-        pip install -U wheel
-        pip install -r requirements.txt
-    )
+if [ ! -d ${VENV_PATH} ]; then
+    ${PROJ_ROOT}/bin/create_venv.sh
 fi
 
 export VIRTUAL_ENV_DISABLE_PROMPT=1
-source venv/bin/activate
+source ${VENV_PATH}/bin/activate
 
 # ----------------------------
 # Invoke tab-completion

diff --git a/docker/faabric-base.dockerfile b/docker/faabric-base.dockerfile
@@ -1,15 +1,22 @@
 FROM ubuntu:20.04
 
-RUN apt-get update
-RUN apt-get install -y software-properties-common gpg wget curl
-RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add -
-RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add -
-RUN add-apt-repository -y -n "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main"
-RUN add-apt-repository -y -n "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main"
-RUN add-apt-repository -y -n "deb https://apt.kitware.com/ubuntu/ focal main"
-RUN add-apt-repository -y -n ppa:ubuntu-toolchain-r/test
+# Configure APT repositories
+RUN apt update \
+    && apt upgrade -y \
+    && apt install -y \
+        curl \
+        gpg \
+        software-properties-common \
+        wget \
+    && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add - \
+    && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add - \
+    && add-apt-repository -y -n "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" \
+    && add-apt-repository -y -n "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" \
+    && add-apt-repository -y -n "deb https://apt.kitware.com/ubuntu/ focal main" \
+    && add-apt-repository -y -n ppa:ubuntu-toolchain-r/test
 
-RUN apt update -y && apt install -y \
+# Install APT packages
+RUN apt update && apt install -y \
     autoconf \
     automake \
     build-essential \
@@ -49,11 +56,12 @@ RUN apt update -y && apt install -y \
     sudo \
     unzip
 
-RUN curl -s -L -o /tmp/conan-latest.deb https://github.com/conan-io/conan/releases/download/1.52.0/conan-ubuntu-64.deb && sudo dpkg -i /tmp/conan-latest.deb && rm -f /tmp/conan-latest.deb
-
-# Update pip
-RUN pip install -U pip
+# Install Conan
+RUN curl -s -L -o \
+        /tmp/conan-latest.deb https://github.com/conan-io/conan/releases/download/1.52.0/conan-ubuntu-64.deb \
+    && sudo dpkg -i /tmp/conan-latest.deb \
+    && rm -f /tmp/conan-latest.deb
 
 # Tidy up
-RUN apt-get clean autoclean
-RUN apt-get autoremove
+RUN apt clean autoclean -y \
+    && apt autoremove -y
diff --git a/docker/faabric.dockerfile b/docker/faabric.dockerfile
@@ -1,7 +1,8 @@
-FROM faasm/faabric-base:0.3.4
+FROM faasm/faabric-base:0.3.5
 ARG FAABRIC_VERSION
 
 # faabic-base image is not re-built often, so tag may be behind
+SHELL ["/bin/bash", "-c"]
 
 # Flag to say we're in a container
 ENV FAABRIC_DOCKER="on"
@@ -11,25 +12,24 @@ WORKDIR /code
 RUN git clone -b v${FAABRIC_VERSION} https://github.com/faasm/faabric
 
 WORKDIR /code/faabric
-RUN pip3 install -r requirements.txt
-
-# Static build
-RUN inv dev.cmake --build=Release
-RUN inv dev.cc faabric
-RUN inv dev.cc faabric_tests
 
-# Shared build
-RUN inv dev.cmake --shared --build=Release
-RUN inv dev.cc faabric --shared
-RUN inv dev.install faabric --shared
+# Python set-up and code builds
+RUN ./bin/create_venv.sh \
+    && source venv/bin/activate \
+    # Static build
+    && inv dev.cmake --build=Release \
+    && inv dev.cc faabric \
+    && inv dev.cc faabric_tests \
+    # Shared build
+    && inv dev.cmake --shared --build=Release \
+    && inv dev.cc faabric --shared \
+    && inv dev.install faabric --shared
 
 # GDB config, allow loading repo-specific config
-RUN touch /root/.gdbinit
 RUN echo "set auto-load safe-path /" > /root/.gdbinit
 
 # CLI setup
 ENV TERM xterm-256color
-SHELL ["/bin/bash", "-c"]
 
 RUN echo ". /code/faabric/bin/workon.sh" >> ~/.bashrc
 CMD ["/bin/bash", "-l"]
diff --git a/docs/source/development.md b/docs/source/development.md
@@ -129,12 +129,18 @@ docker compose stop
 
 ## Creating a new tag
 
-Create a new branch, then find and replace the current version with the relevant
-bumped version. It should appear in:
+Create a new branch, then bump the code version:
 
-- `VERSION`
-- `.env`
-- `.github/workflows/tests.yml`.
+```bash
+inv git.bump
+```
+
+This will increment the minor version, to bump the code to an arbitrary version
+you can run:
+
+```bash
+inv git.bump --ver=<new_version>
+```
 
 Once done, commit and push, then run:
 
@@ -171,13 +177,17 @@ To build the main container, run:
 source bin/workon.sh
 
 # Build
-inv container.build
+inv docker.build -c faabric [--push]
+```
+
+The base container is not re-built often, and not re-built as part of Github
+Actions. If you ever need to add an APT dependency, or update the Conan
+version, run:
 
-# Push
-inv container.push
+```bash
+source bin/workon.sh
 
-# Build and push
-inv container.build --push
+inv docker.build -c faabric-base [--push]
 ```
 
 ## Publishing a release

diff --git a/tasks/__init__.py b/tasks/__init__.py
@@ -1,16 +1,16 @@
 from invoke import Collection
 
 from . import call
-from . import container
 from . import dev
+from . import docker
 from . import docs
 from . import examples
 from . import git
 
 ns = Collection(
     call,
-    container,
     dev,
+    docker,
     docs,
     examples,
     git,