From 02547bdcc5c609880a19a2d9bebda7d9e72b77f7 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Sun, 17 Mar 2024 10:18:00 -0300 Subject: [PATCH 1/7] feat(apache): switch to pcre2 Signed-off-by: Felipe Zipitria --- apache/Dockerfile | 5 +++-- apache/Dockerfile-alpine | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apache/Dockerfile b/apache/Dockerfile index f2a2157..c58eee3 100644 --- a/apache/Dockerfile +++ b/apache/Dockerfile @@ -17,7 +17,7 @@ RUN set -eux; \ libaprutil1-dev \ libcurl4-gnutls-dev \ libfuzzy-dev \ - libpcre3-dev \ + libpcre2-dev \ libtool \ libxml2-dev \ libyajl-dev \ @@ -31,7 +31,7 @@ RUN set -eux; \ tar -zxvf v${MODSEC2_VERSION}.tar.gz; \ cd ModSecurity-${MODSEC2_VERSION}; \ ./autogen.sh; \ - ./configure --with-yajl --with-ssdeep; \ + ./configure --with-yajl --with-ssdeep --with-pcre2; \ make; \ make install; \ make clean @@ -173,6 +173,7 @@ RUN set -eux; \ libfuzzy2 \ liblua${LUA_VERSION} \ ${LUA_MODULES} \ + libpcre2 \ libxml2 \ libyajl2; \ update-ca-certificates -f; \ diff --git a/apache/Dockerfile-alpine b/apache/Dockerfile-alpine index 843f464..2e7c75e 100644 --- a/apache/Dockerfile-alpine +++ b/apache/Dockerfile-alpine @@ -33,7 +33,7 @@ RUN set -eux; \ make \ openssl \ openssl-dev \ - pcre-dev \ + pcre2-dev \ zlib-dev RUN set -eux; \ @@ -41,7 +41,7 @@ RUN set -eux; \ tar -zxvf v${MODSEC2_VERSION}.tar.gz; \ cd ModSecurity-${MODSEC2_VERSION}; \ ./autogen.sh; \ - ./configure --with-yajl --with-ssdeep --with-lmdb; \ + ./configure --with-yajl --with-ssdeep --with-lmdb --with-pcre2; \ make; \ make install; \ make clean From d8c45465e7493a34b9bcb93d654566a365e716fd Mon Sep 17 00:00:00 2001 From: Max Leske <250711+theseion@users.noreply.github.com> Date: Sun, 2 Mar 2025 15:46:30 +0100 Subject: [PATCH 2/7] chore: specify correct library name --- apache/Dockerfile | 2 +- apache/Dockerfile-alpine | 1 + nginx/Dockerfile | 4 +--- nginx/Dockerfile-alpine | 2 -- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/apache/Dockerfile b/apache/Dockerfile index c58eee3..630111a 100644 --- a/apache/Dockerfile +++ b/apache/Dockerfile @@ -173,7 +173,7 @@ RUN set -eux; \ libfuzzy2 \ liblua${LUA_VERSION} \ ${LUA_MODULES} \ - libpcre2 \ + libpcre2-8-0 \ libxml2 \ libyajl2; \ update-ca-certificates -f; \ diff --git a/apache/Dockerfile-alpine b/apache/Dockerfile-alpine index 2e7c75e..eb55c7f 100644 --- a/apache/Dockerfile-alpine +++ b/apache/Dockerfile-alpine @@ -184,6 +184,7 @@ RUN set -eux; \ ${LUA_MODULES} \ moreutils \ openssl \ + pcre2 \ sed \ tzdata \ yajl; \ diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 3a1d350..d47e102 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -10,8 +10,6 @@ ARG READ_ONLY_FS="false" USER root -# Note: libpcre3-dev (PCRE 1) is required by the build description, -# even though the build will use PCRE2. RUN set -eux; \ echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections; \ apt-get update -qq; \ @@ -24,7 +22,6 @@ RUN set -eux; \ libcurl4-gnutls-dev \ libfuzzy-dev \ liblua${LUA_VERSION}-dev \ - libpcre3-dev \ libpcre2-dev \ libtool \ libxml2-dev \ @@ -243,6 +240,7 @@ RUN set -eux; \ libfuzzy2 \ liblua${LUA_VERSION} \ ${LUA_MODULES} \ + libpcre2-8-0 \ libxml2 \ libyajl2 \ libmaxminddb-dev \ diff --git a/nginx/Dockerfile-alpine b/nginx/Dockerfile-alpine index 9c89f5a..0043a6c 100644 --- a/nginx/Dockerfile-alpine +++ b/nginx/Dockerfile-alpine @@ -35,7 +35,6 @@ RUN set -eux; \ openssl-dev \ patch \ pkgconfig \ - pcre-dev \ pcre2-dev \ yajl-dev \ zlib-dev @@ -240,7 +239,6 @@ RUN set -eux; \ moreutils \ openssl \ tzdata \ - pcre \ pcre2 \ # Alpine needs GNU 'sed' because the 'sed' program shipped with busybox does not support 'z' parameter for separating lines with a 'NUL' character. sed \ From 749eb60116baba72d7452ff7cb14b03f0d47a082 Mon Sep 17 00:00:00 2001 From: Max Leske <250711+theseion@users.noreply.github.com> Date: Sun, 2 Mar 2025 16:02:56 +0100 Subject: [PATCH 3/7] chore: enable alpine images --- docker-bake.hcl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index d18c741..744612b 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -125,13 +125,13 @@ target "apache" { lua_modules = join(" ", lua-modules-debian) tag_base = "apache" }, - # { - # name = "alpine" - # dockerfile = "apache/Dockerfile-alpine" - # image = "docker-image://httpd:${httpd-version}-alpine" - # lua_modules = join(" ", lua-modules-alpine) - # tag_base = "apache-alpine" - # } + { + name = "alpine" + dockerfile = "apache/Dockerfile-alpine" + image = "docker-image://httpd:${httpd-version}-alpine" + lua_modules = join(" ", lua-modules-alpine) + tag_base = "apache-alpine" + } ] } @@ -159,13 +159,13 @@ target "nginx" { lua_modules = join(" ", lua-modules-debian) tag_base = "nginx" }, - # { - # name = "alpine" - # dockerfile = "nginx/Dockerfile-alpine" - # image = "docker-image://nginxinc/nginx-unprivileged:${nginx-version}-alpine" - # lua_modules = join(" ", lua-modules-alpine) - # tag_base = "nginx-alpine" - # } + { + name = "alpine" + dockerfile = "nginx/Dockerfile-alpine" + image = "docker-image://nginxinc/nginx-unprivileged:${nginx-version}-alpine" + lua_modules = join(" ", lua-modules-alpine) + tag_base = "nginx-alpine" + } ], read-only-fs = [ { From 0be91fd9cc68e6fdbbb915520253a3b8062961ec Mon Sep 17 00:00:00 2001 From: Max Leske <250711+theseion@users.noreply.github.com> Date: Sun, 2 Mar 2025 17:18:16 +0100 Subject: [PATCH 4/7] debugging --- .github/workflows/verifyimage.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verifyimage.yml b/.github/workflows/verifyimage.yml index 052c6f7..b6ccb58 100644 --- a/.github/workflows/verifyimage.yml +++ b/.github/workflows/verifyimage.yml @@ -134,6 +134,7 @@ jobs: backend="${{ contains(matrix.target, 'apache') && 'httpd' || 'nginx' }}" mkdir -p "tests/logs/${modsec_version}/{nginx,apache2}" docker compose -f ./tests/docker-compose.yml up -d "${modsec_version}" + sleep 10 docker compose -f ./tests/docker-compose.yml logs if ! [ "$(docker inspect ${modsec_version} --format='{{.State.Running}}')" = "true" ]; then echo "Web server failed to start. Aborting." @@ -145,4 +146,5 @@ jobs: -d tests/regression/tests \ --log-file "tests/logs/${modsec_version}/error.log" \ --overrides "tests/regression/${backend}-overrides.yaml" \ - --show-failures-only + --show-failures-only \ + --trace From 2b7b423e61188243d5d7b5f3aeb086a34bbf7666 Mon Sep 17 00:00:00 2001 From: Max Leske <250711+theseion@users.noreply.github.com> Date: Sun, 9 Mar 2025 16:24:04 +0100 Subject: [PATCH 5/7] chore: parameterize compilation flags --- .github/workflows/verifyimage.yml | 3 +-- apache/Dockerfile | 3 ++- apache/Dockerfile-alpine | 3 ++- docker-bake.hcl | 10 ++++++++++ nginx/Dockerfile | 3 ++- nginx/Dockerfile-alpine | 3 ++- 6 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verifyimage.yml b/.github/workflows/verifyimage.yml index b6ccb58..cc50a0d 100644 --- a/.github/workflows/verifyimage.yml +++ b/.github/workflows/verifyimage.yml @@ -146,5 +146,4 @@ jobs: -d tests/regression/tests \ --log-file "tests/logs/${modsec_version}/error.log" \ --overrides "tests/regression/${backend}-overrides.yaml" \ - --show-failures-only \ - --trace + --show-failures-only diff --git a/apache/Dockerfile b/apache/Dockerfile index 630111a..8d03a18 100644 --- a/apache/Dockerfile +++ b/apache/Dockerfile @@ -3,6 +3,7 @@ ARG HTTPD_VERSION="n/a" FROM image AS build ARG MODSEC2_VERSION="n/a" +ARG MODSEC2_FLAGS="n/a" ARG LUA_VERSION="n/a" RUN set -eux; \ @@ -31,7 +32,7 @@ RUN set -eux; \ tar -zxvf v${MODSEC2_VERSION}.tar.gz; \ cd ModSecurity-${MODSEC2_VERSION}; \ ./autogen.sh; \ - ./configure --with-yajl --with-ssdeep --with-pcre2; \ + ./configure ${MODSEC2_FLAGS}; \ make; \ make install; \ make clean diff --git a/apache/Dockerfile-alpine b/apache/Dockerfile-alpine index eb55c7f..ebd30d9 100644 --- a/apache/Dockerfile-alpine +++ b/apache/Dockerfile-alpine @@ -3,6 +3,7 @@ ARG HTTPD_VERSION="n/a" FROM image AS build ARG MODSEC2_VERSION="n/a" +ARG MODSEC2_FLAGS="n/a" ARG LUA_VERSION="n/a" # see https://httpd.apache.org/docs/2.4/install.html#requirements @@ -41,7 +42,7 @@ RUN set -eux; \ tar -zxvf v${MODSEC2_VERSION}.tar.gz; \ cd ModSecurity-${MODSEC2_VERSION}; \ ./autogen.sh; \ - ./configure --with-yajl --with-ssdeep --with-lmdb --with-pcre2; \ + ./configure ${MODSEC2_FLAGS}; \ make; \ make install; \ make clean diff --git a/docker-bake.hcl b/docker-bake.hcl index 744612b..968f345 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -4,11 +4,19 @@ variable "modsec3-version" { default = "3.0.14" } +variable "modsec3-flags" { + default = "--with-yajl --with-ssdeep --with-lmdb --with-pcre2 --with-maxmind --enable-silent-rules" +} + variable "modsec2-version" { # renovate: depName=ModSecurity2 packageName=owasp-modsecurity/ModSecurity datasource=github-releases default = "2.9.8" } +variable "modsec2-flags" { + default = "--with-yajl --with-ssdeep --with-pcre2" +} + variable "crs-version" { # renovate: depName=coreruleset/coreruleset datasource=github-releases default = "4.12.0" @@ -109,7 +117,9 @@ target "platforms-base" { args = { CRS_RELEASE = "${crs-version}" MODSEC2_VERSION = "${modsec2-version}" + MODSEC2_FLAGS = modsec2-flags MODSEC3_VERSION = "${modsec3-version}" + MODSEC3_FLAGS = modsec3-flags LUA_VERSION = "${lua-version}" LMDB_VERSION = "${lmdb-version}" } diff --git a/nginx/Dockerfile b/nginx/Dockerfile index d47e102..dc583ac 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -2,6 +2,7 @@ FROM image AS build ARG NGINX_VERSION="n/a" ARG MODSEC3_VERSION="n/a" +ARG MODSEC3_FLAGS="n/a" ARG LMDB_VERSION="n/a" ARG LUA_VERSION="n/a" ARG NGINX_DYNAMIC_MODULES="n/a" @@ -50,7 +51,7 @@ RUN set -eux; \ sed -ie "s/i386-linux-gnu/${ARCH}/g" build/ssdeep.m4; \ sed -ie "s/i386-linux-gnu/${ARCH}/g" build/pcre2.m4; \ ./build.sh; \ - ./configure --with-yajl --with-ssdeep --with-pcre2 --with-maxmind --enable-silent-rules; \ + ./configure ${MODSEC3_FLAGS}; \ make -j$(nproc) install; \ strip /usr/local/modsecurity/lib/lib*.so* diff --git a/nginx/Dockerfile-alpine b/nginx/Dockerfile-alpine index 0043a6c..699430d 100644 --- a/nginx/Dockerfile-alpine +++ b/nginx/Dockerfile-alpine @@ -2,6 +2,7 @@ FROM image AS build ARG NGINX_VERSION="n/a" ARG MODSEC3_VERSION="n/a" +ARG MODSEC3_FLAGS="n/a" ARG LUA_VERSION="n/a" ARG NGINX_DYNAMIC_MODULES="n/a" ARG NGINX_HOME="n/a" @@ -48,7 +49,7 @@ RUN set -eux; \ sed -ie "s/i386-linux-gnu/${ARCH}/g" build/ssdeep.m4; \ sed -ie "s/i386-linux-gnu/${ARCH}/g" build/pcre2.m4; \ ./build.sh; \ - ./configure --with-yajl --with-ssdeep --with-lmdb --with-pcre2 --with-maxmind --enable-silent-rules; \ + ./configure ${MODSEC3_FLAGS}; \ make -j$(nproc) install; \ strip /usr/local/modsecurity/lib/lib*.so* From e52d587156aac98a4008e313d6cba5c583d3822e Mon Sep 17 00:00:00 2001 From: Max Leske <250711+theseion@users.noreply.github.com> Date: Sun, 9 Mar 2025 18:53:02 +0100 Subject: [PATCH 6/7] chore: also upload test run error logs --- .github/workflows/configure-rules-for-test.sh | 2 +- .github/workflows/verifyimage.yml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/configure-rules-for-test.sh b/.github/workflows/configure-rules-for-test.sh index d29e285..b158e2f 100755 --- a/.github/workflows/configure-rules-for-test.sh +++ b/.github/workflows/configure-rules-for-test.sh @@ -22,7 +22,7 @@ while read -r line; do test_value="$(cut -d'|' -f5 <<< "${line}")" if ! grep -q "${var_name}" "${readme_file}"; then - echo "Failed to find environment variable `${var_name}` in README." + echo "Failed to find environment variable '${var_name}' in README." exit 1 fi diff --git a/.github/workflows/verifyimage.yml b/.github/workflows/verifyimage.yml index cc50a0d..43e970f 100644 --- a/.github/workflows/verifyimage.yml +++ b/.github/workflows/verifyimage.yml @@ -147,3 +147,12 @@ jobs: --log-file "tests/logs/${modsec_version}/error.log" \ --overrides "tests/regression/${backend}-overrides.yaml" \ --show-failures-only + + - name: Upload logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ matrix.target }}-error.log + path: "crs/tests/logs/${{ contains(matrix.target, 'apache') && 'modsec2-apache' || 'modsec3-nginx' }}/error.log" + retention-days: 7 + overwrite: true \ No newline at end of file From 60f2816d392e683699116b6448ea1f48ddb6b6b3 Mon Sep 17 00:00:00 2001 From: Max Leske <250711+theseion@users.noreply.github.com> Date: Tue, 11 Mar 2025 06:52:20 +0100 Subject: [PATCH 7/7] chore: exclude nginx-alpine builds from test runs nginx-alpine builds don't flush the log reliably. Refs https://github.com/coreruleset/go-ftw/issues/473 --- .github/workflows/verifyimage.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/verifyimage.yml b/.github/workflows/verifyimage.yml index 43e970f..dbefa59 100644 --- a/.github/workflows/verifyimage.yml +++ b/.github/workflows/verifyimage.yml @@ -128,6 +128,8 @@ jobs: 's#image: owasp/modsecurity-crs:${{ contains(matrix.target, 'apache') && 'apache' || 'nginx' }}.*#image: ${{ matrix.target }}-verification#' \ crs/tests/docker-compose.yml - name: Run CRS tests for ${{ matrix.target }} + # Log flushing isn't reliable enough for go-ftw (https://github.com/coreruleset/go-ftw/issues/473) + if: ${{!contains(matrix.target, 'nginx-alpine')}} run: | cd crs modsec_version="${{ contains(matrix.target, 'apache') && 'modsec2-apache' || 'modsec3-nginx' }}"