coreruleset · theseion · Jul 1, 2024 · Nov 9, 2023 · Nov 9, 2023 · Nov 11, 2023
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -1,8 +1,8 @@
 blank_issues_enabled: false
 contact_links:
   - name: Help and support
-    url:  https://security.stackexchange.com/questions/tagged/owasp-crs
+    url: https://security.stackexchange.com/questions/tagged/owasp-crs
     about: For help and support please go here.
   - name: OWASP Core Rule Set mailing list
-    url:  https://groups.google.com/a/owasp.org/g/modsecurity-core-rule-set-project
+    url: https://groups.google.com/a/owasp.org/g/modsecurity-core-rule-set-project
     about: Ask general usage questions and participate in discussions on the CRS.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -16,7 +16,7 @@ on:
 
 # Pin tool versions to prevent problems
 env:
-  GO_FTW_VERSION: '0.6.4'
+  GO_FTW_VERSION: '1.0.3'
 
 jobs:
   regression:
@@ -36,16 +36,22 @@ jobs:
             -p "ftw_${{ env.GO_FTW_VERSION }}_linux_amd64.tar.gz" -O - | tar -xzvf - ftw
 
       - name: "Run tests for ${{ matrix.modsec_version }}"
-        env:
-          FTW_LOGFILE: './tests/logs/${{ matrix.modsec_version }}/error.log'
         run: |
           mkdir -p "tests/logs/${{ matrix.modsec_version }}/{nginx,apache2}"
           chmod -R o+rw "tests/logs"
           docker-compose -f ./tests/docker-compose.yml up -d "${{ matrix.modsec_version }}"
           docker-compose -f ./tests/docker-compose.yml logs
-          [ "$(docker inspect ${{ matrix.modsec_version }} --format='{{.State.Running}}')" = "true" ]
+          if ! [ "$(docker inspect ${{ matrix.modsec_version }} --format='{{.State.Running}}')" = "true" ]; then
+            echo "Web server failed to start. Aborting."
+            exit 1
+          fi
+
           ./ftw check -d tests/regression/tests
-          ./ftw run -d tests/regression/tests --show-failures-only
+          ./ftw run \
+            -d tests/regression/tests \
+            --log-file "tests/logs/${{ matrix.modsec_version }}/error.log" \
+            --overrides tests/regression/httpd-overrides.yaml \
+            --show-failures-only
 
       - name: "Change permissions if failed"
         if: failure()

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -372,23 +372,23 @@ Documentation on how to run the CRS test suite can be found in the [online docum
 Example of a simple *positive test*:
 
 ```yaml
-- test_title: 932230-26
+- test_id: 26
   desc: "Unix command injection"
   stages:
-    - stage:
-        input:
-          dest_addr: 127.0.0.1
-          headers:
-            Host: localhost
-            User-Agent: "OWASP CRS test agent"
-            Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
-          method: POST
-          port: 80
-          uri: "/post"
-          data: "var=` /bin/cat /etc/passwd`"
-          version: HTTP/1.1
-        output:
-          log_contains: id "932230"
+    - input:
+        dest_addr: 127.0.0.1
+        headers:
+          Host: localhost
+          User-Agent: "OWASP CRS test agent"
+          Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+        method: POST
+        port: 80
+        uri: "/post"
+        data: "var=` /bin/cat /etc/passwd`"
+        version: HTTP/1.1
+      output:
+        log:
+          expect_ids: [932230]
 ```
 
 This test will succeed if the log output contains `id "932230"`, which would indicate that the rule in question matched and generated an alert.
@@ -402,21 +402,21 @@ The rule's description field, `desc`, is important. It should describe what is b
 Example of a simple *negative test*:
 
 ```yaml
-- test_title: 932260-4
+- test_id: 4
   stages:
-    - stage:
-        input:
-          dest_addr: "127.0.0.1"
-          method: "POST"
-          port: 80
-          headers:
-            User-Agent: "OWASP CRS test agent"
-            Host: "localhost"
-            Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
-          data: 'foo=ping pong tables'
-          uri: '/post'
-        output:
-          no_log_contains: id "932260"
+    - input:
+        dest_addr: "127.0.0.1"
+        method: "POST"
+        port: 80
+        headers:
+          User-Agent: "OWASP CRS test agent"
+          Host: "localhost"
+          Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+        data: 'foo=ping pong tables'
+        uri: '/post'
+      output:
+        log:
+          no_expect_ids: [932260]
 ```
 
 This test will succeed if the log output does **not** contain `id "932260"`, which would indicate that the rule in question did **not** match and so did **not** generate an alert.

diff --git a/tests/regression/README.md b/tests/regression/README.md
@@ -15,22 +15,22 @@ The OWASP CRS project was part of the effort to develop the Web Application Fire
 ---
 meta:
   author: "csanders-git"
-  enabled: true
   name: "911100.yaml"
-  description: "Description"
+  description: "A meaningful description"
+  rule_id: 911100
 tests:
-  - test_title: 911100-1
+  - test_id: 1
     stages:
-      - stage:
-          input:
-            dest_addr: "127.0.0.1"
-            port: 80
-            headers:
-              User-Agent: "OWASP CRS test agent"
-              Host: "localhost"
-              Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
-          output:
-            no_log_contains: "id \"911100\""
+      - input:
+          dest_addr: "127.0.0.1"
+          port: 80
+          headers:
+            User-Agent: "OWASP CRS test agent"
+            Host: "localhost"
+            Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+        output:
+          log:
+            expect_id: 911100
 ```
 
 For go-ftw, please check out from [go-ftw releases page](https://github.com/coreruleset/go-ftw/releases).

diff --git a/tests/regression/coraza-overrides.yaml b/tests/regression/coraza-overrides.yaml
@@ -0,0 +1,52 @@
+version: "v0.0.0"
+meta:
+  engine: "coraza"
+  platform: "go"
+  annotations:
+    - purpose: "CRS test suite"
+test_overrides:
+  - rule_id: 920100
+    test_ids: [4]
+    reason: 'Invalid uri, Coraza not reached - 404 page not found'
+    output:
+      status: 404
+  - rule_id: 920100
+    test_ids: [5]
+    reason: 'Invalid uri, Coraza not reached - 404 page not found'
+    output:
+      status: 404
+  - rule_id: 920100
+    test_ids: [8]
+    reason: 'Go/http allows a colon in the path. Test expects status 400 or 403 (Apache behaviour)'
+    output:
+      status: 200
+      log:
+        expect_ids: [920100]
+  - rule_id: 920270
+    test_ids: [4]
+    reason: 'Rule works, log contains 920270. Test expects status 400 (Apache behaviour)'
+    output:
+      status: 200
+  - rule_id: 920272
+    test_ids: [5]
+    reason: 'Rule works, log contains 920272. Test expects status 400 (Apache behaviour)'
+    output:
+      status: 200
+  - rule_id: 920290
+    test_ids: [1]
+    reason: 'Rule works, log contains 920290. Test expects status 400 (Apache behaviour)'
+    output:
+      status: 200
+  - rule_id: 920430
+    test_ids: [8]
+    reason: 'Go/http does not allow HTTP/3.0 - 505 HTTP Version Not Supported'
+    output:
+      status: 505
+      log:
+        no_expect_ids: [920430]
+  - rule_id: 932200
+    test_ids: [13]
+    reason: 'wip'
+  - rule_id: 934131
+    test_ids: [5, 7]
+    reason: 'TODO: check why we are failing to deobfuscate these payloads'
diff --git a/tests/regression/httpd-overrides.yaml b/tests/regression/httpd-overrides.yaml
@@ -0,0 +1,35 @@
+version: "v0.0.0"
+meta:
+  engine: "ModSecurity v2"
+  platform: "httpd"
+  annotations:
+    - purpose: "CRS test suite"
+test_overrides:
+  - rule_id: 920360
+    test_ids: [1]
+    reason: Requires ARG_NAME_LENGTH to be set to an appropriately low value
+    output:
+      status: 200
+      log:
+        no_expect_ids: [920360]
+  - rule_id: 920370
+    test_ids: [1]
+    reason: Doesn't trigger. Needs to be fixed (https://github.com/coreruleset/coreruleset/issues/3745)
+    output:
+      status: 200
+      log:
+        no_expect_ids: [920370]
+  - rule_id: 920380
+    test_ids: [1]
+    reason: Requires MAX_NUM_ARGS to be set to a sufficiently low value
+    output:
+      status: 200
+      log:
+        no_expect_ids: [920380]
+  - rule_id: 920390
+    test_ids: [1]
+    reason: Exceeds PCRE limits, currently segfaults on the CI
+    output:
+      expect_error: true
+      log:
+        no_expect_ids: [920390]