google · rw1nkler · Oct 24, 2023 · Oct 31, 2023 · Oct 24, 2023 · Oct 24, 2023
diff --git a/.github/workflows/modules-zstd.yml b/.github/workflows/modules-zstd.yml
@@ -0,0 +1,73 @@
+name: Modules (ZSTD)
+on:
+  # Avoid triggering on pushes to /all/ open PR branches.
+  push:
+    branches:
+      - main
+    paths:
+      - 'xls/modules/zstd/**'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'xls/modules/zstd/**'
+  # This lets us trigger manually from the UI.
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Test ZSTD module (opt)
+    runs-on:
+      labels: ubuntu-22.04-64core
+    timeout-minutes: 600
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Restore Nightly Bazel Cache
+        uses: actions/cache/restore@v4
+        with:
+          path: "~/.cache/bazel"
+          key: bazel-cache-nightly-${{ runner.os }}-${{ github.sha }}
+          restore-keys: bazel-cache-nightly-${{ runner.os }}-
+
+      - name: Install dependencies via apt
+        run: sudo apt-get install python3-distutils python3-dev python-is-python3 libtinfo5  build-essential liblapack-dev libblas-dev gfortran
+
+      - name: Bazel Build Tools (opt)
+        run: |
+          bazel build -c opt --test_output=errors -- //xls/dslx:interpreter_main //xls/dslx/ir_convert:ir_converter_main //xls/tools:opt_main //xls/tools:codegen_main
+
+      - name: Build ZSTD Module (opt)
+        run: |
+          bazel build -c opt --test_output=errors -- //xls/modules/zstd:all
+
+      - name: Test ZSTD Module - DSLX Tests (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_dslx_test", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Test ZSTD Module - CC Tests (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_cc_test", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build ZSTD verilog targets (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel build -c opt -- $(bazel query 'filter(".*_verilog", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build and run ZSTD IR benchmark rules (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel run -c opt -- $(bazel query 'filter(".*_ir_benchmark", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build and run synthesis benchmarks of the ZSTD module (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel run -c opt -- $(bazel query 'filter(".*_benchmark_synth", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build ZSTD place and route targets (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel build -c opt -- $(bazel query 'filter(".*_place_and_route", kind(rule, //xls/modules/zstd/...))')
diff --git a/dependency_support/com_github_facebook_zstd/bundled.BUILD.bazel b/dependency_support/com_github_facebook_zstd/bundled.BUILD.bazel
@@ -1,4 +1,4 @@
-# Copyright 2023 The XLS Authors
+# Copyright 2024 The XLS Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,38 +12,203 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-package(default_visibility = ["//visibility:public"])
+""" Builds zstd.
+"""
 
-licenses(["notice"])
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
 
-# Builds everything together similarly to the zstd cmake file:
-# https://github.com/facebook/zstd/blob/dev/build/cmake/lib/CMakeLists.txt
-# but with legacy support and defines as in the zstd BUCK file:
-# https://github.com/facebook/zstd/blob/dev/lib/BUCK
+package(default_visibility = ["//visibility:public"])
 
-cc_library(
-    name = "zstd",
+filegroup(
+    name = "common_sources",
     srcs = glob([
-        "lib/common/*.h",
         "lib/common/*.c",
-        "lib/compress/*.h",
+        "lib/common/*.h",
+    ]),
+)
+
+filegroup(
+    name = "compress_sources",
+    srcs = glob([
         "lib/compress/*.c",
-        "lib/decompress/*.h",
+        "lib/compress/*.h",
+    ]),
+)
+
+filegroup(
+    name = "decompress_sources",
+    srcs = glob([
         "lib/decompress/*.c",
-        "lib/decompress/*.S",
-        "lib/deprecated/*.h",
-        "lib/deprecated/*.c",
-        "lib/dictBuilder/*.h",
+        "lib/decompress/*.h",
+    ]) + select({
+        "@platforms//os:windows": [],
+        "//conditions:default": glob(["lib/decompress/*.S"]),
+    }),
+)
+
+filegroup(
+    name = "dictbuilder_sources",
+    srcs = glob([
         "lib/dictBuilder/*.c",
-        "lib/legacy/*.h",
-        "lib/legacy/*.c",
+        "lib/dictBuilder/*.h",
     ]),
+)
+
+cc_library(
+    name = "zstd",
+    srcs = [
+        ":common_sources",
+        ":compress_sources",
+        ":decompress_sources",
+        ":dictbuilder_sources",
+    ],
     hdrs = [
+        "lib/zdict.h",
         "lib/zstd.h",
+        "lib/zstd_errors.h",
+    ],
+    includes = ["lib"],
+    linkopts = ["-pthread"],
+    linkstatic = True,
+    local_defines = [
+        "XXH_NAMESPACE=ZSTD_",
+        "ZSTD_MULTITHREAD",
+        "ZSTD_BUILD_SHARED=OFF",
+        "ZSTD_BUILD_STATIC=ON",
+    ] + select({
+        "@platforms//os:windows": ["ZSTD_DISABLE_ASM"],
+        "//conditions:default": [],
+    }),
+)
+
+cc_binary(
+    name = "zstd_cli",
+    srcs = glob(
+        include = [
+            "programs/*.c",
+            "programs/*.h",
+        ],
+        exclude = [
+            "programs/datagen.c",
+            "programs/datagen.h",
+            "programs/platform.h",
+            "programs/util.h",
+        ],
+    ),
+    deps = [
+        ":datagen",
+        ":util",
+        ":zstd",
+    ],
+)
+
+cc_library(
+    name = "util",
+    srcs = [
+        "programs/platform.h",
+        "programs/util.c",
+    ],
+    hdrs = [
+        "lib/common/compiler.h",
+        "lib/common/debug.h",
+        "lib/common/mem.h",
+        "lib/common/portability_macros.h",
+        "lib/common/zstd_deps.h",
+        "programs/util.h",
+    ],
+)
+
+cc_library(
+    name = "datagen",
+    srcs = [
+        "programs/datagen.c",
+        "programs/platform.h",
+    ],
+    hdrs = ["programs/datagen.h"],
+    deps = [":util"],
+)
+
+cc_binary(
+    name = "datagen_cli",
+    srcs = [
+        "programs/lorem.c",
+        "programs/lorem.h",
+        "tests/datagencli.c",
+        "tests/loremOut.c",
+        "tests/loremOut.h",
+    ],
+    includes = [
+        "programs",
+        "tests",
+    ],
+    deps = [":datagen"],
+)
+
+cc_test(
+    name = "fullbench",
+    srcs = [
+        "lib/decompress/zstd_decompress_internal.h",
+        "programs/benchfn.c",
+        "programs/benchfn.h",
+        "programs/benchzstd.c",
+        "programs/benchzstd.h",
+        "programs/lorem.c",
+        "programs/lorem.h",
+        "programs/timefn.c",
+        "programs/timefn.h",
+        "tests/fullbench.c",
+        "tests/loremOut.c",
+        "tests/loremOut.h",
+    ],
+    copts = select({
+        "@platforms//os:windows": [],
+        "//conditions:default": ["-Wno-deprecated-declarations"],
+    }),
+    includes = [
+        "lib/common",
+        "programs",
+        "tests",
+    ],
+    deps = [
+        ":datagen",
+        ":zstd",
+    ],
+)
+
+# NOTE: Required because of direct zstd_compress.c include in decodecorpus sources
+cc_library(
+    name = "decodecorpus_includes",
+    hdrs = [
+        "lib/compress/zstd_compress.c",
+    ],
+)
+
+cc_binary(
+    name = "decodecorpus",
+    srcs = [
+        "tests/decodecorpus.c",
+    ] + glob(
+        [
+            "programs/*.c",
+            "programs/*.h",
+        ],
+        exclude = [
+            "programs/zstdcli.c",
+        ],
+    ),
+    deps = [
+        ":zstd",
+        ":decodecorpus_includes",
+    ],
+    includes = [
+        "lib/",
+        "lib/common/",
+        "lib/compress/",
+        "lib/dictBuilder/",
+        "lib/deprecated/",
+        "programs/",
     ],
-    strip_include_prefix = "lib",
     local_defines = [
-        "ZSTD_LEGACY_SUPPORT=4",
         "XXH_NAMESPACE=ZSTD_",
     ],
     visibility = ["//visibility:public"],