10000 modules/zstd: Add ZstdDecoder top level proc by lpawelcz · Pull Request #1315 · google/xls · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

modules/zstd: Add ZstdDecoder top level proc #1315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 16, 2024
Merged

modules/zstd: Add ZstdDecoder top level proc #1315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9af92d4
dependency_support: Remove net_zstd
lpawelcz Jul 19, 2024
42c7ff4
dependency_support: Reintroduce com_github_facebook_zstd
lpawelcz Jun 27, 2024
db9adc3
dependency_support: rename com_github_facebook -> zstd
lpawelcz Aug 8, 2024
347e8db
dependency_support: bump zstd version to fdfb2aff
lpawelcz Jul 19, 2024
76e650a
modules: Add ZSTD Decoder module
lpawelcz Oct 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions .github/workflows/modules-zstd.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
name: Modules (ZSTD)
on:
# Avoid triggering on pushes to /all/ open PR branches.
push:
branches:
- main
paths:
- 'xls/modules/zstd/**'
pull_request:
branches:
- main
paths:
- 'xls/modules/zstd/**'
# This lets us trigger manually from the UI.
workflow_dispatch:

jobs:
test:
name: Test ZSTD module (opt)
runs-on:
labels: ubuntu-22.04-64core
timeout-minutes: 600
continue-on-error: true
steps:
- uses: actions/checkout@v2

- name: Restore Nightly Bazel Cache
uses: actions/cache/restore@v4
with:
path: "~/.cache/bazel"
key: bazel-cache-nightly-${{ runner.os }}-${{ github.sha }}
restore-keys: bazel-cache-nightly-${{ runner.os }}-

- name: Install dependencies via apt
run: sudo apt-get install python3-distutils python3-dev python-is-python3 libtinfo5 build-essential liblapack-dev libblas-dev gfortran

- name: Bazel Build Tools (opt)
run: |
bazel build -c opt --test_output=errors -- //xls/dslx:interpreter_main //xls/dslx/ir_convert:ir_converter_main //xls/tools:opt_main //xls/tools:codegen_main

- name: Build ZSTD Module (opt)
run: |
bazel build -c opt --test_output=errors -- //xls/modules/zstd:all

- name: Test ZSTD Module - DSLX Tests (opt)
if: ${{ !cancelled() }}
run: |
bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_dslx_test", kind(rule, //xls/modules/zstd/...))')

- name: Test ZSTD Module - CC Tests (opt)
if: ${{ !cancelled() }}
run: |
bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_cc_test", kind(rule, //xls/modules/zstd/...))')

- name: Build ZSTD verilog targets (opt)
if: ${{ !cancelled() }}
run: |
bazel build -c opt -- $(bazel query 'filter(".*_verilog", kind(rule, //xls/modules/zstd/...))')

- name: Build and run ZSTD IR benchmark rules (opt)
if: ${{ !cancelled() }}
run: |
bazel run -c opt -- $(bazel query 'filter(".*_ir_benchmark", kind(rule, //xls/modules/zstd/...))')

- name: Build and run synthesis benchmarks of the ZSTD module (opt)
if: ${{ !cancelled() }}
run: |
bazel run -c opt -- $(bazel query 'filter(".*_benchmark_synth", kind(rule, //xls/modules/zstd/...))')

- name: Build ZSTD place and route targets (opt)
if: ${{ !cancelled() }}
run: |
bazel build -c opt -- $(bazel query 'filter(".*_place_and_route", kind(rule, //xls/modules/zstd/...))')
205 changes: 185 additions & 20 deletions dependency_support/com_github_facebook_zstd/bundled.BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023 The XLS Authors
# Copyright 2024 The XLS Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -12,38 +12,203 @@
# See the License for the specific language governing permissions and
# limitations under the License.

package(default_visibility = ["//visibility:public"])
""" Builds zstd.
"""

licenses(["notice"])
load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")

# Builds everything together similarly to the zstd cmake file:
# https://github.com/facebook/zstd/blob/dev/build/cmake/lib/CMakeLists.txt
# but with legacy support and defines as in the zstd BUCK file:
# https://github.com/facebook/zstd/blob/dev/lib/BUCK
package(default_visibility = ["//visibility:public"])

cc_library(
name = "zstd",
filegroup(
name = "common_sources",
srcs = glob([
"lib/common/*.h",
"lib/common/*.c",
"lib/compress/*.h",
"lib/common/*.h",
]),
)

filegroup(
name = "compress_sources",
srcs = glob([
"lib/compress/*.c",
"lib/decompress/*.h",
"lib/compress/*.h",
]),
)

filegroup(
name = "decompress_sources",
srcs = glob([
"lib/decompress/*.c",
"lib/decompress/*.S",
"lib/deprecated/*.h",
"lib/deprecated/*.c",
"lib/dictBuilder/*.h",
"lib/decompress/*.h",
]) + select({
"@platforms//os:windows": [],
"//conditions:default": glob(["lib/decompress/*.S"]),
}),
)

filegroup(
name = "dictbuilder_sources",
srcs = glob([
"lib/dictBuilder/*.c",
"lib/legacy/*.h",
"lib/legacy/*.c",
"lib/dictBuilder/*.h",
]),
)

cc_library(
name = "zstd",
srcs = [
":common_sources",
":compress_sources",
":decompress_sources",
":dictbuilder_sources",
],
hdrs = [
"lib/zdict.h",
"lib/zstd.h",
"lib/zstd_errors.h",
],
includes = ["lib"],
linkopts = ["-pthread"],
linkstatic = True,
local_defines = [
"XXH_NAMESPACE=ZSTD_",
"ZSTD_MULTITHREAD",
"ZSTD_BUILD_SHARED=OFF",
"ZSTD_BUILD_STATIC=ON",
] + select({
"@platforms//os:windows": ["ZSTD_DISABLE_ASM"],
"//conditions:default": [],
}),
)

cc_binary(
name = "zstd_cli",
srcs = glob(
include = [
"programs/*.c",
"programs/*.h",
],
exclude = [
"programs/datagen.c",
"programs/datagen.h",
"programs/platform.h",
"programs/util.h",
],
),
deps = [
":datagen",
":util",
":zstd",
],
)

cc_library(
name = "util",
srcs = [
"programs/platform.h",
"programs/util.c",
],
hdrs = [
"lib/common/compiler.h",
"lib/common/debug.h",
"lib/common/mem.h",
"lib/common/portability_macros.h",
"lib/common/zstd_deps.h",
"programs/util.h",
],
)

cc_library(
name = "datagen",
srcs = [
"programs/datagen.c",
"programs/platform.h",
],
hdrs = ["programs/datagen.h"],
deps = [":util"],
)

cc_binary(
name = "datagen_cli",
srcs = [
"programs/lorem.c",
"programs/lorem.h",
"tests/datagencli.c",
"tests/loremOut.c",
"tests/loremOut.h",
],
includes = [
"programs",
"tests",
],
deps = [":datagen"],
)

cc_test(
name = "fullbench",
srcs = [
"lib/decompress/zstd_decompress_internal.h",
"programs/benchfn.c",
"programs/benchfn.h",
"programs/benchzstd.c",
"programs/benchzstd.h",
"programs/lorem.c",
"programs/lorem.h",
"programs/timefn.c",
"programs/timefn.h",
"tests/fullbench.c",
"tests/loremOut.c",
"tests/loremOut.h",
],
copts = select({
"@platforms//os:windows": [],
"//conditions:default": ["-Wno-deprecated-declarations"],
}),
includes = [
"lib/common",
"programs",
"tests",
],
deps = [
":datagen",
":zstd",
],
)

# NOTE: Required because of direct zstd_compress.c include in decodecorpus sources
cc_library(
name = "decodecorpus_includes",
hdrs = [
"lib/compress/zstd_compress.c",
],
)

cc_binary(
name = "decodecorpus",
srcs = [
"tests/decodecorpus.c",
] + glob(
[
"programs/*.c",
"programs/*.h",
],
exclude = [
"programs/zstdcli.c",
],
),
deps = [
":zstd",
":decodecorpus_includes",
],
includes = [
"lib/",
"lib/common/",
"lib/compress/",
"lib/dictBuilder/",
"lib/deprecated/",
"programs/",
],
strip_include_prefix = "lib",
local_defines = [
"ZSTD_LEGACY_SUPPORT=4",
"XXH_NAMESPACE=ZSTD_",
],
visibility = ["//visibility:public"],
Expand Down
23 changes: 14 additions & 9 deletions dependency_support/load_external.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,15 +192,6 @@ def load_external_repositories():
url = "https://github.com/google/riegeli/archive/cb68d579f108c96831b6a7815da43ff24b4e5242.tar.gz",
)

# Needed by fuzztest. Release 2024-03-30, current as of 2024-06-26
http_archive(
name = "net_zstd",
build_file = "@com_google_riegeli//third_party:net_zstd.BUILD",
sha256 = "30f35f71c1203369dc979ecde0400ffea93c27391bfd2ac5a9715d2173d92ff7",
strip_prefix = "zstd-1.5.6/lib",
urls = ["https://github.com/facebook/zstd/archive/v1.5.6.tar.gz"],
)

# Needed by fuzztest. Release 2024-05-21, current as of 2024-06-26
http_archive(
name = "snappy",
Expand Down Expand Up @@ -331,3 +322,17 @@ def load_external_repositories():
urls = ["https://github.com/bazelbuild/rules_pkg/releases/download/1.0.0/rules_pkg-1.0.0.tar.gz"],
sha256 = "cad05f864a32799f6f9022891de91ac78f30e0fa07dc68abac92a628121b5b11",
)

# Used in C++ tests of the ZSTD Module
# Transitive dependency of fuzztest (required by riegeli in fuzztest workspace)
# Version fdfb2aff released on 2024-07-31
# https://github.com/facebook/zstd/commit/fdfb2aff39dc498372d8c9e5f2330b692fea9794
# Updated 2024-08-08
http_archive(
name = "zstd",
sha256 = "9ace5a1b3c477048c6e034fe88d2abb5d1402ced199cae8e9eef32fdc32204df",
strip_prefix = "zstd-fdfb2aff39dc498372d8c9e5f2330b692fea9794",
urls = ["https://github.com/facebook/zstd/archive/fdfb2aff39dc498372d8c9e5f2330b692fea9794.zip"],
build_file = "//dependency_support/com_github_facebook_zstd:bundled.BUILD.bazel",
)

13 changes: 5 additions & 8 deletions xls/examples/ram.x
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ pub fn ReadWordReq<NUM_PARTITIONS:u32, ADDR_WIDTH:u32>(addr:uN[ADDR_WIDTH]) ->
}

// Behavior of reads and writes to the same address in the same "tick".
enum SimultaneousReadWriteBehavior : u2 {
pub enum SimultaneousReadWriteBehavior : u2 {
// The read shows the contents at the address before the write.
READ_BEFORE_WRITE = 0,
// The read shows the contents at the address after the write.
Expand Down Expand Up @@ -160,7 +160,7 @@ fn write_word_test() {

// Function to compute num partitions (e.g. mask width) for a data_width-wide
// word divided into word_partition_size-chunks.
fn num_partitions(word_partition_size: u32, data_width: u32) -> u32 {
pub fn num_partitions(word_partition_size: u32, data_width: u32) -> u32 {
match word_partition_size {
u32:0 => u32:0,
_ => (word_partition_size + data_width - u32:1) / word_partition_size,
Expand Down Expand Up @@ -251,7 +251,7 @@ proc RamModel<DATA_WIDTH:u32, SIZE:u32, WORD_PARTITION_SIZE:u32={u32:0},
if read_req_valid && ASSERT_VALID_READ {
let mem_initialized_as_bits =
std::convert_to_bits_msb0(array_rev(mem_initialized[read_req.addr]));
assert_eq(read_req.mask & !mem_initialized_as_bits, uN[NUM_PARTITIONS]:0)
assert!(read_req.mask & !mem_initialized_as_bits == uN[NUM_PARTITIONS]:0, "memory_not_initialized")
} else { () };

let (value_to_write, written_mem_initialized) = write_word(
Expand All @@ -265,11 +265,8 @@ proc RamModel<DATA_WIDTH:u32, SIZE:u32, WORD_PARTITION_SIZE:u32={u32:0},
match SIMULTANEOUS_READ_WRITE_BEHAVIOR {
SimultaneousReadWriteBehavior::READ_BEFORE_WRITE => mem[read_req.addr],
SimultaneousReadWriteBehavior::WRITE_BEFORE_READ => value_to_write,
SimultaneousReadWriteBehavior::ASSERT_NO_CONFLICT => {
// Assertion failure, we have a conflicting read and write.
assert_eq(true, false);
mem[read_req.addr] // Need to return something.
},
SimultaneousReadWriteBehavior::ASSERT_NO_CONFLICT => fail!("conflicting_read_and_write", mem[read_req.addr]),
_ => fail!("impossible_case", uN[DATA_WIDTH]:0),
}
} else { mem[read_req.addr] };
let read_resp_value = ReadResp<DATA_WIDTH> {
Expand Down
Loading
Loading
0